aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorIgor Makunin <igor.makunin@gmail.com>2022-02-10 16:49:35 +0300
committerDaniil Cherednik <dcherednik@yandex-team.ru>2022-02-10 16:49:35 +0300
commitc617191a3b33c0f5e1be6390361dbe540775d158 (patch)
tree5d5cb817648f650d76cf1076100726fd9b8448e8
parentf7a438035f19b364b5479caf536990e82b174f38 (diff)
downloadydb-c617191a3b33c0f5e1be6390361dbe540775d158.tar.gz
Restoring authorship annotation for Igor Makunin <igor.makunin@gmail.com>. Commit 2 of 2.
-rw-r--r--library/cpp/http/server/response_ut.cpp26
-rw-r--r--library/cpp/monlib/counters/histogram.cpp6
-rw-r--r--library/cpp/monlib/counters/histogram.h2
-rw-r--r--library/cpp/monlib/counters/histogram_ut.cpp4
-rw-r--r--library/cpp/monlib/counters/timer.h2
-rw-r--r--library/cpp/monlib/metrics/metric.h8
-rw-r--r--library/cpp/monlib/metrics/timer.h56
-rw-r--r--library/cpp/monlib/metrics/timer_ut.cpp206
-rw-r--r--library/cpp/monlib/metrics/ut/ya.make2
-rw-r--r--ydb/core/base/appdata.h2
-rw-r--r--ydb/core/base/board_lookup.cpp6
-rw-r--r--ydb/core/base/events.h6
-rw-r--r--ydb/core/base/localdb.cpp4
-rw-r--r--ydb/core/base/localdb.h4
-rw-r--r--ydb/core/base/path.cpp10
-rw-r--r--ydb/core/base/path.h2
-rw-r--r--ydb/core/driver_lib/run/kikimr_services_initializers.cpp32
-rw-r--r--ydb/core/driver_lib/run/run.cpp2
-rw-r--r--ydb/core/engine/minikql/minikql_engine_host.cpp134
-rw-r--r--ydb/core/engine/minikql/minikql_engine_host.h36
-rw-r--r--ydb/core/engine/mkql_proto.cpp2
-rw-r--r--ydb/core/grpc_services/grpc_request_proxy.cpp88
-rw-r--r--ydb/core/grpc_services/grpc_request_proxy.h10
-rw-r--r--ydb/core/grpc_services/rpc_execute_data_query.cpp8
-rw-r--r--ydb/core/grpc_services/rpc_kqp_base.cpp8
-rw-r--r--ydb/core/grpc_services/rpc_kqp_base.h8
-rw-r--r--ydb/core/grpc_services/rpc_read_columns.cpp14
-rw-r--r--ydb/core/grpc_services/rpc_stream_execute_scan_query.cpp484
-rw-r--r--ydb/core/kqp/common/kqp_common.h152
-rw-r--r--ydb/core/kqp/common/kqp_gateway.h34
-rw-r--r--ydb/core/kqp/common/kqp_resolve.cpp290
-rw-r--r--ydb/core/kqp/common/kqp_resolve.h58
-rw-r--r--ydb/core/kqp/common/kqp_ru_calc.cpp22
-rw-r--r--ydb/core/kqp/common/kqp_transform.cpp2
-rw-r--r--ydb/core/kqp/common/kqp_transform.h192
-rw-r--r--ydb/core/kqp/common/kqp_tx_info.h30
-rw-r--r--ydb/core/kqp/common/kqp_yql.cpp206
-rw-r--r--ydb/core/kqp/common/kqp_yql.h48
-rw-r--r--ydb/core/kqp/compile/kqp_compile.cpp314
-rw-r--r--ydb/core/kqp/compile/kqp_compile.h2
-rw-r--r--ydb/core/kqp/compile/kqp_mkql_compiler.cpp70
-rw-r--r--ydb/core/kqp/compile/kqp_mkql_compiler.h2
-rw-r--r--ydb/core/kqp/compute_actor/kqp_compute_actor.cpp26
-rw-r--r--ydb/core/kqp/compute_actor/kqp_compute_actor.h12
-rw-r--r--ydb/core/kqp/compute_actor/kqp_compute_actor_impl.h86
-rw-r--r--ydb/core/kqp/compute_actor/kqp_pure_compute_actor.cpp578
-rw-r--r--ydb/core/kqp/compute_actor/kqp_scan_compute_actor.cpp1662
-rw-r--r--ydb/core/kqp/compute_actor/ya.make6
-rw-r--r--ydb/core/kqp/counters/kqp_counters.cpp50
-rw-r--r--ydb/core/kqp/counters/kqp_counters.h82
-rw-r--r--ydb/core/kqp/executer/kqp_data_executer.cpp3238
-rw-r--r--ydb/core/kqp/executer/kqp_executer.h98
-rw-r--r--ydb/core/kqp/executer/kqp_executer_impl.cpp218
-rw-r--r--ydb/core/kqp/executer/kqp_executer_impl.h976
-rw-r--r--ydb/core/kqp/executer/kqp_executer_stats.cpp510
-rw-r--r--ydb/core/kqp/executer/kqp_executer_stats.h132
-rw-r--r--ydb/core/kqp/executer/kqp_literal_executer.cpp806
-rw-r--r--ydb/core/kqp/executer/kqp_locks_helper.cpp150
-rw-r--r--ydb/core/kqp/executer/kqp_locks_helper.h24
-rw-r--r--ydb/core/kqp/executer/kqp_partition_helper.cpp1172
-rw-r--r--ydb/core/kqp/executer/kqp_partition_helper.h76
-rw-r--r--ydb/core/kqp/executer/kqp_planner.cpp566
-rw-r--r--ydb/core/kqp/executer/kqp_planner.h126
-rw-r--r--ydb/core/kqp/executer/kqp_planner_strategy.cpp260
-rw-r--r--ydb/core/kqp/executer/kqp_planner_strategy.h64
-rw-r--r--ydb/core/kqp/executer/kqp_result_channel.cpp174
-rw-r--r--ydb/core/kqp/executer/kqp_result_channel.h24
-rw-r--r--ydb/core/kqp/executer/kqp_scan_executer.cpp924
-rw-r--r--ydb/core/kqp/executer/kqp_shards_resolver.cpp252
-rw-r--r--ydb/core/kqp/executer/kqp_shards_resolver.h18
-rw-r--r--ydb/core/kqp/executer/kqp_table_resolver.cpp588
-rw-r--r--ydb/core/kqp/executer/kqp_table_resolver.h14
-rw-r--r--ydb/core/kqp/executer/kqp_tasks_graph.cpp958
-rw-r--r--ydb/core/kqp/executer/kqp_tasks_graph.h138
-rw-r--r--ydb/core/kqp/executer/kqp_tasks_validate.cpp24
-rw-r--r--ydb/core/kqp/executer/kqp_tasks_validate.h4
-rw-r--r--ydb/core/kqp/executer/ut/kqp_executer_ut.cpp12
-rw-r--r--ydb/core/kqp/executer/ut/ya.make2
-rw-r--r--ydb/core/kqp/executer/ya.make22
-rw-r--r--ydb/core/kqp/expr_nodes/kqp_expr_nodes.json148
-rw-r--r--ydb/core/kqp/host/kqp_host.cpp580
-rw-r--r--ydb/core/kqp/host/kqp_host.h34
-rw-r--r--ydb/core/kqp/host/kqp_host_impl.h30
-rw-r--r--ydb/core/kqp/host/kqp_ne_helper.cpp120
-rw-r--r--ydb/core/kqp/host/kqp_ne_helper.h20
-rw-r--r--ydb/core/kqp/host/kqp_run_data.cpp144
-rw-r--r--ydb/core/kqp/host/kqp_run_physical.cpp128
-rw-r--r--ydb/core/kqp/host/kqp_run_physical.h20
-rw-r--r--ydb/core/kqp/host/kqp_run_prepared.cpp2
-rw-r--r--ydb/core/kqp/host/kqp_run_scan.cpp76
-rw-r--r--ydb/core/kqp/host/kqp_runner.cpp242
-rw-r--r--ydb/core/kqp/host/ya.make2
-rw-r--r--ydb/core/kqp/kqp.h84
-rw-r--r--ydb/core/kqp/kqp_compile_actor.cpp242
-rw-r--r--ydb/core/kqp/kqp_compile_request.cpp2
-rw-r--r--ydb/core/kqp/kqp_compile_service.cpp184
-rw-r--r--ydb/core/kqp/kqp_compute.h12
-rw-r--r--ydb/core/kqp/kqp_default_settings.txt20
-rw-r--r--ydb/core/kqp/kqp_ic_gateway.cpp192
-rw-r--r--ydb/core/kqp/kqp_impl.h14
-rw-r--r--ydb/core/kqp/kqp_response.cpp10
-rw-r--r--ydb/core/kqp/kqp_worker_actor.cpp766
-rw-r--r--ydb/core/kqp/node/kqp_node.cpp1060
-rw-r--r--ydb/core/kqp/node/kqp_node.h132
-rw-r--r--ydb/core/kqp/node/kqp_node_state.h356
-rw-r--r--ydb/core/kqp/node/kqp_node_ut.cpp1302
-rw-r--r--ydb/core/kqp/node/ut/ya.make38
-rw-r--r--ydb/core/kqp/node/ya.make28
-rw-r--r--ydb/core/kqp/opt/kqp_opt.cpp8
-rw-r--r--ydb/core/kqp/opt/kqp_opt.h12
-rw-r--r--ydb/core/kqp/opt/kqp_opt_build_txs.cpp1034
-rw-r--r--ydb/core/kqp/opt/kqp_opt_effects.cpp176
-rw-r--r--ydb/core/kqp/opt/kqp_opt_impl.h24
-rw-r--r--ydb/core/kqp/opt/kqp_opt_join.cpp194
-rw-r--r--ydb/core/kqp/opt/kqp_opt_kql.cpp84
-rw-r--r--ydb/core/kqp/opt/kqp_opt_phy_check.cpp244
-rw-r--r--ydb/core/kqp/opt/kqp_opt_phy_finalize.cpp376
-rw-r--r--ydb/core/kqp/opt/logical/kqp_opt_log.cpp18
-rw-r--r--ydb/core/kqp/opt/logical/kqp_opt_log_join.cpp688
-rw-r--r--ydb/core/kqp/opt/logical/kqp_opt_log_rules.h18
-rw-r--r--ydb/core/kqp/opt/logical/kqp_opt_log_sqlin.cpp260
-rw-r--r--ydb/core/kqp/opt/logical/ya.make4
-rw-r--r--ydb/core/kqp/opt/physical/kqp_opt_phy.cpp30
-rw-r--r--ydb/core/kqp/opt/physical/kqp_opt_phy_build_stage.cpp158
-rw-r--r--ydb/core/kqp/opt/ya.make8
-rw-r--r--ydb/core/kqp/prepare/kqp_prepare.h20
-rw-r--r--ydb/core/kqp/prepare/kqp_prepare_impl.h2
-rw-r--r--ydb/core/kqp/prepare/kqp_query_analyze.cpp2
-rw-r--r--ydb/core/kqp/prepare/kqp_query_exec.cpp42
-rw-r--r--ydb/core/kqp/prepare/kqp_query_finalize.cpp24
-rw-r--r--ydb/core/kqp/prepare/kqp_query_plan.cpp46
-rw-r--r--ydb/core/kqp/prepare/kqp_query_plan.h2
-rw-r--r--ydb/core/kqp/prepare/kqp_query_rewrite.cpp2
-rw-r--r--ydb/core/kqp/prepare/kqp_query_simplify.cpp66
-rw-r--r--ydb/core/kqp/prepare/kqp_query_substitute.cpp2
-rw-r--r--ydb/core/kqp/prepare/kqp_type_ann.cpp886
-rw-r--r--ydb/core/kqp/provider/ya.make6
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_datasink.cpp14
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_datasource.cpp20
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_exec.cpp22
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_gateway.h34
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp8
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_opt.cpp22
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_opt_build.cpp50
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_opt_join.cpp504
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_opt_range.cpp234
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_opt_utils.cpp140
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_opt_utils.h26
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_provider.cpp2
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_provider.h32
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_provider_impl.h28
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_query_traits.cpp82
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_query_traits.h62
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_settings.cpp60
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_settings.h26
-rw-r--r--ydb/core/kqp/proxy/kqp_proxy_service.cpp10
-rw-r--r--ydb/core/kqp/rm/kqp_resource_estimation.cpp130
-rw-r--r--ydb/core/kqp/rm/kqp_resource_estimation.h76
-rw-r--r--ydb/core/kqp/rm/kqp_resource_estimation_ut.cpp164
-rw-r--r--ydb/core/kqp/rm/kqp_rm.cpp1840
-rw-r--r--ydb/core/kqp/rm/kqp_rm.h196
-rw-r--r--ydb/core/kqp/rm/kqp_rm_ut.cpp742
-rw-r--r--ydb/core/kqp/rm/kqp_snapshot_manager.cpp388
-rw-r--r--ydb/core/kqp/rm/kqp_snapshot_manager.h86
-rw-r--r--ydb/core/kqp/rm/ut/ya.make38
-rw-r--r--ydb/core/kqp/rm/ya.make32
-rw-r--r--ydb/core/kqp/runtime/kqp_channel_storage.cpp478
-rw-r--r--ydb/core/kqp/runtime/kqp_channel_storage.h20
-rw-r--r--ydb/core/kqp/runtime/kqp_compute.cpp172
-rw-r--r--ydb/core/kqp/runtime/kqp_compute.h58
-rw-r--r--ydb/core/kqp/runtime/kqp_effects.cpp2
-rw-r--r--ydb/core/kqp/runtime/kqp_output_stream.cpp52
-rw-r--r--ydb/core/kqp/runtime/kqp_program_builder.cpp74
-rw-r--r--ydb/core/kqp/runtime/kqp_program_builder.h14
-rw-r--r--ydb/core/kqp/runtime/kqp_read_table.cpp62
-rw-r--r--ydb/core/kqp/runtime/kqp_read_table.h4
-rw-r--r--ydb/core/kqp/runtime/kqp_runtime_impl.h12
-rw-r--r--ydb/core/kqp/runtime/kqp_scan_data.cpp128
-rw-r--r--ydb/core/kqp/runtime/kqp_scan_data.h98
-rw-r--r--ydb/core/kqp/runtime/kqp_spilling.h100
-rw-r--r--ydb/core/kqp/runtime/kqp_spilling_file.cpp1878
-rw-r--r--ydb/core/kqp/runtime/kqp_spilling_file.h30
-rw-r--r--ydb/core/kqp/runtime/kqp_spilling_file_ut.cpp824
-rw-r--r--ydb/core/kqp/runtime/kqp_tasks_runner.cpp396
-rw-r--r--ydb/core/kqp/runtime/kqp_tasks_runner.h132
-rw-r--r--ydb/core/kqp/runtime/kqp_transport.cpp430
-rw-r--r--ydb/core/kqp/runtime/kqp_transport.h68
-rw-r--r--ydb/core/kqp/runtime/ut/ya.make32
-rw-r--r--ydb/core/kqp/runtime/ya.make14
-rw-r--r--ydb/core/kqp/ut/common/kqp_ut_common.cpp330
-rw-r--r--ydb/core/kqp/ut/common/kqp_ut_common.h114
-rw-r--r--ydb/core/kqp/ut/fat/kqp_force_newengine_ut.cpp1724
-rw-r--r--ydb/core/kqp/ut/fat/ya.make58
-rw-r--r--ydb/core/kqp/ut/kqp_acl_ut.cpp20
-rw-r--r--ydb/core/kqp/ut/kqp_explain_ut.cpp14
-rw-r--r--ydb/core/kqp/ut/kqp_flip_join_ut.cpp1124
-rw-r--r--ydb/core/kqp/ut/kqp_index_lookup_join_ut.cpp328
-rw-r--r--ydb/core/kqp/ut/kqp_indexes_ut.cpp730
-rw-r--r--ydb/core/kqp/ut/kqp_join_ut.cpp1164
-rw-r--r--ydb/core/kqp/ut/kqp_limits_ut.cpp226
-rw-r--r--ydb/core/kqp/ut/kqp_locks_ut.cpp66
-rw-r--r--ydb/core/kqp/ut/kqp_merge_connection_ut.cpp2
-rw-r--r--ydb/core/kqp/ut/kqp_mvcc_ut.cpp118
-rw-r--r--ydb/core/kqp/ut/kqp_newengine_effects_ut.cpp1018
-rw-r--r--ydb/core/kqp/ut/kqp_newengine_flowcontrol_ut.cpp356
-rw-r--r--ydb/core/kqp/ut/kqp_newengine_ut.cpp1712
-rw-r--r--ydb/core/kqp/ut/kqp_olap_ut.cpp4
-rw-r--r--ydb/core/kqp/ut/kqp_params_ut.cpp82
-rw-r--r--ydb/core/kqp/ut/kqp_pragma_ut.cpp16
-rw-r--r--ydb/core/kqp/ut/kqp_query_ut.cpp388
-rw-r--r--ydb/core/kqp/ut/kqp_scan_spilling_ut.cpp180
-rw-r--r--ydb/core/kqp/ut/kqp_scan_ut.cpp2110
-rw-r--r--ydb/core/kqp/ut/kqp_scripting_ut.cpp14
-rw-r--r--ydb/core/kqp/ut/kqp_sort_ut.cpp490
-rw-r--r--ydb/core/kqp/ut/kqp_sqlin_ut.cpp1530
-rw-r--r--ydb/core/kqp/ut/kqp_stats_ut.cpp40
-rw-r--r--ydb/core/kqp/ut/kqp_sys_col_ut.cpp94
-rw-r--r--ydb/core/kqp/ut/kqp_sys_view_ut.cpp94
-rw-r--r--ydb/core/kqp/ut/kqp_table_predicate_ut.cpp514
-rw-r--r--ydb/core/kqp/ut/kqp_tx_ut.cpp330
-rw-r--r--ydb/core/kqp/ut/kqp_write_ut.cpp120
-rw-r--r--ydb/core/kqp/ut/ya.make18
-rw-r--r--ydb/core/persqueue/cluster_tracker.cpp28
-rw-r--r--ydb/core/protos/config.proto90
-rw-r--r--ydb/core/protos/counters_datashard.proto8
-rw-r--r--ydb/core/protos/kqp.proto284
-rw-r--r--ydb/core/protos/kqp_physical.proto62
-rw-r--r--ydb/core/protos/kqp_query_settings.proto2
-rw-r--r--ydb/core/protos/kqp_stats.proto102
-rw-r--r--ydb/core/protos/services.proto20
-rw-r--r--ydb/core/protos/sys_view.proto18
-rw-r--r--ydb/core/protos/tx_datashard.proto174
-rw-r--r--ydb/core/scheme/scheme_tablecell.h4
-rw-r--r--ydb/core/scheme/scheme_tabledefs.h44
-rw-r--r--ydb/core/sys_view/common/scan_actor_base_impl.h28
-rw-r--r--ydb/core/sys_view/nodes/nodes.cpp2
-rw-r--r--ydb/core/sys_view/partition_stats/partition_stats.cpp4
-rw-r--r--ydb/core/sys_view/query_stats/query_stats.cpp4
-rw-r--r--ydb/core/sys_view/service/sysview_service.cpp22
-rw-r--r--ydb/core/sys_view/tablets/tablets.cpp2
-rw-r--r--ydb/core/sys_view/ut_kqp.cpp10
-rw-r--r--ydb/core/tablet/resource_broker.cpp934
-rw-r--r--ydb/core/tablet/resource_broker.h102
-rw-r--r--ydb/core/tablet/resource_broker_impl.h156
-rw-r--r--ydb/core/tablet/resource_broker_ut.cpp270
-rw-r--r--ydb/core/tablet_flat/flat_exec_seat.h2
-rw-r--r--ydb/core/tablet_flat/flat_executor.cpp4
-rw-r--r--ydb/core/tablet_flat/tablet_flat_executor.h18
-rw-r--r--ydb/core/tablet_flat/test/libs/exec/runner.h2
-rw-r--r--ydb/core/testlib/actors/test_runtime.cpp2
-rw-r--r--ydb/core/testlib/basics/appdata.h2
-rw-r--r--ydb/core/testlib/basics/services.cpp2
-rw-r--r--ydb/core/testlib/minikql_compile.h28
-rw-r--r--ydb/core/testlib/test_client.cpp20
-rw-r--r--ydb/core/testlib/test_client.h4
-rw-r--r--ydb/core/tx/columnshard/columnshard__scan.cpp16
-rw-r--r--ydb/core/tx/datashard/build_data_tx_out_rs_unit.cpp10
-rw-r--r--ydb/core/tx/datashard/build_kqp_data_tx_out_rs_unit.cpp184
-rw-r--r--ydb/core/tx/datashard/datashard.cpp2
-rw-r--r--ydb/core/tx/datashard/datashard__engine_host.cpp126
-rw-r--r--ydb/core/tx/datashard/datashard__engine_host.h52
-rw-r--r--ydb/core/tx/datashard/datashard__kqp_scan.cpp226
-rw-r--r--ydb/core/tx/datashard/datashard_active_transaction.cpp198
-rw-r--r--ydb/core/tx/datashard/datashard_active_transaction.h10
-rw-r--r--ydb/core/tx/datashard/datashard_impl.h20
-rw-r--r--ydb/core/tx/datashard/datashard_kqp.cpp986
-rw-r--r--ydb/core/tx/datashard/datashard_kqp.h54
-rw-r--r--ydb/core/tx/datashard/datashard_kqp_compute.cpp138
-rw-r--r--ydb/core/tx/datashard/datashard_kqp_compute.h46
-rw-r--r--ydb/core/tx/datashard/datashard_kqp_delete_rows.cpp22
-rw-r--r--ydb/core/tx/datashard/datashard_kqp_lookup_table.cpp110
-rw-r--r--ydb/core/tx/datashard/datashard_kqp_read_table.cpp148
-rw-r--r--ydb/core/tx/datashard/datashard_kqp_upsert_rows.cpp30
-rw-r--r--ydb/core/tx/datashard/datashard_pipeline.cpp6
-rw-r--r--ydb/core/tx/datashard/datashard_pipeline.h28
-rw-r--r--ydb/core/tx/datashard/datashard_ut_common.h30
-rw-r--r--ydb/core/tx/datashard/datashard_ut_common_kqp.h24
-rw-r--r--ydb/core/tx/datashard/datashard_ut_kqp.cpp942
-rw-r--r--ydb/core/tx/datashard/datashard_ut_kqp_errors.cpp696
-rw-r--r--ydb/core/tx/datashard/datashard_ut_kqp_scan.cpp90
-rw-r--r--ydb/core/tx/datashard/datashard_ut_order.cpp828
-rw-r--r--ydb/core/tx/datashard/execute_data_tx_unit.cpp26
-rw-r--r--ydb/core/tx/datashard/execute_kqp_data_tx_unit.cpp234
-rw-r--r--ydb/core/tx/datashard/execute_kqp_scan_tx_unit.cpp14
-rw-r--r--ydb/core/tx/datashard/prepare_kqp_data_tx_in_rs_unit.cpp20
-rw-r--r--ydb/core/tx/datashard/range_ops.cpp54
-rw-r--r--ydb/core/tx/datashard/range_ops.h6
-rw-r--r--ydb/core/viewer/json_query.h2
-rw-r--r--ydb/core/ydb_convert/ydb_convert.cpp22
-rw-r--r--ydb/core/yq/libs/result_formatter/result_formatter_ut.cpp12
-rw-r--r--ydb/library/mkql_proto/mkql_proto.cpp76
-rw-r--r--ydb/library/mkql_proto/protos/minikql.proto12
-rw-r--r--ydb/library/yql/ast/yql_expr.cpp48
-rw-r--r--ydb/library/yql/ast/yql_expr.h6
-rw-r--r--ydb/library/yql/core/common_opt/yql_co_extr_members.cpp14
-rw-r--r--ydb/library/yql/core/common_opt/yql_co_simple1.cpp628
-rw-r--r--ydb/library/yql/core/common_opt/yql_co_sqlin.h22
-rw-r--r--ydb/library/yql/core/expr_nodes/yql_expr_nodes.json84
-rw-r--r--ydb/library/yql/core/issue/protos/issue_id.proto2
-rw-r--r--ydb/library/yql/core/issue/yql_issue.txt8
-rw-r--r--ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.cpp70
-rw-r--r--ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.h2
-rw-r--r--ydb/library/yql/core/type_ann/type_ann_core.cpp108
-rw-r--r--ydb/library/yql/core/type_ann/type_ann_join.cpp2
-rw-r--r--ydb/library/yql/core/type_ann/type_ann_list.cpp16
-rw-r--r--ydb/library/yql/core/yql_expr_type_annotation.cpp12
-rw-r--r--ydb/library/yql/core/yql_expr_type_annotation.h2
-rw-r--r--ydb/library/yql/core/yql_opt_utils.cpp8
-rw-r--r--ydb/library/yql/core/yql_opt_utils.h2
-rw-r--r--ydb/library/yql/dq/actors/compute/dq_checkpoints.h16
-rw-r--r--ydb/library/yql/dq/actors/compute/dq_compute_actor.cpp100
-rw-r--r--ydb/library/yql/dq/actors/compute/dq_compute_actor.h220
-rw-r--r--ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.cpp1412
-rw-r--r--ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.h344
-rw-r--r--ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.cpp22
-rw-r--r--ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.h20
-rw-r--r--ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h1164
-rw-r--r--ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h2
-rw-r--r--ydb/library/yql/dq/actors/compute/dq_compute_actor_stats.cpp234
-rw-r--r--ydb/library/yql/dq/actors/compute/ya.make2
-rw-r--r--ydb/library/yql/dq/actors/dq.cpp2
-rw-r--r--ydb/library/yql/dq/actors/dq.h74
-rw-r--r--ydb/library/yql/dq/actors/dq_events_ids.h68
-rw-r--r--ydb/library/yql/dq/actors/protos/dq_events.proto78
-rw-r--r--ydb/library/yql/dq/actors/protos/dq_legacy_stats.proto582
-rw-r--r--ydb/library/yql/dq/actors/protos/dq_stats.proto446
-rw-r--r--ydb/library/yql/dq/actors/protos/ya.make2
-rw-r--r--ydb/library/yql/dq/actors/ya.make8
-rw-r--r--ydb/library/yql/dq/common/dq_common.cpp2
-rw-r--r--ydb/library/yql/dq/common/dq_value.cpp2
-rw-r--r--ydb/library/yql/dq/common/dq_value.h18
-rw-r--r--ydb/library/yql/dq/common/ya.make4
-rw-r--r--ydb/library/yql/dq/comp_nodes/yql_common_dq_factory.cpp2
-rw-r--r--ydb/library/yql/dq/connections.md228
-rw-r--r--ydb/library/yql/dq/expr_nodes/dq_expr_nodes.json62
-rw-r--r--ydb/library/yql/dq/opt/dq_opt.cpp78
-rw-r--r--ydb/library/yql/dq/opt/dq_opt.h28
-rw-r--r--ydb/library/yql/dq/opt/dq_opt_build.cpp176
-rw-r--r--ydb/library/yql/dq/opt/dq_opt_build.h14
-rw-r--r--ydb/library/yql/dq/opt/dq_opt_join.cpp688
-rw-r--r--ydb/library/yql/dq/opt/dq_opt_log.cpp80
-rw-r--r--ydb/library/yql/dq/opt/dq_opt_peephole.cpp286
-rw-r--r--ydb/library/yql/dq/opt/dq_opt_phy.cpp524
-rw-r--r--ydb/library/yql/dq/opt/dq_opt_phy.h44
-rw-r--r--ydb/library/yql/dq/opt/dq_opt_phy_finalizing.cpp572
-rw-r--r--ydb/library/yql/dq/opt/dq_opt_phy_finalizing.h4
-rw-r--r--ydb/library/yql/dq/opt/ya.make2
-rw-r--r--ydb/library/yql/dq/proto/dq_tasks.proto20
-rw-r--r--ydb/library/yql/dq/proto/dq_transport.proto2
-rw-r--r--ydb/library/yql/dq/runtime/dq_channel_storage.h58
-rw-r--r--ydb/library/yql/dq/runtime/dq_columns_resolve.cpp2
-rw-r--r--ydb/library/yql/dq/runtime/dq_columns_resolve.h10
-rw-r--r--ydb/library/yql/dq/runtime/dq_compute.h8
-rw-r--r--ydb/library/yql/dq/runtime/dq_input.h8
-rw-r--r--ydb/library/yql/dq/runtime/dq_input_channel.cpp52
-rw-r--r--ydb/library/yql/dq/runtime/dq_input_channel.h52
-rw-r--r--ydb/library/yql/dq/runtime/dq_input_impl.h36
-rw-r--r--ydb/library/yql/dq/runtime/dq_input_producer.cpp112
-rw-r--r--ydb/library/yql/dq/runtime/dq_input_producer.h18
-rw-r--r--ydb/library/yql/dq/runtime/dq_output.h10
-rw-r--r--ydb/library/yql/dq/runtime/dq_output_channel.cpp1630
-rw-r--r--ydb/library/yql/dq/runtime/dq_output_channel.h122
-rw-r--r--ydb/library/yql/dq/runtime/dq_output_channel_ut.cpp1000
-rw-r--r--ydb/library/yql/dq/runtime/dq_output_consumer.cpp342
-rw-r--r--ydb/library/yql/dq/runtime/dq_output_consumer.h58
-rw-r--r--ydb/library/yql/dq/runtime/dq_sink.cpp46
-rw-r--r--ydb/library/yql/dq/runtime/dq_sink.h6
-rw-r--r--ydb/library/yql/dq/runtime/dq_source.cpp12
-rw-r--r--ydb/library/yql/dq/runtime/dq_source.h4
-rw-r--r--ydb/library/yql/dq/runtime/dq_tasks_runner.cpp922
-rw-r--r--ydb/library/yql/dq/runtime/dq_tasks_runner.h166
-rw-r--r--ydb/library/yql/dq/runtime/dq_transport.cpp498
-rw-r--r--ydb/library/yql/dq/runtime/dq_transport.h58
-rw-r--r--ydb/library/yql/dq/runtime/ut/ut_helper.cpp2
-rw-r--r--ydb/library/yql/dq/runtime/ut/ut_helper.h122
-rw-r--r--ydb/library/yql/dq/runtime/ut/ya.make36
-rw-r--r--ydb/library/yql/dq/runtime/ya.make16
-rw-r--r--ydb/library/yql/dq/tasks/dq_connection_builder.h52
-rw-r--r--ydb/library/yql/dq/tasks/dq_task_program.cpp22
-rw-r--r--ydb/library/yql/dq/tasks/dq_task_program.h4
-rw-r--r--ydb/library/yql/dq/tasks/dq_tasks_graph.h8
-rw-r--r--ydb/library/yql/dq/type_ann/dq_type_ann.cpp232
-rw-r--r--ydb/library/yql/dq/type_ann/dq_type_ann.h8
-rw-r--r--ydb/library/yql/dq/ya.make2
-rw-r--r--ydb/library/yql/minikql/aligned_page_pool.cpp22
-rw-r--r--ydb/library/yql/minikql/aligned_page_pool.h16
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp4
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_skip.cpp4
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_todict.cpp566
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_todict.h6
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_todict_ut.cpp114
-rw-r--r--ydb/library/yql/minikql/computation/mkql_computation_node_holders.h32
-rw-r--r--ydb/library/yql/minikql/mkql_alloc.h4
-rw-r--r--ydb/library/yql/minikql/mkql_program_builder.cpp96
-rw-r--r--ydb/library/yql/minikql/mkql_program_builder.h10
-rw-r--r--ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp82
-rw-r--r--ydb/library/yql/providers/common/mkql/yql_provider_mkql.h2
-rw-r--r--ydb/library/yql/providers/dq/actors/compute_actor.cpp10
-rw-r--r--ydb/library/yql/providers/dq/actors/result_receiver.cpp6
-rw-r--r--ydb/library/yql/providers/dq/actors/task_controller.cpp100
-rw-r--r--ydb/library/yql/providers/dq/api/protos/task_command_executor.proto16
-rw-r--r--ydb/library/yql/providers/dq/counters/counters.h4
-rw-r--r--ydb/library/yql/providers/dq/opt/dqs_opt.cpp22
-rw-r--r--ydb/library/yql/providers/dq/opt/logical_optimize.cpp2
-rw-r--r--ydb/library/yql/providers/dq/opt/physical_optimize.cpp34
-rw-r--r--ydb/library/yql/providers/dq/planner/dqs_task_graph.h2
-rw-r--r--ydb/library/yql/providers/dq/planner/execution_planner.cpp20
-rw-r--r--ydb/library/yql/providers/dq/planner/execution_planner.h2
-rw-r--r--ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp22
-rw-r--r--ydb/library/yql/providers/dq/provider/yql_dq_datasink_type_ann.cpp2
-rw-r--r--ydb/library/yql/providers/dq/runtime/task_command_executor.cpp84
-rw-r--r--ydb/library/yql/providers/dq/task_runner/tasks_runner_local.cpp62
-rw-r--r--ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.cpp138
-rw-r--r--ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.h20
-rw-r--r--ydb/library/yql/utils/log/log_component.h10
-rw-r--r--ydb/public/api/protos/ydb_experimental.proto30
-rw-r--r--ydb/public/api/protos/ydb_query_stats.proto6
-rw-r--r--ydb/public/lib/experimental/ydb_experimental.cpp54
-rw-r--r--ydb/public/lib/experimental/ydb_experimental.h54
-rw-r--r--ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp4
-rw-r--r--ydb/public/lib/yson_value/ydb_yson_value.cpp16
-rw-r--r--ydb/public/sdk/cpp/client/ydb_table/table.cpp16
-rw-r--r--ydb/public/sdk/cpp/client/ydb_table/table.h16
-rw-r--r--ydb/services/ydb/ydb_table_ut.cpp108
-rw-r--r--ydb/services/ydb/ydb_ut.cpp8
-rw-r--r--ydb/tests/functional/api/test_isolation.py272
-rw-r--r--ydb/tests/functional/canonical/sql/topsort/topsort.sql2
-rw-r--r--ydb/tests/functional/canonical/sql/topsort/topsort_multi_range.sql2
-rw-r--r--ydb/tests/functional/canonical/test_sql.py8
-rw-r--r--ydb/tests/functional/suite_tests/canondata/result.json2
-rw-r--r--ydb/tests/tools/ydb_serializable/__main__.py4
-rw-r--r--ydb/tests/tools/ydb_serializable/lib/__init__.py80
432 files changed, 38457 insertions, 38457 deletions
diff --git a/library/cpp/http/server/response_ut.cpp b/library/cpp/http/server/response_ut.cpp
index 8569839db9..73e2112ad3 100644
--- a/library/cpp/http/server/response_ut.cpp
+++ b/library/cpp/http/server/response_ut.cpp
@@ -126,17 +126,17 @@ Y_UNIT_TEST_SUITE(TestHttpResponse) {
UNIT_ASSERT_EQUAL(&resp, &resp.SetContent("some stuff"));
UNIT_ASSERT_EQUAL(&resp, &resp.SetContent("some other stuff").SetContentType("text/plain"));
}
-
- Y_UNIT_TEST(TestSetContentType) {
- const char* EXPECTED = "HTTP/1.1 200 Ok\r\n"
- "Content-Type: text/xml\r\n"
- "Content-Length: 28\r\n"
- "\r\n"
- "<xml><tag value=\"1\" /></xml>";
- THttpResponse resp;
- resp.SetContent("<xml><tag value=\"1\" /></xml>")
- .SetContentType("application/json")
- .SetContentType("text/xml");
- UNIT_ASSERT_STRINGS_EQUAL(ToString(resp), EXPECTED);
- }
+
+ Y_UNIT_TEST(TestSetContentType) {
+ const char* EXPECTED = "HTTP/1.1 200 Ok\r\n"
+ "Content-Type: text/xml\r\n"
+ "Content-Length: 28\r\n"
+ "\r\n"
+ "<xml><tag value=\"1\" /></xml>";
+ THttpResponse resp;
+ resp.SetContent("<xml><tag value=\"1\" /></xml>")
+ .SetContentType("application/json")
+ .SetContentType("text/xml");
+ UNIT_ASSERT_STRINGS_EQUAL(ToString(resp), EXPECTED);
+ }
}
diff --git a/library/cpp/monlib/counters/histogram.cpp b/library/cpp/monlib/counters/histogram.cpp
index 41d8a19372..46cf4e6ec8 100644
--- a/library/cpp/monlib/counters/histogram.cpp
+++ b/library/cpp/monlib/counters/histogram.cpp
@@ -12,8 +12,8 @@ namespace NMonitoring {
<< ", 95%: " << Percentile95
<< ", 98%: " << Percentile98
<< ", 99%: " << Percentile99
- << ", 99.9%: " << Percentile999
- << ", count: " << TotalCount;
+ << ", 99.9%: " << Percentile999
+ << ", count: " << TotalCount;
}
void THdrHistogram::TakeSnaphot(THistogramSnapshot* snapshot) {
@@ -30,7 +30,7 @@ namespace NMonitoring {
snapshot->Percentile98 = Data_.GetValueAtPercentile(98.0);
snapshot->Percentile99 = Data_.GetValueAtPercentile(99.0);
snapshot->Percentile999 = Data_.GetValueAtPercentile(99.9);
- snapshot->TotalCount = Data_.GetTotalCount();
+ snapshot->TotalCount = Data_.GetTotalCount();
// cleanup histogram data
Data_.Reset();
diff --git a/library/cpp/monlib/counters/histogram.h b/library/cpp/monlib/counters/histogram.h
index ef4d50d369..96361b0023 100644
--- a/library/cpp/monlib/counters/histogram.h
+++ b/library/cpp/monlib/counters/histogram.h
@@ -21,7 +21,7 @@ namespace NMonitoring {
i64 Percentile98;
i64 Percentile99;
i64 Percentile999;
- i64 TotalCount;
+ i64 TotalCount;
void Print(IOutputStream* out) const;
};
diff --git a/library/cpp/monlib/counters/histogram_ut.cpp b/library/cpp/monlib/counters/histogram_ut.cpp
index c45a6237ac..5a0800505a 100644
--- a/library/cpp/monlib/counters/histogram_ut.cpp
+++ b/library/cpp/monlib/counters/histogram_ut.cpp
@@ -41,7 +41,7 @@ Y_UNIT_TEST_SUITE(THistorgamTest) {
UNIT_ASSERT_EQUAL(snapshot.Percentile98, 5);
UNIT_ASSERT_EQUAL(snapshot.Percentile99, 5);
UNIT_ASSERT_EQUAL(snapshot.Percentile999, 5);
-
- UNIT_ASSERT_EQUAL(snapshot.TotalCount, 14);
+
+ UNIT_ASSERT_EQUAL(snapshot.TotalCount, 14);
}
}
diff --git a/library/cpp/monlib/counters/timer.h b/library/cpp/monlib/counters/timer.h
index 87bf3c720b..03dfb35337 100644
--- a/library/cpp/monlib/counters/timer.h
+++ b/library/cpp/monlib/counters/timer.h
@@ -111,7 +111,7 @@ namespace NMonitoring {
template <typename TFunc>
void Measure(TFunc&& fn) {
using TClock = std::chrono::high_resolution_clock;
-
+
auto start = TClock::now();
Y_SCOPE_EXIT(this, start) {
diff --git a/library/cpp/monlib/metrics/metric.h b/library/cpp/monlib/metrics/metric.h
index fe97d2b475..b8ce12d753 100644
--- a/library/cpp/monlib/metrics/metric.h
+++ b/library/cpp/monlib/metrics/metric.h
@@ -371,13 +371,13 @@ namespace NMonitoring {
}
void Accept(TInstant time, IMetricConsumer* consumer) const override {
- consumer->OnHistogram(time, TakeSnapshot());
+ consumer->OnHistogram(time, TakeSnapshot());
}
IHistogramSnapshotPtr TakeSnapshot() const override {
- return Collector_->Snapshot();
- }
-
+ return Collector_->Snapshot();
+ }
+
void Reset() override {
Collector_->Reset();
}
diff --git a/library/cpp/monlib/metrics/timer.h b/library/cpp/monlib/metrics/timer.h
index b45cc0bc9b..5c4e26e37b 100644
--- a/library/cpp/monlib/metrics/timer.h
+++ b/library/cpp/monlib/metrics/timer.h
@@ -1,29 +1,29 @@
-#pragma once
-
+#pragma once
+
#include "metric.h"
-
+
#include <util/generic/typetraits.h>
-#include <chrono>
-
-
-namespace NMonitoring {
-
- /**
- * A timing scope to record elapsed time since creation.
- */
+#include <chrono>
+
+
+namespace NMonitoring {
+
+ /**
+ * A timing scope to record elapsed time since creation.
+ */
template <typename TMetric,
- typename Resolution = std::chrono::milliseconds,
- typename Clock = std::chrono::high_resolution_clock>
+ typename Resolution = std::chrono::milliseconds,
+ typename Clock = std::chrono::high_resolution_clock>
class TMetricTimerScope {
- public:
+ public:
explicit TMetricTimerScope(TMetric* metric)
: Metric_(metric)
- , StartTime_(Clock::now())
- {
+ , StartTime_(Clock::now())
+ {
Y_ENSURE(Metric_);
- }
-
+ }
+
TMetricTimerScope(TMetricTimerScope&) = delete;
TMetricTimerScope& operator=(const TMetricTimerScope&) = delete;
@@ -45,7 +45,7 @@ namespace NMonitoring {
return;
}
- auto duration = std::chrono::duration_cast<Resolution>(Clock::now() - StartTime_).count();
+ auto duration = std::chrono::duration_cast<Resolution>(Clock::now() - StartTime_).count();
if constexpr (std::is_same<TMetric, TGauge>::value) {
Metric_->Set(duration);
} else if constexpr (std::is_same<TMetric, TIntGauge>::value) {
@@ -56,13 +56,13 @@ namespace NMonitoring {
Metric_->Add(duration);
} else if constexpr (std::is_same<TMetric, THistogram>::value) {
Metric_->Record(duration);
- } else {
+ } else {
static_assert(TDependentFalse<TMetric>, "Not supported metric type");
- }
+ }
Metric_ = nullptr;
- }
-
+ }
+
~TMetricTimerScope() {
if (Metric_ == nullptr) {
return;
@@ -71,11 +71,11 @@ namespace NMonitoring {
Record();
}
- private:
+ private:
TMetric* Metric_{nullptr};
- typename Clock::time_point StartTime_;
- };
-
+ typename Clock::time_point StartTime_;
+ };
+
/**
* @brief A class that is supposed to use to measure execution time of an asynchronuous operation.
*
@@ -124,4 +124,4 @@ namespace NMonitoring {
TFutureFriendlyTimer<TMetric> FutureTimer(TMetric* metric) {
return TFutureFriendlyTimer<TMetric>{metric};
}
-}
+}
diff --git a/library/cpp/monlib/metrics/timer_ut.cpp b/library/cpp/monlib/metrics/timer_ut.cpp
index 9827edcda0..c244a8c9e1 100644
--- a/library/cpp/monlib/metrics/timer_ut.cpp
+++ b/library/cpp/monlib/metrics/timer_ut.cpp
@@ -1,120 +1,120 @@
-#include "timer.h"
-
+#include "timer.h"
+
#include <library/cpp/testing/unittest/registar.h>
#include <library/cpp/threading/future/async.h>
#include <library/cpp/threading/future/future.h>
-
-using namespace NMonitoring;
+
+using namespace NMonitoring;
using namespace NThreading;
-
-Y_UNIT_TEST_SUITE(TTimerTest) {
-
- using namespace std::chrono;
-
- struct TTestClock {
- using time_point = time_point<high_resolution_clock>;
-
- static time_point TimePoint;
-
- static time_point now() {
- return TimePoint;
- }
- };
-
- TTestClock::time_point TTestClock::TimePoint;
-
-
- Y_UNIT_TEST(Gauge) {
- TTestClock::TimePoint = TTestClock::time_point::min();
-
- TGauge gauge(0);
- {
+
+Y_UNIT_TEST_SUITE(TTimerTest) {
+
+ using namespace std::chrono;
+
+ struct TTestClock {
+ using time_point = time_point<high_resolution_clock>;
+
+ static time_point TimePoint;
+
+ static time_point now() {
+ return TimePoint;
+ }
+ };
+
+ TTestClock::time_point TTestClock::TimePoint;
+
+
+ Y_UNIT_TEST(Gauge) {
+ TTestClock::TimePoint = TTestClock::time_point::min();
+
+ TGauge gauge(0);
+ {
TMetricTimerScope<TGauge, milliseconds, TTestClock> t{&gauge};
- TTestClock::TimePoint += milliseconds(10);
- }
- UNIT_ASSERT_EQUAL(10, gauge.Get());
-
- {
+ TTestClock::TimePoint += milliseconds(10);
+ }
+ UNIT_ASSERT_EQUAL(10, gauge.Get());
+
+ {
TMetricTimerScope<TGauge, milliseconds, TTestClock> t{&gauge};
- TTestClock::TimePoint += milliseconds(20);
- }
- UNIT_ASSERT_EQUAL(20, gauge.Get());
- }
-
- Y_UNIT_TEST(IntGauge) {
- TTestClock::TimePoint = TTestClock::time_point::min();
-
- TIntGauge gauge(0);
- {
+ TTestClock::TimePoint += milliseconds(20);
+ }
+ UNIT_ASSERT_EQUAL(20, gauge.Get());
+ }
+
+ Y_UNIT_TEST(IntGauge) {
+ TTestClock::TimePoint = TTestClock::time_point::min();
+
+ TIntGauge gauge(0);
+ {
TMetricTimerScope<TIntGauge, milliseconds, TTestClock> t{&gauge};
- TTestClock::TimePoint += milliseconds(10);
- }
- UNIT_ASSERT_EQUAL(10, gauge.Get());
-
- {
+ TTestClock::TimePoint += milliseconds(10);
+ }
+ UNIT_ASSERT_EQUAL(10, gauge.Get());
+
+ {
TMetricTimerScope<TIntGauge, milliseconds, TTestClock> t{&gauge};
- TTestClock::TimePoint += milliseconds(20);
- }
- UNIT_ASSERT_EQUAL(20, gauge.Get());
- }
-
- Y_UNIT_TEST(CounterNew) {
- TTestClock::TimePoint = TTestClock::time_point::min();
-
+ TTestClock::TimePoint += milliseconds(20);
+ }
+ UNIT_ASSERT_EQUAL(20, gauge.Get());
+ }
+
+ Y_UNIT_TEST(CounterNew) {
+ TTestClock::TimePoint = TTestClock::time_point::min();
+
TCounter counter(0);
- {
+ {
TMetricTimerScope<TCounter, milliseconds, TTestClock> t{&counter};
- TTestClock::TimePoint += milliseconds(10);
- }
- UNIT_ASSERT_EQUAL(10, counter.Get());
-
- {
+ TTestClock::TimePoint += milliseconds(10);
+ }
+ UNIT_ASSERT_EQUAL(10, counter.Get());
+
+ {
TMetricTimerScope<TCounter, milliseconds, TTestClock> t{&counter};
- TTestClock::TimePoint += milliseconds(20);
- }
- UNIT_ASSERT_EQUAL(30, counter.Get());
- }
-
- Y_UNIT_TEST(Rate) {
- TTestClock::TimePoint = TTestClock::time_point::min();
-
- TRate rate(0);
- {
+ TTestClock::TimePoint += milliseconds(20);
+ }
+ UNIT_ASSERT_EQUAL(30, counter.Get());
+ }
+
+ Y_UNIT_TEST(Rate) {
+ TTestClock::TimePoint = TTestClock::time_point::min();
+
+ TRate rate(0);
+ {
TMetricTimerScope<TRate, milliseconds, TTestClock> t{&rate};
- TTestClock::TimePoint += milliseconds(10);
- }
- UNIT_ASSERT_EQUAL(10, rate.Get());
-
- {
+ TTestClock::TimePoint += milliseconds(10);
+ }
+ UNIT_ASSERT_EQUAL(10, rate.Get());
+
+ {
TMetricTimerScope<TRate, milliseconds, TTestClock> t{&rate};
- TTestClock::TimePoint += milliseconds(20);
- }
- UNIT_ASSERT_EQUAL(30, rate.Get());
- }
-
- Y_UNIT_TEST(Histogram) {
- TTestClock::TimePoint = TTestClock::time_point::min();
-
- auto assertHistogram = [](const TVector<ui64>& expected, IHistogramSnapshotPtr snapshot) {
- UNIT_ASSERT_EQUAL(expected.size(), snapshot->Count());
- for (size_t i = 0; i < expected.size(); ++i) {
- UNIT_ASSERT_EQUAL(expected[i], snapshot->Value(i));
- }
- };
-
- THistogram histogram(ExplicitHistogram({10, 20, 30}), true);
- {
+ TTestClock::TimePoint += milliseconds(20);
+ }
+ UNIT_ASSERT_EQUAL(30, rate.Get());
+ }
+
+ Y_UNIT_TEST(Histogram) {
+ TTestClock::TimePoint = TTestClock::time_point::min();
+
+ auto assertHistogram = [](const TVector<ui64>& expected, IHistogramSnapshotPtr snapshot) {
+ UNIT_ASSERT_EQUAL(expected.size(), snapshot->Count());
+ for (size_t i = 0; i < expected.size(); ++i) {
+ UNIT_ASSERT_EQUAL(expected[i], snapshot->Value(i));
+ }
+ };
+
+ THistogram histogram(ExplicitHistogram({10, 20, 30}), true);
+ {
TMetricTimerScope<THistogram, milliseconds, TTestClock> t{&histogram};
- TTestClock::TimePoint += milliseconds(5);
- }
- assertHistogram({1, 0, 0, 0}, histogram.TakeSnapshot());
-
- {
+ TTestClock::TimePoint += milliseconds(5);
+ }
+ assertHistogram({1, 0, 0, 0}, histogram.TakeSnapshot());
+
+ {
TMetricTimerScope<THistogram, milliseconds, TTestClock> t{&histogram};
- TTestClock::TimePoint += milliseconds(15);
- }
- assertHistogram({1, 1, 0, 0}, histogram.TakeSnapshot());
- }
+ TTestClock::TimePoint += milliseconds(15);
+ }
+ assertHistogram({1, 1, 0, 0}, histogram.TakeSnapshot());
+ }
Y_UNIT_TEST(Moving) {
TTestClock::TimePoint = TTestClock::time_point::min();
@@ -154,4 +154,4 @@ Y_UNIT_TEST_SUITE(TTimerTest) {
UNIT_ASSERT_EQUAL(counter.Get(), 5);
}
-}
+}
diff --git a/library/cpp/monlib/metrics/ut/ya.make b/library/cpp/monlib/metrics/ut/ya.make
index 698920571f..aec9974fbd 100644
--- a/library/cpp/monlib/metrics/ut/ya.make
+++ b/library/cpp/monlib/metrics/ut/ya.make
@@ -15,7 +15,7 @@ SRCS(
metric_sub_registry_ut.cpp
metric_value_ut.cpp
summary_collector_ut.cpp
- timer_ut.cpp
+ timer_ut.cpp
)
RESOURCE(
diff --git a/ydb/core/base/appdata.h b/ydb/core/base/appdata.h
index 064df6c347..c666f7468c 100644
--- a/ydb/core/base/appdata.h
+++ b/ydb/core/base/appdata.h
@@ -148,7 +148,7 @@ struct TAppData {
NKikimrConfig::TCompactionConfig CompactionConfig;
bool EnforceUserTokenRequirement = false;
bool AllowHugeKeyValueDeletes = true; // delete when all clients limit deletes per request
- bool EnableKqpSpilling = false;
+ bool EnableKqpSpilling = false;
bool AllowShadowDataInSchemeShardForTests = false;
bool EnableMvccSnapshotWithLegacyDomainRoot = false;
TVector<TString> AdministrationAllowedSIDs; // users/groups which allowed to perform administrative tasks
diff --git a/ydb/core/base/board_lookup.cpp b/ydb/core/base/board_lookup.cpp
index 8de88a4508..890b88f6b9 100644
--- a/ydb/core/base/board_lookup.cpp
+++ b/ydb/core/base/board_lookup.cpp
@@ -27,7 +27,7 @@ class TBoardLookupActor : public TActorBootstrapped<TBoardLookupActor> {
enum class EReplicaState {
Unknown,
- NotAvailable,
+ NotAvailable,
NoInfo,
Ready,
};
@@ -140,7 +140,7 @@ class TBoardLookupActor : public TActorBootstrapped<TBoardLookupActor> {
const ui32 nodeId = ev->Get()->NodeId;
for (auto &replica : Replicas) {
if (replica.Replica.NodeId() == nodeId && replica.State == EReplicaState::Unknown) {
- replica.State = EReplicaState::NotAvailable;
+ replica.State = EReplicaState::NotAvailable;
++Stats.Replied;
++Stats.NoInfo;
}
@@ -159,7 +159,7 @@ class TBoardLookupActor : public TActorBootstrapped<TBoardLookupActor> {
auto &replica = Replicas[idx];
if (replica.State != EReplicaState::Unknown)
return;
- replica.State = EReplicaState::NotAvailable;
+ replica.State = EReplicaState::NotAvailable;
++Stats.Replied;
++Stats.NoInfo;
diff --git a/ydb/core/base/events.h b/ydb/core/base/events.h
index 4c3ec71e0d..f5fedfe19b 100644
--- a/ydb/core/base/events.h
+++ b/ydb/core/base/events.h
@@ -65,7 +65,7 @@ struct TKikimrEvents : TEvents {
ES_PQ_L2_CACHE, //4142
ES_TOKEN_BUILDER,
ES_TICKET_PARSER,
- ES_KQP = NYql::NDq::TDqEvents::ES_DQ_COMPUTE_KQP_COMPATIBLE, // 4145
+ ES_KQP = NYql::NDq::TDqEvents::ES_DQ_COMPUTE_KQP_COMPATIBLE, // 4145
ES_BLACKBOX_VALIDATOR,
ES_SELF_PING,
ES_PIPECACHE,
@@ -121,7 +121,7 @@ struct TKikimrEvents : TEvents {
ES_FILESTORE,
ES_FILESTORE_PRIVATE,
ES_YDB_METERING,
- ES_IMPORT_SERVICE, // 4200
+ ES_IMPORT_SERVICE, // 4200
ES_TX_OLAPSHARD,
ES_TX_COLUMNSHARD,
ES_CROSSREF,
@@ -133,7 +133,7 @@ struct TKikimrEvents : TEvents {
ES_DATASTREAMS_PROXY,
ES_IAM_TOKEN_SERVICE,
ES_HEALTH_CHECK,
- ES_DQ = NYql::NDq::TDqEvents::ES_DQ_COMPUTE, // 4212
+ ES_DQ = NYql::NDq::TDqEvents::ES_DQ_COMPUTE, // 4212
ES_YQ, // 4213
ES_CHANGE_EXCHANGE,
ES_DATABASE_SERVICE, //4215
diff --git a/ydb/core/base/localdb.cpp b/ydb/core/base/localdb.cpp
index d0ca9751a7..d07ee26fc1 100644
--- a/ydb/core/base/localdb.cpp
+++ b/ydb/core/base/localdb.cpp
@@ -260,8 +260,8 @@ const TString UnknownTaskName = "unknown";
const TString TransactionTaskName = "transaction";
const TString ScanTaskName = "scan";
const TString BackgroundCompactionTaskName = "background_compaction";
-const TString KqpResourceManagerTaskName = "kqp_query";
-const TString KqpResourceManagerQueue = "queue_kqp_resource_manager";
+const TString KqpResourceManagerTaskName = "kqp_query";
+const TString KqpResourceManagerQueue = "queue_kqp_resource_manager";
const TString LegacyQueueIdTaskNamePrefix = "compaction_gen";
}}
diff --git a/ydb/core/base/localdb.h b/ydb/core/base/localdb.h
index 39ca9cb76e..8679cb286f 100644
--- a/ydb/core/base/localdb.h
+++ b/ydb/core/base/localdb.h
@@ -144,8 +144,8 @@ extern const TString UnknownTaskName;
extern const TString TransactionTaskName;
extern const TString ScanTaskName;
extern const TString BackgroundCompactionTaskName;
-extern const TString KqpResourceManagerTaskName;
-extern const TString KqpResourceManagerQueue;
+extern const TString KqpResourceManagerTaskName;
+extern const TString KqpResourceManagerQueue;
extern const TString LegacyQueueIdTaskNamePrefix;
}}
diff --git a/ydb/core/base/path.cpp b/ydb/core/base/path.cpp
index 6526847429..0cfb16f99d 100644
--- a/ydb/core/base/path.cpp
+++ b/ydb/core/base/path.cpp
@@ -73,14 +73,14 @@ ui32 CanonizedPathLen(const TVector<TString>& path) {
TStringBuf ExtractDomain(const TString& path) noexcept {
auto domain = TStringBuf(path);
- return ExtractDomain(domain);
-}
-
-TStringBuf ExtractDomain(TStringBuf path) noexcept {
+ return ExtractDomain(domain);
+}
+
+TStringBuf ExtractDomain(TStringBuf path) noexcept {
// coherence with SplitPath and JoinPath that allow no / leading path
path.SkipPrefix(TStringBuf("/"));
- return path.Before('/');
+ return path.Before('/');
}
bool IsEqualPaths(const TString& l, const TString& r) noexcept {
diff --git a/ydb/core/base/path.h b/ydb/core/base/path.h
index 421a3c253d..c2b93d2baf 100644
--- a/ydb/core/base/path.h
+++ b/ydb/core/base/path.h
@@ -12,7 +12,7 @@ TString CanonizePath(const TString &path);
TString CanonizePath(const TVector<TString>& path);
ui32 CanonizedPathLen(const TVector<TString>& path);
TStringBuf ExtractDomain(const TString& path) noexcept;
-TStringBuf ExtractDomain(TStringBuf path) noexcept;
+TStringBuf ExtractDomain(TStringBuf path) noexcept;
TStringBuf ExtractBase(const TString& path) noexcept;
TStringBuf ExtractParent(const TString& path) noexcept;
bool IsEqualPaths(const TString& l, const TString& r) noexcept;
diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp
index 146ab6bb7e..819c1478d1 100644
--- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp
+++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp
@@ -1103,7 +1103,7 @@ void TResourceBrokerInitializer::InitializeServices(
}
auto counters = GetServiceCounters(appData->Counters, "tablets");
- TActorSetupCmd actorSetup = { NResourceBroker::CreateResourceBrokerActor(config, counters),
+ TActorSetupCmd actorSetup = { NResourceBroker::CreateResourceBrokerActor(config, counters),
TMailboxType::ReadAsFilled, appData->UserPoolId };
setup->LocalServices.push_back(std::make_pair(NResourceBroker::MakeResourceBrokerID(), actorSetup));
}
@@ -1507,7 +1507,7 @@ void TGRpcServicesInitializer::InitializeServices(NActors::TActorSystemSetup* se
}
if (!IsServiceInitialized(setup, NGRpcService::CreateGRpcRequestProxyId())) {
- auto grpcReqProxy = NGRpcService::CreateGRpcRequestProxy(Config);
+ auto grpcReqProxy = NGRpcService::CreateGRpcRequestProxy(Config);
setup->LocalServices.push_back(std::pair<TActorId,
TActorSetupCmd>(NGRpcService::CreateGRpcRequestProxyId(),
TActorSetupCmd(grpcReqProxy, TMailboxType::ReadAsFilled,
@@ -1949,22 +1949,22 @@ void TKqpServiceInitializer::InitializeServices(NActors::TActorSystemSetup* setu
}
if (enableKqp) {
- NKikimrKqp::TKqpSetting enableSpilling;
- enableSpilling.SetName("_KqpEnableSpilling");
- enableSpilling.SetValue(appData->EnableKqpSpilling ? "true" : "false");
- settings.emplace_back(std::move(enableSpilling));
-
- // Crate resource manager
- auto rm = NKqp::CreateKqpResourceManagerActor(Config.GetTableServiceConfig().GetResourceManager(), nullptr);
- setup->LocalServices.push_back(std::make_pair(
- NKqp::MakeKqpRmServiceID(NodeId),
- TActorSetupCmd(rm, TMailboxType::HTSwap, appData->UserPoolId)));
-
+ NKikimrKqp::TKqpSetting enableSpilling;
+ enableSpilling.SetName("_KqpEnableSpilling");
+ enableSpilling.SetValue(appData->EnableKqpSpilling ? "true" : "false");
+ settings.emplace_back(std::move(enableSpilling));
+
+ // Crate resource manager
+ auto rm = NKqp::CreateKqpResourceManagerActor(Config.GetTableServiceConfig().GetResourceManager(), nullptr);
+ setup->LocalServices.push_back(std::make_pair(
+ NKqp::MakeKqpRmServiceID(NodeId),
+ TActorSetupCmd(rm, TMailboxType::HTSwap, appData->UserPoolId)));
+
auto proxy = NKqp::CreateKqpProxyService(Config.GetLogConfig(), Config.GetTableServiceConfig(),
std::move(settings), Factories->QueryReplayBackendFactory);
- setup->LocalServices.push_back(std::make_pair(
- NKqp::MakeKqpProxyID(NodeId),
- TActorSetupCmd(proxy, TMailboxType::HTSwap, appData->UserPoolId)));
+ setup->LocalServices.push_back(std::make_pair(
+ NKqp::MakeKqpProxyID(NodeId),
+ TActorSetupCmd(proxy, TMailboxType::HTSwap, appData->UserPoolId)));
}
}
diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp
index b84b7199e9..a4f74aa4e0 100644
--- a/ydb/core/driver_lib/run/run.cpp
+++ b/ydb/core/driver_lib/run/run.cpp
@@ -203,7 +203,7 @@ public:
appData->FeatureFlags = Config.GetFeatureFlags();
appData->AllowHugeKeyValueDeletes = Config.GetFeatureFlags().GetAllowHugeKeyValueDeletes();
- appData->EnableKqpSpilling = Config.GetTableServiceConfig().GetSpillingServiceConfig().GetLocalFileConfig().GetEnable();
+ appData->EnableKqpSpilling = Config.GetTableServiceConfig().GetSpillingServiceConfig().GetLocalFileConfig().GetEnable();
appData->CompactionConfig = Config.GetCompactionConfig();
}
diff --git a/ydb/core/engine/minikql/minikql_engine_host.cpp b/ydb/core/engine/minikql/minikql_engine_host.cpp
index ecc4233961..36c35f32cc 100644
--- a/ydb/core/engine/minikql/minikql_engine_host.cpp
+++ b/ydb/core/engine/minikql/minikql_engine_host.cpp
@@ -332,7 +332,7 @@ NUdf::TUnboxedValue TEngineHost::SelectRow(const TTableId& tableId, const TArray
rowBytes = std::max(rowBytes, (ui64)8);
Counters.SelectRowBytes += rowBytes;
- Counters.SelectRowRows++;
+ Counters.SelectRowRows++;
return std::move(rowResult);
}
@@ -964,72 +964,72 @@ NUdf::TUnboxedValue GetCellValue(const TCell& cell, NScheme::TTypeId type) {
return NUdf::TUnboxedValue();
}
- switch (type) {
- case NYql::NProto::TypeIds::Bool:
- return NUdf::TUnboxedValuePod(cell.AsValue<bool>());
- case NYql::NProto::TypeIds::Int8:
- return NUdf::TUnboxedValuePod(cell.AsValue<i8>());
- case NYql::NProto::TypeIds::Uint8:
- return NUdf::TUnboxedValuePod(cell.AsValue<ui8>());
- case NYql::NProto::TypeIds::Int16:
- return NUdf::TUnboxedValuePod(cell.AsValue<i16>());
- case NYql::NProto::TypeIds::Uint16:
- return NUdf::TUnboxedValuePod(cell.AsValue<ui16>());
- case NYql::NProto::TypeIds::Int32:
- return NUdf::TUnboxedValuePod(cell.AsValue<i32>());
- case NYql::NProto::TypeIds::Uint32:
- return NUdf::TUnboxedValuePod(cell.AsValue<ui32>());
- case NYql::NProto::TypeIds::Int64:
- return NUdf::TUnboxedValuePod(cell.AsValue<i64>());
- case NYql::NProto::TypeIds::Uint64:
- return NUdf::TUnboxedValuePod(cell.AsValue<ui64>());
- case NYql::NProto::TypeIds::Float:
- return NUdf::TUnboxedValuePod(cell.AsValue<float>());
- case NYql::NProto::TypeIds::Double:
- return NUdf::TUnboxedValuePod(cell.AsValue<double>());
-
- case NYql::NProto::TypeIds::Decimal: {
- // Decimal values are stored as 128-bit integers in Kikimr but
- // should be stored as 120-bit embedded value in UnboxedValue.
- NYql::NDecimal::TInt128 val;
- std::memcpy(reinterpret_cast<char*>(&val), cell.Data(), cell.Size());
- return NUdf::TUnboxedValuePod(val);
- }
-
- case NYql::NProto::TypeIds::Date: {
- NUdf::TDataType<NUdf::TDate>::TLayout v = cell.AsValue<ui16>();
- return NUdf::TUnboxedValuePod(v);
- }
- case NYql::NProto::TypeIds::Datetime: {
- NUdf::TDataType<NUdf::TDatetime>::TLayout v = cell.AsValue<ui32>();
- return NUdf::TUnboxedValuePod(v);
- }
- case NYql::NProto::TypeIds::Timestamp: {
- NUdf::TDataType<NUdf::TTimestamp>::TLayout v = cell.AsValue<ui64>();
- return NUdf::TUnboxedValuePod(v);
- }
- case NYql::NProto::TypeIds::Interval: {
- NUdf::TDataType<NUdf::TInterval>::TLayout v = cell.AsValue<i64>();
- return NUdf::TUnboxedValuePod(v);
- }
-
- case NYql::NProto::TypeIds::TzDate:
- case NYql::NProto::TypeIds::TzDatetime:
- case NYql::NProto::TypeIds::TzTimestamp:
- // TODO: create TUnboxedValue and call .SetTimezoneId()
-
- case NYql::NProto::TypeIds::String:
- case NYql::NProto::TypeIds::Utf8:
- case NYql::NProto::TypeIds::Yson:
- case NYql::NProto::TypeIds::Json:
- case NYql::NProto::TypeIds::Uuid:
- case NYql::NProto::TypeIds::JsonDocument:
- case NYql::NProto::TypeIds::DyNumber:
- return MakeString(NUdf::TStringRef(cell.Data(), cell.Size()));
-
- default:
- Y_VERIFY_DEBUG(false, "Unsupported type: %" PRIu16, type);
- return MakeString(NUdf::TStringRef(cell.Data(), cell.Size()));
+ switch (type) {
+ case NYql::NProto::TypeIds::Bool:
+ return NUdf::TUnboxedValuePod(cell.AsValue<bool>());
+ case NYql::NProto::TypeIds::Int8:
+ return NUdf::TUnboxedValuePod(cell.AsValue<i8>());
+ case NYql::NProto::TypeIds::Uint8:
+ return NUdf::TUnboxedValuePod(cell.AsValue<ui8>());
+ case NYql::NProto::TypeIds::Int16:
+ return NUdf::TUnboxedValuePod(cell.AsValue<i16>());
+ case NYql::NProto::TypeIds::Uint16:
+ return NUdf::TUnboxedValuePod(cell.AsValue<ui16>());
+ case NYql::NProto::TypeIds::Int32:
+ return NUdf::TUnboxedValuePod(cell.AsValue<i32>());
+ case NYql::NProto::TypeIds::Uint32:
+ return NUdf::TUnboxedValuePod(cell.AsValue<ui32>());
+ case NYql::NProto::TypeIds::Int64:
+ return NUdf::TUnboxedValuePod(cell.AsValue<i64>());
+ case NYql::NProto::TypeIds::Uint64:
+ return NUdf::TUnboxedValuePod(cell.AsValue<ui64>());
+ case NYql::NProto::TypeIds::Float:
+ return NUdf::TUnboxedValuePod(cell.AsValue<float>());
+ case NYql::NProto::TypeIds::Double:
+ return NUdf::TUnboxedValuePod(cell.AsValue<double>());
+
+ case NYql::NProto::TypeIds::Decimal: {
+ // Decimal values are stored as 128-bit integers in Kikimr but
+ // should be stored as 120-bit embedded value in UnboxedValue.
+ NYql::NDecimal::TInt128 val;
+ std::memcpy(reinterpret_cast<char*>(&val), cell.Data(), cell.Size());
+ return NUdf::TUnboxedValuePod(val);
+ }
+
+ case NYql::NProto::TypeIds::Date: {
+ NUdf::TDataType<NUdf::TDate>::TLayout v = cell.AsValue<ui16>();
+ return NUdf::TUnboxedValuePod(v);
+ }
+ case NYql::NProto::TypeIds::Datetime: {
+ NUdf::TDataType<NUdf::TDatetime>::TLayout v = cell.AsValue<ui32>();
+ return NUdf::TUnboxedValuePod(v);
+ }
+ case NYql::NProto::TypeIds::Timestamp: {
+ NUdf::TDataType<NUdf::TTimestamp>::TLayout v = cell.AsValue<ui64>();
+ return NUdf::TUnboxedValuePod(v);
+ }
+ case NYql::NProto::TypeIds::Interval: {
+ NUdf::TDataType<NUdf::TInterval>::TLayout v = cell.AsValue<i64>();
+ return NUdf::TUnboxedValuePod(v);
+ }
+
+ case NYql::NProto::TypeIds::TzDate:
+ case NYql::NProto::TypeIds::TzDatetime:
+ case NYql::NProto::TypeIds::TzTimestamp:
+ // TODO: create TUnboxedValue and call .SetTimezoneId()
+
+ case NYql::NProto::TypeIds::String:
+ case NYql::NProto::TypeIds::Utf8:
+ case NYql::NProto::TypeIds::Yson:
+ case NYql::NProto::TypeIds::Json:
+ case NYql::NProto::TypeIds::Uuid:
+ case NYql::NProto::TypeIds::JsonDocument:
+ case NYql::NProto::TypeIds::DyNumber:
+ return MakeString(NUdf::TStringRef(cell.Data(), cell.Size()));
+
+ default:
+ Y_VERIFY_DEBUG(false, "Unsupported type: %" PRIu16, type);
+ return MakeString(NUdf::TStringRef(cell.Data(), cell.Size()));
}
}
diff --git a/ydb/core/engine/minikql/minikql_engine_host.h b/ydb/core/engine/minikql/minikql_engine_host.h
index cda9a37c88..012ee6891b 100644
--- a/ydb/core/engine/minikql/minikql_engine_host.h
+++ b/ydb/core/engine/minikql/minikql_engine_host.h
@@ -18,7 +18,7 @@ struct TEngineHostCounters {
ui64 NUpdateRow = 0;
ui64 NEraseRow = 0;
- ui64 SelectRowRows = 0;
+ ui64 SelectRowRows = 0;
ui64 SelectRowBytes = 0;
ui64 SelectRangeRows = 0;
ui64 SelectRangeBytes = 0;
@@ -28,29 +28,29 @@ struct TEngineHostCounters {
ui64 InvisibleRowSkips = 0;
- TEngineHostCounters& operator+=(const TEngineHostCounters& other) {
- NSelectRow += other.NSelectRow;
- NSelectRange += other.NSelectRange;
- NUpdateRow += other.NUpdateRow;
- NEraseRow += other.NEraseRow;
- SelectRowRows += other.SelectRowRows;
- SelectRowBytes += other.SelectRowBytes;
- SelectRangeRows += other.SelectRangeRows;
- SelectRangeBytes += other.SelectRangeBytes;
- SelectRangeDeletedRowSkips += other.SelectRangeDeletedRowSkips;
- UpdateRowBytes += other.UpdateRowBytes;
- EraseRowBytes += other.EraseRowBytes;
- InvisibleRowSkips += other.InvisibleRowSkips;
- return *this;
- }
-
+ TEngineHostCounters& operator+=(const TEngineHostCounters& other) {
+ NSelectRow += other.NSelectRow;
+ NSelectRange += other.NSelectRange;
+ NUpdateRow += other.NUpdateRow;
+ NEraseRow += other.NEraseRow;
+ SelectRowRows += other.SelectRowRows;
+ SelectRowBytes += other.SelectRowBytes;
+ SelectRangeRows += other.SelectRangeRows;
+ SelectRangeBytes += other.SelectRangeBytes;
+ SelectRangeDeletedRowSkips += other.SelectRangeDeletedRowSkips;
+ UpdateRowBytes += other.UpdateRowBytes;
+ EraseRowBytes += other.EraseRowBytes;
+ InvisibleRowSkips += other.InvisibleRowSkips;
+ return *this;
+ }
+
TString ToString() const {
return TStringBuilder()
<< "{NSelectRow: " << NSelectRow
<< ", NSelectRange: " << NSelectRange
<< ", NUpdateRow: " << NUpdateRow
<< ", NEraseRow: " << NEraseRow
- << ", SelectRowRows: " << SelectRowRows
+ << ", SelectRowRows: " << SelectRowRows
<< ", SelectRowBytes: " << SelectRowBytes
<< ", SelectRangeRows: " << SelectRangeRows
<< ", SelectRangeBytes: " << SelectRangeBytes
diff --git a/ydb/core/engine/mkql_proto.cpp b/ydb/core/engine/mkql_proto.cpp
index 18981c61ec..3dd053e372 100644
--- a/ydb/core/engine/mkql_proto.cpp
+++ b/ydb/core/engine/mkql_proto.cpp
@@ -128,7 +128,7 @@ NUdf::TUnboxedValue ImportValueFromProto(TType* type, const Ydb::Value& value, c
auto dictType = static_cast<TDictType*>(type);
auto keyType = dictType->GetKeyType();
auto payloadType = dictType->GetPayloadType();
- auto dictBuilder = factory.NewDict(dictType, NUdf::TDictFlags::EDictKind::Hashed);
+ auto dictBuilder = factory.NewDict(dictType, NUdf::TDictFlags::EDictKind::Hashed);
for (const auto& x : value.pairs()) {
dictBuilder->Add(
diff --git a/ydb/core/grpc_services/grpc_request_proxy.cpp b/ydb/core/grpc_services/grpc_request_proxy.cpp
index 1eeb4d3a65..a1e7accb87 100644
--- a/ydb/core/grpc_services/grpc_request_proxy.cpp
+++ b/ydb/core/grpc_services/grpc_request_proxy.cpp
@@ -289,7 +289,7 @@ private:
std::unordered_map<TString, TActorId> Subscribers;
THashSet<TSubDomainKey> SubDomainKeys;
bool AllowYdbRequestsWithoutDatabase = true;
- NKikimrConfig::TAppConfig AppConfig;
+ NKikimrConfig::TAppConfig AppConfig;
TActorId SchemeCache;
bool DynamicNode = false;
TString RootDatabase;
@@ -299,13 +299,13 @@ private:
void TGRpcRequestProxyImpl::Bootstrap(const TActorContext& ctx) {
ctx.Send(MakeTenantPoolRootID(), new TEvents::TEvSubscribe());
AllowYdbRequestsWithoutDatabase = AppData(ctx)->FeatureFlags.GetAllowYdbRequestsWithoutDatabase();
-
- // Subscribe for TableService config changes
- ui32 tableServiceConfigKind = (ui32) NKikimrConsole::TConfigItem::TableServiceConfigItem;
- Send(NConsole::MakeConfigsDispatcherID(SelfId().NodeId()),
- new NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest({tableServiceConfigKind}),
- IEventHandle::FlagTrackDelivery);
-
+
+ // Subscribe for TableService config changes
+ ui32 tableServiceConfigKind = (ui32) NKikimrConsole::TConfigItem::TableServiceConfigItem;
+ Send(NConsole::MakeConfigsDispatcherID(SelfId().NodeId()),
+ new NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest({tableServiceConfigKind}),
+ IEventHandle::FlagTrackDelivery);
+
Send(MakeTxProxyID(), new TEvTxUserProxy::TEvGetProxyServicesRequest);
auto nodeID = SelfId().NodeId();
@@ -387,43 +387,43 @@ void TGRpcRequestProxyImpl::HandleRefreshToken(TRefreshTokenImpl::TPtr& ev, cons
NYql::TIssues()});
}
-void TGRpcRequestProxyImpl::HandleConfig(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse::TPtr&) {
- LOG_INFO(*TlsActivationContext, NKikimrServices::GRPC_SERVER, "Subscribed for config changes");
-}
-
-void TGRpcRequestProxyImpl::HandleConfig(NConsole::TEvConsole::TEvConfigNotificationRequest::TPtr& ev) {
- auto &event = ev->Get()->Record;
-
- AppConfig.Swap(event.MutableConfig());
- LOG_INFO(*TlsActivationContext, NKikimrServices::GRPC_SERVER, "Updated app config");
-
- auto responseEv = MakeHolder<NConsole::TEvConsole::TEvConfigNotificationResponse>(event);
- Send(ev->Sender, responseEv.Release(), IEventHandle::FlagTrackDelivery, ev->Cookie);
-}
-
+void TGRpcRequestProxyImpl::HandleConfig(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse::TPtr&) {
+ LOG_INFO(*TlsActivationContext, NKikimrServices::GRPC_SERVER, "Subscribed for config changes");
+}
+
+void TGRpcRequestProxyImpl::HandleConfig(NConsole::TEvConsole::TEvConfigNotificationRequest::TPtr& ev) {
+ auto &event = ev->Get()->Record;
+
+ AppConfig.Swap(event.MutableConfig());
+ LOG_INFO(*TlsActivationContext, NKikimrServices::GRPC_SERVER, "Updated app config");
+
+ auto responseEv = MakeHolder<NConsole::TEvConsole::TEvConfigNotificationResponse>(event);
+ Send(ev->Sender, responseEv.Release(), IEventHandle::FlagTrackDelivery, ev->Cookie);
+}
+
void TGRpcRequestProxyImpl::HandleProxyService(TEvTxUserProxy::TEvGetProxyServicesResponse::TPtr& ev) {
SchemeCache = ev->Get()->Services.SchemeCache;
LOG_DEBUG(*TlsActivationContext, NKikimrServices::GRPC_SERVER,
"Got proxy service configuration");
}
-void TGRpcRequestProxyImpl::HandleUndelivery(TEvents::TEvUndelivered::TPtr& ev) {
- switch (ev->Get()->SourceType) {
- case NConsole::TEvConfigsDispatcher::EvSetConfigSubscriptionRequest:
- LOG_CRIT(*TlsActivationContext, NKikimrServices::GRPC_SERVER,
- "Failed to deliver subscription request to config dispatcher");
- break;
- case NConsole::TEvConsole::EvConfigNotificationResponse:
- LOG_ERROR(*TlsActivationContext, NKikimrServices::GRPC_SERVER,
- "Failed to deliver config notification response");
- break;
- default:
- LOG_ERROR(*TlsActivationContext, NKikimrServices::GRPC_SERVER,
- "Undelivered event with unexpected source type: %d", ev->Get()->SourceType);
- break;
- }
-}
-
+void TGRpcRequestProxyImpl::HandleUndelivery(TEvents::TEvUndelivered::TPtr& ev) {
+ switch (ev->Get()->SourceType) {
+ case NConsole::TEvConfigsDispatcher::EvSetConfigSubscriptionRequest:
+ LOG_CRIT(*TlsActivationContext, NKikimrServices::GRPC_SERVER,
+ "Failed to deliver subscription request to config dispatcher");
+ break;
+ case NConsole::TEvConsole::EvConfigNotificationResponse:
+ LOG_ERROR(*TlsActivationContext, NKikimrServices::GRPC_SERVER,
+ "Failed to deliver config notification response");
+ break;
+ default:
+ LOG_ERROR(*TlsActivationContext, NKikimrServices::GRPC_SERVER,
+ "Undelivered event with unexpected source type: %d", ev->Get()->SourceType);
+ break;
+ }
+}
+
bool TGRpcRequestProxyImpl::IsAuthStateOK(const IRequestProxyCtx& ctx) {
const auto& state = ctx.GetAuthState();
return state.State == NGrpc::TAuthState::AS_OK ||
@@ -547,9 +547,9 @@ void TGRpcRequestProxyImpl::StateFunc(TAutoPtr<IEventHandle>& ev, const TActorCo
switch (ev->GetTypeRewrite()) {
HFunc(TEvTenantPool::TEvTenantPoolStatus, HandlePoolStatus);
hFunc(TEvTxUserProxy::TEvGetProxyServicesResponse, HandleProxyService);
- hFunc(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse, HandleConfig);
- hFunc(NConsole::TEvConsole::TEvConfigNotificationRequest, HandleConfig);
- hFunc(TEvents::TEvUndelivered, HandleUndelivery);
+ hFunc(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse, HandleConfig);
+ hFunc(NConsole::TEvConsole::TEvConfigNotificationRequest, HandleConfig);
+ hFunc(TEvents::TEvUndelivered, HandleUndelivery);
HFunc(TSchemeBoardEvents::TEvNotifyUpdate, HandleSchemeBoard);
hFunc(TSchemeBoardEvents::TEvNotifyDelete, HandleSchemeBoard);
@@ -678,8 +678,8 @@ void TGRpcRequestProxyImpl::StateFunc(TAutoPtr<IEventHandle>& ev, const TActorCo
}
}
-IActor* CreateGRpcRequestProxy(const NKikimrConfig::TAppConfig& appConfig) {
- return new TGRpcRequestProxyImpl(appConfig);
+IActor* CreateGRpcRequestProxy(const NKikimrConfig::TAppConfig& appConfig) {
+ return new TGRpcRequestProxyImpl(appConfig);
}
} // namespace NGRpcService
diff --git a/ydb/core/grpc_services/grpc_request_proxy.h b/ydb/core/grpc_services/grpc_request_proxy.h
index dc67785b38..97315f6e9f 100644
--- a/ydb/core/grpc_services/grpc_request_proxy.h
+++ b/ydb/core/grpc_services/grpc_request_proxy.h
@@ -9,10 +9,10 @@
#include <util/generic/ptr.h>
#include <util/generic/vector.h>
-namespace NKikimrConfig {
-class TAppConfig;
-}
-
+namespace NKikimrConfig {
+class TAppConfig;
+}
+
namespace NKikimr {
struct TAppData;
@@ -20,7 +20,7 @@ struct TAppData;
namespace NGRpcService {
TString DatabaseFromDomain(const TAppData* appdata);
-IActor* CreateGRpcRequestProxy(const NKikimrConfig::TAppConfig& appConfig);
+IActor* CreateGRpcRequestProxy(const NKikimrConfig::TAppConfig& appConfig);
class TGRpcRequestProxy : public IFacilityProvider {
public:
diff --git a/ydb/core/grpc_services/rpc_execute_data_query.cpp b/ydb/core/grpc_services/rpc_execute_data_query.cpp
index a4b8566650..317a3488ce 100644
--- a/ydb/core/grpc_services/rpc_execute_data_query.cpp
+++ b/ydb/core/grpc_services/rpc_execute_data_query.cpp
@@ -33,7 +33,7 @@ public:
TBase::Bootstrap(ctx);
this->Become(&TExecuteDataQueryRPC::StateWork);
- Proceed(ctx);
+ Proceed(ctx);
}
void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) {
@@ -43,7 +43,7 @@ public:
}
}
- void Proceed(const TActorContext& ctx) {
+ void Proceed(const TActorContext& ctx) {
const auto req = GetProtoRequest();
const auto traceId = Request_->GetTraceId();
const auto requestType = Request_->GetRequestType();
@@ -76,7 +76,7 @@ public:
ConvertYdbParamsToMiniKQLParams(req->parameters(), params);
ev->Record.MutableRequest()->MutableParameters()->CopyFrom(params);
} catch (const std::exception& ex) {
- auto issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, "Failed to parse query parameters.");
+ auto issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, "Failed to parse query parameters.");
issue.AddSubIssue(MakeIntrusive<NYql::TIssue>(NYql::ExceptionToIssue(ex)));
NYql::TIssues issues;
@@ -238,7 +238,7 @@ public:
ReplyWithResult(Ydb::StatusIds::SUCCESS, issueMessage, *queryResult, ctx);
} else {
- return OnQueryResponseErrorWithTxMeta(record, ctx);
+ return OnQueryResponseErrorWithTxMeta(record, ctx);
}
}
};
diff --git a/ydb/core/grpc_services/rpc_kqp_base.cpp b/ydb/core/grpc_services/rpc_kqp_base.cpp
index d4a5e8ca2e..414543f574 100644
--- a/ydb/core/grpc_services/rpc_kqp_base.cpp
+++ b/ydb/core/grpc_services/rpc_kqp_base.cpp
@@ -48,10 +48,10 @@ void FillQueryStats(Ydb::TableStats::QueryStats& queryStats, const NKikimrKqp::T
return a.name() < b.name();
});
- NKqpProto::TKqpExecutionExtraStats executionExtraStats;
- if (exec.HasExtra() && exec.GetExtra().UnpackTo(&executionExtraStats)) {
- toPhase.set_affected_shards(executionExtraStats.GetAffectedShards());
- }
+ NKqpProto::TKqpExecutionExtraStats executionExtraStats;
+ if (exec.HasExtra() && exec.GetExtra().UnpackTo(&executionExtraStats)) {
+ toPhase.set_affected_shards(executionExtraStats.GetAffectedShards());
+ }
}
if (kqpStats.HasCompilation()) {
diff --git a/ydb/core/grpc_services/rpc_kqp_base.h b/ydb/core/grpc_services/rpc_kqp_base.h
index efbeeec9fe..2811a49e78 100644
--- a/ydb/core/grpc_services/rpc_kqp_base.h
+++ b/ydb/core/grpc_services/rpc_kqp_base.h
@@ -41,14 +41,14 @@ inline TString GetTransactionModeName(const Ydb::Table::TransactionSettings& set
}
}
-inline NYql::NDqProto::EDqStatsMode GetKqpStatsMode(Ydb::Table::QueryStatsCollection::Mode mode) {
+inline NYql::NDqProto::EDqStatsMode GetKqpStatsMode(Ydb::Table::QueryStatsCollection::Mode mode) {
switch (mode) {
case Ydb::Table::QueryStatsCollection::STATS_COLLECTION_BASIC:
- return NYql::NDqProto::DQ_STATS_MODE_BASIC;
+ return NYql::NDqProto::DQ_STATS_MODE_BASIC;
case Ydb::Table::QueryStatsCollection::STATS_COLLECTION_FULL:
- return NYql::NDqProto::DQ_STATS_MODE_PROFILE;
+ return NYql::NDqProto::DQ_STATS_MODE_PROFILE;
default:
- return NYql::NDqProto::DQ_STATS_MODE_NONE;
+ return NYql::NDqProto::DQ_STATS_MODE_NONE;
}
}
diff --git a/ydb/core/grpc_services/rpc_read_columns.cpp b/ydb/core/grpc_services/rpc_read_columns.cpp
index 88af19e146..4baae606a9 100644
--- a/ydb/core/grpc_services/rpc_read_columns.cpp
+++ b/ydb/core/grpc_services/rpc_read_columns.cpp
@@ -317,7 +317,7 @@ private:
SysViewScanActor = ctx.Register(tableScanActor.Release());
- auto ackEv = MakeHolder<NKqp::TEvKqpCompute::TEvScanDataAck>(0);
+ auto ackEv = MakeHolder<NKqp::TEvKqpCompute::TEvScanDataAck>(0);
ctx.Send(SysViewScanActor, ackEv.Release());
}
@@ -374,22 +374,22 @@ private:
return ReplyWithError(Ydb::StatusIds::INTERNAL_ERROR, "Arrow format not supported yet", ctx);
}
- for (auto&& row : ev->Get()->Rows) {
+ for (auto&& row : ev->Get()->Rows) {
++rowsExtracted;
- if (row.size() != keyColumnCount + ValueColumnTypes.size()) {
+ if (row.size() != keyColumnCount + ValueColumnTypes.size()) {
return ReplyWithError(Ydb::StatusIds::INTERNAL_ERROR,
"System view row format doesn't match the schema",
ctx);
}
- TDbTupleRef rowKey(KeyColumnTypes.data(), row.data(), keyColumnCount);
+ TDbTupleRef rowKey(KeyColumnTypes.data(), row.data(), keyColumnCount);
if (!skippedBeforeMinKey) {
int cmp = CompareTypedCellVectors(MinKey.GetCells().data(), rowKey.Cells().data(),
KeyColumnTypes.data(),
MinKey.GetCells().size(), rowKey.Cells().size());
- // Skip rows before MinKey just in case (because currently sys view scan ignores key range)
+ // Skip rows before MinKey just in case (because currently sys view scan ignores key range)
if (cmp > 0 || (cmp == 0 && !MinKeyInclusive)) {
LOG_DEBUG_S(ctx, NKikimrServices::MSGBUS_REQUEST, "Skipped rows by sys view scan");
continue;
@@ -398,7 +398,7 @@ private:
}
}
- TDbTupleRef rowValues(ValueColumnTypes.data(), row.data() + keyColumnCount, row.size() - keyColumnCount);
+ TDbTupleRef rowValues(ValueColumnTypes.data(), row.data() + keyColumnCount, row.size() - keyColumnCount);
BlockBuilder->AddRow(rowKey, rowValues);
++SysViewRowsReceived;
@@ -408,7 +408,7 @@ private:
}
}
- auto ackEv = MakeHolder<NKqp::TEvKqpCompute::TEvScanDataAck>(0);
+ auto ackEv = MakeHolder<NKqp::TEvKqpCompute::TEvScanDataAck>(0);
ctx.Send(ev->Sender, ackEv.Release());
bool done =
diff --git a/ydb/core/grpc_services/rpc_stream_execute_scan_query.cpp b/ydb/core/grpc_services/rpc_stream_execute_scan_query.cpp
index 430b7a2be3..5f95f263c6 100644
--- a/ydb/core/grpc_services/rpc_stream_execute_scan_query.cpp
+++ b/ydb/core/grpc_services/rpc_stream_execute_scan_query.cpp
@@ -1,6 +1,6 @@
#include "grpc_request_proxy.h"
-#include "rpc_common.h"
+#include "rpc_common.h"
#include "rpc_kqp_base.h"
#include <ydb/core/actorlib_impl/long_timer.h>
@@ -14,7 +14,7 @@
#include <ydb/core/ydb_convert/ydb_convert.h>
#include <library/cpp/actors/core/actor_bootstrapped.h>
-
+
namespace NKikimr {
namespace NGRpcService {
@@ -39,31 +39,31 @@ struct TParseRequestError {
, Issues(issues) {}
};
-bool NeedReportStats(const Ydb::Experimental::ExecuteStreamQueryRequest& req) {
- switch (req.profile_mode()) {
- case Experimental::ExecuteStreamQueryRequest_ProfileMode_PROFILE_MODE_UNSPECIFIED:
- case Experimental::ExecuteStreamQueryRequest_ProfileMode_NONE:
- return false;
-
- case Experimental::ExecuteStreamQueryRequest_ProfileMode_BASIC:
- case Experimental::ExecuteStreamQueryRequest_ProfileMode_FULL:
- return true;
-
- case Experimental::ExecuteStreamQueryRequest_ProfileMode_ExecuteStreamQueryRequest_ProfileMode_INT_MIN_SENTINEL_DO_NOT_USE_:
- case Experimental::ExecuteStreamQueryRequest_ProfileMode_ExecuteStreamQueryRequest_ProfileMode_INT_MAX_SENTINEL_DO_NOT_USE_:
- YQL_ENSURE(false);
- }
-}
-
-bool NeedReportStats(const Ydb::Table::ExecuteScanQueryRequest& req) {
- switch (req.mode()) {
- case ExecuteScanQueryRequest_Mode_MODE_UNSPECIFIED:
- return false;
-
- case ExecuteScanQueryRequest_Mode_MODE_EXPLAIN:
+bool NeedReportStats(const Ydb::Experimental::ExecuteStreamQueryRequest& req) {
+ switch (req.profile_mode()) {
+ case Experimental::ExecuteStreamQueryRequest_ProfileMode_PROFILE_MODE_UNSPECIFIED:
+ case Experimental::ExecuteStreamQueryRequest_ProfileMode_NONE:
+ return false;
+
+ case Experimental::ExecuteStreamQueryRequest_ProfileMode_BASIC:
+ case Experimental::ExecuteStreamQueryRequest_ProfileMode_FULL:
return true;
- case ExecuteScanQueryRequest_Mode_MODE_EXEC:
+ case Experimental::ExecuteStreamQueryRequest_ProfileMode_ExecuteStreamQueryRequest_ProfileMode_INT_MIN_SENTINEL_DO_NOT_USE_:
+ case Experimental::ExecuteStreamQueryRequest_ProfileMode_ExecuteStreamQueryRequest_ProfileMode_INT_MAX_SENTINEL_DO_NOT_USE_:
+ YQL_ENSURE(false);
+ }
+}
+
+bool NeedReportStats(const Ydb::Table::ExecuteScanQueryRequest& req) {
+ switch (req.mode()) {
+ case ExecuteScanQueryRequest_Mode_MODE_UNSPECIFIED:
+ return false;
+
+ case ExecuteScanQueryRequest_Mode_MODE_EXPLAIN:
+ return true;
+
+ case ExecuteScanQueryRequest_Mode_MODE_EXEC:
switch (req.collect_stats()) {
case Ydb::Table::QueryStatsCollection::STATS_COLLECTION_BASIC:
case Ydb::Table::QueryStatsCollection::STATS_COLLECTION_FULL:
@@ -87,8 +87,8 @@ bool NeedReportPlan(const Ydb::Table::ExecuteScanQueryRequest& req) {
return false;
case ExecuteScanQueryRequest_Mode_MODE_EXPLAIN:
- return true;
-
+ return true;
+
case ExecuteScanQueryRequest_Mode_MODE_EXEC:
switch (req.collect_stats()) {
case Ydb::Table::QueryStatsCollection::STATS_COLLECTION_FULL:
@@ -100,12 +100,12 @@ bool NeedReportPlan(const Ydb::Table::ExecuteScanQueryRequest& req) {
return false;
- case ExecuteScanQueryRequest_Mode_ExecuteScanQueryRequest_Mode_INT_MIN_SENTINEL_DO_NOT_USE_:
- case ExecuteScanQueryRequest_Mode_ExecuteScanQueryRequest_Mode_INT_MAX_SENTINEL_DO_NOT_USE_:
- YQL_ENSURE(false);
- }
-}
-
+ case ExecuteScanQueryRequest_Mode_ExecuteScanQueryRequest_Mode_INT_MIN_SENTINEL_DO_NOT_USE_:
+ case ExecuteScanQueryRequest_Mode_ExecuteScanQueryRequest_Mode_INT_MAX_SENTINEL_DO_NOT_USE_:
+ YQL_ENSURE(false);
+ }
+}
+
bool FillKqpParameters(const ::google::protobuf::Map<TString, Ydb::TypedValue>& input,
NKikimrMiniKQL::TParams& output, TParseRequestError& error)
{
@@ -151,13 +151,13 @@ bool FillKqpRequest(const Ydb::Experimental::ExecuteStreamQueryRequest& req, NKi
switch (req.profile_mode()) {
case Ydb::Experimental::ExecuteStreamQueryRequest_ProfileMode_PROFILE_MODE_UNSPECIFIED:
case Ydb::Experimental::ExecuteStreamQueryRequest_ProfileMode_NONE:
- kqpRequest.MutableRequest()->SetStatsMode(NYql::NDqProto::DQ_STATS_MODE_NONE);
+ kqpRequest.MutableRequest()->SetStatsMode(NYql::NDqProto::DQ_STATS_MODE_NONE);
break;
case Ydb::Experimental::ExecuteStreamQueryRequest_ProfileMode_BASIC:
- kqpRequest.MutableRequest()->SetStatsMode(NYql::NDqProto::DQ_STATS_MODE_BASIC);
+ kqpRequest.MutableRequest()->SetStatsMode(NYql::NDqProto::DQ_STATS_MODE_BASIC);
break;
case Ydb::Experimental::ExecuteStreamQueryRequest_ProfileMode_FULL:
- kqpRequest.MutableRequest()->SetStatsMode(NYql::NDqProto::DQ_STATS_MODE_PROFILE);
+ kqpRequest.MutableRequest()->SetStatsMode(NYql::NDqProto::DQ_STATS_MODE_PROFILE);
break;
default:
YQL_ENSURE(false, "Unknown profile_mode "
@@ -208,7 +208,7 @@ bool FillKqpRequest(const Ydb::Table::ExecuteScanQueryRequest& req, NKikimrKqp::
case Query::kId: {
NYql::TIssues issues;
issues.AddIssue(MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR,
- "Specifying query by ID not supported in scan execution."));
+ "Specifying query by ID not supported in scan execution."));
error = TParseRequestError(Ydb::StatusIds::BAD_REQUEST, issues);
return false;
}
@@ -225,7 +225,7 @@ bool FillKqpRequest(const Ydb::Table::ExecuteScanQueryRequest& req, NKikimrKqp::
}
bool FillProfile(Ydb::Experimental::ExecuteStreamQueryResponse& response,
- const NYql::NDqProto::TDqExecutionStats& profile)
+ const NYql::NDqProto::TDqExecutionStats& profile)
{
response.set_status(Ydb::StatusIds::SUCCESS);
response.mutable_result()->set_profile(profile.Utf8DebugString());
@@ -233,7 +233,7 @@ bool FillProfile(Ydb::Experimental::ExecuteStreamQueryResponse& response,
}
bool FillProfile(Ydb::Table::ExecuteScanQueryPartialResponse& response,
- const NYql::NDqProto::TDqExecutionStats& profile)
+ const NYql::NDqProto::TDqExecutionStats& profile)
{
Y_UNUSED(response);
Y_UNUSED(profile);
@@ -242,11 +242,11 @@ bool FillProfile(Ydb::Table::ExecuteScanQueryPartialResponse& response,
template<typename TRequestEv, typename TResponse>
class TStreamExecuteScanQueryRPC : public TActorBootstrapped<TStreamExecuteScanQueryRPC<TRequestEv, TResponse>> {
-private:
- enum EWakeupTag : ui64 {
- ClientLostTag = 1,
- TimeoutTag = 2
- };
+private:
+ enum EWakeupTag : ui64 {
+ ClientLostTag = 1,
+ TimeoutTag = 2
+ };
public:
static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
@@ -254,50 +254,50 @@ public:
}
TStreamExecuteScanQueryRPC(TRequestEv* request, ui64 rpcBufferSize)
- : Request_(request)
+ : Request_(request)
, RpcBufferSize_(rpcBufferSize) {}
void Bootstrap(const TActorContext &ctx) {
this->Become(&TStreamExecuteScanQueryRPC::StateWork);
-
- const auto& cfg = AppData(ctx)->StreamingConfig.GetOutputStreamConfig();
-
- InactiveClientTimeout_ = TDuration::FromValue(cfg.GetInactiveClientTimeout());
- if (InactiveClientTimeout_) {
- SetTimeoutTimer(InactiveClientTimeout_, ctx);
- }
-
- LastDataStreamTimestamp_ = TAppData::TimeProvider->Now();
-
+
+ const auto& cfg = AppData(ctx)->StreamingConfig.GetOutputStreamConfig();
+
+ InactiveClientTimeout_ = TDuration::FromValue(cfg.GetInactiveClientTimeout());
+ if (InactiveClientTimeout_) {
+ SetTimeoutTimer(InactiveClientTimeout_, ctx);
+ }
+
+ LastDataStreamTimestamp_ = TAppData::TimeProvider->Now();
+
auto selfId = this->SelfId();
- auto as = TActivationContext::ActorSystem();
-
- Request_->SetClientLostAction([selfId, as]() {
- as->Send(selfId, new TEvents::TEvWakeup(EWakeupTag::ClientLostTag));
- });
-
- Request_->SetStreamingNotify([selfId, as](size_t left) {
- as->Send(selfId, new TRpcServices::TEvGrpcNextReply(left));
- });
-
+ auto as = TActivationContext::ActorSystem();
+
+ Request_->SetClientLostAction([selfId, as]() {
+ as->Send(selfId, new TEvents::TEvWakeup(EWakeupTag::ClientLostTag));
+ });
+
+ Request_->SetStreamingNotify([selfId, as](size_t left) {
+ as->Send(selfId, new TRpcServices::TEvGrpcNextReply(left));
+ });
+
Proceed(ctx);
}
-private:
+private:
void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) {
switch (ev->GetTypeRewrite()) {
- HFunc(TEvents::TEvWakeup, Handle);
- HFunc(TRpcServices::TEvGrpcNextReply, Handle);
+ HFunc(TEvents::TEvWakeup, Handle);
+ HFunc(TRpcServices::TEvGrpcNextReply, Handle);
HFunc(NKqp::TEvKqp::TEvQueryResponse, Handle);
HFunc(NKqp::TEvKqp::TEvProcessResponse, Handle);
- HFunc(NKqp::TEvKqp::TEvAbortExecution, Handle);
+ HFunc(NKqp::TEvKqp::TEvAbortExecution, Handle);
HFunc(NKqp::TEvKqpExecuter::TEvStreamData, Handle);
- HFunc(NKqp::TEvKqpExecuter::TEvStreamProfile, Handle);
- HFunc(NKqp::TEvKqpExecuter::TEvExecuterProgress, Handle);
+ HFunc(NKqp::TEvKqpExecuter::TEvStreamProfile, Handle);
+ HFunc(NKqp::TEvKqpExecuter::TEvExecuterProgress, Handle);
default: {
- auto issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, TStringBuilder()
+ auto issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, TStringBuilder()
<< "Unexpected event received in TStreamExecuteScanQueryRPC::StateWork: " << ev->GetTypeRewrite());
- return ReplyFinishStream(Ydb::StatusIds::INTERNAL_ERROR, issue, ctx);
+ return ReplyFinishStream(Ydb::StatusIds::INTERNAL_ERROR, issue, ctx);
}
}
}
@@ -322,54 +322,54 @@ private:
return ReplyFinishStream(parseError.Status, parseError.Issues, ctx);
}
- if (!ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release())) {
+ if (!ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release())) {
NYql::TIssues issues;
- issues.AddIssue(MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, "Internal error"));
- ReplyFinishStream(Ydb::StatusIds::INTERNAL_ERROR, issues, ctx);
- }
+ issues.AddIssue(MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, "Internal error"));
+ ReplyFinishStream(Ydb::StatusIds::INTERNAL_ERROR, issues, ctx);
+ }
+ }
+
+ void Handle(TEvents::TEvWakeup::TPtr& ev, const TActorContext& ctx) {
+ switch ((EWakeupTag) ev->Get()->Tag) {
+ case EWakeupTag::ClientLostTag:
+ return HandleClientLost(ctx);
+ case EWakeupTag::TimeoutTag:
+ return HandleTimeout(ctx);
+ }
}
- void Handle(TEvents::TEvWakeup::TPtr& ev, const TActorContext& ctx) {
- switch ((EWakeupTag) ev->Get()->Tag) {
- case EWakeupTag::ClientLostTag:
- return HandleClientLost(ctx);
- case EWakeupTag::TimeoutTag:
- return HandleTimeout(ctx);
- }
- }
-
- void Handle(TRpcServices::TEvGrpcNextReply::TPtr& ev, const TActorContext& ctx) {
+ void Handle(TRpcServices::TEvGrpcNextReply::TPtr& ev, const TActorContext& ctx) {
LOG_DEBUG_S(ctx, NKikimrServices::RPC_REQUEST, this->SelfId() << " NextReply"
- << ", left: " << ev->Get()->LeftInQueue
- << ", queue: " << GRpcResponsesSizeQueue_.size()
- << ", used memory: " << GRpcResponsesSize_
- << ", buffer size: " << RpcBufferSize_);
-
- while (GRpcResponsesSizeQueue_.size() > ev->Get()->LeftInQueue) {
- GRpcResponsesSize_ -= GRpcResponsesSizeQueue_.front();
- GRpcResponsesSizeQueue_.pop();
- }
- Y_VERIFY_DEBUG(GRpcResponsesSizeQueue_.empty() == (GRpcResponsesSize_ == 0));
- LastDataStreamTimestamp_ = TAppData::TimeProvider->Now();
-
- if (WaitOnSeqNo_ && RpcBufferSize_ > GRpcResponsesSize_) {
- ui64 freeSpace = RpcBufferSize_ - GRpcResponsesSize_;
-
+ << ", left: " << ev->Get()->LeftInQueue
+ << ", queue: " << GRpcResponsesSizeQueue_.size()
+ << ", used memory: " << GRpcResponsesSize_
+ << ", buffer size: " << RpcBufferSize_);
+
+ while (GRpcResponsesSizeQueue_.size() > ev->Get()->LeftInQueue) {
+ GRpcResponsesSize_ -= GRpcResponsesSizeQueue_.front();
+ GRpcResponsesSizeQueue_.pop();
+ }
+ Y_VERIFY_DEBUG(GRpcResponsesSizeQueue_.empty() == (GRpcResponsesSize_ == 0));
+ LastDataStreamTimestamp_ = TAppData::TimeProvider->Now();
+
+ if (WaitOnSeqNo_ && RpcBufferSize_ > GRpcResponsesSize_) {
+ ui64 freeSpace = RpcBufferSize_ - GRpcResponsesSize_;
+
LOG_DEBUG_S(ctx, NKikimrServices::RPC_REQUEST, this->SelfId() << " Send stream data ack"
- << ", seqNo: " << *WaitOnSeqNo_
- << ", freeSpace: " << freeSpace
- << ", to: " << ExecuterActorId_);
-
- auto resp = MakeHolder<NKqp::TEvKqpExecuter::TEvStreamDataAck>();
- resp->Record.SetSeqNo(*WaitOnSeqNo_);
- resp->Record.SetFreeSpace(freeSpace);
-
+ << ", seqNo: " << *WaitOnSeqNo_
+ << ", freeSpace: " << freeSpace
+ << ", to: " << ExecuterActorId_);
+
+ auto resp = MakeHolder<NKqp::TEvKqpExecuter::TEvStreamDataAck>();
+ resp->Record.SetSeqNo(*WaitOnSeqNo_);
+ resp->Record.SetFreeSpace(freeSpace);
+
ctx.Send(ExecuterActorId_, resp.Release());
-
- WaitOnSeqNo_.Clear();
- }
- }
-
+
+ WaitOnSeqNo_.Clear();
+ }
+ }
+
void Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) {
auto& record = ev->Get()->Record.GetRef();
@@ -390,7 +390,7 @@ private:
if (reportStats) {
if (kqpResponse.HasQueryStats()) {
for (const auto& execStats: ExecutionProfiles_) {
- record.MutableResponse()->MutableQueryStats()->AddExecutions()->Swap(execStats.get());
+ record.MutableResponse()->MutableQueryStats()->AddExecutions()->Swap(execStats.get());
}
record.MutableResponse()->SetQueryPlan(reportPlan
@@ -440,130 +440,130 @@ private:
ReplyFinishStream(kqpResponse.GetYdbStatus(), issues, ctx);
}
- void Handle(NKqp::TEvKqp::TEvAbortExecution::TPtr& ev, const TActorContext& ctx) {
+ void Handle(NKqp::TEvKqp::TEvAbortExecution::TPtr& ev, const TActorContext& ctx) {
auto& record = ev->Get()->Record;
-
+
LOG_DEBUG_S(ctx, NKikimrServices::RPC_REQUEST, this->SelfId() << " Got abort execution event, from: "
- << ev->Sender << ", code: " << Ydb::StatusIds::StatusCode_Name(record.GetStatusCode())
- << ", message: " << record.GetMessage());
-
- NYql::TIssue issue(record.GetMessage());
- ReplyFinishStream(record.GetStatusCode(), issue, ctx);
- }
-
- void Handle(NKqp::TEvKqpExecuter::TEvStreamData::TPtr& ev, const TActorContext& ctx) {
+ << ev->Sender << ", code: " << Ydb::StatusIds::StatusCode_Name(record.GetStatusCode())
+ << ", message: " << record.GetMessage());
+
+ NYql::TIssue issue(record.GetMessage());
+ ReplyFinishStream(record.GetStatusCode(), issue, ctx);
+ }
+
+ void Handle(NKqp::TEvKqpExecuter::TEvStreamData::TPtr& ev, const TActorContext& ctx) {
TResponse response;
response.set_status(StatusIds::SUCCESS);
- response.mutable_result()->mutable_result_set()->Swap(ev->Get()->Record.MutableResultSet());
+ response.mutable_result()->mutable_result_set()->Swap(ev->Get()->Record.MutableResultSet());
TString out;
Y_PROTOBUF_SUPPRESS_NODISCARD response.SerializeToString(&out);
- GRpcResponsesSizeQueue_.push(out.size());
- GRpcResponsesSize_ += out.size();
-
+ GRpcResponsesSizeQueue_.push(out.size());
+ GRpcResponsesSize_ += out.size();
+
Request_->SendSerializedResult(std::move(out), StatusIds::SUCCESS);
-
- ui64 freeSpace = GRpcResponsesSize_ < RpcBufferSize_
- ? RpcBufferSize_ - GRpcResponsesSize_
- : 0;
-
- if (freeSpace == 0) {
- WaitOnSeqNo_ = ev->Get()->Record.GetSeqNo();
- }
-
+
+ ui64 freeSpace = GRpcResponsesSize_ < RpcBufferSize_
+ ? RpcBufferSize_ - GRpcResponsesSize_
+ : 0;
+
+ if (freeSpace == 0) {
+ WaitOnSeqNo_ = ev->Get()->Record.GetSeqNo();
+ }
+
LOG_DEBUG_S(ctx, NKikimrServices::RPC_REQUEST, this->SelfId() << " Send stream data ack"
- << ", seqNo: " << ev->Get()->Record.GetSeqNo()
- << ", freeSpace: " << freeSpace
- << ", to: " << ev->Sender
- << ", queue: " << GRpcResponsesSizeQueue_.size());
-
- auto resp = MakeHolder<NKqp::TEvKqpExecuter::TEvStreamDataAck>();
- resp->Record.SetSeqNo(ev->Get()->Record.GetSeqNo());
- resp->Record.SetFreeSpace(freeSpace);
-
+ << ", seqNo: " << ev->Get()->Record.GetSeqNo()
+ << ", freeSpace: " << freeSpace
+ << ", to: " << ev->Sender
+ << ", queue: " << GRpcResponsesSizeQueue_.size());
+
+ auto resp = MakeHolder<NKqp::TEvKqpExecuter::TEvStreamDataAck>();
+ resp->Record.SetSeqNo(ev->Get()->Record.GetSeqNo());
+ resp->Record.SetFreeSpace(freeSpace);
+
ctx.Send(ev->Sender, resp.Release());
}
- void Handle(NKqp::TEvKqpExecuter::TEvStreamProfile::TPtr& ev, const TActorContext&) {
- auto req = Request_->GetProtoRequest();
- if (!NeedReportStats(*req)) {
- return;
- }
-
- // every TKqpExecuter sends its own profile
- auto profile = std::make_unique<NYql::NDqProto::TDqExecutionStats>();
- profile->Swap(ev->Get()->Record.MutableProfile());
- ExecutionProfiles_.emplace_back(std::move(profile));
- }
-
- void Handle(NKqp::TEvKqpExecuter::TEvExecuterProgress::TPtr& ev, const TActorContext& ctx) {
+ void Handle(NKqp::TEvKqpExecuter::TEvStreamProfile::TPtr& ev, const TActorContext&) {
+ auto req = Request_->GetProtoRequest();
+ if (!NeedReportStats(*req)) {
+ return;
+ }
+
+ // every TKqpExecuter sends its own profile
+ auto profile = std::make_unique<NYql::NDqProto::TDqExecutionStats>();
+ profile->Swap(ev->Get()->Record.MutableProfile());
+ ExecutionProfiles_.emplace_back(std::move(profile));
+ }
+
+ void Handle(NKqp::TEvKqpExecuter::TEvExecuterProgress::TPtr& ev, const TActorContext& ctx) {
ExecuterActorId_ = ActorIdFromProto(ev->Get()->Record.GetExecuterActorId());
LOG_DEBUG_S(ctx, NKikimrServices::RPC_REQUEST, this->SelfId() << " ExecuterActorId: " << ExecuterActorId_);
- }
-
-private:
- void SetTimeoutTimer(TDuration timeout, const TActorContext& ctx) {
+ }
+
+private:
+ void SetTimeoutTimer(TDuration timeout, const TActorContext& ctx) {
LOG_DEBUG_S(ctx, NKikimrServices::RPC_REQUEST, this->SelfId() << " Set stream timeout timer for " << timeout);
-
+
auto *ev = new IEventHandle(this->SelfId(), this->SelfId(), new TEvents::TEvWakeup(EWakeupTag::TimeoutTag));
- TimeoutTimerCookieHolder_.Reset(ISchedulerCookie::Make2Way());
- CreateLongTimer(ctx, timeout, ev, 0, TimeoutTimerCookieHolder_.Get());
- }
-
- void HandleClientLost(const TActorContext& ctx) {
- LOG_WARN_S(ctx, NKikimrServices::RPC_REQUEST, "Client lost, send abort event to executer " << ExecuterActorId_);
-
- if (ExecuterActorId_) {
+ TimeoutTimerCookieHolder_.Reset(ISchedulerCookie::Make2Way());
+ CreateLongTimer(ctx, timeout, ev, 0, TimeoutTimerCookieHolder_.Get());
+ }
+
+ void HandleClientLost(const TActorContext& ctx) {
+ LOG_WARN_S(ctx, NKikimrServices::RPC_REQUEST, "Client lost, send abort event to executer " << ExecuterActorId_);
+
+ if (ExecuterActorId_) {
auto abortEv = TEvKqp::TEvAbortExecution::Aborted("Client lost"); // any status code can be here
-
+
ctx.Send(ExecuterActorId_, abortEv.Release());
- }
-
- // We must try to finish stream otherwise grpc will not free allocated memory
- // If stream already scheduled to be finished (ReplyFinishStream already called)
- // this call do nothing but Die will be called after reply to grpc
- auto issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR,
- "Client should not see this message, if so... may the force be with you");
- ReplyFinishStream(StatusIds::INTERNAL_ERROR, issue, ctx);
- }
-
- void HandleTimeout(const TActorContext& ctx) {
- TInstant now = TAppData::TimeProvider->Now();
- TDuration timeout;
-
- if (InactiveClientTimeout_ && GRpcResponsesSizeQueue_.size() > 0) {
- TDuration processTime = now - LastDataStreamTimestamp_;
- if (processTime >= InactiveClientTimeout_) {
+ }
+
+ // We must try to finish stream otherwise grpc will not free allocated memory
+ // If stream already scheduled to be finished (ReplyFinishStream already called)
+ // this call do nothing but Die will be called after reply to grpc
+ auto issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR,
+ "Client should not see this message, if so... may the force be with you");
+ ReplyFinishStream(StatusIds::INTERNAL_ERROR, issue, ctx);
+ }
+
+ void HandleTimeout(const TActorContext& ctx) {
+ TInstant now = TAppData::TimeProvider->Now();
+ TDuration timeout;
+
+ if (InactiveClientTimeout_ && GRpcResponsesSizeQueue_.size() > 0) {
+ TDuration processTime = now - LastDataStreamTimestamp_;
+ if (processTime >= InactiveClientTimeout_) {
auto message = TStringBuilder() << this->SelfId() << " Client cannot process data in " << processTime
- << " which exceeds client timeout " << InactiveClientTimeout_;
- LOG_WARN_S(ctx, NKikimrServices::RPC_REQUEST, message);
-
- if (ExecuterActorId_) {
- auto timeoutEv = MakeHolder<TEvKqp::TEvAbortExecution>(Ydb::StatusIds::TIMEOUT, "Client timeout");
-
+ << " which exceeds client timeout " << InactiveClientTimeout_;
+ LOG_WARN_S(ctx, NKikimrServices::RPC_REQUEST, message);
+
+ if (ExecuterActorId_) {
+ auto timeoutEv = MakeHolder<TEvKqp::TEvAbortExecution>(Ydb::StatusIds::TIMEOUT, "Client timeout");
+
ctx.Send(ExecuterActorId_, timeoutEv.Release());
- }
-
- auto issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, message);
- return ReplyFinishStream(StatusIds::TIMEOUT, issue, ctx);
- }
- TDuration remain = InactiveClientTimeout_ - processTime;
- timeout = timeout ? Min(timeout, remain) : remain;
- }
-
- if (timeout) {
- SetTimeoutTimer(timeout, ctx);
- }
- }
-
- void ReplyFinishStream(Ydb::StatusIds::StatusCode status, const NYql::TIssue& issue, const TActorContext& ctx) {
- google::protobuf::RepeatedPtrField<TYdbIssueMessageType> issuesMessage;
- NYql::IssueToMessage(issue, issuesMessage.Add());
-
- ReplyFinishStream(status, issuesMessage, ctx);
- }
-
+ }
+
+ auto issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, message);
+ return ReplyFinishStream(StatusIds::TIMEOUT, issue, ctx);
+ }
+ TDuration remain = InactiveClientTimeout_ - processTime;
+ timeout = timeout ? Min(timeout, remain) : remain;
+ }
+
+ if (timeout) {
+ SetTimeoutTimer(timeout, ctx);
+ }
+ }
+
+ void ReplyFinishStream(Ydb::StatusIds::StatusCode status, const NYql::TIssue& issue, const TActorContext& ctx) {
+ google::protobuf::RepeatedPtrField<TYdbIssueMessageType> issuesMessage;
+ NYql::IssueToMessage(issue, issuesMessage.Add());
+
+ ReplyFinishStream(status, issuesMessage, ctx);
+ }
+
void ReplyFinishStream(Ydb::StatusIds::StatusCode status, const NYql::TIssues& issues, const TActorContext& ctx) {
google::protobuf::RepeatedPtrField<TYdbIssueMessageType> issuesMessage;
for (auto& issue : issues) {
@@ -575,11 +575,11 @@ private:
}
void ReplyFinishStream(Ydb::StatusIds::StatusCode status,
- const google::protobuf::RepeatedPtrField<TYdbIssueMessageType>& message, const TActorContext& ctx)
+ const google::protobuf::RepeatedPtrField<TYdbIssueMessageType>& message, const TActorContext& ctx)
{
- LOG_INFO_S(ctx, NKikimrServices::RPC_REQUEST, "Finish grpc stream, status: "
- << Ydb::StatusIds::StatusCode_Name(status));
-
+ LOG_INFO_S(ctx, NKikimrServices::RPC_REQUEST, "Finish grpc stream, status: "
+ << Ydb::StatusIds::StatusCode_Name(status));
+
// Skip sending empty result in case of success status - simplify client logic
if (status != Ydb::StatusIds::SUCCESS) {
TString out;
@@ -588,36 +588,36 @@ private:
response.mutable_issues()->CopyFrom(message);
Y_PROTOBUF_SUPPRESS_NODISCARD response.SerializeToString(&out);
Request_->SendSerializedResult(std::move(out), status);
- } else {
- for (auto& profile : ExecutionProfiles_) {
+ } else {
+ for (auto& profile : ExecutionProfiles_) {
TResponse response;
if (!FillProfile(response, *profile)) {
break;
}
-
- TString out;
+
+ TString out;
Y_PROTOBUF_SUPPRESS_NODISCARD response.SerializeToString(&out);
- Request_->SendSerializedResult(std::move(out), status);
- }
+ Request_->SendSerializedResult(std::move(out), status);
+ }
}
Request_->FinishStream();
this->PassAway();
- }
+ }
private:
std::unique_ptr<TRequestEv> Request_;
- const ui64 RpcBufferSize_;
-
- TDuration InactiveClientTimeout_;
- TQueue<ui64> GRpcResponsesSizeQueue_;
- ui64 GRpcResponsesSize_ = 0;
- TInstant LastDataStreamTimestamp_;
- TMaybe<ui64> WaitOnSeqNo_;
-
- TSchedulerCookieHolder TimeoutTimerCookieHolder_;
-
- TVector<std::unique_ptr<NYql::NDqProto::TDqExecutionStats>> ExecutionProfiles_;
+ const ui64 RpcBufferSize_;
+
+ TDuration InactiveClientTimeout_;
+ TQueue<ui64> GRpcResponsesSizeQueue_;
+ ui64 GRpcResponsesSize_ = 0;
+ TInstant LastDataStreamTimestamp_;
+ TMaybe<ui64> WaitOnSeqNo_;
+
+ TSchedulerCookieHolder TimeoutTimerCookieHolder_;
+
+ TVector<std::unique_ptr<NYql::NDqProto::TDqExecutionStats>> ExecutionProfiles_;
TActorId ExecuterActorId_;
};
@@ -630,7 +630,7 @@ void TGRpcRequestProxy::Handle(TEvExperimentalStreamQueryRequest::TPtr& ev, cons
}
void TGRpcRequestProxy::Handle(TEvStreamExecuteScanQueryRequest::TPtr& ev, const TActorContext& ctx) {
- ui64 rpcBufferSize = GetAppConfig().GetTableServiceConfig().GetResourceManager().GetChannelBufferSize();
+ ui64 rpcBufferSize = GetAppConfig().GetTableServiceConfig().GetResourceManager().GetChannelBufferSize();
ctx.Register(new TStreamExecuteScanQueryRPC<TEvStreamExecuteScanQueryRequest,
Ydb::Table::ExecuteScanQueryPartialResponse>(ev->Release().Release(), rpcBufferSize));
}
diff --git a/ydb/core/kqp/common/kqp_common.h b/ydb/core/kqp/common/kqp_common.h
index 4b295d9914..f9379ad129 100644
--- a/ydb/core/kqp/common/kqp_common.h
+++ b/ydb/core/kqp/common/kqp_common.h
@@ -23,7 +23,7 @@ struct TKqpEvents {
EvCompileRequest,
EvCompileResponse,
EvCompileInvalidateRequest,
- EvAbortExecution = NYql::NDq::TDqEvents::EDqEvents::EvAbortExecution,
+ EvAbortExecution = NYql::NDq::TDqEvents::EDqEvents::EvAbortExecution,
EvInitiateShutdownRequest,
EvInitiateSessionShutdown,
EvContinueShutdown,
@@ -31,91 +31,91 @@ struct TKqpEvents {
EvDataQueryStreamPartAck,
EvRecompileRequest,
};
-
- static_assert (EvCompileInvalidateRequest + 1 == EvAbortExecution);
+
+ static_assert (EvCompileInvalidateRequest + 1 == EvAbortExecution);
};
-
+
struct TKqpExecuterEvents {
enum EKqpExecuterEvents {
EvTxRequest = EventSpaceBegin(TKikimrEvents::ES_KQP) + 100,
EvTxResponse,
- EvStreamData,
- EvStreamProfile,
- EvProgress,
- EvStreamDataAck,
+ EvStreamData,
+ EvStreamProfile,
+ EvProgress,
+ EvStreamDataAck,
EvTableResolveStatus,
EvShardsResolveStatus
};
};
-struct TKqpSnapshotEvents {
- enum EKqpSnapshotEvents {
- EvCreateSnapshotRequest = EventSpaceBegin(TKikimrEvents::ES_KQP) + 150,
- EvCreateSnapshotResponse,
+struct TKqpSnapshotEvents {
+ enum EKqpSnapshotEvents {
+ EvCreateSnapshotRequest = EventSpaceBegin(TKikimrEvents::ES_KQP) + 150,
+ EvCreateSnapshotResponse,
EvDiscardSnapshot
- };
-};
-
-struct TKqpComputeEvents {
- enum EKqpComputeEvents {
- Unused0 = NYql::NDq::TDqComputeEvents::EDqComputeEvents::Unused0,
- EvState = NYql::NDq::TDqComputeEvents::EDqComputeEvents::EvState,
- EvResumeExecution = NYql::NDq::TDqComputeEvents::EDqComputeEvents::EvResumeExecution,
- EvChannelData = NYql::NDq::TDqComputeEvents::EDqComputeEvents::EvChannelData,
- EvScanData,
- EvScanDataAck,
- EvChannelsInfo = NYql::NDq::TDqComputeEvents::EDqComputeEvents::EvChannelsInfo,
- EvChannelDataAck = NYql::NDq::TDqComputeEvents::EDqComputeEvents::EvChannelDataAck,
- EvScanError,
- EvKillScanTablet = NYql::NDq::TDqComputeEvents::EDqComputeEvents::Unused1,
- EvRetryChannelData = NYql::NDq::TDqComputeEvents::EDqComputeEvents::EvRetryChannelData,
- EvRetryChannelDataAck = NYql::NDq::TDqComputeEvents::EDqComputeEvents::EvRetryChannelDataAck,
- EvScanInitActor,
- EvRemoteScanData,
- EvRemoteScanDataAck,
- };
-
- static_assert(Unused0 == EventSpaceBegin(TKikimrEvents::ES_KQP) + 200);
- static_assert(EvState == Unused0 + 1);
- static_assert(EvResumeExecution == EvState + 1);
- static_assert(EvChannelData == EvResumeExecution + 1);
- static_assert(EvScanData == EvChannelData + 1);
- static_assert(EvScanDataAck == EvScanData + 1);
- static_assert(EvChannelsInfo == EvScanDataAck + 1);
- static_assert(EvChannelDataAck == EvChannelsInfo + 1);
- static_assert(EvScanError == EvChannelDataAck + 1);
- static_assert(EvKillScanTablet == EvScanError + 1);
- static_assert(EvRetryChannelData == EvKillScanTablet + 1);
- static_assert(EvRetryChannelDataAck == EvRetryChannelData + 1);
- static_assert(EvScanInitActor == EvRetryChannelDataAck + 1);
-};
-
-struct TKqpResourceManagerEvents {
- enum EKqpResourceManagerEvents {
- EvStartComputeTasks = EventSpaceBegin(TKikimrEvents::ES_KQP) + 300,
- EvStartComputeTasksFailure,
- EvStartedComputeTasks,
- EvFinishComputeTask,
- EvCancelComputeTasks,
- Unused0,
- EvStartDsComputeTasks,
- Unused3, // EvEstimateResourcesRequest,
- Unused4, // EvEstimateResourcesResponse,
- Unused1, // EvAllocateTaskResourcesRequest, extra resources allocation
- Unused2, // EvAllocateTaskResourcesResponse
- };
-};
-
-struct TKqpSpillingEvents {
- enum EKqpSpillingEvents {
- EvWrite = EventSpaceBegin(TKikimrEvents::ES_KQP) + 400,
- EvWriteResult,
- EvRead,
- EvReadResult,
- EvError,
- };
-};
-
-} // namespace NKqp
+ };
+};
+
+struct TKqpComputeEvents {
+ enum EKqpComputeEvents {
+ Unused0 = NYql::NDq::TDqComputeEvents::EDqComputeEvents::Unused0,
+ EvState = NYql::NDq::TDqComputeEvents::EDqComputeEvents::EvState,
+ EvResumeExecution = NYql::NDq::TDqComputeEvents::EDqComputeEvents::EvResumeExecution,
+ EvChannelData = NYql::NDq::TDqComputeEvents::EDqComputeEvents::EvChannelData,
+ EvScanData,
+ EvScanDataAck,
+ EvChannelsInfo = NYql::NDq::TDqComputeEvents::EDqComputeEvents::EvChannelsInfo,
+ EvChannelDataAck = NYql::NDq::TDqComputeEvents::EDqComputeEvents::EvChannelDataAck,
+ EvScanError,
+ EvKillScanTablet = NYql::NDq::TDqComputeEvents::EDqComputeEvents::Unused1,
+ EvRetryChannelData = NYql::NDq::TDqComputeEvents::EDqComputeEvents::EvRetryChannelData,
+ EvRetryChannelDataAck = NYql::NDq::TDqComputeEvents::EDqComputeEvents::EvRetryChannelDataAck,
+ EvScanInitActor,
+ EvRemoteScanData,
+ EvRemoteScanDataAck,
+ };
+
+ static_assert(Unused0 == EventSpaceBegin(TKikimrEvents::ES_KQP) + 200);
+ static_assert(EvState == Unused0 + 1);
+ static_assert(EvResumeExecution == EvState + 1);
+ static_assert(EvChannelData == EvResumeExecution + 1);
+ static_assert(EvScanData == EvChannelData + 1);
+ static_assert(EvScanDataAck == EvScanData + 1);
+ static_assert(EvChannelsInfo == EvScanDataAck + 1);
+ static_assert(EvChannelDataAck == EvChannelsInfo + 1);
+ static_assert(EvScanError == EvChannelDataAck + 1);
+ static_assert(EvKillScanTablet == EvScanError + 1);
+ static_assert(EvRetryChannelData == EvKillScanTablet + 1);
+ static_assert(EvRetryChannelDataAck == EvRetryChannelData + 1);
+ static_assert(EvScanInitActor == EvRetryChannelDataAck + 1);
+};
+
+struct TKqpResourceManagerEvents {
+ enum EKqpResourceManagerEvents {
+ EvStartComputeTasks = EventSpaceBegin(TKikimrEvents::ES_KQP) + 300,
+ EvStartComputeTasksFailure,
+ EvStartedComputeTasks,
+ EvFinishComputeTask,
+ EvCancelComputeTasks,
+ Unused0,
+ EvStartDsComputeTasks,
+ Unused3, // EvEstimateResourcesRequest,
+ Unused4, // EvEstimateResourcesResponse,
+ Unused1, // EvAllocateTaskResourcesRequest, extra resources allocation
+ Unused2, // EvAllocateTaskResourcesResponse
+ };
+};
+
+struct TKqpSpillingEvents {
+ enum EKqpSpillingEvents {
+ EvWrite = EventSpaceBegin(TKikimrEvents::ES_KQP) + 400,
+ EvWriteResult,
+ EvRead,
+ EvReadResult,
+ EvError,
+ };
+};
+
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/common/kqp_gateway.h b/ydb/core/kqp/common/kqp_gateway.h
index e3797008ba..1e06beca39 100644
--- a/ydb/core/kqp/common/kqp_gateway.h
+++ b/ydb/core/kqp/common/kqp_gateway.h
@@ -106,19 +106,19 @@ public:
struct TExecPhysicalRequest : private TMoveOnly {
TVector<TPhysicalTxData> Transactions;
- TVector<NYql::NDq::TMkqlValueRef> Locks;
- bool ValidateLocks = false;
- bool EraseLocks = false;
+ TVector<NYql::NDq::TMkqlValueRef> Locks;
+ bool ValidateLocks = false;
+ bool EraseLocks = false;
TMaybe<ui64> AcquireLocksTxId;
- TDuration Timeout;
- TMaybe<TDuration> CancelAfter;
- ui32 MaxComputeActors = 10'000;
- ui32 MaxAffectedShards = 0;
- ui64 TotalReadSizeLimitBytes = 0;
- ui64 MkqlMemoryLimit = 0; // old engine compatibility
- ui64 PerShardKeysSizeLimitBytes = 0;
- NYql::NDqProto::EDqStatsMode StatsMode = NYql::NDqProto::DQ_STATS_MODE_NONE;
- bool DisableLlvmForUdfStages = false;
+ TDuration Timeout;
+ TMaybe<TDuration> CancelAfter;
+ ui32 MaxComputeActors = 10'000;
+ ui32 MaxAffectedShards = 0;
+ ui64 TotalReadSizeLimitBytes = 0;
+ ui64 MkqlMemoryLimit = 0; // old engine compatibility
+ ui64 PerShardKeysSizeLimitBytes = 0;
+ NYql::NDqProto::EDqStatsMode StatsMode = NYql::NDqProto::DQ_STATS_MODE_NONE;
+ bool DisableLlvmForUdfStages = false;
bool LlvmEnabled = true;
TKqpSnapshot Snapshot = TKqpSnapshot();
NKikimrKqp::EIsolationLevel IsolationLevel = NKikimrKqp::ISOLATION_LEVEL_UNDEFINED;
@@ -130,7 +130,7 @@ public:
};
struct TAstQuerySettings {
- NYql::NDqProto::EDqStatsMode StatsMode = NYql::NDqProto::DQ_STATS_MODE_NONE;
+ NYql::NDqProto::EDqStatsMode StatsMode = NYql::NDqProto::DQ_STATS_MODE_NONE;
};
public:
@@ -152,7 +152,7 @@ public:
virtual NThreading::TFuture<TKqpSnapshotHandle> AcquireMvccSnapshot(TDuration queryTimeout) = 0;
/* Physical */
- virtual NThreading::TFuture<TExecPhysicalResult> ExecutePhysical(TExecPhysicalRequest&& request,
+ virtual NThreading::TFuture<TExecPhysicalResult> ExecutePhysical(TExecPhysicalRequest&& request,
const NActors::TActorId& target) = 0;
virtual NThreading::TFuture<TExecPhysicalResult> ExecuteScanQuery(TExecPhysicalRequest&& request,
@@ -180,11 +180,11 @@ public:
virtual NThreading::TFuture<TQueryResult> StreamExecScanQueryAst(const TString& cluster, const TString& query,
TKqpParamsMap&& params, const TAstQuerySettings& settings, const NActors::TActorId& target) = 0;
-public:
- virtual TInstant GetCurrentTime() const = 0;
+public:
+ virtual TInstant GetCurrentTime() const = 0;
};
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
template<>
diff --git a/ydb/core/kqp/common/kqp_resolve.cpp b/ydb/core/kqp/common/kqp_resolve.cpp
index 9ba5e27c75..b4abafc86c 100644
--- a/ydb/core/kqp/common/kqp_resolve.cpp
+++ b/ydb/core/kqp/common/kqp_resolve.cpp
@@ -2,13 +2,13 @@
#include <ydb/core/kqp/provider/yql_kikimr_gateway.h>
-// #define DBG_TRACE
-
-#ifdef DBG_TRACE
+// #define DBG_TRACE
+
+#ifdef DBG_TRACE
#include <ydb/core/base/appdata.h>
#include <ydb/core/tx/datashard/range_ops.h>
-#endif
-
+#endif
+
namespace NKikimr {
namespace NKqp {
@@ -46,144 +46,144 @@ TTableId MakeTableId(const NKqpProto::TKqpPhyTable& table) {
return tableId;
}
-TVector<TPartitionWithRange> GetKeyRangePartitions(const TTableRange& range,
- const TVector<TKeyDesc::TPartitionInfo>& partitions, const TVector<NUdf::TDataTypeId>& keyColumnTypes)
-{
- auto it = std::lower_bound(partitions.begin(), partitions.end(), true,
- [&range, &keyColumnTypes](const auto& partition, bool) {
- const int cmp = CompareBorders<true, false>(partition.Range->EndKeyPrefix.GetCells(), range.From,
- partition.Range->IsInclusive || partition.Range->IsPoint, range.InclusiveFrom || range.Point,
- keyColumnTypes);
-
- return (cmp < 0);
- });
-
- MKQL_ENSURE_S(it != partitions.end());
- auto affectedPartitions = std::distance(it, partitions.end());
-
-#ifdef DBG_TRACE
- auto& typeRegistry = *AppData()->TypeRegistry;
- Cerr << (TStringBuilder() << "-- read range: " << DebugPrintRange(keyColumnTypes, range, typeRegistry)
- << ", affected partitions: " << affectedPartitions << Endl);
-#endif
-
- TVector<TPartitionWithRange> rangePartitions;
- rangePartitions.reserve(affectedPartitions);
- do {
-#ifdef DBG_TRACE
- Cerr << (TStringBuilder() << "-- add partition: "
- << DebugPrintPartitionInfo(*it, keyColumnTypes, typeRegistry)
- << " (total: " << partitions.size() << ")" << Endl);
-#endif
-
- if (range.Point) {
- TPartitionWithRange ret(it);
- if (!it->Range->IsPoint) {
- ret.PointOrRange = TSerializedCellVec(TSerializedCellVec::Serialize(range.From));
- } else {
- ret.FullRange.emplace(TSerializedTableRange(range));
- }
- rangePartitions.emplace_back(std::move(ret));
-
-#ifdef DBG_TRACE
-{
- auto& x = rangePartitions.back();
- TStringBuilder sb;
- sb << "-- added point: ";
- if (x.FullRange.has_value()) {
- sb << "FULL " << DebugPrintRange(keyColumnTypes, x.FullRange->ToTableRange(), typeRegistry);
- } else {
- sb << "point " << DebugPrintPoint(keyColumnTypes, std::get<TSerializedCellVec>(x.PointOrRange).GetCells(), typeRegistry);
- }
- sb << Endl;
- Cerr << sb;
-}
-#endif
- break;
- }
-
- TConstArrayRef<TCell> fromValues;
- TConstArrayRef<TCell> toValues;
- bool inclusiveFrom;
- bool inclusiveTo;
- int fullRange = 0;
-
- if (rangePartitions.empty()) {
- fromValues = range.From;
- inclusiveFrom = range.InclusiveFrom;
- if (it == partitions.begin() && !fromValues.empty() && fromValues.begin()->IsNull()) {
- fullRange = 1;
- }
- } else {
- fromValues = rangePartitions.back().PartitionInfo->Range->EndKeyPrefix.GetCells();
- inclusiveFrom = !rangePartitions.back().PartitionInfo->Range->IsInclusive;
- fullRange = 1;
- }
-
- const int prevCmp = CompareBorders<true, true>(it->Range->EndKeyPrefix.GetCells(), range.To,
- it->Range->IsPoint || it->Range->IsInclusive, range.InclusiveTo, keyColumnTypes);
-
- if (prevCmp > 0) {
- toValues = range.To;
- inclusiveTo = range.InclusiveTo;
- } else {
- toValues = it->Range->EndKeyPrefix.GetCells();
- inclusiveTo = it->Range->IsInclusive;
- fullRange += 1;
- }
-
- bool point = false;
- if (inclusiveFrom && inclusiveTo && fromValues.size() == keyColumnTypes.size()) {
- if (toValues.empty()) {
- point = !fromValues.back().IsNull();
- } else if (fromValues.size() == toValues.size()) {
- point = CompareTypedCellVectors(fromValues.data(), toValues.data(), keyColumnTypes.data(), keyColumnTypes.size()) == 0;
- }
- }
-
- rangePartitions.emplace_back(TPartitionWithRange(it));
-
- if (point) {
- if (fullRange == 2) {
- rangePartitions.back().FullRange.emplace(TSerializedTableRange(fromValues, true, fromValues, true));
- rangePartitions.back().FullRange->Point = true;
- } else {
- rangePartitions.back().PointOrRange = TSerializedCellVec(TSerializedCellVec::Serialize(fromValues));
- }
- } else {
- auto r = TTableRange(fromValues, inclusiveFrom, toValues, inclusiveTo);
- if (fullRange == 2) {
- rangePartitions.back().FullRange.emplace(TSerializedTableRange(r));
- } else {
- rangePartitions.back().PointOrRange = TSerializedTableRange(r);
- }
- }
-
-#ifdef DBG_TRACE
-{
- auto& x = rangePartitions.back();
- TStringBuilder sb;
- sb << "-- added range: ";
- if (x.FullRange.has_value()) {
- sb << "FULL " << DebugPrintRange(keyColumnTypes, x.FullRange->ToTableRange(), typeRegistry);
- } else if (std::holds_alternative<TSerializedCellVec>(x.PointOrRange)) {
- sb << "point " << DebugPrintPoint(keyColumnTypes, std::get<TSerializedCellVec>(x.PointOrRange).GetCells(), typeRegistry);
- } else {
- sb << DebugPrintRange(keyColumnTypes, std::get<TSerializedTableRange>(x.PointOrRange).ToTableRange(), typeRegistry);
- }
- sb << Endl;
- Cerr << sb;
-}
-#endif
-
- if (prevCmp >= 0) {
- break;
- }
- } while (++it != partitions.end());
-
- return rangePartitions;
-}
-
-#undef DBG_TRACE
-} // namespace NKqp
+TVector<TPartitionWithRange> GetKeyRangePartitions(const TTableRange& range,
+ const TVector<TKeyDesc::TPartitionInfo>& partitions, const TVector<NUdf::TDataTypeId>& keyColumnTypes)
+{
+ auto it = std::lower_bound(partitions.begin(), partitions.end(), true,
+ [&range, &keyColumnTypes](const auto& partition, bool) {
+ const int cmp = CompareBorders<true, false>(partition.Range->EndKeyPrefix.GetCells(), range.From,
+ partition.Range->IsInclusive || partition.Range->IsPoint, range.InclusiveFrom || range.Point,
+ keyColumnTypes);
+
+ return (cmp < 0);
+ });
+
+ MKQL_ENSURE_S(it != partitions.end());
+ auto affectedPartitions = std::distance(it, partitions.end());
+
+#ifdef DBG_TRACE
+ auto& typeRegistry = *AppData()->TypeRegistry;
+ Cerr << (TStringBuilder() << "-- read range: " << DebugPrintRange(keyColumnTypes, range, typeRegistry)
+ << ", affected partitions: " << affectedPartitions << Endl);
+#endif
+
+ TVector<TPartitionWithRange> rangePartitions;
+ rangePartitions.reserve(affectedPartitions);
+ do {
+#ifdef DBG_TRACE
+ Cerr << (TStringBuilder() << "-- add partition: "
+ << DebugPrintPartitionInfo(*it, keyColumnTypes, typeRegistry)
+ << " (total: " << partitions.size() << ")" << Endl);
+#endif
+
+ if (range.Point) {
+ TPartitionWithRange ret(it);
+ if (!it->Range->IsPoint) {
+ ret.PointOrRange = TSerializedCellVec(TSerializedCellVec::Serialize(range.From));
+ } else {
+ ret.FullRange.emplace(TSerializedTableRange(range));
+ }
+ rangePartitions.emplace_back(std::move(ret));
+
+#ifdef DBG_TRACE
+{
+ auto& x = rangePartitions.back();
+ TStringBuilder sb;
+ sb << "-- added point: ";
+ if (x.FullRange.has_value()) {
+ sb << "FULL " << DebugPrintRange(keyColumnTypes, x.FullRange->ToTableRange(), typeRegistry);
+ } else {
+ sb << "point " << DebugPrintPoint(keyColumnTypes, std::get<TSerializedCellVec>(x.PointOrRange).GetCells(), typeRegistry);
+ }
+ sb << Endl;
+ Cerr << sb;
+}
+#endif
+ break;
+ }
+
+ TConstArrayRef<TCell> fromValues;
+ TConstArrayRef<TCell> toValues;
+ bool inclusiveFrom;
+ bool inclusiveTo;
+ int fullRange = 0;
+
+ if (rangePartitions.empty()) {
+ fromValues = range.From;
+ inclusiveFrom = range.InclusiveFrom;
+ if (it == partitions.begin() && !fromValues.empty() && fromValues.begin()->IsNull()) {
+ fullRange = 1;
+ }
+ } else {
+ fromValues = rangePartitions.back().PartitionInfo->Range->EndKeyPrefix.GetCells();
+ inclusiveFrom = !rangePartitions.back().PartitionInfo->Range->IsInclusive;
+ fullRange = 1;
+ }
+
+ const int prevCmp = CompareBorders<true, true>(it->Range->EndKeyPrefix.GetCells(), range.To,
+ it->Range->IsPoint || it->Range->IsInclusive, range.InclusiveTo, keyColumnTypes);
+
+ if (prevCmp > 0) {
+ toValues = range.To;
+ inclusiveTo = range.InclusiveTo;
+ } else {
+ toValues = it->Range->EndKeyPrefix.GetCells();
+ inclusiveTo = it->Range->IsInclusive;
+ fullRange += 1;
+ }
+
+ bool point = false;
+ if (inclusiveFrom && inclusiveTo && fromValues.size() == keyColumnTypes.size()) {
+ if (toValues.empty()) {
+ point = !fromValues.back().IsNull();
+ } else if (fromValues.size() == toValues.size()) {
+ point = CompareTypedCellVectors(fromValues.data(), toValues.data(), keyColumnTypes.data(), keyColumnTypes.size()) == 0;
+ }
+ }
+
+ rangePartitions.emplace_back(TPartitionWithRange(it));
+
+ if (point) {
+ if (fullRange == 2) {
+ rangePartitions.back().FullRange.emplace(TSerializedTableRange(fromValues, true, fromValues, true));
+ rangePartitions.back().FullRange->Point = true;
+ } else {
+ rangePartitions.back().PointOrRange = TSerializedCellVec(TSerializedCellVec::Serialize(fromValues));
+ }
+ } else {
+ auto r = TTableRange(fromValues, inclusiveFrom, toValues, inclusiveTo);
+ if (fullRange == 2) {
+ rangePartitions.back().FullRange.emplace(TSerializedTableRange(r));
+ } else {
+ rangePartitions.back().PointOrRange = TSerializedTableRange(r);
+ }
+ }
+
+#ifdef DBG_TRACE
+{
+ auto& x = rangePartitions.back();
+ TStringBuilder sb;
+ sb << "-- added range: ";
+ if (x.FullRange.has_value()) {
+ sb << "FULL " << DebugPrintRange(keyColumnTypes, x.FullRange->ToTableRange(), typeRegistry);
+ } else if (std::holds_alternative<TSerializedCellVec>(x.PointOrRange)) {
+ sb << "point " << DebugPrintPoint(keyColumnTypes, std::get<TSerializedCellVec>(x.PointOrRange).GetCells(), typeRegistry);
+ } else {
+ sb << DebugPrintRange(keyColumnTypes, std::get<TSerializedTableRange>(x.PointOrRange).ToTableRange(), typeRegistry);
+ }
+ sb << Endl;
+ Cerr << sb;
+}
+#endif
+
+ if (prevCmp >= 0) {
+ break;
+ }
+ } while (++it != partitions.end());
+
+ return rangePartitions;
+}
+
+#undef DBG_TRACE
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/common/kqp_resolve.h b/ydb/core/kqp/common/kqp_resolve.h
index 2fbe36f181..517d1bc451 100644
--- a/ydb/core/kqp/common/kqp_resolve.h
+++ b/ydb/core/kqp/common/kqp_resolve.h
@@ -30,7 +30,7 @@ public:
TString Path;
TMap<TString, TColumn> Columns;
TVector<TString> KeyColumns;
- TVector<NUdf::TDataTypeId> KeyColumnTypes;
+ TVector<NUdf::TDataTypeId> KeyColumnTypes;
ETableKind TableKind = ETableKind::Unknown;
};
@@ -38,11 +38,11 @@ public:
return TablesById.FindPtr(id);
}
- const TTable* FindTablePtr(const TTableId& id) const {
- return TablesById.FindPtr(id);
- }
-
- TTable& GetOrAddTable(const TTableId& id, const TStringBuf path) {
+ const TTable* FindTablePtr(const TTableId& id) const {
+ return TablesById.FindPtr(id);
+ }
+
+ TTable& GetOrAddTable(const TTableId& id, const TStringBuf path) {
auto& table = TablesById[id];
if (table.Path.empty()) {
@@ -76,8 +76,8 @@ TVector<TCell> MakeKeyCells(const NKikimr::NUdf::TUnboxedValue& value, const TVe
const TVector<ui32>& keyColumnIndices, const NMiniKQL::TTypeEnvironment& typeEnv, bool copyValues);
template<typename TList, typename TRangeFunc>
-size_t FindKeyPartitionIndex(const TVector<TCell>& key, const TList& partitions,
- const TVector<NUdf::TDataTypeId>& keyColumnTypes, const TRangeFunc& rangeFunc)
+size_t FindKeyPartitionIndex(const TVector<TCell>& key, const TList& partitions,
+ const TVector<NUdf::TDataTypeId>& keyColumnTypes, const TRangeFunc& rangeFunc)
{
auto it = std::lower_bound(partitions.begin(), partitions.end(), key,
[&keyColumnTypes, &rangeFunc](const auto& partition, const auto& key) {
@@ -90,32 +90,32 @@ size_t FindKeyPartitionIndex(const TVector<TCell>& key, const TList& partitions,
MKQL_ENSURE_S(it != partitions.end());
- return std::distance(partitions.begin(), it);
+ return std::distance(partitions.begin(), it);
}
-template<typename TList, typename TRangeFunc>
-size_t FindKeyPartitionIndex(const NMiniKQL::TTypeEnvironment& typeEnv, const NKikimr::NUdf::TUnboxedValue& value,
- const TList& partitions, const TVector<NUdf::TDataTypeId>& keyColumnTypes, const TVector<ui32>& keyColumnIndices,
- const TRangeFunc& rangeFunc)
+template<typename TList, typename TRangeFunc>
+size_t FindKeyPartitionIndex(const NMiniKQL::TTypeEnvironment& typeEnv, const NKikimr::NUdf::TUnboxedValue& value,
+ const TList& partitions, const TVector<NUdf::TDataTypeId>& keyColumnTypes, const TVector<ui32>& keyColumnIndices,
+ const TRangeFunc& rangeFunc)
{
- auto key = MakeKeyCells(value, keyColumnTypes, keyColumnIndices, typeEnv, /* copyValues */ true);
+ auto key = MakeKeyCells(value, keyColumnTypes, keyColumnIndices, typeEnv, /* copyValues */ true);
- return FindKeyPartitionIndex(key, partitions, keyColumnTypes, rangeFunc);
-}
+ return FindKeyPartitionIndex(key, partitions, keyColumnTypes, rangeFunc);
+}
+
+using TSerializedPointOrRange = std::variant<TSerializedCellVec, TSerializedTableRange>;
-using TSerializedPointOrRange = std::variant<TSerializedCellVec, TSerializedTableRange>;
-
-struct TPartitionWithRange {
- const TKeyDesc::TPartitionInfo* PartitionInfo;
- TSerializedPointOrRange PointOrRange;
- std::optional<TSerializedTableRange> FullRange;
-
- TPartitionWithRange(const TKeyDesc::TPartitionInfo* partitionInfo)
- : PartitionInfo(partitionInfo) {}
-};
+struct TPartitionWithRange {
+ const TKeyDesc::TPartitionInfo* PartitionInfo;
+ TSerializedPointOrRange PointOrRange;
+ std::optional<TSerializedTableRange> FullRange;
+
+ TPartitionWithRange(const TKeyDesc::TPartitionInfo* partitionInfo)
+ : PartitionInfo(partitionInfo) {}
+};
-TVector<TPartitionWithRange> GetKeyRangePartitions(const TTableRange& range,
- const TVector<TKeyDesc::TPartitionInfo>& partitions, const TVector<NUdf::TDataTypeId>& keyColumnTypes);
+TVector<TPartitionWithRange> GetKeyRangePartitions(const TTableRange& range,
+ const TVector<TKeyDesc::TPartitionInfo>& partitions, const TVector<NUdf::TDataTypeId>& keyColumnTypes);
template<typename TList, typename TRangeFunc>
void SortPartitions(TList& partitions, const TVector<NUdf::TDataTypeId>& keyColumnTypes, const TRangeFunc& rangeFunc) {
@@ -135,5 +135,5 @@ void SortPartitions(TList& partitions, const TVector<NUdf::TDataTypeId>& keyColu
TTableId MakeTableId(const NYql::NNodes::TKqpTable& node);
TTableId MakeTableId(const NKqpProto::TKqpPhyTable& table);
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/common/kqp_ru_calc.cpp b/ydb/core/kqp/common/kqp_ru_calc.cpp
index e3e42636e5..a58f57f8e2 100644
--- a/ydb/core/kqp/common/kqp_ru_calc.cpp
+++ b/ydb/core/kqp/common/kqp_ru_calc.cpp
@@ -26,9 +26,9 @@ ui64 CalcReadIORu(const TTableStat& stat) {
class TIoReadStat: public TTableStat {
public:
- void Add(const NYql::NDqProto::TDqTableStats& tableAggr) {
- Rows += tableAggr.GetReadRows();
- Bytes += tableAggr.GetReadBytes();
+ void Add(const NYql::NDqProto::TDqTableStats& tableAggr) {
+ Rows += tableAggr.GetReadRows();
+ Bytes += tableAggr.GetReadBytes();
}
ui64 CalcRu() const {
@@ -38,10 +38,10 @@ public:
class TIoWriteStat: public TTableStat {
public:
- void Add(const NYql::NDqProto::TDqTableStats& tableAggr) {
- Rows += tableAggr.GetWriteRows();
- Rows += tableAggr.GetEraseRows();
- Bytes += tableAggr.GetWriteBytes();
+ void Add(const NYql::NDqProto::TDqTableStats& tableAggr) {
+ Rows += tableAggr.GetWriteRows();
+ Rows += tableAggr.GetEraseRows();
+ Bytes += tableAggr.GetWriteBytes();
}
ui64 CalcRu() const {
@@ -65,9 +65,9 @@ ui64 CalcRequestUnit(const NKqpProto::TKqpStatsQuery& stats) {
TIoWriteStat totalWriteStat;
for (const auto& exec : stats.GetExecutions()) {
- totalCpuTime += TDuration::MicroSeconds(exec.GetCpuTimeUs());
+ totalCpuTime += TDuration::MicroSeconds(exec.GetCpuTimeUs());
- for (auto& table : exec.GetTables()) {
+ for (auto& table : exec.GetTables()) {
totalReadStat.Add(table);
}
}
@@ -84,9 +84,9 @@ ui64 CalcRequestUnit(const NKqpProto::TKqpStatsQuery& stats) {
}
ui64 CalcRequestUnit(const TProgressStatEntry& stats) {
- auto ioRu = CalcReadIORu(stats.ReadIOStat);
+ auto ioRu = CalcReadIORu(stats.ReadIOStat);
- return std::max(std::max(CpuTimeToUnit(stats.ComputeTime), ioRu), (ui64)1);
+ return std::max(std::max(CpuTimeToUnit(stats.ComputeTime), ioRu), (ui64)1);
}
} // namespace NRuCalc
diff --git a/ydb/core/kqp/common/kqp_transform.cpp b/ydb/core/kqp/common/kqp_transform.cpp
index 0054b0acd3..3653c78d67 100644
--- a/ydb/core/kqp/common/kqp_transform.cpp
+++ b/ydb/core/kqp/common/kqp_transform.cpp
@@ -30,5 +30,5 @@ void TLogExprTransformer::LogExpr(const TExprNode& input, TExprContext& ctx, con
YQL_CVLOG(level, component) << description << ":\n" << KqpExprToPrettyString(input, ctx);
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/common/kqp_transform.h b/ydb/core/kqp/common/kqp_transform.h
index 2e59473b89..457c424801 100644
--- a/ydb/core/kqp/common/kqp_transform.h
+++ b/ydb/core/kqp/common/kqp_transform.h
@@ -92,67 +92,67 @@ struct TDeferredEffect {
: PhysicalTx(physicalTx) {}
};
-class TKqpTransactionContext;
-
-struct TDeferredEffects {
-public:
- bool Empty() const {
- return DeferredEffects.empty();
- }
-
- std::optional<TKqpTransactionInfo::EEngine> GetEngine() const {
- return Engine;
- }
-
- ui64 Size() const {
- return DeferredEffects.size();
- }
-
- decltype(auto) begin() const {
- return DeferredEffects.begin();
- }
-
- decltype(auto) end() const {
- return DeferredEffects.end();
- }
-
-private:
- [[nodiscard]]
- bool Add(const NYql::NNodes::TExprBase& node) {
- if (Engine.has_value() && *Engine != TKqpTransactionInfo::EEngine::OldEngine) {
- return false;
- }
- Engine.emplace(TKqpTransactionInfo::EEngine::OldEngine);
- DeferredEffects.emplace_back(node);
- return true;
- }
-
- [[nodiscard]]
- bool Add(const NKqpProto::TKqpPhyTx& physicalTx, TParamValueMap&& params) {
- if (Engine.has_value() && *Engine != TKqpTransactionInfo::EEngine::NewEngine) {
- return false;
- }
- Engine.emplace(TKqpTransactionInfo::EEngine::NewEngine);
- DeferredEffects.emplace_back(physicalTx);
- DeferredEffects.back().Params = std::move(params);
- return true;
- }
-
- void Clear() {
- DeferredEffects.clear();
- Engine.reset();
- }
-
-private:
- TVector<TDeferredEffect> DeferredEffects;
- std::optional<TKqpTransactionInfo::EEngine> Engine;
-
- friend class TKqpTransactionContext;
-};
-
+class TKqpTransactionContext;
+
+struct TDeferredEffects {
+public:
+ bool Empty() const {
+ return DeferredEffects.empty();
+ }
+
+ std::optional<TKqpTransactionInfo::EEngine> GetEngine() const {
+ return Engine;
+ }
+
+ ui64 Size() const {
+ return DeferredEffects.size();
+ }
+
+ decltype(auto) begin() const {
+ return DeferredEffects.begin();
+ }
+
+ decltype(auto) end() const {
+ return DeferredEffects.end();
+ }
+
+private:
+ [[nodiscard]]
+ bool Add(const NYql::NNodes::TExprBase& node) {
+ if (Engine.has_value() && *Engine != TKqpTransactionInfo::EEngine::OldEngine) {
+ return false;
+ }
+ Engine.emplace(TKqpTransactionInfo::EEngine::OldEngine);
+ DeferredEffects.emplace_back(node);
+ return true;
+ }
+
+ [[nodiscard]]
+ bool Add(const NKqpProto::TKqpPhyTx& physicalTx, TParamValueMap&& params) {
+ if (Engine.has_value() && *Engine != TKqpTransactionInfo::EEngine::NewEngine) {
+ return false;
+ }
+ Engine.emplace(TKqpTransactionInfo::EEngine::NewEngine);
+ DeferredEffects.emplace_back(physicalTx);
+ DeferredEffects.back().Params = std::move(params);
+ return true;
+ }
+
+ void Clear() {
+ DeferredEffects.clear();
+ Engine.reset();
+ }
+
+private:
+ TVector<TDeferredEffect> DeferredEffects;
+ std::optional<TKqpTransactionInfo::EEngine> Engine;
+
+ friend class TKqpTransactionContext;
+};
+
class TKqpTransactionContext : public NYql::TKikimrTransactionContextBase {
public:
- explicit TKqpTransactionContext(bool implicit)
+ explicit TKqpTransactionContext(bool implicit)
: Implicit(implicit)
, ParamsState(MakeIntrusive<TParamsState>())
{
@@ -164,26 +164,26 @@ public:
return TStringBuilder() << ParamNamePrefix << (++ParamsState->LastIndex);
}
- void ClearDeferredEffects() {
- DeferredEffects.Clear();
- }
-
- [[nodiscard]]
- bool AddDeferredEffect(const NYql::NNodes::TExprBase& node) {
- return DeferredEffects.Add(node);
- }
-
- [[nodiscard]]
- bool AddDeferredEffect(const NKqpProto::TKqpPhyTx& physicalTx, TParamValueMap&& params) {
- return DeferredEffects.Add(physicalTx, std::move(params));
- }
-
+ void ClearDeferredEffects() {
+ DeferredEffects.Clear();
+ }
+
+ [[nodiscard]]
+ bool AddDeferredEffect(const NYql::NNodes::TExprBase& node) {
+ return DeferredEffects.Add(node);
+ }
+
+ [[nodiscard]]
+ bool AddDeferredEffect(const NKqpProto::TKqpPhyTx& physicalTx, TParamValueMap&& params) {
+ return DeferredEffects.Add(physicalTx, std::move(params));
+ }
+
const IKqpGateway::TKqpSnapshot& GetSnapshot() const {
return SnapshotHandle.Snapshot;
}
void Finish() final {
- YQL_ENSURE(DeferredEffects.Empty());
+ YQL_ENSURE(DeferredEffects.Empty());
YQL_ENSURE(!Locks.HasLocks());
FinishTime = TInstant::Now();
@@ -211,28 +211,28 @@ public:
void Reset() {
TKikimrTransactionContextBase::Reset();
- DeferredEffects.Clear();
+ DeferredEffects.Clear();
ParamsState = MakeIntrusive<TParamsState>();
SnapshotHandle.Snapshot = IKqpGateway::TKqpSnapshot::InvalidSnapshot;
- ForceNewEngineSettings = {};
+ ForceNewEngineSettings = {};
+ }
+
+ TKqpTransactionInfo GetInfo() const;
+
+ void ForceOldEngine() {
+ YQL_ENSURE(DeferredEffects.Empty());
+ YQL_ENSURE(!ForceNewEngineSettings.ForcedNewEngine.has_value());
+ ForceNewEngineSettings.ForcedNewEngine = false;
+ }
+
+ void ForceNewEngine(ui32 percent, ui32 level) {
+ YQL_ENSURE(DeferredEffects.Empty());
+ YQL_ENSURE(!ForceNewEngineSettings.ForcedNewEngine.has_value());
+ ForceNewEngineSettings.ForcedNewEngine = true;
+ ForceNewEngineSettings.ForceNewEnginePercent = percent;
+ ForceNewEngineSettings.ForceNewEngineLevel = level;
}
- TKqpTransactionInfo GetInfo() const;
-
- void ForceOldEngine() {
- YQL_ENSURE(DeferredEffects.Empty());
- YQL_ENSURE(!ForceNewEngineSettings.ForcedNewEngine.has_value());
- ForceNewEngineSettings.ForcedNewEngine = false;
- }
-
- void ForceNewEngine(ui32 percent, ui32 level) {
- YQL_ENSURE(DeferredEffects.Empty());
- YQL_ENSURE(!ForceNewEngineSettings.ForcedNewEngine.has_value());
- ForceNewEngineSettings.ForcedNewEngine = true;
- ForceNewEngineSettings.ForceNewEnginePercent = percent;
- ForceNewEngineSettings.ForceNewEngineLevel = level;
- }
-
public:
struct TParamsState : public TThrRefBase {
TParamValueMap Values;
@@ -252,12 +252,12 @@ public:
TKqpTxLocks Locks;
- TDeferredEffects DeferredEffects;
+ TDeferredEffects DeferredEffects;
TIntrusivePtr<TParamsState> ParamsState;
IKqpGateway::TKqpSnapshotHandle SnapshotHandle;
-
- TKqpForceNewEngineState ForceNewEngineSettings;
+
+ TKqpForceNewEngineState ForceNewEngineSettings;
};
class TLogExprTransformer {
@@ -284,5 +284,5 @@ private:
NYql::NLog::ELevel Level;
};
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/common/kqp_tx_info.h b/ydb/core/kqp/common/kqp_tx_info.h
index 1f136b3fe8..a3e57414ae 100644
--- a/ydb/core/kqp/common/kqp_tx_info.h
+++ b/ydb/core/kqp/common/kqp_tx_info.h
@@ -2,17 +2,17 @@
#include <util/datetime/base.h>
-#include <optional>
-
+#include <optional>
+
namespace NKikimr {
namespace NKqp {
-struct TKqpForceNewEngineState {
- ui32 ForceNewEnginePercent = 0;
- ui32 ForceNewEngineLevel = 0;
- std::optional<bool> ForcedNewEngine;
-};
-
+struct TKqpForceNewEngineState {
+ ui32 ForceNewEnginePercent = 0;
+ ui32 ForceNewEngineLevel = 0;
+ std::optional<bool> ForcedNewEngine;
+};
+
struct TKqpTransactionInfo {
enum class EKind {
Pure,
@@ -27,19 +27,19 @@ struct TKqpTransactionInfo {
Aborted
};
- enum class EEngine {
- OldEngine,
- NewEngine
- };
-
+ enum class EEngine {
+ OldEngine,
+ NewEngine
+ };
+
public:
EStatus Status;
EKind Kind;
- std::optional<EEngine> TxEngine;
+ std::optional<EEngine> TxEngine;
TDuration TotalDuration;
TDuration ServerDuration;
ui32 QueriesCount = 0;
- TKqpForceNewEngineState ForceNewEngineState;
+ TKqpForceNewEngineState ForceNewEngineState;
};
} // namespace NKqp
diff --git a/ydb/core/kqp/common/kqp_yql.cpp b/ydb/core/kqp/common/kqp_yql.cpp
index 80793834e7..4cd92ee10e 100644
--- a/ydb/core/kqp/common/kqp_yql.cpp
+++ b/ydb/core/kqp/common/kqp_yql.cpp
@@ -1,7 +1,7 @@
#include "kqp_yql.h"
#include <ydb/library/yql/core/yql_expr_type_annotation.h>
-
+
namespace NYql {
using namespace NKikimr;
@@ -95,8 +95,8 @@ TKqpPhyTxSettings TKqpPhyTxSettings::Parse(const TKqpPhysicalTx& node) {
if (name == TypeSettingName) {
YQL_ENSURE(tuple.Value().Maybe<TCoAtom>());
settings.Type = GetPhysicalTxType(tuple.Value().Cast<TCoAtom>().Value());
- } else if (name == WithEffectsSettingName) {
- settings.WithEffects = true;
+ } else if (name == WithEffectsSettingName) {
+ settings.WithEffects = true;
}
}
@@ -105,21 +105,21 @@ TKqpPhyTxSettings TKqpPhyTxSettings::Parse(const TKqpPhysicalTx& node) {
NNodes::TCoNameValueTupleList TKqpPhyTxSettings::BuildNode(TExprContext& ctx, TPositionHandle pos) const {
TVector<TCoNameValueTuple> settings;
- settings.reserve(2);
+ settings.reserve(2);
if (Type) {
- settings.emplace_back(Build<TCoNameValueTuple>(ctx, pos)
+ settings.emplace_back(Build<TCoNameValueTuple>(ctx, pos)
.Name().Build(TypeSettingName)
.Value<TCoAtom>().Build(PhysicalTxTypeToString(*Type))
.Done());
}
- if (WithEffects) {
- settings.emplace_back(Build<TCoNameValueTuple>(ctx, pos)
- .Name().Build(WithEffectsSettingName)
- .Done());
- }
-
+ if (WithEffects) {
+ settings.emplace_back(Build<TCoNameValueTuple>(ctx, pos)
+ .Name().Build(WithEffectsSettingName)
+ .Done());
+ }
+
return Build<TCoNameValueTupleList>(ctx, pos)
.Add(settings)
.Done();
@@ -129,30 +129,30 @@ namespace {
template <typename TKqlReadOperation>
TKqpReadTableSettings ParseInternal(const TKqlReadOperation& node) {
- TKqpReadTableSettings settings;
-
- for (const auto& tuple : node.Settings()) {
- TStringBuf name = tuple.Name().Value();
+ TKqpReadTableSettings settings;
+
+ for (const auto& tuple : node.Settings()) {
+ TStringBuf name = tuple.Name().Value();
if (name == TKqpReadTableSettings::SkipNullKeysSettingName) {
YQL_ENSURE(tuple.Value().template Maybe<TCoAtomList>());
for (const auto& key : tuple.Value().template Cast<TCoAtomList>()) {
- settings.SkipNullKeys.emplace_back(TString(key.Value()));
- }
+ settings.SkipNullKeys.emplace_back(TString(key.Value()));
+ }
} else if (name == TKqpReadTableSettings::ItemsLimitSettingName) {
- YQL_ENSURE(tuple.Value().IsValid());
- settings.ItemsLimit = tuple.Value().Cast().Ptr();
+ YQL_ENSURE(tuple.Value().IsValid());
+ settings.ItemsLimit = tuple.Value().Cast().Ptr();
} else if (name == TKqpReadTableSettings::ReverseSettingName) {
- YQL_ENSURE(tuple.Ref().ChildrenSize() == 1);
- settings.Reverse = true;
- } else {
- YQL_ENSURE(false, "Unknown KqpReadTable setting name '" << name << "'");
- }
- }
-
- return settings;
-}
-
+ YQL_ENSURE(tuple.Ref().ChildrenSize() == 1);
+ settings.Reverse = true;
+ } else {
+ YQL_ENSURE(false, "Unknown KqpReadTable setting name '" << name << "'");
+ }
+ }
+
+ return settings;
+}
+
} // anonymous namespace end
TKqpReadTableSettings TKqpReadTableSettings::Parse(const TKqlReadTableBase& node) {
@@ -163,58 +163,58 @@ TKqpReadTableSettings TKqpReadTableSettings::Parse(const TKqlReadTableRangesBase
return ParseInternal(node);
}
-NNodes::TCoNameValueTupleList TKqpReadTableSettings::BuildNode(TExprContext& ctx, TPositionHandle pos) const {
- TVector<TCoNameValueTuple> settings;
- settings.reserve(3);
-
- if (!SkipNullKeys.empty()) {
- TVector<TExprNodePtr> keys;
- keys.reserve(SkipNullKeys.size());
- for (auto& key: SkipNullKeys) {
- keys.emplace_back(ctx.NewAtom(pos, key));
- }
-
- settings.emplace_back(
- Build<TCoNameValueTuple>(ctx, pos)
- .Name()
- .Build(SkipNullKeysSettingName)
- .Value<TCoAtomList>()
- .Add(keys)
- .Build()
- .Done());
- }
-
- if (ItemsLimit) {
- settings.emplace_back(
- Build<TCoNameValueTuple>(ctx, pos)
- .Name()
- .Build(ItemsLimitSettingName)
- .Value(ItemsLimit)
- .Done());
- }
-
- if (Reverse) {
- settings.emplace_back(
- Build<TCoNameValueTuple>(ctx, pos)
- .Name()
- .Build(ReverseSettingName)
- .Done());
- }
-
- return Build<TCoNameValueTupleList>(ctx, pos)
- .Add(settings)
- .Done();
-}
-
-void TKqpReadTableSettings::AddSkipNullKey(const TString& key) {
- for (auto& k : SkipNullKeys) {
- if (k == key) {
- return;
- }
- }
- SkipNullKeys.emplace_back(key);
-}
-
+NNodes::TCoNameValueTupleList TKqpReadTableSettings::BuildNode(TExprContext& ctx, TPositionHandle pos) const {
+ TVector<TCoNameValueTuple> settings;
+ settings.reserve(3);
+
+ if (!SkipNullKeys.empty()) {
+ TVector<TExprNodePtr> keys;
+ keys.reserve(SkipNullKeys.size());
+ for (auto& key: SkipNullKeys) {
+ keys.emplace_back(ctx.NewAtom(pos, key));
+ }
+
+ settings.emplace_back(
+ Build<TCoNameValueTuple>(ctx, pos)
+ .Name()
+ .Build(SkipNullKeysSettingName)
+ .Value<TCoAtomList>()
+ .Add(keys)
+ .Build()
+ .Done());
+ }
+
+ if (ItemsLimit) {
+ settings.emplace_back(
+ Build<TCoNameValueTuple>(ctx, pos)
+ .Name()
+ .Build(ItemsLimitSettingName)
+ .Value(ItemsLimit)
+ .Done());
+ }
+
+ if (Reverse) {
+ settings.emplace_back(
+ Build<TCoNameValueTuple>(ctx, pos)
+ .Name()
+ .Build(ReverseSettingName)
+ .Done());
+ }
+
+ return Build<TCoNameValueTupleList>(ctx, pos)
+ .Add(settings)
+ .Done();
+}
+
+void TKqpReadTableSettings::AddSkipNullKey(const TString& key) {
+ for (auto& k : SkipNullKeys) {
+ if (k == key) {
+ return;
+ }
+ }
+ SkipNullKeys.emplace_back(key);
+}
+
TKqpUpsertRowsSettings TKqpUpsertRowsSettings::Parse(const TKqpUpsertRows& node) {
TKqpUpsertRowsSettings settings;
@@ -342,8 +342,8 @@ TString KqpExprToPrettyString(const TExprNode& expr, TExprContext& ctx) {
TString exprText = exprStream.Str();
return exprText;
- } catch (const std::exception& e) {
- return TStringBuilder() << "Failed to render expression to pretty string: " << e.what();
+ } catch (const std::exception& e) {
+ return TStringBuilder() << "Failed to render expression to pretty string: " << e.what();
}
}
@@ -351,24 +351,24 @@ TString KqpExprToPrettyString(const TExprBase& expr, TExprContext& ctx) {
return KqpExprToPrettyString(expr.Ref(), ctx);
}
-TString PrintKqpStageOnly(const TDqStageBase& stage, TExprContext& ctx) {
- if (stage.Inputs().Empty()) {
- return KqpExprToPrettyString(stage, ctx);
- }
-
- TNodeOnNodeOwnedMap replaces;
- for (ui64 i = 0; i < stage.Inputs().Size(); ++i) {
- auto input = stage.Inputs().Item(i);
- auto param = Build<TCoParameter>(ctx, input.Pos())
- .Name().Build(TStringBuilder() << "stage_input_" << i)
- .Type(ExpandType(input.Pos(), *input.Ref().GetTypeAnn(), ctx))
- .Done();
-
- replaces[input.Raw()] = param.Ptr();
- }
-
- auto newStage = ctx.ReplaceNodes(stage.Ptr(), replaces);
- return KqpExprToPrettyString(TExprBase(newStage), ctx);
-}
-
+TString PrintKqpStageOnly(const TDqStageBase& stage, TExprContext& ctx) {
+ if (stage.Inputs().Empty()) {
+ return KqpExprToPrettyString(stage, ctx);
+ }
+
+ TNodeOnNodeOwnedMap replaces;
+ for (ui64 i = 0; i < stage.Inputs().Size(); ++i) {
+ auto input = stage.Inputs().Item(i);
+ auto param = Build<TCoParameter>(ctx, input.Pos())
+ .Name().Build(TStringBuilder() << "stage_input_" << i)
+ .Type(ExpandType(input.Pos(), *input.Ref().GetTypeAnn(), ctx))
+ .Done();
+
+ replaces[input.Raw()] = param.Ptr();
+ }
+
+ auto newStage = ctx.ReplaceNodes(stage.Ptr(), replaces);
+ return KqpExprToPrettyString(TExprBase(newStage), ctx);
+}
+
} // namespace NYql
diff --git a/ydb/core/kqp/common/kqp_yql.h b/ydb/core/kqp/common/kqp_yql.h
index 47474065e2..7de6cffe39 100644
--- a/ydb/core/kqp/common/kqp_yql.h
+++ b/ydb/core/kqp/common/kqp_yql.h
@@ -15,8 +15,8 @@ enum class EPhysicalQueryType {
};
struct TKqpPhyQuerySettings {
- static constexpr std::string_view TypeSettingName = "type"sv;
- std::optional<EPhysicalQueryType> Type;
+ static constexpr std::string_view TypeSettingName = "type"sv;
+ std::optional<EPhysicalQueryType> Type;
static TKqpPhyQuerySettings Parse(const NNodes::TKqpPhysicalQuery& node);
NNodes::TCoNameValueTupleList BuildNode(TExprContext& ctx, TPositionHandle pos) const;
@@ -31,33 +31,33 @@ enum class EPhysicalTxType {
struct TKqpPhyTxSettings {
static constexpr TStringBuf TypeSettingName = "type";
- std::optional<EPhysicalTxType> Type;
+ std::optional<EPhysicalTxType> Type;
+
+ static constexpr std::string_view WithEffectsSettingName = "with_effects"sv;
+ bool WithEffects = false;
- static constexpr std::string_view WithEffectsSettingName = "with_effects"sv;
- bool WithEffects = false;
-
static TKqpPhyTxSettings Parse(const NNodes::TKqpPhysicalTx& node);
NNodes::TCoNameValueTupleList BuildNode(TExprContext& ctx, TPositionHandle pos) const;
};
-struct TKqpReadTableSettings {
+struct TKqpReadTableSettings {
static constexpr TStringBuf SkipNullKeysSettingName = "SkipNullKeys";
- static constexpr TStringBuf ItemsLimitSettingName = "ItemsLimit";
- static constexpr TStringBuf ReverseSettingName = "Reverse";
-
- TVector<TString> SkipNullKeys;
- TExprNode::TPtr ItemsLimit;
- bool Reverse = false;
-
- void AddSkipNullKey(const TString& key);
- void SetItemsLimit(const TExprNode::TPtr& expr) { ItemsLimit = expr; }
- void SetReverse() { Reverse = true; }
-
- static TKqpReadTableSettings Parse(const NNodes::TKqlReadTableBase& node);
+ static constexpr TStringBuf ItemsLimitSettingName = "ItemsLimit";
+ static constexpr TStringBuf ReverseSettingName = "Reverse";
+
+ TVector<TString> SkipNullKeys;
+ TExprNode::TPtr ItemsLimit;
+ bool Reverse = false;
+
+ void AddSkipNullKey(const TString& key);
+ void SetItemsLimit(const TExprNode::TPtr& expr) { ItemsLimit = expr; }
+ void SetReverse() { Reverse = true; }
+
+ static TKqpReadTableSettings Parse(const NNodes::TKqlReadTableBase& node);
static TKqpReadTableSettings Parse(const NNodes::TKqlReadTableRangesBase& node);
- NNodes::TCoNameValueTupleList BuildNode(TExprContext& ctx, TPositionHandle pos) const;
-};
-
+ NNodes::TCoNameValueTupleList BuildNode(TExprContext& ctx, TPositionHandle pos) const;
+};
+
struct TKqpUpsertRowsSettings {
static constexpr TStringBuf InplaceSettingName = "Inplace";
@@ -91,6 +91,6 @@ struct TKqpReadTableExplainPrompt {
TString KqpExprToPrettyString(const TExprNode& expr, TExprContext& ctx);
TString KqpExprToPrettyString(const NNodes::TExprBase& expr, TExprContext& ctx);
-TString PrintKqpStageOnly(const NNodes::TDqStageBase& stage, TExprContext& ctx);
-
+TString PrintKqpStageOnly(const NNodes::TDqStageBase& stage, TExprContext& ctx);
+
} // namespace NYql
diff --git a/ydb/core/kqp/compile/kqp_compile.cpp b/ydb/core/kqp/compile/kqp_compile.cpp
index 0f885f5f90..c7d305ab46 100644
--- a/ydb/core/kqp/compile/kqp_compile.cpp
+++ b/ydb/core/kqp/compile/kqp_compile.cpp
@@ -108,7 +108,7 @@ void FillKeyBound(const TVarArgCallable<TExprBase>& bound, NKqpProto::TKqpPhyKey
if (bound.Maybe<TKqlKeyInc>()) {
boundProto.SetIsInclusive(true);
} else if (bound.Maybe<TKqlKeyExc>()) {
- boundProto.SetIsInclusive(false);
+ boundProto.SetIsInclusive(false);
} else {
YQL_ENSURE(false, "Unexpected key bound type: " << bound.CallableName());
}
@@ -140,28 +140,28 @@ void FillKeyRange(const TKqlKeyRange& range, NKqpProto::TKqpPhyKeyRange& rangePr
}
template <typename TReader, typename TProto>
-void FillReadRange(const TReader& read, const TKikimrTableMetadata& tableMeta, TProto& readProto) {
+void FillReadRange(const TReader& read, const TKikimrTableMetadata& tableMeta, TProto& readProto) {
FillKeyRange(read.Range(), *readProto.MutableKeyRange());
-
- auto settings = TKqpReadTableSettings::Parse(read);
-
- readProto.MutableSkipNullKeys()->Resize(tableMeta.KeyColumnNames.size(), false);
- for (const auto& key : settings.SkipNullKeys) {
- size_t keyIndex = FindIndex(tableMeta.KeyColumnNames, key);
- YQL_ENSURE(keyIndex != NPOS);
- readProto.MutableSkipNullKeys()->Set(keyIndex, true);
- }
-
- if (settings.ItemsLimit) {
- TExprBase expr(settings.ItemsLimit);
- if (expr.template Maybe<TCoParameter>()) {
- readProto.MutableItemsLimit()->SetParamName(TString(expr.template Cast<TCoParameter>().Name().Value()));
- } else {
- YQL_ENSURE(false, "Unexpected ItemsLimit callable " << expr.Ref().Content());
- }
- }
-
- readProto.SetReverse(settings.Reverse);
+
+ auto settings = TKqpReadTableSettings::Parse(read);
+
+ readProto.MutableSkipNullKeys()->Resize(tableMeta.KeyColumnNames.size(), false);
+ for (const auto& key : settings.SkipNullKeys) {
+ size_t keyIndex = FindIndex(tableMeta.KeyColumnNames, key);
+ YQL_ENSURE(keyIndex != NPOS);
+ readProto.MutableSkipNullKeys()->Set(keyIndex, true);
+ }
+
+ if (settings.ItemsLimit) {
+ TExprBase expr(settings.ItemsLimit);
+ if (expr.template Maybe<TCoParameter>()) {
+ readProto.MutableItemsLimit()->SetParamName(TString(expr.template Cast<TCoParameter>().Name().Value()));
+ } else {
+ YQL_ENSURE(false, "Unexpected ItemsLimit callable " << expr.Ref().Content());
+ }
+ }
+
+ readProto.SetReverse(settings.Reverse);
}
template <typename TReader, typename TProto>
@@ -211,99 +211,99 @@ void FillEffectRows(const TEffectCallable& callable, TEffectProto& proto, bool i
}
void FillLookup(const TKqpLookupTable& lookup, NKqpProto::TKqpPhyOpLookup& lookupProto) {
- auto maybeList = lookup.LookupKeys().Maybe<TCoIterator>().List();
+ auto maybeList = lookup.LookupKeys().Maybe<TCoIterator>().List();
YQL_ENSURE(maybeList, "Expected iterator as lookup input, got: " << lookup.LookupKeys().Ref().Content());
-
- if (auto maybeParam = maybeList.Cast().Maybe<TCoParameter>()) {
- lookupProto.MutableKeysValue()->MutableParamValue()->SetParamName(maybeParam.Cast().Name().StringValue());
+
+ if (auto maybeParam = maybeList.Cast().Maybe<TCoParameter>()) {
+ lookupProto.MutableKeysValue()->MutableParamValue()->SetParamName(maybeParam.Cast().Name().StringValue());
} else if (auto maybeAsList = maybeList.Cast().Maybe<TCoAsList>()) {
- auto asList = maybeAsList.Cast();
- auto proto = lookupProto.MutableKeysValue()->MutableRowsList();
-
- for (auto row : asList) {
- YQL_ENSURE(row.Maybe<TCoAsStruct>(), "" << row.Ref().Dump());
- auto asStruct = row.Cast<TCoAsStruct>();
- auto protoRow = proto->AddRows();
- auto& protoRowColumns = *protoRow->MutableColumns();
-
- auto* structType = asStruct.Ref().GetTypeAnn()->Cast<TStructExprType>();
-
- for (auto item : asStruct) {
- auto tuple = item.Cast<TCoNameValueTuple>();
- auto columnName = tuple.Name().StringValue();
- auto& protoColumn = protoRowColumns[columnName];
-
- if (auto maybeParam = tuple.Value().Maybe<TCoParameter>()) {
- protoColumn.MutableParamValue()->SetParamName(maybeParam.Cast().Name().StringValue());
- } else {
- YQL_ENSURE(tuple.Value().Maybe<TCoDataCtor>(), "" << tuple.Value().Ref().Dump());
- auto dataCtor = tuple.Value().Cast<TCoDataCtor>();
-
- auto value = dataCtor.Literal().Value();
-
- auto& protoType = *protoColumn.MutableLiteralValue()->MutableType();
- auto& protoValue = *protoColumn.MutableLiteralValue()->MutableValue();
-
- auto type = structType->FindItemType(columnName);
- auto slot = type->Cast<TDataExprType>()->GetSlot();
- auto typeId = NKikimr::NUdf::GetDataTypeInfo(slot).TypeId;
-
- YQL_ENSURE(NScheme::NTypeIds::IsYqlType(typeId) && NSchemeShard::IsAllowedKeyType(typeId));
-
- protoType.SetKind(NKikimrMiniKQL::ETypeKind::Data);
- protoType.MutableData()->SetScheme(typeId);
-
- switch (slot) {
- case EDataSlot::Bool:
- protoValue.SetBool(FromString<bool>(value));
- break;
- case EDataSlot::Uint8:
- case EDataSlot::Uint32:
- case EDataSlot::Date:
- case EDataSlot::Datetime:
- protoValue.SetUint32(FromString<ui32>(value));
- break;
- case EDataSlot::Int32:
- protoValue.SetInt32(FromString<i32>(value));
- break;
- case EDataSlot::Int64:
- case EDataSlot::Interval:
- protoValue.SetInt64(FromString<i64>(value));
- break;
- case EDataSlot::Uint64:
- case EDataSlot::Timestamp:
- protoValue.SetUint64(FromString<ui64>(value));
- break;
- case EDataSlot::String:
- case EDataSlot::DyNumber:
- protoValue.SetBytes(value.Data(), value.Size());
- break;
- case EDataSlot::Utf8:
- protoValue.SetText(ToString(value));
- break;
- case EDataSlot::Decimal: {
- const auto paramsDataType = type->Cast<TDataExprParamsType>();
- auto precision = FromString<ui8>(paramsDataType->GetParamOne());
- auto scale = FromString<ui8>(paramsDataType->GetParamTwo());
- protoType.MutableData()->MutableDecimalParams()->SetPrecision(precision);
- protoType.MutableData()->MutableDecimalParams()->SetScale(scale);
-
- auto v = NDecimal::FromString(dataCtor.Cast<TCoDecimal>().Literal().Value(), precision, scale);
- const auto p = reinterpret_cast<ui8*>(&v);
- protoValue.SetLow128(*reinterpret_cast<ui64*>(p));
- protoValue.SetHi128(*reinterpret_cast<ui64*>(p + 8));
- break;
- }
-
- default:
- YQL_ENSURE(false, "Unexpected type slot " << slot);
- }
- }
- }
- }
+ auto asList = maybeAsList.Cast();
+ auto proto = lookupProto.MutableKeysValue()->MutableRowsList();
+
+ for (auto row : asList) {
+ YQL_ENSURE(row.Maybe<TCoAsStruct>(), "" << row.Ref().Dump());
+ auto asStruct = row.Cast<TCoAsStruct>();
+ auto protoRow = proto->AddRows();
+ auto& protoRowColumns = *protoRow->MutableColumns();
+
+ auto* structType = asStruct.Ref().GetTypeAnn()->Cast<TStructExprType>();
+
+ for (auto item : asStruct) {
+ auto tuple = item.Cast<TCoNameValueTuple>();
+ auto columnName = tuple.Name().StringValue();
+ auto& protoColumn = protoRowColumns[columnName];
+
+ if (auto maybeParam = tuple.Value().Maybe<TCoParameter>()) {
+ protoColumn.MutableParamValue()->SetParamName(maybeParam.Cast().Name().StringValue());
+ } else {
+ YQL_ENSURE(tuple.Value().Maybe<TCoDataCtor>(), "" << tuple.Value().Ref().Dump());
+ auto dataCtor = tuple.Value().Cast<TCoDataCtor>();
+
+ auto value = dataCtor.Literal().Value();
+
+ auto& protoType = *protoColumn.MutableLiteralValue()->MutableType();
+ auto& protoValue = *protoColumn.MutableLiteralValue()->MutableValue();
+
+ auto type = structType->FindItemType(columnName);
+ auto slot = type->Cast<TDataExprType>()->GetSlot();
+ auto typeId = NKikimr::NUdf::GetDataTypeInfo(slot).TypeId;
+
+ YQL_ENSURE(NScheme::NTypeIds::IsYqlType(typeId) && NSchemeShard::IsAllowedKeyType(typeId));
+
+ protoType.SetKind(NKikimrMiniKQL::ETypeKind::Data);
+ protoType.MutableData()->SetScheme(typeId);
+
+ switch (slot) {
+ case EDataSlot::Bool:
+ protoValue.SetBool(FromString<bool>(value));
+ break;
+ case EDataSlot::Uint8:
+ case EDataSlot::Uint32:
+ case EDataSlot::Date:
+ case EDataSlot::Datetime:
+ protoValue.SetUint32(FromString<ui32>(value));
+ break;
+ case EDataSlot::Int32:
+ protoValue.SetInt32(FromString<i32>(value));
+ break;
+ case EDataSlot::Int64:
+ case EDataSlot::Interval:
+ protoValue.SetInt64(FromString<i64>(value));
+ break;
+ case EDataSlot::Uint64:
+ case EDataSlot::Timestamp:
+ protoValue.SetUint64(FromString<ui64>(value));
+ break;
+ case EDataSlot::String:
+ case EDataSlot::DyNumber:
+ protoValue.SetBytes(value.Data(), value.Size());
+ break;
+ case EDataSlot::Utf8:
+ protoValue.SetText(ToString(value));
+ break;
+ case EDataSlot::Decimal: {
+ const auto paramsDataType = type->Cast<TDataExprParamsType>();
+ auto precision = FromString<ui8>(paramsDataType->GetParamOne());
+ auto scale = FromString<ui8>(paramsDataType->GetParamTwo());
+ protoType.MutableData()->MutableDecimalParams()->SetPrecision(precision);
+ protoType.MutableData()->MutableDecimalParams()->SetScale(scale);
+
+ auto v = NDecimal::FromString(dataCtor.Cast<TCoDecimal>().Literal().Value(), precision, scale);
+ const auto p = reinterpret_cast<ui8*>(&v);
+ protoValue.SetLow128(*reinterpret_cast<ui64*>(p));
+ protoValue.SetHi128(*reinterpret_cast<ui64*>(p + 8));
+ break;
+ }
+
+ default:
+ YQL_ENSURE(false, "Unexpected type slot " << slot);
+ }
+ }
+ }
+ }
} else {
YQL_ENSURE(false, "Unexpected lookup input: " << maybeList.Cast().Ref().Content());
- }
+ }
}
void FillOlapProgram(const TCoLambda& process, const TKikimrTableMetadata& tableMeta,
@@ -313,11 +313,11 @@ void FillOlapProgram(const TCoLambda& process, const TKikimrTableMetadata& table
}
void FillConnection(const TDqConnection& connection, const TMap<ui64, ui32>& stagesMap,
- NKqpProto::TKqpPhyConnection& connectionProto, TExprContext& ctx)
+ NKqpProto::TKqpPhyConnection& connectionProto, TExprContext& ctx)
{
auto inputStageIndex = stagesMap.FindPtr(connection.Output().Stage().Ref().UniqueId());
- YQL_ENSURE(inputStageIndex, "stage #" << connection.Output().Stage().Ref().UniqueId() << " not found in stages map: "
- << PrintKqpStageOnly(connection.Output().Stage(), ctx));
+ YQL_ENSURE(inputStageIndex, "stage #" << connection.Output().Stage().Ref().UniqueId() << " not found in stages map: "
+ << PrintKqpStageOnly(connection.Output().Stage(), ctx));
auto outputIndex = FromString<ui32>(connection.Output().Index().Value());
@@ -329,8 +329,8 @@ void FillConnection(const TDqConnection& connection, const TMap<ui64, ui32>& sta
return;
}
- if (auto maybeShuffle = connection.Maybe<TDqCnHashShuffle>()) {
- auto& shuffleProto = *connectionProto.MutableHashShuffle();
+ if (auto maybeShuffle = connection.Maybe<TDqCnHashShuffle>()) {
+ auto& shuffleProto = *connectionProto.MutableHashShuffle();
for (const auto& keyColumn : maybeShuffle.Cast().KeyColumns()) {
shuffleProto.AddKeyColumns(TString(keyColumn));
}
@@ -453,15 +453,15 @@ private:
auto connection = input.Cast<TDqConnection>();
auto& protoInput = *stageProto.AddInputs();
- FillConnection(connection, stagesMap, protoInput, ctx);
+ FillConnection(connection, stagesMap, protoInput, ctx);
}
bool hasSort = false;
bool hasMapJoin = false;
- bool hasUdf = false;
+ bool hasUdf = false;
VisitExpr(stage.Program().Ptr(), [&](const TExprNode::TPtr& exprNode) {
TExprBase node(exprNode);
- if (auto maybeReadTable = node.Maybe<TKqpWideReadTable>()) {
+ if (auto maybeReadTable = node.Maybe<TKqpWideReadTable>()) {
auto readTable = maybeReadTable.Cast();
auto tableMeta = TablesData->ExistingTable(Cluster, readTable.Table().Path()).Metadata;
YQL_ENSURE(tableMeta);
@@ -469,7 +469,7 @@ private:
auto& tableOp = *stageProto.AddTableOps();
FillTable(readTable.Table(), *tableOp.MutableTable());
FillColumns(readTable.Columns(), *tableMeta, tableOp, true);
- FillReadRange(readTable, *tableMeta, *tableOp.MutableReadRange());
+ FillReadRange(readTable, *tableMeta, *tableOp.MutableReadRange());
} else if (auto maybeLookupTable = node.Maybe<TKqpLookupTable>()) {
auto lookupTable = maybeLookupTable.Cast();
auto tableMeta = TablesData->ExistingTable(Cluster, lookupTable.Table().Path()).Metadata;
@@ -525,11 +525,11 @@ private:
} else if (node.Maybe<TCoMapJoinCore>()) {
hasMapJoin = true;
} else if (node.Maybe<TCoUdf>()) {
- hasUdf = true;
- } else {
- YQL_ENSURE(!node.Maybe<TKqpReadTable>());
- }
-
+ hasUdf = true;
+ } else {
+ YQL_ENSURE(!node.Maybe<TKqpReadTable>());
+ }
+
return true;
});
@@ -554,14 +554,14 @@ private:
auto paramsType = NDq::CollectParameters(stage.Program(), ctx);
auto programBytecode = NDq::BuildProgram(stage.Program(), *paramsType, *KqlCompiler, TypeEnv, FuncRegistry,
- ctx, {});
+ ctx, {});
auto& programProto = *stageProto.MutableProgram();
programProto.SetRuntimeVersion(NYql::NDqProto::ERuntimeVersion::RUNTIME_VERSION_YQL_1_0);
programProto.SetRaw(programBytecode);
programProto.MutableSettings()->SetHasMapJoin(hasMapJoin);
programProto.MutableSettings()->SetHasSort(hasSort);
- programProto.MutableSettings()->SetHasUdf(hasUdf);
+ programProto.MutableSettings()->SetHasUdf(hasUdf);
for (auto member : paramsType->GetItems()) {
auto paramName = TString(member->GetName());
@@ -577,20 +577,20 @@ private:
YQL_ENSURE(txSettings.Type);
txProto.SetType(GetPhyTxType(*txSettings.Type));
- bool hasEffectStage = false;
-
+ bool hasEffectStage = false;
+
TMap<ui64, ui32> stagesMap;
for (const auto& stage : tx.Stages()) {
- auto* protoStage = txProto.AddStages();
- CompileStage(stage, *protoStage, ctx, stagesMap);
- hasEffectStage |= protoStage->GetIsEffectsStage();
+ auto* protoStage = txProto.AddStages();
+ CompileStage(stage, *protoStage, ctx, stagesMap);
+ hasEffectStage |= protoStage->GetIsEffectsStage();
stagesMap[stage.Ref().UniqueId()] = txProto.StagesSize() - 1;
}
- YQL_ENSURE(hasEffectStage == txSettings.WithEffects);
-
- txProto.SetHasEffects(txSettings.WithEffects);
-
+ YQL_ENSURE(hasEffectStage == txSettings.WithEffects);
+
+ txProto.SetHasEffects(txSettings.WithEffects);
+
for (const auto& paramBinding : tx.ParamBindings()) {
TString paramName(paramBinding.Name().Value());
const auto& binding = paramBinding.Binding();
@@ -619,12 +619,12 @@ private:
TProgramBuilder pgmBuilder(TypeEnv, FuncRegistry);
for (const auto& resultNode : tx.Results()) {
- YQL_ENSURE(resultNode.Maybe<TDqConnection>(), "" << NCommon::ExprToPrettyString(ctx, tx.Ref()));
+ YQL_ENSURE(resultNode.Maybe<TDqConnection>(), "" << NCommon::ExprToPrettyString(ctx, tx.Ref()));
auto connection = resultNode.Cast<TDqConnection>();
auto& resultProto = *txProto.AddResults();
auto& connectionProto = *resultProto.MutableConnection();
- FillConnection(connection, stagesMap, connectionProto, ctx);
+ FillConnection(connection, stagesMap, connectionProto, ctx);
const TTypeAnnotationNode* itemType = nullptr;
switch (connectionProto.GetTypeCase()) {
@@ -648,24 +648,24 @@ private:
YQL_ENSURE(type);
ExportTypeToProto(type, *resultProto.MutableItemType());
-
- TMaybeNode<TCoAtomList> maybeColumnHints;
- if (connection.Maybe<TDqCnResult>()) {
- maybeColumnHints = connection.Cast<TDqCnResult>().ColumnHints();
- } else if (connection.Maybe<TDqCnValue>()) {
- // no column hints
- } else {
- YQL_ENSURE(false, "Unexpected tx result connection type " << connection.CallableName());
- }
-
- if (maybeColumnHints) {
- auto columnHints = maybeColumnHints.Cast();
- auto& columnHintsProto = *resultProto.MutableColumnHints();
- columnHintsProto.Reserve(columnHints.Size());
- for (const auto& columnHint : columnHints) {
- columnHintsProto.Add(TString(columnHint.Value()));
- }
- }
+
+ TMaybeNode<TCoAtomList> maybeColumnHints;
+ if (connection.Maybe<TDqCnResult>()) {
+ maybeColumnHints = connection.Cast<TDqCnResult>().ColumnHints();
+ } else if (connection.Maybe<TDqCnValue>()) {
+ // no column hints
+ } else {
+ YQL_ENSURE(false, "Unexpected tx result connection type " << connection.CallableName());
+ }
+
+ if (maybeColumnHints) {
+ auto columnHints = maybeColumnHints.Cast();
+ auto& columnHintsProto = *resultProto.MutableColumnHints();
+ columnHintsProto.Reserve(columnHints.Size());
+ for (const auto& columnHint : columnHints) {
+ columnHintsProto.Add(TString(columnHint.Value()));
+ }
+ }
}
}
@@ -687,5 +687,5 @@ TIntrusivePtr<IKqpQueryCompiler> CreateKqpQueryCompiler(const TString& cluster,
return MakeIntrusive<TKqpQueryCompiler>(cluster, tablesData, funcRegistry);
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/compile/kqp_compile.h b/ydb/core/kqp/compile/kqp_compile.h
index 3ced73f61f..774bb3cc53 100644
--- a/ydb/core/kqp/compile/kqp_compile.h
+++ b/ydb/core/kqp/compile/kqp_compile.h
@@ -19,5 +19,5 @@ public:
TIntrusivePtr<IKqpQueryCompiler> CreateKqpQueryCompiler(const TString& cluster,
const TIntrusivePtr<NYql::TKikimrTablesData> tablesData, const NMiniKQL::IFunctionRegistry& funcRegistry);
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/compile/kqp_mkql_compiler.cpp b/ydb/core/kqp/compile/kqp_mkql_compiler.cpp
index 50cf81d85c..0413044698 100644
--- a/ydb/core/kqp/compile/kqp_mkql_compiler.cpp
+++ b/ydb/core/kqp/compile/kqp_mkql_compiler.cpp
@@ -54,26 +54,26 @@ TVector<TKqpTableColumn> GetKqpColumns(const TKikimrTableMetadata& table, const
return GetKqpColumns(table, columnNames, allowSystemColumns);
}
-TSmallVec<bool> GetSkipNullKeys(const TKqpReadTableSettings& settings, const TKikimrTableMetadata& tableMeta) {
- TSmallVec<bool> skipNullKeys(tableMeta.KeyColumnNames.size(), false);
-
- for (const auto& key : settings.SkipNullKeys) {
- size_t keyIndex = FindIndex(tableMeta.KeyColumnNames, key);
- YQL_ENSURE(keyIndex != NPOS);
- skipNullKeys[keyIndex] = true;
- }
-
- return skipNullKeys;
-}
-
-NMiniKQL::TType* CreateColumnType(NUdf::TDataTypeId typeId, const TKqlCompileContext& ctx) {
- if (typeId == NUdf::TDataType<NUdf::TDecimal>::Id) {
- return ctx.PgmBuilder().NewDecimalType(22, 9);
- } else {
- return ctx.PgmBuilder().NewDataType(typeId);
- }
-}
-
+TSmallVec<bool> GetSkipNullKeys(const TKqpReadTableSettings& settings, const TKikimrTableMetadata& tableMeta) {
+ TSmallVec<bool> skipNullKeys(tableMeta.KeyColumnNames.size(), false);
+
+ for (const auto& key : settings.SkipNullKeys) {
+ size_t keyIndex = FindIndex(tableMeta.KeyColumnNames, key);
+ YQL_ENSURE(keyIndex != NPOS);
+ skipNullKeys[keyIndex] = true;
+ }
+
+ return skipNullKeys;
+}
+
+NMiniKQL::TType* CreateColumnType(NUdf::TDataTypeId typeId, const TKqlCompileContext& ctx) {
+ if (typeId == NUdf::TDataType<NUdf::TDecimal>::Id) {
+ return ctx.PgmBuilder().NewDecimalType(22, 9);
+ } else {
+ return ctx.PgmBuilder().NewDataType(typeId);
+ }
+}
+
void ValidateColumnType(const TTypeAnnotationNode* type, NUdf::TDataTypeId columnTypeId) {
YQL_ENSURE(type);
bool isOptional;
@@ -248,7 +248,7 @@ TIntrusivePtr<IMkqlCallableCompiler> CreateKqlCompiler(const TKqlCompileContext&
TKqpKeyRanges ranges = MakeComputedKeyRanges(readTable, ctx, buildCtx);
- // Return type depends on the process program, so it is built explicitly.
+ // Return type depends on the process program, so it is built explicitly.
TStringStream errorStream;
auto returnType = NCommon::BuildType(*readTable.Ref().GetTypeAnn(), ctx.PgmBuilder(), errorStream);
YQL_ENSURE(returnType, "Failed to build type: " << errorStream.Str());
@@ -257,7 +257,7 @@ TIntrusivePtr<IMkqlCallableCompiler> CreateKqlCompiler(const TKqlCompileContext&
// in physical plan directly to executer. Read callables in MKQL only used to associate
// input stream of the graph with the external scans, so it doesn't make much sense to pass
// the process program through callable.
- // We anyway move to explicit sources as external nodes in KQP program, so all the information
+ // We anyway move to explicit sources as external nodes in KQP program, so all the information
// about read settings will be passed in a side channel, not the program.
auto result = ctx.PgmBuilder().KqpWideReadTableRanges(
MakeTableId(readTable.Table()),
@@ -347,20 +347,20 @@ TIntrusivePtr<IMkqlCallableCompiler> CreateKqlCompiler(const TKqlCompileContext&
return result;
});
- compiler->AddCallable(TKqpEnsure::CallableName(),
- [&ctx](const TExprNode& node, TMkqlBuildContext& buildCtx) {
- TKqpEnsure ensure(&node);
-
- const auto value = MkqlBuildExpr(ensure.Value().Ref(), buildCtx);
- const auto predicate = MkqlBuildExpr(ensure.Predicate().Ref(), buildCtx);
- const auto issueCode = buildCtx.ProgramBuilder.NewDataLiteral<ui32>(FromString(ensure.IssueCode().Value()));
- const auto message = MkqlBuildExpr(ensure.Message().Ref(), buildCtx);
-
- return ctx.PgmBuilder().KqpEnsure(value, predicate, issueCode, message);
- });
-
+ compiler->AddCallable(TKqpEnsure::CallableName(),
+ [&ctx](const TExprNode& node, TMkqlBuildContext& buildCtx) {
+ TKqpEnsure ensure(&node);
+
+ const auto value = MkqlBuildExpr(ensure.Value().Ref(), buildCtx);
+ const auto predicate = MkqlBuildExpr(ensure.Predicate().Ref(), buildCtx);
+ const auto issueCode = buildCtx.ProgramBuilder.NewDataLiteral<ui32>(FromString(ensure.IssueCode().Value()));
+ const auto message = MkqlBuildExpr(ensure.Message().Ref(), buildCtx);
+
+ return ctx.PgmBuilder().KqpEnsure(value, predicate, issueCode, message);
+ });
+
return compiler;
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/compile/kqp_mkql_compiler.h b/ydb/core/kqp/compile/kqp_mkql_compiler.h
index ad60d6509a..9d89a0188d 100644
--- a/ydb/core/kqp/compile/kqp_mkql_compiler.h
+++ b/ydb/core/kqp/compile/kqp_mkql_compiler.h
@@ -33,5 +33,5 @@ private:
TIntrusivePtr<NYql::NCommon::IMkqlCallableCompiler> CreateKqlCompiler(const TKqlCompileContext& ctx);
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/compute_actor/kqp_compute_actor.cpp b/ydb/core/kqp/compute_actor/kqp_compute_actor.cpp
index 2734d41863..780f865e9e 100644
--- a/ydb/core/kqp/compute_actor/kqp_compute_actor.cpp
+++ b/ydb/core/kqp/compute_actor/kqp_compute_actor.cpp
@@ -22,21 +22,21 @@ TComputationNodeFactory GetKqpActorComputeFactory(TKqpScanComputeContext* comput
return compute;
}
- auto name = callable.GetType()->GetName();
-
- if (name == "KqpWideReadTable"sv) {
- return WrapKqpScanWideReadTable(callable, ctx, *computeCtx);
+ auto name = callable.GetType()->GetName();
+
+ if (name == "KqpWideReadTable"sv) {
+ return WrapKqpScanWideReadTable(callable, ctx, *computeCtx);
+ }
+
+ if (name == "KqpWideReadTableRanges"sv) {
+ return WrapKqpScanWideReadTableRanges(callable, ctx, *computeCtx);
+ }
+
+ // only for _pure_ compute actors!
+ if (name == "KqpEnsure"sv) {
+ return WrapKqpEnsure(callable, ctx);
}
- if (name == "KqpWideReadTableRanges"sv) {
- return WrapKqpScanWideReadTableRanges(callable, ctx, *computeCtx);
- }
-
- // only for _pure_ compute actors!
- if (name == "KqpEnsure"sv) {
- return WrapKqpEnsure(callable, ctx);
- }
-
return nullptr;
};
}
diff --git a/ydb/core/kqp/compute_actor/kqp_compute_actor.h b/ydb/core/kqp/compute_actor/kqp_compute_actor.h
index 6f2f0ab0f4..02b7b98029 100644
--- a/ydb/core/kqp/compute_actor/kqp_compute_actor.h
+++ b/ydb/core/kqp/compute_actor/kqp_compute_actor.h
@@ -7,8 +7,8 @@
namespace NKikimr {
namespace NMiniKQL {
-class TKqpScanComputeContext;
-
+class TKqpScanComputeContext;
+
TComputationNodeFactory GetKqpActorComputeFactory(TKqpScanComputeContext* computeCtx);
} // namespace NMiniKQL
@@ -17,13 +17,13 @@ namespace NKqp {
IActor* CreateKqpComputeActor(const TActorId& executerId, ui64 txId, NYql::NDqProto::TDqTask&& task,
NYql::NDq::IDqSourceActorFactory::TPtr sourceActorFactory, NYql::NDq::IDqSinkActorFactory::TPtr sinkActorFactory,
- const NYql::NDq::TComputeRuntimeSettings& settings, const NYql::NDq::TComputeMemoryLimits& memoryLimits);
+ const NYql::NDq::TComputeRuntimeSettings& settings, const NYql::NDq::TComputeMemoryLimits& memoryLimits);
-IActor* CreateKqpScanComputeActor(const NKikimrKqp::TKqpSnapshot& snapshot, const TActorId& executerId, ui64 txId,
+IActor* CreateKqpScanComputeActor(const NKikimrKqp::TKqpSnapshot& snapshot, const TActorId& executerId, ui64 txId,
NYql::NDqProto::TDqTask&& task, NYql::NDq::IDqSourceActorFactory::TPtr sourceActorFactory, NYql::NDq::IDqSinkActorFactory::TPtr sinkActorFactory,
const NYql::NDq::TComputeRuntimeSettings& settings, const NYql::NDq::TComputeMemoryLimits& memoryLimits,
TIntrusivePtr<TKqpCounters> counters);
-
-
+
+
} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/compute_actor/kqp_compute_actor_impl.h b/ydb/core/kqp/compute_actor/kqp_compute_actor_impl.h
index e1c78620ae..b32537d951 100644
--- a/ydb/core/kqp/compute_actor/kqp_compute_actor_impl.h
+++ b/ydb/core/kqp/compute_actor/kqp_compute_actor_impl.h
@@ -1,47 +1,47 @@
-#pragma once
-
-#include "kqp_compute_actor.h"
-
+#pragma once
+
+#include "kqp_compute_actor.h"
+
#include <ydb/core/kqp/runtime/kqp_channel_storage.h>
#include <ydb/core/kqp/runtime/kqp_tasks_runner.h>
-
-
-namespace NKikimr {
-namespace NKqp {
-
-using namespace NYql;
-using namespace NYql::NDq;
-
-class TKqpTaskRunnerExecutionContext : public IDqTaskRunnerExecutionContext {
-public:
- TKqpTaskRunnerExecutionContext(ui64 txId, bool withSpilling, IDqChannelStorage::TWakeUpCallback&& wakeUp,
- const TActorContext& ctx)
- : TxId(txId)
- , WakeUp(std::move(wakeUp))
- , Ctx(ctx)
- , WithSpilling(withSpilling) {}
-
- IDqOutputConsumer::TPtr CreateOutputConsumer(const NDqProto::TTaskOutput& outputDesc,
- const NMiniKQL::TType* type, NUdf::IApplyContext* applyCtx, const NMiniKQL::TTypeEnvironment& typeEnv,
+
+
+namespace NKikimr {
+namespace NKqp {
+
+using namespace NYql;
+using namespace NYql::NDq;
+
+class TKqpTaskRunnerExecutionContext : public IDqTaskRunnerExecutionContext {
+public:
+ TKqpTaskRunnerExecutionContext(ui64 txId, bool withSpilling, IDqChannelStorage::TWakeUpCallback&& wakeUp,
+ const TActorContext& ctx)
+ : TxId(txId)
+ , WakeUp(std::move(wakeUp))
+ , Ctx(ctx)
+ , WithSpilling(withSpilling) {}
+
+ IDqOutputConsumer::TPtr CreateOutputConsumer(const NDqProto::TTaskOutput& outputDesc,
+ const NMiniKQL::TType* type, NUdf::IApplyContext* applyCtx, const NMiniKQL::TTypeEnvironment& typeEnv,
TVector<IDqOutput::TPtr>&& outputs) const override
- {
+ {
return KqpBuildOutputConsumer(outputDesc, type, applyCtx, typeEnv, std::move(outputs));
- }
-
- IDqChannelStorage::TPtr CreateChannelStorage(ui64 channelId) const override {
- if (WithSpilling) {
- return CreateKqpChannelStorage(TxId, channelId, WakeUp, Ctx);
- } else {
- return nullptr;
- }
- }
-
-private:
- const ui64 TxId;
- const IDqChannelStorage::TWakeUpCallback WakeUp;
- const TActorContext& Ctx;
- const bool WithSpilling;
-};
-
-} // namespace NKqp
-} // namespace NKikimr
+ }
+
+ IDqChannelStorage::TPtr CreateChannelStorage(ui64 channelId) const override {
+ if (WithSpilling) {
+ return CreateKqpChannelStorage(TxId, channelId, WakeUp, Ctx);
+ } else {
+ return nullptr;
+ }
+ }
+
+private:
+ const ui64 TxId;
+ const IDqChannelStorage::TWakeUpCallback WakeUp;
+ const TActorContext& Ctx;
+ const bool WithSpilling;
+};
+
+} // namespace NKqp
+} // namespace NKikimr
diff --git a/ydb/core/kqp/compute_actor/kqp_pure_compute_actor.cpp b/ydb/core/kqp/compute_actor/kqp_pure_compute_actor.cpp
index 2f7af1203e..750207da8e 100644
--- a/ydb/core/kqp/compute_actor/kqp_pure_compute_actor.cpp
+++ b/ydb/core/kqp/compute_actor/kqp_pure_compute_actor.cpp
@@ -1,247 +1,247 @@
-#include "kqp_compute_actor.h"
-#include "kqp_compute_actor_impl.h"
-
+#include "kqp_compute_actor.h"
+#include "kqp_compute_actor_impl.h"
+
#include <ydb/core/base/appdata.h>
#include <ydb/core/protos/tx_datashard.pb.h>
#include <ydb/core/kqp/runtime/kqp_compute.h>
#include <ydb/core/kqp/runtime/kqp_scan_data.h>
#include <ydb/core/sys_view/scan.h>
#include <ydb/core/util/yverify_stream.h>
-
+
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h>
-
-
-namespace NKikimr {
-namespace NKqp {
-
-namespace {
-
-using namespace NYql;
-using namespace NYql::NDq;
-
-inline bool IsDebugLogEnabled(const TActorSystem* actorSystem) {
- auto* settings = actorSystem->LoggerSettings();
- return settings && settings->Satisfies(NActors::NLog::EPriority::PRI_DEBUG, NKikimrServices::KQP_TASKS_RUNNER);
-}
-
-class TKqpComputeActor : public TDqComputeActorBase<TKqpComputeActor> {
- using TBase = TDqComputeActorBase<TKqpComputeActor>;
-
-public:
- static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
- return NKikimrServices::TActivity::KQP_COMPUTE_ACTOR;
- }
-
- TKqpComputeActor(const TActorId& executerId, ui64 txId, NDqProto::TDqTask&& task,
- IDqSourceActorFactory::TPtr sourceActorFactory, IDqSinkActorFactory::TPtr sinkActorFactory,
+
+
+namespace NKikimr {
+namespace NKqp {
+
+namespace {
+
+using namespace NYql;
+using namespace NYql::NDq;
+
+inline bool IsDebugLogEnabled(const TActorSystem* actorSystem) {
+ auto* settings = actorSystem->LoggerSettings();
+ return settings && settings->Satisfies(NActors::NLog::EPriority::PRI_DEBUG, NKikimrServices::KQP_TASKS_RUNNER);
+}
+
+class TKqpComputeActor : public TDqComputeActorBase<TKqpComputeActor> {
+ using TBase = TDqComputeActorBase<TKqpComputeActor>;
+
+public:
+ static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
+ return NKikimrServices::TActivity::KQP_COMPUTE_ACTOR;
+ }
+
+ TKqpComputeActor(const TActorId& executerId, ui64 txId, NDqProto::TDqTask&& task,
+ IDqSourceActorFactory::TPtr sourceActorFactory, IDqSinkActorFactory::TPtr sinkActorFactory,
const TComputeRuntimeSettings& settings, const TComputeMemoryLimits& memoryLimits)
- : TBase(executerId, txId, std::move(task), std::move(sourceActorFactory), std::move(sinkActorFactory), settings, memoryLimits)
- , ComputeCtx(settings.StatsMode)
- {
- if (GetTask().GetMeta().Is<NKikimrTxDataShard::TKqpTransaction::TScanTaskMeta>()) {
- Meta.ConstructInPlace();
- YQL_ENSURE(GetTask().GetMeta().UnpackTo(Meta.Get()), "Invalid task meta: " << GetTask().GetMeta().DebugString());
- YQL_ENSURE(Meta->GetReads().size() == 1);
- YQL_ENSURE(!Meta->GetReads()[0].GetKeyRanges().empty());
- YQL_ENSURE(!Meta->GetTable().GetSysViewInfo().empty());
- }
- }
-
- void DoBootstrap() {
- const TActorSystem* actorSystem = TlsActivationContext->ActorSystem();
-
- TLogFunc logger;
- if (IsDebugLogEnabled(actorSystem)) {
- logger = [actorSystem, txId = this->GetTxId(), taskId = GetTask().GetId()] (const TString& message) {
- LOG_DEBUG_S(*actorSystem, NKikimrServices::KQP_TASKS_RUNNER, "TxId: " << txId
- << ", task: " << taskId << ": " << message);
- };
- }
-
- TDqTaskRunnerContext execCtx;
-
- execCtx.FuncRegistry = AppData()->FunctionRegistry;
- execCtx.RandomProvider = TAppData::RandomProvider.Get();
- execCtx.TimeProvider = TAppData::TimeProvider.Get();
- execCtx.ComputeCtx = &ComputeCtx;
- execCtx.ComputationFactory = NMiniKQL::GetKqpActorComputeFactory(&ComputeCtx);
- execCtx.ApplyCtx = nullptr;
- execCtx.Alloc = nullptr;
- execCtx.TypeEnv = nullptr;
-
- TDqTaskRunnerSettings settings;
- settings.CollectBasicStats = RuntimeSettings.StatsMode >= NYql::NDqProto::DQ_STATS_MODE_BASIC;
- settings.CollectProfileStats = RuntimeSettings.StatsMode >= NYql::NDqProto::DQ_STATS_MODE_PROFILE;
- settings.OptLLVM = GetUseLLVM() ? "--compile-options=disable-opt" : "OFF";
- settings.AllowGeneratorsInUnboxedValues = false;
-
- auto taskRunner = MakeDqTaskRunner(execCtx, settings, logger);
- SetTaskRunner(taskRunner);
-
- auto wakeup = [this]{ ContinueExecute(); };
- try {
- PrepareTaskRunner(TKqpTaskRunnerExecutionContext(std::get<ui64>(TxId), RuntimeSettings.UseSpilling,
- std::move(wakeup), TlsActivationContext->AsActorContext()));
- } catch (const NMiniKQL::TKqpEnsureFail& e) {
- InternalError((TIssuesIds::EIssueCode) e.GetCode(), e.GetMessage());
- return;
- }
-
- TSmallVec<NMiniKQL::TKqpScanComputeContext::TColumn> columns;
- TSerializedTableRange* rangeRef = nullptr;
-
- TSerializedTableRange coveringRange;
- if (Meta) {
- YQL_ENSURE(ComputeCtx.GetTableScans().empty());
-
- ComputeCtx.AddTableScan(0, *Meta, GetStatsMode());
- ScanData = &ComputeCtx.GetTableScan(0);
-
- columns.reserve(Meta->ColumnsSize());
+ : TBase(executerId, txId, std::move(task), std::move(sourceActorFactory), std::move(sinkActorFactory), settings, memoryLimits)
+ , ComputeCtx(settings.StatsMode)
+ {
+ if (GetTask().GetMeta().Is<NKikimrTxDataShard::TKqpTransaction::TScanTaskMeta>()) {
+ Meta.ConstructInPlace();
+ YQL_ENSURE(GetTask().GetMeta().UnpackTo(Meta.Get()), "Invalid task meta: " << GetTask().GetMeta().DebugString());
+ YQL_ENSURE(Meta->GetReads().size() == 1);
+ YQL_ENSURE(!Meta->GetReads()[0].GetKeyRanges().empty());
+ YQL_ENSURE(!Meta->GetTable().GetSysViewInfo().empty());
+ }
+ }
+
+ void DoBootstrap() {
+ const TActorSystem* actorSystem = TlsActivationContext->ActorSystem();
+
+ TLogFunc logger;
+ if (IsDebugLogEnabled(actorSystem)) {
+ logger = [actorSystem, txId = this->GetTxId(), taskId = GetTask().GetId()] (const TString& message) {
+ LOG_DEBUG_S(*actorSystem, NKikimrServices::KQP_TASKS_RUNNER, "TxId: " << txId
+ << ", task: " << taskId << ": " << message);
+ };
+ }
+
+ TDqTaskRunnerContext execCtx;
+
+ execCtx.FuncRegistry = AppData()->FunctionRegistry;
+ execCtx.RandomProvider = TAppData::RandomProvider.Get();
+ execCtx.TimeProvider = TAppData::TimeProvider.Get();
+ execCtx.ComputeCtx = &ComputeCtx;
+ execCtx.ComputationFactory = NMiniKQL::GetKqpActorComputeFactory(&ComputeCtx);
+ execCtx.ApplyCtx = nullptr;
+ execCtx.Alloc = nullptr;
+ execCtx.TypeEnv = nullptr;
+
+ TDqTaskRunnerSettings settings;
+ settings.CollectBasicStats = RuntimeSettings.StatsMode >= NYql::NDqProto::DQ_STATS_MODE_BASIC;
+ settings.CollectProfileStats = RuntimeSettings.StatsMode >= NYql::NDqProto::DQ_STATS_MODE_PROFILE;
+ settings.OptLLVM = GetUseLLVM() ? "--compile-options=disable-opt" : "OFF";
+ settings.AllowGeneratorsInUnboxedValues = false;
+
+ auto taskRunner = MakeDqTaskRunner(execCtx, settings, logger);
+ SetTaskRunner(taskRunner);
+
+ auto wakeup = [this]{ ContinueExecute(); };
+ try {
+ PrepareTaskRunner(TKqpTaskRunnerExecutionContext(std::get<ui64>(TxId), RuntimeSettings.UseSpilling,
+ std::move(wakeup), TlsActivationContext->AsActorContext()));
+ } catch (const NMiniKQL::TKqpEnsureFail& e) {
+ InternalError((TIssuesIds::EIssueCode) e.GetCode(), e.GetMessage());
+ return;
+ }
+
+ TSmallVec<NMiniKQL::TKqpScanComputeContext::TColumn> columns;
+ TSerializedTableRange* rangeRef = nullptr;
+
+ TSerializedTableRange coveringRange;
+ if (Meta) {
+ YQL_ENSURE(ComputeCtx.GetTableScans().empty());
+
+ ComputeCtx.AddTableScan(0, *Meta, GetStatsMode());
+ ScanData = &ComputeCtx.GetTableScan(0);
+
+ columns.reserve(Meta->ColumnsSize());
for (const auto& column : ScanData->GetColumns()) {
- NMiniKQL::TKqpScanComputeContext::TColumn c;
- c.Tag = column.Tag;
- c.Type = column.Type;
- columns.emplace_back(std::move(c));
- }
-
- const auto& protoRanges = Meta->GetReads()[0].GetKeyRanges();
- if (protoRanges.size() == 1) {
- coveringRange.Load(protoRanges[0]);
- if (!protoRanges[0].HasTo()) {
- coveringRange.To = coveringRange.From;
- coveringRange.FromInclusive = coveringRange.ToInclusive = true;
- }
- } else {
- // TODO: FIX IT, verify here?
- auto from = TSerializedTableRange(*protoRanges.begin());
- auto to = TSerializedTableRange(*protoRanges.rbegin());
- coveringRange.From = from.From;
- coveringRange.FromInclusive = from.FromInclusive;
- coveringRange.To = protoRanges.rbegin()->HasTo() ? to.To : to.From;
- coveringRange.ToInclusive = protoRanges.rbegin()->HasTo() ? to.ToInclusive : true;
- }
-
- rangeRef = &coveringRange;
- }
-
- if (ScanData) {
- ScanData->TaskId = GetTask().GetId();
- ScanData->TableReader = CreateKqpTableReader(*ScanData);
-
- auto scanActor = NSysView::CreateSystemViewScan(SelfId(), 0, ScanData->TableId, rangeRef->ToTableRange(), columns);
-
- if (!scanActor) {
- InternalError(TIssuesIds::DEFAULT_ERROR, TStringBuilder()
- << "Failed to create system view scan, table id: " << ScanData->TableId);
- return;
- }
-
- SysViewActorId = Register(scanActor.Release());
- Send(SysViewActorId, new TEvKqpCompute::TEvScanDataAck(MemoryLimits.ScanBufferSize));
- }
-
- ContinueExecute();
- Become(&TKqpComputeActor::StateFunc);
- }
-
- STFUNC(StateFunc) {
- try {
- switch (ev->GetTypeRewrite()) {
- hFunc(TEvKqpCompute::TEvScanInitActor, HandleExecute);
- hFunc(TEvKqpCompute::TEvScanData, HandleExecute);
- hFunc(TEvKqpCompute::TEvScanError, HandleExecute);
- default:
- StateFuncBase(ev, ctx);
- }
- } catch (const TMemoryLimitExceededException& e) {
- InternalError(TIssuesIds::KIKIMR_PRECONDITION_FAILED, TStringBuilder()
- << "Mkql memory limit exceeded, limit: " << MkqlMemoryLimit
- << ", host: " << HostName()
- << ", canAllocateExtraMemory: " << CanAllocateExtraMemory);
- } catch (const NMiniKQL::TKqpEnsureFail& e) {
- InternalError((TIssuesIds::EIssueCode) e.GetCode(), e.GetMessage());
- } catch (const yexception& e) {
- InternalError(TIssuesIds::DEFAULT_ERROR, e.what());
- }
-
- ReportEventElapsedTime();
- }
-
-protected:
- ui64 CalcMkqlMemoryLimit() override {
- return TBase::CalcMkqlMemoryLimit() + ComputeCtx.GetTableScans().size() * MemoryLimits.ScanBufferSize;
- }
-
-public:
- void FillExtraStats(NDqProto::TDqComputeActorStats* dst, bool last) {
- if (last && SysViewActorId && ScanData && dst->TasksSize() > 0) {
- YQL_ENSURE(dst->TasksSize() == 1);
-
- auto* taskStats = dst->MutableTasks(0);
- auto* tableStats = taskStats->AddTables();
-
- tableStats->SetTablePath(ScanData->TablePath);
-
- if (auto* x = ScanData->BasicStats.get()) {
- tableStats->SetReadRows(x->Rows);
- tableStats->SetReadBytes(x->Bytes);
- // TODO: CpuTime
- }
-
- if (auto* x = ScanData->ProfileStats.get()) {
- NKqpProto::TKqpScanTableExtraStats tableExtraStats;
- // protoScanStats->SetIScanStartTimeMs()
- // protoScanStats->SetIScanFinishTimeMs();
- tableExtraStats.SetIScanCpuTimeUs(x->ScanCpuTime.MicroSeconds());
- tableExtraStats.SetIScanWaitTimeUs(x->ScanWaitTime.MicroSeconds());
- tableExtraStats.SetIScanPageFaults(x->PageFaults);
-
- tableExtraStats.SetMessages(x->Messages);
- tableExtraStats.SetMessagesByPageFault(x->MessagesByPageFault);
-
- tableStats->MutableExtra()->PackFrom(tableExtraStats);
- }
- }
- }
-
-private:
- void PassAway() override {
- if (SysViewActorId) {
- Send(SysViewActorId, new TEvents::TEvPoison);
- }
-
- if (TaskRunner) {
- if (TaskRunner->IsAllocatorAttached()) {
- ComputeCtx.Clear();
- } else {
- auto guard = TaskRunner->BindAllocator(MkqlMemoryLimit);
- ComputeCtx.Clear();
- }
- }
-
- TBase::PassAway();
- }
-
-private:
- void HandleExecute(TEvKqpCompute::TEvScanInitActor::TPtr& ev) {
- Y_VERIFY_DEBUG(ScanData);
-
- auto& msg = ev->Get()->Record;
-
- Y_VERIFY_DEBUG(SysViewActorId == ActorIdFromProto(msg.GetScanActorId()));
-
- CA_LOG_D("Got sysview scan initial event, scan actor: " << SysViewActorId << ", scanId: 0");
- Send(ev->Sender, new TEvKqpCompute::TEvScanDataAck(GetMemoryLimits().ScanBufferSize));
- return;
- }
-
- void HandleExecute(TEvKqpCompute::TEvScanData::TPtr& ev) {
- Y_VERIFY_DEBUG(ScanData);
- Y_VERIFY_DEBUG(SysViewActorId == ev->Sender);
-
- auto& msg = *ev->Get();
-
+ NMiniKQL::TKqpScanComputeContext::TColumn c;
+ c.Tag = column.Tag;
+ c.Type = column.Type;
+ columns.emplace_back(std::move(c));
+ }
+
+ const auto& protoRanges = Meta->GetReads()[0].GetKeyRanges();
+ if (protoRanges.size() == 1) {
+ coveringRange.Load(protoRanges[0]);
+ if (!protoRanges[0].HasTo()) {
+ coveringRange.To = coveringRange.From;
+ coveringRange.FromInclusive = coveringRange.ToInclusive = true;
+ }
+ } else {
+ // TODO: FIX IT, verify here?
+ auto from = TSerializedTableRange(*protoRanges.begin());
+ auto to = TSerializedTableRange(*protoRanges.rbegin());
+ coveringRange.From = from.From;
+ coveringRange.FromInclusive = from.FromInclusive;
+ coveringRange.To = protoRanges.rbegin()->HasTo() ? to.To : to.From;
+ coveringRange.ToInclusive = protoRanges.rbegin()->HasTo() ? to.ToInclusive : true;
+ }
+
+ rangeRef = &coveringRange;
+ }
+
+ if (ScanData) {
+ ScanData->TaskId = GetTask().GetId();
+ ScanData->TableReader = CreateKqpTableReader(*ScanData);
+
+ auto scanActor = NSysView::CreateSystemViewScan(SelfId(), 0, ScanData->TableId, rangeRef->ToTableRange(), columns);
+
+ if (!scanActor) {
+ InternalError(TIssuesIds::DEFAULT_ERROR, TStringBuilder()
+ << "Failed to create system view scan, table id: " << ScanData->TableId);
+ return;
+ }
+
+ SysViewActorId = Register(scanActor.Release());
+ Send(SysViewActorId, new TEvKqpCompute::TEvScanDataAck(MemoryLimits.ScanBufferSize));
+ }
+
+ ContinueExecute();
+ Become(&TKqpComputeActor::StateFunc);
+ }
+
+ STFUNC(StateFunc) {
+ try {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvKqpCompute::TEvScanInitActor, HandleExecute);
+ hFunc(TEvKqpCompute::TEvScanData, HandleExecute);
+ hFunc(TEvKqpCompute::TEvScanError, HandleExecute);
+ default:
+ StateFuncBase(ev, ctx);
+ }
+ } catch (const TMemoryLimitExceededException& e) {
+ InternalError(TIssuesIds::KIKIMR_PRECONDITION_FAILED, TStringBuilder()
+ << "Mkql memory limit exceeded, limit: " << MkqlMemoryLimit
+ << ", host: " << HostName()
+ << ", canAllocateExtraMemory: " << CanAllocateExtraMemory);
+ } catch (const NMiniKQL::TKqpEnsureFail& e) {
+ InternalError((TIssuesIds::EIssueCode) e.GetCode(), e.GetMessage());
+ } catch (const yexception& e) {
+ InternalError(TIssuesIds::DEFAULT_ERROR, e.what());
+ }
+
+ ReportEventElapsedTime();
+ }
+
+protected:
+ ui64 CalcMkqlMemoryLimit() override {
+ return TBase::CalcMkqlMemoryLimit() + ComputeCtx.GetTableScans().size() * MemoryLimits.ScanBufferSize;
+ }
+
+public:
+ void FillExtraStats(NDqProto::TDqComputeActorStats* dst, bool last) {
+ if (last && SysViewActorId && ScanData && dst->TasksSize() > 0) {
+ YQL_ENSURE(dst->TasksSize() == 1);
+
+ auto* taskStats = dst->MutableTasks(0);
+ auto* tableStats = taskStats->AddTables();
+
+ tableStats->SetTablePath(ScanData->TablePath);
+
+ if (auto* x = ScanData->BasicStats.get()) {
+ tableStats->SetReadRows(x->Rows);
+ tableStats->SetReadBytes(x->Bytes);
+ // TODO: CpuTime
+ }
+
+ if (auto* x = ScanData->ProfileStats.get()) {
+ NKqpProto::TKqpScanTableExtraStats tableExtraStats;
+ // protoScanStats->SetIScanStartTimeMs()
+ // protoScanStats->SetIScanFinishTimeMs();
+ tableExtraStats.SetIScanCpuTimeUs(x->ScanCpuTime.MicroSeconds());
+ tableExtraStats.SetIScanWaitTimeUs(x->ScanWaitTime.MicroSeconds());
+ tableExtraStats.SetIScanPageFaults(x->PageFaults);
+
+ tableExtraStats.SetMessages(x->Messages);
+ tableExtraStats.SetMessagesByPageFault(x->MessagesByPageFault);
+
+ tableStats->MutableExtra()->PackFrom(tableExtraStats);
+ }
+ }
+ }
+
+private:
+ void PassAway() override {
+ if (SysViewActorId) {
+ Send(SysViewActorId, new TEvents::TEvPoison);
+ }
+
+ if (TaskRunner) {
+ if (TaskRunner->IsAllocatorAttached()) {
+ ComputeCtx.Clear();
+ } else {
+ auto guard = TaskRunner->BindAllocator(MkqlMemoryLimit);
+ ComputeCtx.Clear();
+ }
+ }
+
+ TBase::PassAway();
+ }
+
+private:
+ void HandleExecute(TEvKqpCompute::TEvScanInitActor::TPtr& ev) {
+ Y_VERIFY_DEBUG(ScanData);
+
+ auto& msg = ev->Get()->Record;
+
+ Y_VERIFY_DEBUG(SysViewActorId == ActorIdFromProto(msg.GetScanActorId()));
+
+ CA_LOG_D("Got sysview scan initial event, scan actor: " << SysViewActorId << ", scanId: 0");
+ Send(ev->Sender, new TEvKqpCompute::TEvScanDataAck(GetMemoryLimits().ScanBufferSize));
+ return;
+ }
+
+ void HandleExecute(TEvKqpCompute::TEvScanData::TPtr& ev) {
+ Y_VERIFY_DEBUG(ScanData);
+ Y_VERIFY_DEBUG(SysViewActorId == ev->Sender);
+
+ auto& msg = *ev->Get();
+
ui64 bytes = 0;
ui64 rowsCount = 0;
{
@@ -264,65 +264,65 @@ private:
}
}
}
-
+
CA_LOG_D("Got sysview scandata, rows: " << rowsCount << ", bytes: " << bytes
- << ", finished: " << msg.Finished << ", from: " << SysViewActorId);
-
- if (msg.Finished) {
- CA_LOG_D("Finishing rows buffer");
+ << ", finished: " << msg.Finished << ", from: " << SysViewActorId);
+
+ if (msg.Finished) {
+ CA_LOG_D("Finishing rows buffer");
ScanData->Finish();
- }
-
- if (Y_UNLIKELY(ScanData->ProfileStats)) {
- ScanData->ProfileStats->Messages++;
- ScanData->ProfileStats->ScanCpuTime += msg.CpuTime; // TODO: not implemented yet
- ScanData->ProfileStats->ScanWaitTime += msg.WaitTime;
- if (msg.PageFault) {
- ScanData->ProfileStats->PageFaults += msg.PageFaults;
- ScanData->ProfileStats->MessagesByPageFault++;
- }
- }
-
+ }
+
+ if (Y_UNLIKELY(ScanData->ProfileStats)) {
+ ScanData->ProfileStats->Messages++;
+ ScanData->ProfileStats->ScanCpuTime += msg.CpuTime; // TODO: not implemented yet
+ ScanData->ProfileStats->ScanWaitTime += msg.WaitTime;
+ if (msg.PageFault) {
+ ScanData->ProfileStats->PageFaults += msg.PageFaults;
+ ScanData->ProfileStats->MessagesByPageFault++;
+ }
+ }
+
ui64 freeSpace = GetMemoryLimits().ScanBufferSize > ScanData->GetStoredBytes()
? GetMemoryLimits().ScanBufferSize - ScanData->GetStoredBytes()
- : 0;
-
- if (freeSpace > 0) {
- CA_LOG_D("Send scan data ack, freeSpace: " << freeSpace);
-
- Send(SysViewActorId, new TEvKqpCompute::TEvScanDataAck(freeSpace));
- }
-
- DoExecute();
- }
-
- void HandleExecute(TEvKqpCompute::TEvScanError::TPtr& ev) {
- Y_VERIFY_DEBUG(ScanData);
-
- TIssues issues;
- Ydb::StatusIds::StatusCode status = ev->Get()->Record.GetStatus();
- IssuesFromMessage(ev->Get()->Record.GetIssues(), issues);
-
+ : 0;
+
+ if (freeSpace > 0) {
+ CA_LOG_D("Send scan data ack, freeSpace: " << freeSpace);
+
+ Send(SysViewActorId, new TEvKqpCompute::TEvScanDataAck(freeSpace));
+ }
+
+ DoExecute();
+ }
+
+ void HandleExecute(TEvKqpCompute::TEvScanError::TPtr& ev) {
+ Y_VERIFY_DEBUG(ScanData);
+
+ TIssues issues;
+ Ydb::StatusIds::StatusCode status = ev->Get()->Record.GetStatus();
+ IssuesFromMessage(ev->Get()->Record.GetIssues(), issues);
+
State = NDqProto::COMPUTE_STATE_FAILURE;
ReportStateAndMaybeDie(status, issues);
- }
-
-private:
- NMiniKQL::TKqpScanComputeContext ComputeCtx;
- TMaybe<NKikimrTxDataShard::TKqpTransaction::TScanTaskMeta> Meta;
- NMiniKQL::TKqpScanComputeContext::TScanData* ScanData = nullptr;
- TActorId SysViewActorId;
-};
-
-} // anonymous namespace
-
-IActor* CreateKqpComputeActor(const TActorId& executerId, ui64 txId, NDqProto::TDqTask&& task,
+ }
+
+private:
+ NMiniKQL::TKqpScanComputeContext ComputeCtx;
+ TMaybe<NKikimrTxDataShard::TKqpTransaction::TScanTaskMeta> Meta;
+ NMiniKQL::TKqpScanComputeContext::TScanData* ScanData = nullptr;
+ TActorId SysViewActorId;
+};
+
+} // anonymous namespace
+
+IActor* CreateKqpComputeActor(const TActorId& executerId, ui64 txId, NDqProto::TDqTask&& task,
IDqSourceActorFactory::TPtr sourceActorFactory, IDqSinkActorFactory::TPtr sinkActorFactory,
const TComputeRuntimeSettings& settings, const TComputeMemoryLimits& memoryLimits)
-{
- return new TKqpComputeActor(executerId, txId, std::move(task), std::move(sourceActorFactory),
- std::move(sinkActorFactory), settings, memoryLimits);
-}
-
-} // namespace NKqp
-} // namespace NKikimr
+{
+ return new TKqpComputeActor(executerId, txId, std::move(task), std::move(sourceActorFactory),
+ std::move(sinkActorFactory), settings, memoryLimits);
+}
+
+} // namespace NKqp
+} // namespace NKikimr
diff --git a/ydb/core/kqp/compute_actor/kqp_scan_compute_actor.cpp b/ydb/core/kqp/compute_actor/kqp_scan_compute_actor.cpp
index b1581e1e18..f8e3cd0c77 100644
--- a/ydb/core/kqp/compute_actor/kqp_scan_compute_actor.cpp
+++ b/ydb/core/kqp/compute_actor/kqp_scan_compute_actor.cpp
@@ -1,6 +1,6 @@
-#include "kqp_compute_actor.h"
-#include "kqp_compute_actor_impl.h"
-
+#include "kqp_compute_actor.h"
+#include "kqp_compute_actor_impl.h"
+
#include <ydb/core/actorlib_impl/long_timer.h>
#include <ydb/core/base/tablet_pipecache.h>
#include <ydb/core/kqp/runtime/kqp_channel_storage.h>
@@ -15,22 +15,22 @@
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h>
#include <ydb/library/yql/public/issue/yql_issue.h>
-
-#include <library/cpp/actors/core/interconnect.h>
-
-#include <util/generic/deque.h>
-
+
+#include <library/cpp/actors/core/interconnect.h>
+
+#include <util/generic/deque.h>
+
namespace NKikimr {
namespace NKqp {
namespace {
-using namespace NYql;
-using namespace NYql::NDq;
-
-bool IsDebugLogEnabled(const TActorSystem* actorSystem, NActors::NLog::EComponent component) {
+using namespace NYql;
+using namespace NYql::NDq;
+
+bool IsDebugLogEnabled(const TActorSystem* actorSystem, NActors::NLog::EComponent component) {
auto* settings = actorSystem->LoggerSettings();
- return settings && settings->Satisfies(NActors::NLog::EPriority::PRI_DEBUG, component);
+ return settings && settings->Satisfies(NActors::NLog::EPriority::PRI_DEBUG, component);
}
TString DebugPrintRanges(TConstArrayRef<NScheme::TTypeId> types,
@@ -47,132 +47,132 @@ TString DebugPrintRanges(TConstArrayRef<NScheme::TTypeId> types,
return out;
}
-static constexpr TDuration MIN_RETRY_DELAY = TDuration::MilliSeconds(250);
-static constexpr TDuration MAX_RETRY_DELAY = TDuration::Seconds(2);
-static constexpr ui64 MAX_SHARD_RETRIES = 5; // retry after: 0, 250, 500, 1000, 2000
-static constexpr ui64 MAX_TOTAL_SHARD_RETRIES = 20;
-static constexpr ui64 MAX_SHARD_RESOLVES = 3;
+static constexpr TDuration MIN_RETRY_DELAY = TDuration::MilliSeconds(250);
+static constexpr TDuration MAX_RETRY_DELAY = TDuration::Seconds(2);
+static constexpr ui64 MAX_SHARD_RETRIES = 5; // retry after: 0, 250, 500, 1000, 2000
+static constexpr ui64 MAX_TOTAL_SHARD_RETRIES = 20;
+static constexpr ui64 MAX_SHARD_RESOLVES = 3;
static constexpr TDuration RL_MAX_BATCH_DELAY = TDuration::Seconds(50);
-
-
-class TKqpScanComputeActor : public TDqComputeActorBase<TKqpScanComputeActor> {
- using TBase = TDqComputeActorBase<TKqpScanComputeActor>;
-
- struct TEvPrivate {
- enum EEv {
- EvRetryShard = EventSpaceBegin(TKikimrEvents::ES_PRIVATE),
- };
-
- struct TEvRetryShard : public TEventLocal<TEvRetryShard, EvRetryShard> {};
- };
-
+
+
+class TKqpScanComputeActor : public TDqComputeActorBase<TKqpScanComputeActor> {
+ using TBase = TDqComputeActorBase<TKqpScanComputeActor>;
+
+ struct TEvPrivate {
+ enum EEv {
+ EvRetryShard = EventSpaceBegin(TKikimrEvents::ES_PRIVATE),
+ };
+
+ struct TEvRetryShard : public TEventLocal<TEvRetryShard, EvRetryShard> {};
+ };
+
public:
static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
- return NKikimrServices::TActivity::KQP_SCAN_COMPUTE_ACTOR;
+ return NKikimrServices::TActivity::KQP_SCAN_COMPUTE_ACTOR;
}
- TKqpScanComputeActor(const NKikimrKqp::TKqpSnapshot& snapshot, const TActorId& executerId, ui64 txId,
+ TKqpScanComputeActor(const NKikimrKqp::TKqpSnapshot& snapshot, const TActorId& executerId, ui64 txId,
NDqProto::TDqTask&& task, IDqSourceActorFactory::TPtr sourceActorFactory, IDqSinkActorFactory::TPtr sinkActorFactory,
const TComputeRuntimeSettings& settings, const TComputeMemoryLimits& memoryLimits, TIntrusivePtr<TKqpCounters> counters)
- : TBase(executerId, txId, std::move(task), std::move(sourceActorFactory), std::move(sinkActorFactory), settings, memoryLimits)
- , ComputeCtx(settings.StatsMode)
+ : TBase(executerId, txId, std::move(task), std::move(sourceActorFactory), std::move(sinkActorFactory), settings, memoryLimits)
+ , ComputeCtx(settings.StatsMode)
, Snapshot(snapshot)
- , Counters(counters)
- {
- YQL_ENSURE(GetTask().GetMeta().UnpackTo(&Meta), "Invalid task meta: " << GetTask().GetMeta().DebugString());
- YQL_ENSURE(!Meta.GetReads().empty());
+ , Counters(counters)
+ {
+ YQL_ENSURE(GetTask().GetMeta().UnpackTo(&Meta), "Invalid task meta: " << GetTask().GetMeta().DebugString());
+ YQL_ENSURE(!Meta.GetReads().empty());
YQL_ENSURE(Meta.GetTable().GetTableKind() != (ui32)ETableKind::SysView);
- KeyColumnTypes.assign(Meta.GetKeyColumnTypes().begin(), Meta.GetKeyColumnTypes().end());
- }
-
- void DoBootstrap() {
- NDq::TDqTaskRunnerContext execCtx;
- execCtx.FuncRegistry = AppData()->FunctionRegistry;
- execCtx.ComputeCtx = &ComputeCtx;
- execCtx.ComputationFactory = GetKqpActorComputeFactory(&ComputeCtx);
- execCtx.RandomProvider = TAppData::RandomProvider.Get();
- execCtx.TimeProvider = TAppData::TimeProvider.Get();
- execCtx.ApplyCtx = nullptr;
- execCtx.Alloc = nullptr;
- execCtx.TypeEnv = nullptr;
-
- const TActorSystem* actorSystem = TlsActivationContext->ActorSystem();
-
- NDq::TDqTaskRunnerSettings settings;
- settings.CollectBasicStats = GetStatsMode() >= NYql::NDqProto::DQ_STATS_MODE_BASIC;
- settings.CollectProfileStats = GetStatsMode() >= NYql::NDqProto::DQ_STATS_MODE_PROFILE;
- settings.OptLLVM = GetUseLLVM() ? "--compile-options=disable-opt" : "OFF";
- settings.AllowGeneratorsInUnboxedValues = false;
-
- NDq::TLogFunc logger;
- if (IsDebugLogEnabled(actorSystem, NKikimrServices::KQP_TASKS_RUNNER)) {
- logger = [actorSystem, txId = GetTxId(), taskId = GetTask().GetId()] (const TString& message) {
- LOG_DEBUG_S(*actorSystem, NKikimrServices::KQP_TASKS_RUNNER, "TxId: " << txId
- << ", task: " << taskId << ": " << message);
- };
- }
-
- auto taskRunner = CreateKqpTaskRunner(execCtx, settings, logger);
- SetTaskRunner(taskRunner);
-
- auto wakeup = [this]{ ContinueExecute(); };
+ KeyColumnTypes.assign(Meta.GetKeyColumnTypes().begin(), Meta.GetKeyColumnTypes().end());
+ }
+
+ void DoBootstrap() {
+ NDq::TDqTaskRunnerContext execCtx;
+ execCtx.FuncRegistry = AppData()->FunctionRegistry;
+ execCtx.ComputeCtx = &ComputeCtx;
+ execCtx.ComputationFactory = GetKqpActorComputeFactory(&ComputeCtx);
+ execCtx.RandomProvider = TAppData::RandomProvider.Get();
+ execCtx.TimeProvider = TAppData::TimeProvider.Get();
+ execCtx.ApplyCtx = nullptr;
+ execCtx.Alloc = nullptr;
+ execCtx.TypeEnv = nullptr;
+
+ const TActorSystem* actorSystem = TlsActivationContext->ActorSystem();
+
+ NDq::TDqTaskRunnerSettings settings;
+ settings.CollectBasicStats = GetStatsMode() >= NYql::NDqProto::DQ_STATS_MODE_BASIC;
+ settings.CollectProfileStats = GetStatsMode() >= NYql::NDqProto::DQ_STATS_MODE_PROFILE;
+ settings.OptLLVM = GetUseLLVM() ? "--compile-options=disable-opt" : "OFF";
+ settings.AllowGeneratorsInUnboxedValues = false;
+
+ NDq::TLogFunc logger;
+ if (IsDebugLogEnabled(actorSystem, NKikimrServices::KQP_TASKS_RUNNER)) {
+ logger = [actorSystem, txId = GetTxId(), taskId = GetTask().GetId()] (const TString& message) {
+ LOG_DEBUG_S(*actorSystem, NKikimrServices::KQP_TASKS_RUNNER, "TxId: " << txId
+ << ", task: " << taskId << ": " << message);
+ };
+ }
+
+ auto taskRunner = CreateKqpTaskRunner(execCtx, settings, logger);
+ SetTaskRunner(taskRunner);
+
+ auto wakeup = [this]{ ContinueExecute(); };
PrepareTaskRunner(TKqpTaskRunnerExecutionContext(std::get<ui64>(TxId), RuntimeSettings.UseSpilling, std::move(wakeup),
- TlsActivationContext->AsActorContext()));
-
- ComputeCtx.AddTableScan(0, Meta, GetStatsMode());
- ScanData = &ComputeCtx.GetTableScan(0);
-
- ScanData->TaskId = GetTask().GetId();
- ScanData->TableReader = CreateKqpTableReader(*ScanData);
-
- for (const auto& read : Meta.GetReads()) {
- auto& state = Shards.emplace_back(TShardState(read.GetShardId()));
- state.Ranges.reserve(read.GetKeyRanges().size());
- for (const auto& range : read.GetKeyRanges()) {
- auto& sr = state.Ranges.emplace_back(TSerializedTableRange(range));
- if (!range.HasTo()) {
- sr.To = sr.From;
- sr.FromInclusive = sr.ToInclusive = true;
- }
- }
-
- Y_VERIFY_DEBUG(!state.Ranges.empty());
- }
-
- StartTableScan();
-
- ContinueExecute();
- Become(&TKqpScanComputeActor::StateFunc);
- }
-
- STFUNC(StateFunc) {
- try {
- switch (ev->GetTypeRewrite()) {
- hFunc(TEvKqpCompute::TEvScanInitActor, HandleExecute);
- hFunc(TEvKqpCompute::TEvScanData, HandleExecute);
- hFunc(TEvKqpCompute::TEvScanError, HandleExecute);
- hFunc(TEvPipeCache::TEvDeliveryProblem, HandleExecute);
- hFunc(TEvPrivate::TEvRetryShard, HandleExecute);
- hFunc(TEvTxProxySchemeCache::TEvResolveKeySetResult, HandleExecute);
- hFunc(TEvTxProxySchemeCache::TEvInvalidateTableResult, HandleExecute);
- hFunc(TEvents::TEvUndelivered, HandleExecute);
- hFunc(TEvInterconnect::TEvNodeDisconnected, HandleExecute);
- IgnoreFunc(TEvInterconnect::TEvNodeConnected);
- default:
- StateFuncBase(ev, ctx);
- }
- } catch (const TMemoryLimitExceededException& e) {
- InternalError(TIssuesIds::KIKIMR_PRECONDITION_FAILED, TStringBuilder()
- << "Mkql memory limit exceeded, limit: " << MkqlMemoryLimit
- << ", host: " << HostName() << ", canAllocateExtraMemory: " << CanAllocateExtraMemory);
- } catch (const yexception& e) {
- InternalError(TIssuesIds::DEFAULT_ERROR, e.what());
- }
-
- ReportEventElapsedTime();
- }
-
+ TlsActivationContext->AsActorContext()));
+
+ ComputeCtx.AddTableScan(0, Meta, GetStatsMode());
+ ScanData = &ComputeCtx.GetTableScan(0);
+
+ ScanData->TaskId = GetTask().GetId();
+ ScanData->TableReader = CreateKqpTableReader(*ScanData);
+
+ for (const auto& read : Meta.GetReads()) {
+ auto& state = Shards.emplace_back(TShardState(read.GetShardId()));
+ state.Ranges.reserve(read.GetKeyRanges().size());
+ for (const auto& range : read.GetKeyRanges()) {
+ auto& sr = state.Ranges.emplace_back(TSerializedTableRange(range));
+ if (!range.HasTo()) {
+ sr.To = sr.From;
+ sr.FromInclusive = sr.ToInclusive = true;
+ }
+ }
+
+ Y_VERIFY_DEBUG(!state.Ranges.empty());
+ }
+
+ StartTableScan();
+
+ ContinueExecute();
+ Become(&TKqpScanComputeActor::StateFunc);
+ }
+
+ STFUNC(StateFunc) {
+ try {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvKqpCompute::TEvScanInitActor, HandleExecute);
+ hFunc(TEvKqpCompute::TEvScanData, HandleExecute);
+ hFunc(TEvKqpCompute::TEvScanError, HandleExecute);
+ hFunc(TEvPipeCache::TEvDeliveryProblem, HandleExecute);
+ hFunc(TEvPrivate::TEvRetryShard, HandleExecute);
+ hFunc(TEvTxProxySchemeCache::TEvResolveKeySetResult, HandleExecute);
+ hFunc(TEvTxProxySchemeCache::TEvInvalidateTableResult, HandleExecute);
+ hFunc(TEvents::TEvUndelivered, HandleExecute);
+ hFunc(TEvInterconnect::TEvNodeDisconnected, HandleExecute);
+ IgnoreFunc(TEvInterconnect::TEvNodeConnected);
+ default:
+ StateFuncBase(ev, ctx);
+ }
+ } catch (const TMemoryLimitExceededException& e) {
+ InternalError(TIssuesIds::KIKIMR_PRECONDITION_FAILED, TStringBuilder()
+ << "Mkql memory limit exceeded, limit: " << MkqlMemoryLimit
+ << ", host: " << HostName() << ", canAllocateExtraMemory: " << CanAllocateExtraMemory);
+ } catch (const yexception& e) {
+ InternalError(TIssuesIds::DEFAULT_ERROR, e.what());
+ }
+
+ ReportEventElapsedTime();
+ }
+
void HandleEvWakeup(EEvWakeupTag tag) {
switch (tag) {
case RlSendAllowedTag:
@@ -190,48 +190,48 @@ public:
}
}
- void FillExtraStats(NDqProto::TDqComputeActorStats* dst, bool last) {
- if (last && ScanData && dst->TasksSize() > 0) {
- YQL_ENSURE(dst->TasksSize() == 1);
-
- // NKqpProto::TKqpComputeActorExtraStats extraStats;
-
- auto* taskStats = dst->MutableTasks(0);
- auto* tableStats = taskStats->AddTables();
-
- tableStats->SetTablePath(ScanData->TablePath);
-
- if (auto* x = ScanData->BasicStats.get()) {
- tableStats->SetReadRows(x->Rows);
- tableStats->SetReadBytes(x->Bytes);
- tableStats->SetAffectedPartitions(x->AffectedShards);
- // TODO: CpuTime
- }
-
- if (auto* x = ScanData->ProfileStats.get()) {
- NKqpProto::TKqpScanTableExtraStats tableExtraStats;
- // protoScanStats->SetIScanStartTimeMs()
- // protoScanStats->SetIScanFinishTimeMs();
- tableExtraStats.SetIScanCpuTimeUs(x->ScanCpuTime.MicroSeconds());
- tableExtraStats.SetIScanWaitTimeUs(x->ScanWaitTime.MicroSeconds());
- tableExtraStats.SetIScanPageFaults(x->PageFaults);
-
- tableExtraStats.SetMessages(x->Messages);
- tableExtraStats.SetMessagesByPageFault(x->MessagesByPageFault);
-
- tableStats->MutableExtra()->PackFrom(tableExtraStats);
- }
-
- // dst->MutableExtra()->PackFrom(extraStats);
- }
- }
-
-protected:
- ui64 CalcMkqlMemoryLimit() override {
- return TBase::CalcMkqlMemoryLimit() + ComputeCtx.GetTableScans().size() * MemoryLimits.ScanBufferSize;
- }
-
-private:
+ void FillExtraStats(NDqProto::TDqComputeActorStats* dst, bool last) {
+ if (last && ScanData && dst->TasksSize() > 0) {
+ YQL_ENSURE(dst->TasksSize() == 1);
+
+ // NKqpProto::TKqpComputeActorExtraStats extraStats;
+
+ auto* taskStats = dst->MutableTasks(0);
+ auto* tableStats = taskStats->AddTables();
+
+ tableStats->SetTablePath(ScanData->TablePath);
+
+ if (auto* x = ScanData->BasicStats.get()) {
+ tableStats->SetReadRows(x->Rows);
+ tableStats->SetReadBytes(x->Bytes);
+ tableStats->SetAffectedPartitions(x->AffectedShards);
+ // TODO: CpuTime
+ }
+
+ if (auto* x = ScanData->ProfileStats.get()) {
+ NKqpProto::TKqpScanTableExtraStats tableExtraStats;
+ // protoScanStats->SetIScanStartTimeMs()
+ // protoScanStats->SetIScanFinishTimeMs();
+ tableExtraStats.SetIScanCpuTimeUs(x->ScanCpuTime.MicroSeconds());
+ tableExtraStats.SetIScanWaitTimeUs(x->ScanWaitTime.MicroSeconds());
+ tableExtraStats.SetIScanPageFaults(x->PageFaults);
+
+ tableExtraStats.SetMessages(x->Messages);
+ tableExtraStats.SetMessagesByPageFault(x->MessagesByPageFault);
+
+ tableStats->MutableExtra()->PackFrom(tableExtraStats);
+ }
+
+ // dst->MutableExtra()->PackFrom(extraStats);
+ }
+ }
+
+protected:
+ ui64 CalcMkqlMemoryLimit() override {
+ return TBase::CalcMkqlMemoryLimit() + ComputeCtx.GetTableScans().size() * MemoryLimits.ScanBufferSize;
+ }
+
+private:
void ProcessRlNoResourceAndDie() {
const NYql::TIssue issue = MakeIssue(NKikimrIssues::TIssuesIds::YDB_RESOURCE_USAGE_LIMITED,
"Throughput limit exceeded for query");
@@ -243,63 +243,63 @@ private:
ReportStateAndMaybeDie(Ydb::StatusIds::OVERLOADED, TIssues({issue}));
}
- void HandleExecute(TEvKqpCompute::TEvScanInitActor::TPtr& ev) {
- Y_VERIFY_DEBUG(ScanData);
- Y_VERIFY_DEBUG(!Shards.empty());
-
- auto& msg = ev->Get()->Record;
-
- auto& state = Shards.front();
- auto scanActorId = ActorIdFromProto(msg.GetScanActorId());
-
- CA_LOG_D("Got EvScanInitActor from " << scanActorId << ", gen: " << msg.GetGeneration()
- << ", state: " << ToString(state.State) << ", stateGen: " << state.Generation);
-
- switch (state.State) {
- case EShardState::Starting: {
- if (state.Generation == msg.GetGeneration()) {
- state.State = EShardState::Running;
- state.ActorId = scanActorId;
-
- state.ResetRetry();
-
- AffectedShards.insert(state.TabletId);
-
- SendScanDataAck(state, GetMemoryLimits().ScanBufferSize);
- return;
- }
-
- if (state.Generation > msg.GetGeneration()) {
- TerminateExpiredScan(scanActorId, "Got expired EvScanInitActor, terminate it");
- return;
- }
-
- YQL_ENSURE(false, "Got EvScanInitActor from the future, gen: " << msg.GetGeneration()
- << ", expected: " << state.Generation);
- }
-
- case EShardState::Initial:
- case EShardState::Running:
+ void HandleExecute(TEvKqpCompute::TEvScanInitActor::TPtr& ev) {
+ Y_VERIFY_DEBUG(ScanData);
+ Y_VERIFY_DEBUG(!Shards.empty());
+
+ auto& msg = ev->Get()->Record;
+
+ auto& state = Shards.front();
+ auto scanActorId = ActorIdFromProto(msg.GetScanActorId());
+
+ CA_LOG_D("Got EvScanInitActor from " << scanActorId << ", gen: " << msg.GetGeneration()
+ << ", state: " << ToString(state.State) << ", stateGen: " << state.Generation);
+
+ switch (state.State) {
+ case EShardState::Starting: {
+ if (state.Generation == msg.GetGeneration()) {
+ state.State = EShardState::Running;
+ state.ActorId = scanActorId;
+
+ state.ResetRetry();
+
+ AffectedShards.insert(state.TabletId);
+
+ SendScanDataAck(state, GetMemoryLimits().ScanBufferSize);
+ return;
+ }
+
+ if (state.Generation > msg.GetGeneration()) {
+ TerminateExpiredScan(scanActorId, "Got expired EvScanInitActor, terminate it");
+ return;
+ }
+
+ YQL_ENSURE(false, "Got EvScanInitActor from the future, gen: " << msg.GetGeneration()
+ << ", expected: " << state.Generation);
+ }
+
+ case EShardState::Initial:
+ case EShardState::Running:
case EShardState::PostRunning:
- case EShardState::Resolving: {
- TerminateExpiredScan(scanActorId, "Got unexpected/expired EvScanInitActor, terminate it");
- return;
- }
- }
- }
-
- void HandleExecute(TEvKqpCompute::TEvScanData::TPtr& ev) {
- Y_VERIFY_DEBUG(ScanData);
- Y_VERIFY_DEBUG(!Shards.empty());
-
- auto& msg = *ev->Get();
- auto& state = Shards.front();
-
- switch (state.State) {
+ case EShardState::Resolving: {
+ TerminateExpiredScan(scanActorId, "Got unexpected/expired EvScanInitActor, terminate it");
+ return;
+ }
+ }
+ }
+
+ void HandleExecute(TEvKqpCompute::TEvScanData::TPtr& ev) {
+ Y_VERIFY_DEBUG(ScanData);
+ Y_VERIFY_DEBUG(!Shards.empty());
+
+ auto& msg = *ev->Get();
+ auto& state = Shards.front();
+
+ switch (state.State) {
case EShardState::Running: {
- if (state.Generation == msg.Generation) {
- YQL_ENSURE(state.ActorId == ev->Sender, "expected: " << state.ActorId << ", got: " << ev->Sender);
-
+ if (state.Generation == msg.Generation) {
+ YQL_ENSURE(state.ActorId == ev->Sender, "expected: " << state.ActorId << ", got: " << ev->Sender);
+
TInstant startTime = TActivationContext::Now();
if (ev->Get()->Finished) {
state.State = EShardState::PostRunning;
@@ -374,8 +374,8 @@ private:
if (state.Generation == msg.Generation) {
YQL_ENSURE(state.ActorId == ev->Sender, "expected: " << state.ActorId << ", got: " << ev->Sender);
- LastKey = std::move(msg.LastKey);
-
+ LastKey = std::move(msg.LastKey);
+
ui64 bytes = 0;
ui64 rowsCount = 0;
{
@@ -390,7 +390,7 @@ private:
break;
}
case NKikimrTxDataShard::EScanDataFormat::ARROW: {
- if (msg.ArrowBatch != nullptr) {
+ if (msg.ArrowBatch != nullptr) {
bytes = ScanData->AddRows(*msg.ArrowBatch, state.TabletId, TaskRunner->GetHolderFactory());
rowsCount = msg.ArrowBatch->num_rows();
}
@@ -399,426 +399,426 @@ private:
}
}
- CA_LOG_D("Got EvScanData, rows: " << rowsCount << ", bytes: " << bytes << ", finished: " << msg.Finished
+ CA_LOG_D("Got EvScanData, rows: " << rowsCount << ", bytes: " << bytes << ", finished: " << msg.Finished
<< ", from: " << ev->Sender << ", shards remain: " << Shards.size()
<< ", delayed for: " << latency.SecondsFloat() << " seconds by ratelimitter");
-
+
if (rowsCount == 0 && !msg.Finished && state.State != EShardState::PostRunning) {
- ui64 freeSpace = GetMemoryLimits().ScanBufferSize > ScanData->GetStoredBytes()
- ? GetMemoryLimits().ScanBufferSize - ScanData->GetStoredBytes()
- : 0ul;
- SendScanDataAck(state, freeSpace);
- }
-
- if (msg.Finished) {
- CA_LOG_D("Tablet " << state.TabletId << " scan finished, unlink");
- Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(state.TabletId));
-
- Shards.pop_front();
-
- if (!Shards.empty()) {
- CA_LOG_D("Starting next scan");
- StartTableScan();
- } else {
- CA_LOG_D("Finish scans");
+ ui64 freeSpace = GetMemoryLimits().ScanBufferSize > ScanData->GetStoredBytes()
+ ? GetMemoryLimits().ScanBufferSize - ScanData->GetStoredBytes()
+ : 0ul;
+ SendScanDataAck(state, freeSpace);
+ }
+
+ if (msg.Finished) {
+ CA_LOG_D("Tablet " << state.TabletId << " scan finished, unlink");
+ Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(state.TabletId));
+
+ Shards.pop_front();
+
+ if (!Shards.empty()) {
+ CA_LOG_D("Starting next scan");
+ StartTableScan();
+ } else {
+ CA_LOG_D("Finish scans");
ScanData->Finish();
-
- if (ScanData->BasicStats) {
- ScanData->BasicStats->AffectedShards = AffectedShards.size();
- }
- }
- }
-
- if (Y_UNLIKELY(ScanData->ProfileStats)) {
- ScanData->ProfileStats->Messages++;
- ScanData->ProfileStats->ScanCpuTime += msg.CpuTime;
- ScanData->ProfileStats->ScanWaitTime += msg.WaitTime;
- if (msg.PageFault) {
- ScanData->ProfileStats->PageFaults += msg.PageFaults;
- ScanData->ProfileStats->MessagesByPageFault++;
- }
- }
-
- DoExecute();
-
+
+ if (ScanData->BasicStats) {
+ ScanData->BasicStats->AffectedShards = AffectedShards.size();
+ }
+ }
+ }
+
+ if (Y_UNLIKELY(ScanData->ProfileStats)) {
+ ScanData->ProfileStats->Messages++;
+ ScanData->ProfileStats->ScanCpuTime += msg.CpuTime;
+ ScanData->ProfileStats->ScanWaitTime += msg.WaitTime;
+ if (msg.PageFault) {
+ ScanData->ProfileStats->PageFaults += msg.PageFaults;
+ ScanData->ProfileStats->MessagesByPageFault++;
+ }
+ }
+
+ DoExecute();
+
} else if (state.Generation > msg.Generation) {
- TerminateExpiredScan(ev->Sender, "Cancel expired scan");
+ TerminateExpiredScan(ev->Sender, "Cancel expired scan");
} else {
YQL_ENSURE(false, "EvScanData from the future, expected: " << state.Generation << ", got: " << msg.Generation);
- }
+ }
break;
- }
-
+ }
+
case EShardState::Initial:
case EShardState::Starting:
case EShardState::Resolving: {
- TerminateExpiredScan(ev->Sender, "Cancel unexpected scan");
+ TerminateExpiredScan(ev->Sender, "Cancel unexpected scan");
break;
- }
- }
+ }
+ }
PendingScanData.pop_front();
- }
-
- void HandleExecute(TEvKqpCompute::TEvScanError::TPtr& ev) {
- Y_VERIFY_DEBUG(ScanData);
- Y_VERIFY_DEBUG(!Shards.empty());
-
- auto& msg = ev->Get()->Record;
-
- Ydb::StatusIds::StatusCode status = msg.GetStatus();
- TIssues issues;
- IssuesFromMessage(msg.GetIssues(), issues);
-
- auto& state = Shards.front();
-
- switch (state.State) {
+ }
+
+ void HandleExecute(TEvKqpCompute::TEvScanError::TPtr& ev) {
+ Y_VERIFY_DEBUG(ScanData);
+ Y_VERIFY_DEBUG(!Shards.empty());
+
+ auto& msg = ev->Get()->Record;
+
+ Ydb::StatusIds::StatusCode status = msg.GetStatus();
+ TIssues issues;
+ IssuesFromMessage(msg.GetIssues(), issues);
+
+ auto& state = Shards.front();
+
+ switch (state.State) {
case EShardState::Starting: {
- if (state.Generation == msg.GetGeneration()) {
- CA_LOG_W("Got EvScanError while starting scan, status: " << Ydb::StatusIds_StatusCode_Name(status)
- << ", reason: " << issues.ToString());
-
- bool schemeError = false;
-
- if (status == Ydb::StatusIds::SCHEME_ERROR) {
- schemeError = true;
- } else if (status == Ydb::StatusIds::ABORTED) {
- for (auto& issue : issues) {
- WalkThroughIssues(issue, false, [&schemeError](const TIssue& x, ui16) {
- if (x.IssueCode == TIssuesIds::KIKIMR_SCHEME_MISMATCH) {
- schemeError = true;
- }
- });
- if (schemeError) {
- break;
- }
- }
- }
-
- if (schemeError) {
- ResolveShard(state);
- return;
- }
-
+ if (state.Generation == msg.GetGeneration()) {
+ CA_LOG_W("Got EvScanError while starting scan, status: " << Ydb::StatusIds_StatusCode_Name(status)
+ << ", reason: " << issues.ToString());
+
+ bool schemeError = false;
+
+ if (status == Ydb::StatusIds::SCHEME_ERROR) {
+ schemeError = true;
+ } else if (status == Ydb::StatusIds::ABORTED) {
+ for (auto& issue : issues) {
+ WalkThroughIssues(issue, false, [&schemeError](const TIssue& x, ui16) {
+ if (x.IssueCode == TIssuesIds::KIKIMR_SCHEME_MISMATCH) {
+ schemeError = true;
+ }
+ });
+ if (schemeError) {
+ break;
+ }
+ }
+ }
+
+ if (schemeError) {
+ ResolveShard(state);
+ return;
+ }
+
State = NDqProto::COMPUTE_STATE_FAILURE;
ReportStateAndMaybeDie(status, issues);
- return;
- }
-
- if (state.Generation > msg.GetGeneration()) {
- // expired message
- return;
- }
-
- YQL_ENSURE(false, "Got EvScanError from the future, expected: " << state.Generation
- << ", got: " << msg.GetGeneration());
+ return;
+ }
+
+ if (state.Generation > msg.GetGeneration()) {
+ // expired message
+ return;
+ }
+
+ YQL_ENSURE(false, "Got EvScanError from the future, expected: " << state.Generation
+ << ", got: " << msg.GetGeneration());
break;
- }
-
+ }
+
case EShardState::PostRunning:
case EShardState::Running: {
- if (state.Generation == msg.GetGeneration()) {
- CA_LOG_W("Got EvScanError while running scan, status: " << Ydb::StatusIds_StatusCode_Name(status)
- << ", reason: " << issues.ToString() << ", restart");
-
- state.State = EShardState::Initial;
- state.ActorId = {};
- state.ResetRetry();
- StartTableScan();
- return;
- }
-
- if (state.Generation > msg.GetGeneration()) {
- // expired message
- return;
- }
-
- YQL_ENSURE(false, "Got EvScanError from the future, expected: " << state.Generation
- << ", got: " << msg.GetGeneration());
- }
-
+ if (state.Generation == msg.GetGeneration()) {
+ CA_LOG_W("Got EvScanError while running scan, status: " << Ydb::StatusIds_StatusCode_Name(status)
+ << ", reason: " << issues.ToString() << ", restart");
+
+ state.State = EShardState::Initial;
+ state.ActorId = {};
+ state.ResetRetry();
+ StartTableScan();
+ return;
+ }
+
+ if (state.Generation > msg.GetGeneration()) {
+ // expired message
+ return;
+ }
+
+ YQL_ENSURE(false, "Got EvScanError from the future, expected: " << state.Generation
+ << ", got: " << msg.GetGeneration());
+ }
+
case EShardState::Initial:
case EShardState::Resolving: {
- // do nothing
- return;
- }
- }
- }
-
- void HandleExecute(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) {
- auto& msg = *ev->Get();
- CA_LOG_W("Got EvDeliveryProblem, TabletId: " << msg.TabletId << ", NotDelivered: " << msg.NotDelivered);
-
- if (Shards.empty()) {
- return;
- }
-
- Y_VERIFY_DEBUG(ScanData);
- auto& state = Shards.front();
-
- if (state.TabletId != msg.TabletId) {
- CA_LOG_E("Unknown tablet " << msg.TabletId << ", expected " << state.TabletId);
- return;
- }
-
- switch (state.State) {
+ // do nothing
+ return;
+ }
+ }
+ }
+
+ void HandleExecute(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) {
+ auto& msg = *ev->Get();
+ CA_LOG_W("Got EvDeliveryProblem, TabletId: " << msg.TabletId << ", NotDelivered: " << msg.NotDelivered);
+
+ if (Shards.empty()) {
+ return;
+ }
+
+ Y_VERIFY_DEBUG(ScanData);
+ auto& state = Shards.front();
+
+ if (state.TabletId != msg.TabletId) {
+ CA_LOG_E("Unknown tablet " << msg.TabletId << ", expected " << state.TabletId);
+ return;
+ }
+
+ switch (state.State) {
case EShardState::Starting:
case EShardState::Running: {
- Counters->ScanQueryShardDisconnect->Inc();
-
- if (state.TotalRetries >= MAX_TOTAL_SHARD_RETRIES) {
- CA_LOG_E("TKqpScanComputeActor: broken pipe with tablet " << state.TabletId
- << ", retries limit exceeded (" << state.TotalRetries << ")");
- return InternalError(TIssuesIds::DEFAULT_ERROR, TStringBuilder()
- << "Retries limit with shard " << state.TabletId << " exceeded.");
- }
-
- if (state.RetryAttempt < MAX_SHARD_RETRIES) {
- state.RetryAttempt++;
- state.TotalRetries++;
- state.Generation = ++LastGeneration;
- state.ActorId = {};
- state.State = EShardState::Starting;
- state.SubscribedOnTablet = false;
-
- auto retryDelay = state.CalcRetryDelay();
- if (retryDelay) {
- CA_LOG_W("TKqpScanComputeActor: broken pipe with tablet " << state.TabletId
- << ", restarting scan from last received key " << PrintLastKey()
- << ", attempt #" << state.RetryAttempt << " (total " << state.TotalRetries << ")"
- << " schedule after " << retryDelay);
-
- state.RetryTimer = CreateLongTimer(TlsActivationContext->AsActorContext(), retryDelay,
- new IEventHandle(SelfId(), SelfId(), new TEvPrivate::TEvRetryShard));
- } else {
- CA_LOG_W("TKqpScanComputeActor: broken pipe with tablet " << state.TabletId
- << ", restarting scan from last received key " << PrintLastKey()
- << ", attempt #" << state.RetryAttempt << " (total " << state.TotalRetries << ")");
-
- SendStartScanRequest(state, state.Generation);
- }
-
- return;
- }
-
- ResolveShard(state);
- return;
- }
-
+ Counters->ScanQueryShardDisconnect->Inc();
+
+ if (state.TotalRetries >= MAX_TOTAL_SHARD_RETRIES) {
+ CA_LOG_E("TKqpScanComputeActor: broken pipe with tablet " << state.TabletId
+ << ", retries limit exceeded (" << state.TotalRetries << ")");
+ return InternalError(TIssuesIds::DEFAULT_ERROR, TStringBuilder()
+ << "Retries limit with shard " << state.TabletId << " exceeded.");
+ }
+
+ if (state.RetryAttempt < MAX_SHARD_RETRIES) {
+ state.RetryAttempt++;
+ state.TotalRetries++;
+ state.Generation = ++LastGeneration;
+ state.ActorId = {};
+ state.State = EShardState::Starting;
+ state.SubscribedOnTablet = false;
+
+ auto retryDelay = state.CalcRetryDelay();
+ if (retryDelay) {
+ CA_LOG_W("TKqpScanComputeActor: broken pipe with tablet " << state.TabletId
+ << ", restarting scan from last received key " << PrintLastKey()
+ << ", attempt #" << state.RetryAttempt << " (total " << state.TotalRetries << ")"
+ << " schedule after " << retryDelay);
+
+ state.RetryTimer = CreateLongTimer(TlsActivationContext->AsActorContext(), retryDelay,
+ new IEventHandle(SelfId(), SelfId(), new TEvPrivate::TEvRetryShard));
+ } else {
+ CA_LOG_W("TKqpScanComputeActor: broken pipe with tablet " << state.TabletId
+ << ", restarting scan from last received key " << PrintLastKey()
+ << ", attempt #" << state.RetryAttempt << " (total " << state.TotalRetries << ")");
+
+ SendStartScanRequest(state, state.Generation);
+ }
+
+ return;
+ }
+
+ ResolveShard(state);
+ return;
+ }
+
case EShardState::Initial:
case EShardState::Resolving:
case EShardState::PostRunning: {
- CA_LOG_W("TKqpScanComputeActor: broken pipe with tablet " << state.TabletId
- << ", state: " << (int) state.State);
- return;
- }
- }
- }
-
- void HandleExecute(TEvPrivate::TEvRetryShard::TPtr&) {
- Y_VERIFY_DEBUG(!Shards.empty());
- auto& state = Shards.front();
- SendStartScanRequest(state, state.Generation);
- }
-
- void HandleExecute(TEvTxProxySchemeCache::TEvResolveKeySetResult::TPtr& ev) {
- Y_VERIFY_DEBUG(ScanData);
- Y_VERIFY_DEBUG(!Shards.empty());
-
- auto state = std::move(Shards.front());
- Shards.pop_front();
-
- CA_LOG_D("Get resolve result, unlink from tablet " << state.TabletId);
- Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(state.TabletId));
-
- YQL_ENSURE(state.State == EShardState::Resolving);
-
- CA_LOG_D("Received TEvResolveKeySetResult update for table '" << ScanData->TablePath << "'");
-
- auto* request = ev->Get()->Request.Get();
- if (request->ErrorCount > 0) {
- CA_LOG_E("Resolve request failed for table '" << ScanData->TablePath << "', ErrorCount# " << request->ErrorCount);
-
- TString error;
- TIssuesIds::EIssueCode issueCode = TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE;
-
- for (const auto& x : request->ResultSet) {
- if ((ui32)x.Status < (ui32) NSchemeCache::TSchemeCacheRequest::EStatus::OkScheme) {
- // invalidate table
- Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvInvalidateTable(ScanData->TableId, {}));
-
- switch (x.Status) {
- case NSchemeCache::TSchemeCacheRequest::EStatus::PathErrorNotExist:
- issueCode = TIssuesIds::KIKIMR_SCHEME_ERROR;
- error = TStringBuilder() << "Table '" << ScanData->TablePath << "' not exists.";
- break;
- case NSchemeCache::TSchemeCacheRequest::EStatus::TypeCheckError:
- issueCode = TIssuesIds::KIKIMR_SCHEME_MISMATCH;
- error = TStringBuilder() << "Table '" << ScanData->TablePath << "' scheme changed.";
- break;
- default:
- issueCode = TIssuesIds::KIKIMR_SCHEME_ERROR;
- error = TStringBuilder() << "Unresolved table '" << ScanData->TablePath << "'. Status: " << x.Status;
- break;
- }
- }
- }
-
- return InternalError(issueCode, error);
- }
-
+ CA_LOG_W("TKqpScanComputeActor: broken pipe with tablet " << state.TabletId
+ << ", state: " << (int) state.State);
+ return;
+ }
+ }
+ }
+
+ void HandleExecute(TEvPrivate::TEvRetryShard::TPtr&) {
+ Y_VERIFY_DEBUG(!Shards.empty());
+ auto& state = Shards.front();
+ SendStartScanRequest(state, state.Generation);
+ }
+
+ void HandleExecute(TEvTxProxySchemeCache::TEvResolveKeySetResult::TPtr& ev) {
+ Y_VERIFY_DEBUG(ScanData);
+ Y_VERIFY_DEBUG(!Shards.empty());
+
+ auto state = std::move(Shards.front());
+ Shards.pop_front();
+
+ CA_LOG_D("Get resolve result, unlink from tablet " << state.TabletId);
+ Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(state.TabletId));
+
+ YQL_ENSURE(state.State == EShardState::Resolving);
+
+ CA_LOG_D("Received TEvResolveKeySetResult update for table '" << ScanData->TablePath << "'");
+
+ auto* request = ev->Get()->Request.Get();
+ if (request->ErrorCount > 0) {
+ CA_LOG_E("Resolve request failed for table '" << ScanData->TablePath << "', ErrorCount# " << request->ErrorCount);
+
+ TString error;
+ TIssuesIds::EIssueCode issueCode = TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE;
+
+ for (const auto& x : request->ResultSet) {
+ if ((ui32)x.Status < (ui32) NSchemeCache::TSchemeCacheRequest::EStatus::OkScheme) {
+ // invalidate table
+ Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvInvalidateTable(ScanData->TableId, {}));
+
+ switch (x.Status) {
+ case NSchemeCache::TSchemeCacheRequest::EStatus::PathErrorNotExist:
+ issueCode = TIssuesIds::KIKIMR_SCHEME_ERROR;
+ error = TStringBuilder() << "Table '" << ScanData->TablePath << "' not exists.";
+ break;
+ case NSchemeCache::TSchemeCacheRequest::EStatus::TypeCheckError:
+ issueCode = TIssuesIds::KIKIMR_SCHEME_MISMATCH;
+ error = TStringBuilder() << "Table '" << ScanData->TablePath << "' scheme changed.";
+ break;
+ default:
+ issueCode = TIssuesIds::KIKIMR_SCHEME_ERROR;
+ error = TStringBuilder() << "Unresolved table '" << ScanData->TablePath << "'. Status: " << x.Status;
+ break;
+ }
+ }
+ }
+
+ return InternalError(issueCode, error);
+ }
+
auto keyDesc = std::move(request->ResultSet[0].KeyDescription);
-
- if (keyDesc->Partitions.empty()) {
- TString error = TStringBuilder() << "No partitions to read from '" << ScanData->TablePath << "'";
- CA_LOG_E(error);
- InternalError(TIssuesIds::KIKIMR_SCHEME_ERROR, error);
- return;
- }
-
- const auto& tr = *AppData()->TypeRegistry;
-
- TVector<TShardState> newShards;
- newShards.reserve(keyDesc->Partitions.size());
-
- for (ui64 idx = 0, i = 0; idx < keyDesc->Partitions.size(); ++idx) {
- const auto& partition = keyDesc->Partitions[idx];
-
- TTableRange partitionRange{
- idx == 0 ? state.Ranges.front().From.GetCells() : keyDesc->Partitions[idx - 1].Range->EndKeyPrefix.GetCells(),
- idx == 0 ? state.Ranges.front().FromInclusive : !keyDesc->Partitions[idx - 1].Range->IsInclusive,
- keyDesc->Partitions[idx].Range->EndKeyPrefix.GetCells(),
- keyDesc->Partitions[idx].Range->IsInclusive
- };
-
- CA_LOG_D("Processing resolved ShardId# " << partition.ShardId
- << ", partition range: " << DebugPrintRange(KeyColumnTypes, partitionRange, tr)
- << ", i: " << i << ", state ranges: " << state.Ranges.size());
-
- auto newShard = TShardState(partition.ShardId);
-
- for (ui64 j = i; j < state.Ranges.size(); ++j) {
- CA_LOG_D("Intersect state range #" << j << " " << DebugPrintRange(KeyColumnTypes, state.Ranges[j].ToTableRange(), tr)
- << " with partition range " << DebugPrintRange(KeyColumnTypes, partitionRange, tr));
-
- auto intersection = Intersect(KeyColumnTypes, partitionRange, state.Ranges[j].ToTableRange());
-
- if (!intersection.IsEmptyRange(KeyColumnTypes)) {
- CA_LOG_D("Add range to new shardId: " << partition.ShardId
- << ", range: " << DebugPrintRange(KeyColumnTypes, intersection, tr));
-
- newShard.Ranges.emplace_back(TSerializedTableRange(intersection));
- } else {
- CA_LOG_D("empty intersection");
- if (j > i) {
- i = j - 1;
- }
- break;
- }
- }
-
- if (!newShard.Ranges.empty()) {
- newShards.emplace_back(std::move(newShard));
- }
- }
-
- YQL_ENSURE(!newShards.empty());
-
- for (int i = newShards.ysize() - 1; i >= 0; --i) {
- Shards.emplace_front(std::move(newShards[i]));
- }
-
- if (IsDebugLogEnabled(TlsActivationContext->ActorSystem(), NKikimrServices::KQP_COMPUTE)) {
- TStringBuilder sb;
- sb << "States: ";
- for (auto& st : Shards) {
- sb << st.ToString(KeyColumnTypes) << "; ";
- }
- CA_LOG_D(sb);
- }
-
- StartTableScan();
- }
-
- void HandleExecute(TEvTxProxySchemeCache::TEvInvalidateTableResult::TPtr&) {
+
+ if (keyDesc->Partitions.empty()) {
+ TString error = TStringBuilder() << "No partitions to read from '" << ScanData->TablePath << "'";
+ CA_LOG_E(error);
+ InternalError(TIssuesIds::KIKIMR_SCHEME_ERROR, error);
+ return;
+ }
+
+ const auto& tr = *AppData()->TypeRegistry;
+
+ TVector<TShardState> newShards;
+ newShards.reserve(keyDesc->Partitions.size());
+
+ for (ui64 idx = 0, i = 0; idx < keyDesc->Partitions.size(); ++idx) {
+ const auto& partition = keyDesc->Partitions[idx];
+
+ TTableRange partitionRange{
+ idx == 0 ? state.Ranges.front().From.GetCells() : keyDesc->Partitions[idx - 1].Range->EndKeyPrefix.GetCells(),
+ idx == 0 ? state.Ranges.front().FromInclusive : !keyDesc->Partitions[idx - 1].Range->IsInclusive,
+ keyDesc->Partitions[idx].Range->EndKeyPrefix.GetCells(),
+ keyDesc->Partitions[idx].Range->IsInclusive
+ };
+
+ CA_LOG_D("Processing resolved ShardId# " << partition.ShardId
+ << ", partition range: " << DebugPrintRange(KeyColumnTypes, partitionRange, tr)
+ << ", i: " << i << ", state ranges: " << state.Ranges.size());
+
+ auto newShard = TShardState(partition.ShardId);
+
+ for (ui64 j = i; j < state.Ranges.size(); ++j) {
+ CA_LOG_D("Intersect state range #" << j << " " << DebugPrintRange(KeyColumnTypes, state.Ranges[j].ToTableRange(), tr)
+ << " with partition range " << DebugPrintRange(KeyColumnTypes, partitionRange, tr));
+
+ auto intersection = Intersect(KeyColumnTypes, partitionRange, state.Ranges[j].ToTableRange());
+
+ if (!intersection.IsEmptyRange(KeyColumnTypes)) {
+ CA_LOG_D("Add range to new shardId: " << partition.ShardId
+ << ", range: " << DebugPrintRange(KeyColumnTypes, intersection, tr));
+
+ newShard.Ranges.emplace_back(TSerializedTableRange(intersection));
+ } else {
+ CA_LOG_D("empty intersection");
+ if (j > i) {
+ i = j - 1;
+ }
+ break;
+ }
+ }
+
+ if (!newShard.Ranges.empty()) {
+ newShards.emplace_back(std::move(newShard));
+ }
+ }
+
+ YQL_ENSURE(!newShards.empty());
+
+ for (int i = newShards.ysize() - 1; i >= 0; --i) {
+ Shards.emplace_front(std::move(newShards[i]));
+ }
+
+ if (IsDebugLogEnabled(TlsActivationContext->ActorSystem(), NKikimrServices::KQP_COMPUTE)) {
+ TStringBuilder sb;
+ sb << "States: ";
+ for (auto& st : Shards) {
+ sb << st.ToString(KeyColumnTypes) << "; ";
+ }
+ CA_LOG_D(sb);
+ }
+
+ StartTableScan();
}
- void HandleExecute(TEvents::TEvUndelivered::TPtr& ev) {
- switch (ev->Get()->SourceType) {
- case TEvDataShard::TEvKqpScan::EventType:
- // handled by TEvPipeCache::TEvDeliveryProblem event
- return;
- case TEvKqpCompute::TEvScanDataAck::EventType:
- if (Shards.empty()) {
- return;
- }
- auto& shard = Shards.front();
- if (shard.State == EShardState::Running && ev->Sender == shard.ActorId) {
+ void HandleExecute(TEvTxProxySchemeCache::TEvInvalidateTableResult::TPtr&) {
+ }
+
+ void HandleExecute(TEvents::TEvUndelivered::TPtr& ev) {
+ switch (ev->Get()->SourceType) {
+ case TEvDataShard::TEvKqpScan::EventType:
+ // handled by TEvPipeCache::TEvDeliveryProblem event
+ return;
+ case TEvKqpCompute::TEvScanDataAck::EventType:
+ if (Shards.empty()) {
+ return;
+ }
+ auto& shard = Shards.front();
+ if (shard.State == EShardState::Running && ev->Sender == shard.ActorId) {
CA_LOG_E("TEvScanDataAck lost while running scan, terminate execution. DataShard actor: "
<< shard.ActorId);
- InternalError(TIssuesIds::DEFAULT_ERROR, "Delivery problem: EvScanDataAck lost.");
- } else {
- CA_LOG_D("Skip lost TEvScanDataAck to " << ev->Sender << ", active scan actor: " << shard.ActorId);
- }
- return;
- }
- TBase::HandleExecuteBase(ev);
- }
-
- void HandleExecute(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) {
- auto nodeId = ev->Get()->NodeId;
- CA_LOG_N("Disconnected node " << nodeId);
-
- TrackingNodes.erase(nodeId);
-
- auto& state = Shards.front();
- if (state.ActorId && state.ActorId.NodeId() == nodeId) {
- InternalError(TIssuesIds::DEFAULT_ERROR, TStringBuilder() << "Connection with node " << nodeId << " lost.");
- }
- }
-
-private:
- struct TShardState;
-
- void StartTableScan() {
- YQL_ENSURE(!Shards.empty());
-
- auto& state = Shards.front();
-
- YQL_ENSURE(state.State == EShardState::Initial);
- state.State = EShardState::Starting;
- state.Generation = ++LastGeneration;
- state.ActorId = {};
-
- CA_LOG_D("StartTableScan: '" << ScanData->TablePath << "', shardId: " << state.TabletId << ", gen: " << state.Generation
+ InternalError(TIssuesIds::DEFAULT_ERROR, "Delivery problem: EvScanDataAck lost.");
+ } else {
+ CA_LOG_D("Skip lost TEvScanDataAck to " << ev->Sender << ", active scan actor: " << shard.ActorId);
+ }
+ return;
+ }
+ TBase::HandleExecuteBase(ev);
+ }
+
+ void HandleExecute(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) {
+ auto nodeId = ev->Get()->NodeId;
+ CA_LOG_N("Disconnected node " << nodeId);
+
+ TrackingNodes.erase(nodeId);
+
+ auto& state = Shards.front();
+ if (state.ActorId && state.ActorId.NodeId() == nodeId) {
+ InternalError(TIssuesIds::DEFAULT_ERROR, TStringBuilder() << "Connection with node " << nodeId << " lost.");
+ }
+ }
+
+private:
+ struct TShardState;
+
+ void StartTableScan() {
+ YQL_ENSURE(!Shards.empty());
+
+ auto& state = Shards.front();
+
+ YQL_ENSURE(state.State == EShardState::Initial);
+ state.State = EShardState::Starting;
+ state.Generation = ++LastGeneration;
+ state.ActorId = {};
+
+ CA_LOG_D("StartTableScan: '" << ScanData->TablePath << "', shardId: " << state.TabletId << ", gen: " << state.Generation
<< ", ranges: " << DebugPrintRanges(KeyColumnTypes, GetScanRanges(state)));
- SendStartScanRequest(state, state.Generation);
- }
-
- void SendScanDataAck(TShardState& state, ui64 freeSpace) {
- CA_LOG_D("Send EvScanDataAck to " << state.ActorId << ", freeSpace: " << freeSpace << ", gen: " << state.Generation);
- ui32 flags = IEventHandle::FlagTrackDelivery;
- if (TrackingNodes.insert(state.ActorId.NodeId()).second) {
- flags |= IEventHandle::FlagSubscribeOnSession;
- }
- Send(state.ActorId, new TEvKqpCompute::TEvScanDataAck(freeSpace, state.Generation), flags);
- }
-
- void SendStartScanRequest(TShardState& state, ui32 gen) {
- YQL_ENSURE(state.State == EShardState::Starting);
-
+ SendStartScanRequest(state, state.Generation);
+ }
+
+ void SendScanDataAck(TShardState& state, ui64 freeSpace) {
+ CA_LOG_D("Send EvScanDataAck to " << state.ActorId << ", freeSpace: " << freeSpace << ", gen: " << state.Generation);
+ ui32 flags = IEventHandle::FlagTrackDelivery;
+ if (TrackingNodes.insert(state.ActorId.NodeId()).second) {
+ flags |= IEventHandle::FlagSubscribeOnSession;
+ }
+ Send(state.ActorId, new TEvKqpCompute::TEvScanDataAck(freeSpace, state.Generation), flags);
+ }
+
+ void SendStartScanRequest(TShardState& state, ui32 gen) {
+ YQL_ENSURE(state.State == EShardState::Starting);
+
auto ev = MakeHolder<TEvDataShard::TEvKqpScan>();
- ev->Record.SetLocalPathId(ScanData->TableId.PathId.LocalPathId);
+ ev->Record.SetLocalPathId(ScanData->TableId.PathId.LocalPathId);
for (auto& column: ScanData->GetColumns()) {
- ev->Record.AddColumnTags(column.Tag);
- ev->Record.AddColumnTypes(column.Type);
+ ev->Record.AddColumnTags(column.Tag);
+ ev->Record.AddColumnTypes(column.Type);
}
- ev->Record.MutableSkipNullKeys()->CopyFrom(Meta.GetSkipNullKeys());
-
+ ev->Record.MutableSkipNullKeys()->CopyFrom(Meta.GetSkipNullKeys());
+
auto ranges = GetScanRanges(state);
auto protoRanges = ev->Record.MutableRanges();
protoRanges->Reserve(ranges.size());
-
+
for (auto& range: ranges) {
auto newRange = protoRanges->Add();
range.Serialize(*newRange);
@@ -829,13 +829,13 @@ private:
ev->Record.SetTimeoutMs(RuntimeSettings.Timeout.Get()->MilliSeconds());
}
ev->Record.SetStatsMode(RuntimeSettings.StatsMode);
- ev->Record.SetScanId(0);
+ ev->Record.SetScanId(0);
ev->Record.SetTxId(std::get<ui64>(TxId));
- ev->Record.SetTablePath(ScanData->TablePath);
- ev->Record.SetSchemaVersion(ScanData->TableId.SchemaVersion);
+ ev->Record.SetTablePath(ScanData->TablePath);
+ ev->Record.SetSchemaVersion(ScanData->TableId.SchemaVersion);
+
+ ev->Record.SetGeneration(gen);
- ev->Record.SetGeneration(gen);
-
ev->Record.SetReverse(Meta.GetReverse());
ev->Record.SetItemsLimit(Meta.GetItemsLimit());
@@ -851,13 +851,13 @@ private:
ev->Record.SetDataFormat(Meta.GetDataFormat());
- bool subscribed = std::exchange(state.SubscribedOnTablet, true);
-
- CA_LOG_D("Send EvKqpScan to shardId: " << state.TabletId << ", tablePath: " << ScanData->TablePath
- << ", gen: " << gen << ", subscribe: " << (!subscribed)
+ bool subscribed = std::exchange(state.SubscribedOnTablet, true);
+
+ CA_LOG_D("Send EvKqpScan to shardId: " << state.TabletId << ", tablePath: " << ScanData->TablePath
+ << ", gen: " << gen << ", subscribe: " << (!subscribed)
<< ", range: " << DebugPrintRanges(KeyColumnTypes, GetScanRanges(state)));
- Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvForward(ev.Release(), state.TabletId, !subscribed),
+ Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvForward(ev.Release(), state.TabletId, !subscribed),
IEventHandle::FlagTrackDelivery);
}
@@ -865,8 +865,8 @@ private:
// No any data read previously, return all ranges
if (!LastKey.DataSize()) {
return state.Ranges;
- }
-
+ }
+
// Form new vector. Skip ranges already read.
TVector<TSerializedTableRange> ranges;
ranges.reserve(state.Ranges.size());
@@ -893,246 +893,246 @@ private:
}
return ranges;
- }
-
- TString PrintLastKey() const {
- if (LastKey.empty()) {
- return "<none>";
- }
- return DebugPrintPoint(KeyColumnTypes, LastKey, *AppData()->TypeRegistry);
- }
-
- void TerminateExpiredScan(const TActorId& actorId, TStringBuf msg) {
- CA_LOG_W(msg);
-
- auto abortEv = MakeHolder<TEvKqp::TEvAbortExecution>();
- abortEv->Record.SetStatusCode(Ydb::StatusIds::CANCELLED);
- abortEv->Record.SetMessage("Cancel unexpected/expired scan");
-
- Send(actorId, abortEv.Release());
- }
-
- void ResolveShard(TShardState& state) {
- // resolve shards
- if (state.ResolveAttempt >= MAX_SHARD_RESOLVES) {
- InternalError(TIssuesIds::KIKIMR_SCHEME_ERROR, TStringBuilder()
- << "Table '" << ScanData->TablePath << "' resolve limit exceeded");
- return;
- }
-
- Counters->ScanQueryShardResolve->Inc();
-
- state.State = EShardState::Resolving;
- state.ResolveAttempt++;
- state.SubscribedOnTablet = false;
-
- auto range = TTableRange(state.Ranges.front().From.GetCells(), state.Ranges.front().FromInclusive,
- state.Ranges.back().To.GetCells(), state.Ranges.back().ToInclusive);
-
- TVector<TKeyDesc::TColumnOp> columns;
+ }
+
+ TString PrintLastKey() const {
+ if (LastKey.empty()) {
+ return "<none>";
+ }
+ return DebugPrintPoint(KeyColumnTypes, LastKey, *AppData()->TypeRegistry);
+ }
+
+ void TerminateExpiredScan(const TActorId& actorId, TStringBuf msg) {
+ CA_LOG_W(msg);
+
+ auto abortEv = MakeHolder<TEvKqp::TEvAbortExecution>();
+ abortEv->Record.SetStatusCode(Ydb::StatusIds::CANCELLED);
+ abortEv->Record.SetMessage("Cancel unexpected/expired scan");
+
+ Send(actorId, abortEv.Release());
+ }
+
+ void ResolveShard(TShardState& state) {
+ // resolve shards
+ if (state.ResolveAttempt >= MAX_SHARD_RESOLVES) {
+ InternalError(TIssuesIds::KIKIMR_SCHEME_ERROR, TStringBuilder()
+ << "Table '" << ScanData->TablePath << "' resolve limit exceeded");
+ return;
+ }
+
+ Counters->ScanQueryShardResolve->Inc();
+
+ state.State = EShardState::Resolving;
+ state.ResolveAttempt++;
+ state.SubscribedOnTablet = false;
+
+ auto range = TTableRange(state.Ranges.front().From.GetCells(), state.Ranges.front().FromInclusive,
+ state.Ranges.back().To.GetCells(), state.Ranges.back().ToInclusive);
+
+ TVector<TKeyDesc::TColumnOp> columns;
columns.reserve(ScanData->GetColumns().size());
for (const auto& column : ScanData->GetColumns()) {
- TKeyDesc::TColumnOp op;
- op.Column = column.Tag;
- op.Operation = TKeyDesc::EColumnOperation::Read;
- op.ExpectedType = column.Type;
- columns.emplace_back(std::move(op));
- }
-
- auto keyDesc = MakeHolder<TKeyDesc>(ScanData->TableId, range, TKeyDesc::ERowOperation::Read,
- KeyColumnTypes, columns);
-
- CA_LOG_D("Sending TEvResolveKeySet update for table '" << ScanData->TablePath << "'"
- << ", range: " << DebugPrintRange(KeyColumnTypes, range, *AppData()->TypeRegistry)
- << ", attempt #" << state.ResolveAttempt);
-
- auto request = MakeHolder<NSchemeCache::TSchemeCacheRequest>();
- // Avoid setting DomainOwnerId to reduce possible races with schemeshard migration
- // TODO: request->DatabaseName = ...;
- // TODO: request->UserToken = ...;
+ TKeyDesc::TColumnOp op;
+ op.Column = column.Tag;
+ op.Operation = TKeyDesc::EColumnOperation::Read;
+ op.ExpectedType = column.Type;
+ columns.emplace_back(std::move(op));
+ }
+
+ auto keyDesc = MakeHolder<TKeyDesc>(ScanData->TableId, range, TKeyDesc::ERowOperation::Read,
+ KeyColumnTypes, columns);
+
+ CA_LOG_D("Sending TEvResolveKeySet update for table '" << ScanData->TablePath << "'"
+ << ", range: " << DebugPrintRange(KeyColumnTypes, range, *AppData()->TypeRegistry)
+ << ", attempt #" << state.ResolveAttempt);
+
+ auto request = MakeHolder<NSchemeCache::TSchemeCacheRequest>();
+ // Avoid setting DomainOwnerId to reduce possible races with schemeshard migration
+ // TODO: request->DatabaseName = ...;
+ // TODO: request->UserToken = ...;
request->ResultSet.emplace_back(std::move(keyDesc));
- Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvInvalidateTable(ScanData->TableId, {}));
- Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvResolveKeySet(request));
- }
-
-private:
+ Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvInvalidateTable(ScanData->TableId, {}));
+ Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvResolveKeySet(request));
+ }
+
+private:
struct TScanFreeSpace : public IDestructable {
- ui64 FreeSpace = 0;
- };
+ ui64 FreeSpace = 0;
+ };
THolder<IDestructable> GetSourcesState() override {
- if (ScanData) {
- auto state = MakeHolder<TScanFreeSpace>();
+ if (ScanData) {
+ auto state = MakeHolder<TScanFreeSpace>();
state->FreeSpace = GetMemoryLimits().ScanBufferSize > ScanData->GetStoredBytes()
? GetMemoryLimits().ScanBufferSize - ScanData->GetStoredBytes()
- : 0ul;
- return state;
- }
- return nullptr;
- }
+ : 0ul;
+ return state;
+ }
+ return nullptr;
+ }
void PollSources(THolder<IDestructable> prev) override {
- if (!prev || !ScanData || Shards.empty()) {
- return;
+ if (!prev || !ScanData || Shards.empty()) {
+ return;
}
- auto& state = Shards.front();
-
+ auto& state = Shards.front();
+
ui64 freeSpace = GetMemoryLimits().ScanBufferSize > ScanData->GetStoredBytes()
? GetMemoryLimits().ScanBufferSize - ScanData->GetStoredBytes()
- : 0ul;
- ui64 prevFreeSpace = static_cast<TScanFreeSpace*>(prev.Get())->FreeSpace;
+ : 0ul;
+ ui64 prevFreeSpace = static_cast<TScanFreeSpace*>(prev.Get())->FreeSpace;
CA_LOG_T("Scan over tablet " << state.TabletId << " finished: " << ScanData->IsFinished()
- << ", prevFreeSpace: " << prevFreeSpace << ", freeSpace: " << freeSpace << ", peer: " << state.ActorId);
+ << ", prevFreeSpace: " << prevFreeSpace << ", freeSpace: " << freeSpace << ", peer: " << state.ActorId);
if (!ScanData->IsFinished() && state.State != EShardState::PostRunning
&& prevFreeSpace < freeSpace && state.ActorId)
{
- CA_LOG_T("[poll] Send EvScanDataAck to " << state.ActorId << ", gen: " << state.Generation
- << ", freeSpace: " << freeSpace);
- SendScanDataAck(state, freeSpace);
+ CA_LOG_T("[poll] Send EvScanDataAck to " << state.ActorId << ", gen: " << state.Generation
+ << ", freeSpace: " << freeSpace);
+ SendScanDataAck(state, freeSpace);
}
- }
-
- void TerminateSources(const TString& message, bool success) override {
- if (!ScanData || Shards.empty()) {
- return;
- }
-
- auto prio = success ? NActors::NLog::PRI_DEBUG : NActors::NLog::PRI_WARN;
- auto& state = Shards.front();
- if (state.ActorId) {
- CA_LOG(prio, "Send abort execution event to scan over tablet: " << state.TabletId << ", table: "
- << ScanData->TablePath << ", scan actor: " << state.ActorId << ", message: " << message);
-
- Send(state.ActorId, new TEvKqp::TEvAbortExecution(
- success ? Ydb::StatusIds::SUCCESS : Ydb::StatusIds::ABORTED, message));
- } else {
- CA_LOG(prio, "Table: " << ScanData->TablePath << ", scan has not been started yet");
- }
}
- void PassAway() override {
- Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(0));
- for (ui32 nodeId : TrackingNodes) {
- Send(TActivationContext::InterconnectProxy(nodeId), new TEvents::TEvUnsubscribe());
- }
-
- if (TaskRunner) {
- if (TaskRunner->IsAllocatorAttached()) {
- ComputeCtx.Clear();
- } else {
- auto guard = TaskRunner->BindAllocator(MkqlMemoryLimit);
- ComputeCtx.Clear();
- }
- }
-
- TBase::PassAway();
- }
-
-private:
- NMiniKQL::TKqpScanComputeContext ComputeCtx;
- NKikimrKqp::TKqpSnapshot Snapshot;
- TIntrusivePtr<TKqpCounters> Counters;
- NKikimrTxDataShard::TKqpTransaction::TScanTaskMeta Meta;
- TVector<NScheme::TTypeId> KeyColumnTypes;
-
- NMiniKQL::TKqpScanComputeContext::TScanData* ScanData = nullptr;
-
- TOwnedCellVec LastKey;
+ void TerminateSources(const TString& message, bool success) override {
+ if (!ScanData || Shards.empty()) {
+ return;
+ }
+
+ auto prio = success ? NActors::NLog::PRI_DEBUG : NActors::NLog::PRI_WARN;
+ auto& state = Shards.front();
+ if (state.ActorId) {
+ CA_LOG(prio, "Send abort execution event to scan over tablet: " << state.TabletId << ", table: "
+ << ScanData->TablePath << ", scan actor: " << state.ActorId << ", message: " << message);
+
+ Send(state.ActorId, new TEvKqp::TEvAbortExecution(
+ success ? Ydb::StatusIds::SUCCESS : Ydb::StatusIds::ABORTED, message));
+ } else {
+ CA_LOG(prio, "Table: " << ScanData->TablePath << ", scan has not been started yet");
+ }
+ }
+
+ void PassAway() override {
+ Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(0));
+ for (ui32 nodeId : TrackingNodes) {
+ Send(TActivationContext::InterconnectProxy(nodeId), new TEvents::TEvUnsubscribe());
+ }
+
+ if (TaskRunner) {
+ if (TaskRunner->IsAllocatorAttached()) {
+ ComputeCtx.Clear();
+ } else {
+ auto guard = TaskRunner->BindAllocator(MkqlMemoryLimit);
+ ComputeCtx.Clear();
+ }
+ }
+
+ TBase::PassAway();
+ }
+
+private:
+ NMiniKQL::TKqpScanComputeContext ComputeCtx;
+ NKikimrKqp::TKqpSnapshot Snapshot;
+ TIntrusivePtr<TKqpCounters> Counters;
+ NKikimrTxDataShard::TKqpTransaction::TScanTaskMeta Meta;
+ TVector<NScheme::TTypeId> KeyColumnTypes;
+
+ NMiniKQL::TKqpScanComputeContext::TScanData* ScanData = nullptr;
+
+ TOwnedCellVec LastKey;
TDeque<std::pair<TEvKqpCompute::TEvScanData::TPtr, TInstant>> PendingScanData;
-
+
enum class EShardState {
- Initial,
- Starting,
- Running,
+ Initial,
+ Starting,
+ Running,
PostRunning, //We already recieve all data, we has not processed it yet.
- Resolving,
- };
-
- static std::string_view ToString(EShardState state) {
- switch (state) {
- case EShardState::Initial: return "Initial"sv;
- case EShardState::Starting: return "Starting"sv;
- case EShardState::Running: return "Running"sv;
- case EShardState::Resolving: return "Resolving"sv;
- case EShardState::PostRunning: return "PostRunning"sv;
- }
- }
-
- // scan over datashards
- struct TShardState {
- ui64 TabletId;
- TSmallVec<TSerializedTableRange> Ranges;
-
- EShardState State = EShardState::Initial;
- ui32 Generation = 0;
- bool SubscribedOnTablet = false;
-
- ui32 RetryAttempt = 0;
- ui32 TotalRetries = 0;
- bool AllowInstantRetry = true;
- TDuration LastRetryDelay;
- TActorId RetryTimer;
-
- ui32 ResolveAttempt = 0;
-
- TActorId ActorId;
-
- explicit TShardState(ui64 tabletId)
- : TabletId(tabletId) {}
-
- TDuration CalcRetryDelay() {
- if (std::exchange(AllowInstantRetry, false)) {
- return TDuration::Zero();
- }
-
- if (LastRetryDelay) {
- LastRetryDelay = Min(LastRetryDelay * 2, MAX_RETRY_DELAY);
- } else {
- LastRetryDelay = MIN_RETRY_DELAY;
- }
- return LastRetryDelay;
- }
-
- void ResetRetry() {
- RetryAttempt = 0;
- AllowInstantRetry = true;
- LastRetryDelay = {};
- if (RetryTimer) {
- TlsActivationContext->Send(new IEventHandle(RetryTimer, RetryTimer, new TEvents::TEvPoison));
- RetryTimer = {};
- }
- ResolveAttempt = 0;
- }
-
- TString ToString(TConstArrayRef<NScheme::TTypeId> keyTypes) const {
- TStringBuilder sb;
- sb << "TShardState{ TabletId: " << TabletId << ", State: " << TKqpScanComputeActor::ToString(State)
- << ", Gen: " << Generation << ", Ranges: [";
- for (size_t i = 0; i < Ranges.size(); ++i) {
- sb << "#" << i << ": " << DebugPrintRange(keyTypes, Ranges[i].ToTableRange(), *AppData()->TypeRegistry);
- if (i + 1 != Ranges.size()) {
- sb << ", ";
- }
- }
+ Resolving,
+ };
+
+ static std::string_view ToString(EShardState state) {
+ switch (state) {
+ case EShardState::Initial: return "Initial"sv;
+ case EShardState::Starting: return "Starting"sv;
+ case EShardState::Running: return "Running"sv;
+ case EShardState::Resolving: return "Resolving"sv;
+ case EShardState::PostRunning: return "PostRunning"sv;
+ }
+ }
+
+ // scan over datashards
+ struct TShardState {
+ ui64 TabletId;
+ TSmallVec<TSerializedTableRange> Ranges;
+
+ EShardState State = EShardState::Initial;
+ ui32 Generation = 0;
+ bool SubscribedOnTablet = false;
+
+ ui32 RetryAttempt = 0;
+ ui32 TotalRetries = 0;
+ bool AllowInstantRetry = true;
+ TDuration LastRetryDelay;
+ TActorId RetryTimer;
+
+ ui32 ResolveAttempt = 0;
+
+ TActorId ActorId;
+
+ explicit TShardState(ui64 tabletId)
+ : TabletId(tabletId) {}
+
+ TDuration CalcRetryDelay() {
+ if (std::exchange(AllowInstantRetry, false)) {
+ return TDuration::Zero();
+ }
+
+ if (LastRetryDelay) {
+ LastRetryDelay = Min(LastRetryDelay * 2, MAX_RETRY_DELAY);
+ } else {
+ LastRetryDelay = MIN_RETRY_DELAY;
+ }
+ return LastRetryDelay;
+ }
+
+ void ResetRetry() {
+ RetryAttempt = 0;
+ AllowInstantRetry = true;
+ LastRetryDelay = {};
+ if (RetryTimer) {
+ TlsActivationContext->Send(new IEventHandle(RetryTimer, RetryTimer, new TEvents::TEvPoison));
+ RetryTimer = {};
+ }
+ ResolveAttempt = 0;
+ }
+
+ TString ToString(TConstArrayRef<NScheme::TTypeId> keyTypes) const {
+ TStringBuilder sb;
+ sb << "TShardState{ TabletId: " << TabletId << ", State: " << TKqpScanComputeActor::ToString(State)
+ << ", Gen: " << Generation << ", Ranges: [";
+ for (size_t i = 0; i < Ranges.size(); ++i) {
+ sb << "#" << i << ": " << DebugPrintRange(keyTypes, Ranges[i].ToTableRange(), *AppData()->TypeRegistry);
+ if (i + 1 != Ranges.size()) {
+ sb << ", ";
+ }
+ }
sb << "], "
- << ", RetryAttempt: " << RetryAttempt << ", TotalRetries: " << TotalRetries
- << ", ResolveAttempt: " << ResolveAttempt << ", ActorId: " << ActorId << " }";
- return sb;
- }
- };
- TDeque<TShardState> Shards; // always work with head
-
- ui32 LastGeneration = 0;
- TSet<ui64> AffectedShards;
- THashSet<ui32> TrackingNodes;
+ << ", RetryAttempt: " << RetryAttempt << ", TotalRetries: " << TotalRetries
+ << ", ResolveAttempt: " << ResolveAttempt << ", ActorId: " << ActorId << " }";
+ return sb;
+ }
+ };
+ TDeque<TShardState> Shards; // always work with head
+
+ ui32 LastGeneration = 0;
+ TSet<ui64> AffectedShards;
+ THashSet<ui32> TrackingNodes;
};
-} // anonymous namespace
+} // anonymous namespace
-IActor* CreateKqpScanComputeActor(const NKikimrKqp::TKqpSnapshot& snapshot, const TActorId& executerId, ui64 txId,
+IActor* CreateKqpScanComputeActor(const NKikimrKqp::TKqpSnapshot& snapshot, const TActorId& executerId, ui64 txId,
NDqProto::TDqTask&& task, IDqSourceActorFactory::TPtr sourceActorFactory, IDqSinkActorFactory::TPtr sinkActorFactory,
const TComputeRuntimeSettings& settings, const TComputeMemoryLimits& memoryLimits, TIntrusivePtr<TKqpCounters> counters)
{
diff --git a/ydb/core/kqp/compute_actor/ya.make b/ydb/core/kqp/compute_actor/ya.make
index 7618bdd19c..3a39e3c19a 100644
--- a/ydb/core/kqp/compute_actor/ya.make
+++ b/ydb/core/kqp/compute_actor/ya.make
@@ -7,8 +7,8 @@ OWNER(
SRCS(
kqp_compute_actor.cpp
- kqp_pure_compute_actor.cpp
- kqp_scan_compute_actor.cpp
+ kqp_pure_compute_actor.cpp
+ kqp_scan_compute_actor.cpp
)
PEERDIR(
@@ -21,6 +21,6 @@ PEERDIR(
ydb/library/yql/public/issue
)
-YQL_LAST_ABI_VERSION()
+YQL_LAST_ABI_VERSION()
END()
diff --git a/ydb/core/kqp/counters/kqp_counters.cpp b/ydb/core/kqp/counters/kqp_counters.cpp
index c0d5577b2f..26a28a0249 100644
--- a/ydb/core/kqp/counters/kqp_counters.cpp
+++ b/ydb/core/kqp/counters/kqp_counters.cpp
@@ -140,7 +140,7 @@ void TKqpCountersBase::Init() {
YdbQueryExecuteLatency = YdbGroup->GetNamedHistogram("name",
"table.query.execution.latency_milliseconds", NMonitoring::ExponentialHistogram(20, 2, 1));
-
+
// TODO move to grpc
YdbResponsesLocksInvalidated = YdbGroup->GetSubgroup("issue_type", "optimistic_locks_invalidation")
->GetNamedCounter("name", "api.grpc.response.issues", true);
@@ -379,11 +379,11 @@ void TKqpCountersBase::ReportIssues(const Ydb::Issue::IssueMessage& issue) {
}
void TKqpCountersBase::ReportQueryLatency(NKikimrKqp::EQueryAction action, const TDuration& duration) {
- if (action == NKikimrKqp::QUERY_ACTION_EXECUTE ||
- action == NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED ||
- action == NKikimrKqp::QUERY_ACTION_COMMIT_TX ||
- action == NKikimrKqp::QUERY_ACTION_ROLLBACK_TX)
- {
+ if (action == NKikimrKqp::QUERY_ACTION_EXECUTE ||
+ action == NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED ||
+ action == NKikimrKqp::QUERY_ACTION_COMMIT_TX ||
+ action == NKikimrKqp::QUERY_ACTION_ROLLBACK_TX)
+ {
YdbQueryExecuteLatency->Collect(duration.MilliSeconds());
}
@@ -393,7 +393,7 @@ void TKqpCountersBase::ReportQueryLatency(NKikimrKqp::EQueryAction action, const
}
}
-
+
void TKqpCountersBase::ReportTransaction(const TKqpTransactionInfo& txInfo) {
switch (txInfo.Status) {
case TKqpTransactionInfo::EStatus::Active:
@@ -401,7 +401,7 @@ void TKqpCountersBase::ReportTransaction(const TKqpTransactionInfo& txInfo) {
case TKqpTransactionInfo::EStatus::Aborted:
TxAborted->Inc();
return;
- case TKqpTransactionInfo::EStatus::Committed:
+ case TKqpTransactionInfo::EStatus::Committed:
TxCommited->Inc();
break;
}
@@ -681,7 +681,7 @@ void TKqpCounters::UpdateTxCounters(const TKqpTransactionInfo& txInfo,
byKind->TotalDuration->Collect(txInfo.TotalDuration.MilliSeconds());
byKind->ServerDuration->Collect(txInfo.ServerDuration.MilliSeconds());
- byKind->ClientDuration->Collect((txInfo.TotalDuration - txInfo.ServerDuration).MilliSeconds());
+ byKind->ClientDuration->Collect((txInfo.TotalDuration - txInfo.ServerDuration).MilliSeconds());
byKind->Queries->Collect(txInfo.QueriesCount);
}
@@ -720,7 +720,7 @@ TKqpCounters::TKqpCounters(const NMonitoring::TDynamicCounterPtr& counters, cons
/* Resource Manager */
RmComputeActors = KqpGroup->GetCounter("RM/ComputeActors", false);
RmMemory = KqpGroup->GetCounter("RM/Memory", false);
- RmExternalMemory = KqpGroup->GetCounter("RM/ExternalMemory", false);
+ RmExternalMemory = KqpGroup->GetCounter("RM/ExternalMemory", false);
RmNotEnoughMemory = KqpGroup->GetCounter("RM/NotEnoughMemory", true);
RmNotEnoughComputeActors = KqpGroup->GetCounter("RM/NotEnoughComputeActors", true);
RmExtraMemAllocs = KqpGroup->GetCounter("RM/ExtraMemAllocs", true);
@@ -734,10 +734,10 @@ TKqpCounters::TKqpCounters(const NMonitoring::TDynamicCounterPtr& counters, cons
SpillingTooBigFileErrors = KqpGroup->GetCounter("Spilling/TooBigFileErrors", true);
SpillingNoSpaceErrors = KqpGroup->GetCounter("Spilling/NoSpaceErrors", true);
SpillingIoErrors = KqpGroup->GetCounter("Spilling/IoErrors", true);
-
- /* Scan queries */
- ScanQueryShardDisconnect = KqpGroup->GetCounter("ScanQuery/ShardDisconnect", true);
- ScanQueryShardResolve = KqpGroup->GetCounter("ScanQuery/ShardResolve", true);
+
+ /* Scan queries */
+ ScanQueryShardDisconnect = KqpGroup->GetCounter("ScanQuery/ShardDisconnect", true);
+ ScanQueryShardResolve = KqpGroup->GetCounter("ScanQuery/ShardResolve", true);
ScanQueryRateLimitLatency = KqpGroup->GetHistogram(
"ScanQuery/RateLimitLatency", NMonitoring::ExponentialHistogram(20, 2, 1));
@@ -755,13 +755,13 @@ TKqpCounters::TKqpCounters(const NMonitoring::TDynamicCounterPtr& counters, cons
NewEngineCompatibleComputeCpuTime = KqpGroup->GetCounter("Query/ComputeCpuTime_NECompatible", true);
NewEngineForcedQueryCount = KqpGroup->GetCounter("Query/Count_NEForced", true);
NewEngineCompatibleQueryCount = KqpGroup->GetCounter("Query/Count_NECompatible", true);
-
- LiteralTxTotalTimeHistogram = KqpGroup->GetHistogram(
- "NE/LiteralTxTotalTimeMs", NMonitoring::ExponentialHistogram(10, 2, 1));
- DataTxTotalTimeHistogram = KqpGroup->GetHistogram(
- "NE/DataTxTotalTimeMs", NMonitoring::ExponentialHistogram(20, 2, 1));
- ScanTxTotalTimeHistogram = KqpGroup->GetHistogram(
- "NE/ScanTxTotalTimeMs", NMonitoring::ExponentialHistogram(20, 2, 1));
+
+ LiteralTxTotalTimeHistogram = KqpGroup->GetHistogram(
+ "NE/LiteralTxTotalTimeMs", NMonitoring::ExponentialHistogram(10, 2, 1));
+ DataTxTotalTimeHistogram = KqpGroup->GetHistogram(
+ "NE/DataTxTotalTimeMs", NMonitoring::ExponentialHistogram(20, 2, 1));
+ ScanTxTotalTimeHistogram = KqpGroup->GetHistogram(
+ "NE/ScanTxTotalTimeMs", NMonitoring::ExponentialHistogram(20, 2, 1));
}
NMonitoring::TDynamicCounterPtr TKqpCounters::GetQueryReplayCounters() const {
@@ -914,9 +914,9 @@ void TKqpCounters::ReportTransaction(TKqpDbCountersPtr dbCounters, const TKqpTra
}
if (txInfo.Status == TKqpTransactionInfo::EStatus::Committed) {
UpdateTxCounters(txInfo, TxByKind);
- }
-}
-
+ }
+}
+
void TKqpCounters::ReportSqlVersion(TKqpDbCountersPtr dbCounters, ui16 sqlVersion) {
TKqpCountersBase::ReportSqlVersion(sqlVersion);
if (dbCounters) {
@@ -1134,5 +1134,5 @@ void TKqpCounters::RemoveDbCounters(const TString& database) {
DbCounters.Erase(database);
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/counters/kqp_counters.h b/ydb/core/kqp/counters/kqp_counters.h
index 1988473738..53a8f7d2c2 100644
--- a/ydb/core/kqp/counters/kqp_counters.h
+++ b/ydb/core/kqp/counters/kqp_counters.h
@@ -268,12 +268,12 @@ public:
void ReportSqlVersion(TKqpDbCountersPtr dbCounters, ui16 sqlVersion);
void ReportTransaction(TKqpDbCountersPtr dbCounters, const TKqpTransactionInfo& txInfo);
- // NewEngine
+ // NewEngine
void ReportNewEngineForcedQueryStats(NKikimrKqp::EQueryAction action,
TDuration duration, ui64 computeCpuTime);
void ReportNewEngineCompatibleQueryStats(NKikimrKqp::EQueryAction action,
TDuration duration, ui64 computeCpuTime);
-
+
void ReportWorkerCreated(TKqpDbCountersPtr dbCounters);
void ReportWorkerFinished(TKqpDbCountersPtr dbCounters, TDuration lifeSpan);
void ReportWorkerCleanupLatency(TKqpDbCountersPtr dbCounters, TDuration cleanupTime);
@@ -317,32 +317,32 @@ public:
NMonitoring::TDynamicCounters::TCounterPtr CompileQueryCacheBytes;
NMonitoring::TDynamicCounters::TCounterPtr CompileQueryCacheEvicted;
NMonitoring::TDynamicCounters::TCounterPtr CompileQueueSize;
- NMonitoring::TDynamicCounters::TCounterPtr ForceNewEngineCompileErrors;
-
- // Resource Manager
- NMonitoring::TDynamicCounters::TCounterPtr RmComputeActors;
- NMonitoring::TDynamicCounters::TCounterPtr RmMemory;
- NMonitoring::TDynamicCounters::TCounterPtr RmExternalMemory;
- NMonitoring::TDynamicCounters::TCounterPtr RmNotEnoughMemory;
- NMonitoring::TDynamicCounters::TCounterPtr RmNotEnoughComputeActors;
- NMonitoring::TDynamicCounters::TCounterPtr RmExtraMemAllocs;
- NMonitoring::TDynamicCounters::TCounterPtr RmInternalError;
-
- // Spilling counters
- NMonitoring::TDynamicCounters::TCounterPtr SpillingWriteBlobs;
- NMonitoring::TDynamicCounters::TCounterPtr SpillingReadBlobs;
- NMonitoring::TDynamicCounters::TCounterPtr SpillingStoredBlobs;
- NMonitoring::TDynamicCounters::TCounterPtr SpillingTotalSpaceUsed;
- NMonitoring::TDynamicCounters::TCounterPtr SpillingTooBigFileErrors;
- NMonitoring::TDynamicCounters::TCounterPtr SpillingNoSpaceErrors;
- NMonitoring::TDynamicCounters::TCounterPtr SpillingIoErrors;
-
- // Scan queries counters
- NMonitoring::TDynamicCounters::TCounterPtr ScanQueryShardDisconnect;
- NMonitoring::TDynamicCounters::TCounterPtr ScanQueryShardResolve;
+ NMonitoring::TDynamicCounters::TCounterPtr ForceNewEngineCompileErrors;
+
+ // Resource Manager
+ NMonitoring::TDynamicCounters::TCounterPtr RmComputeActors;
+ NMonitoring::TDynamicCounters::TCounterPtr RmMemory;
+ NMonitoring::TDynamicCounters::TCounterPtr RmExternalMemory;
+ NMonitoring::TDynamicCounters::TCounterPtr RmNotEnoughMemory;
+ NMonitoring::TDynamicCounters::TCounterPtr RmNotEnoughComputeActors;
+ NMonitoring::TDynamicCounters::TCounterPtr RmExtraMemAllocs;
+ NMonitoring::TDynamicCounters::TCounterPtr RmInternalError;
+
+ // Spilling counters
+ NMonitoring::TDynamicCounters::TCounterPtr SpillingWriteBlobs;
+ NMonitoring::TDynamicCounters::TCounterPtr SpillingReadBlobs;
+ NMonitoring::TDynamicCounters::TCounterPtr SpillingStoredBlobs;
+ NMonitoring::TDynamicCounters::TCounterPtr SpillingTotalSpaceUsed;
+ NMonitoring::TDynamicCounters::TCounterPtr SpillingTooBigFileErrors;
+ NMonitoring::TDynamicCounters::TCounterPtr SpillingNoSpaceErrors;
+ NMonitoring::TDynamicCounters::TCounterPtr SpillingIoErrors;
+
+ // Scan queries counters
+ NMonitoring::TDynamicCounters::TCounterPtr ScanQueryShardDisconnect;
+ NMonitoring::TDynamicCounters::TCounterPtr ScanQueryShardResolve;
NMonitoring::THistogramPtr ScanQueryRateLimitLatency;
-
- // NewEngine vs OldEngine
+
+ // NewEngine vs OldEngine
THashMap<NKikimrKqp::EQueryAction, NMonitoring::THistogramPtr> NewEngineForcedQueryLatencies;
THashMap<NKikimrKqp::EQueryAction, NMonitoring::THistogramPtr> NewEngineCompatibleQueryLatencies;
NMonitoring::TDynamicCounters::TCounterPtr NewEngineForcedComputeCpuTime;
@@ -350,11 +350,11 @@ public:
NMonitoring::TDynamicCounters::TCounterPtr NewEngineCompatibleComputeCpuTime;
NMonitoring::TDynamicCounters::TCounterPtr NewEngineCompatibleQueryCount;
- // NewEngine tx duration
- NMonitoring::THistogramPtr LiteralTxTotalTimeHistogram;
- NMonitoring::THistogramPtr DataTxTotalTimeHistogram;
- NMonitoring::THistogramPtr ScanTxTotalTimeHistogram;
-
+ // NewEngine tx duration
+ NMonitoring::THistogramPtr LiteralTxTotalTimeHistogram;
+ NMonitoring::THistogramPtr DataTxTotalTimeHistogram;
+ NMonitoring::THistogramPtr ScanTxTotalTimeHistogram;
+
TAlignedPagePoolCounters AllocCounters;
// db counters
@@ -363,13 +363,13 @@ public:
TActorId DbWatcherActorId;
};
-struct TKqpRequestCounters : public TThrRefBase {
- using TPtr = TIntrusivePtr<TKqpRequestCounters>;
-
- TIntrusivePtr<TKqpCounters> Counters;
- TIntrusivePtr<TKqpDbCounters> DbCounters; // may be null
- TIntrusivePtr<NTxProxy::TTxProxyMon> TxProxyMon; // OldEngine compatibility
-};
-
-} // namespace NKqp
+struct TKqpRequestCounters : public TThrRefBase {
+ using TPtr = TIntrusivePtr<TKqpRequestCounters>;
+
+ TIntrusivePtr<TKqpCounters> Counters;
+ TIntrusivePtr<TKqpDbCounters> DbCounters; // may be null
+ TIntrusivePtr<NTxProxy::TTxProxyMon> TxProxyMon; // OldEngine compatibility
+};
+
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/executer/kqp_data_executer.cpp b/ydb/core/kqp/executer/kqp_data_executer.cpp
index 68516ccd7a..003d4ffa20 100644
--- a/ydb/core/kqp/executer/kqp_data_executer.cpp
+++ b/ydb/core/kqp/executer/kqp_data_executer.cpp
@@ -1,13 +1,13 @@
-#include "kqp_executer.h"
-#include "kqp_executer_impl.h"
-#include "kqp_locks_helper.h"
-#include "kqp_partition_helper.h"
-#include "kqp_planner.h"
-#include "kqp_result_channel.h"
-#include "kqp_table_resolver.h"
-#include "kqp_tasks_validate.h"
-#include "kqp_shards_resolver.h"
-
+#include "kqp_executer.h"
+#include "kqp_executer_impl.h"
+#include "kqp_locks_helper.h"
+#include "kqp_partition_helper.h"
+#include "kqp_planner.h"
+#include "kqp_result_channel.h"
+#include "kqp_table_resolver.h"
+#include "kqp_tasks_validate.h"
+#include "kqp_shards_resolver.h"
+
#include <ydb/core/base/appdata.h>
#include <ydb/core/base/tablet_pipecache.h>
#include <ydb/core/client/minikql_compile/db_key_resolver.h>
@@ -19,120 +19,120 @@
#include <ydb/core/tx/coordinator/coordinator_impl.h>
#include <ydb/core/tx/datashard/datashard.h>
#include <ydb/core/tx/tx_proxy/proxy.h>
-
+
#include <ydb/library/yql/dq/runtime/dq_columns_resolve.h>
#include <ydb/library/yql/dq/tasks/dq_connection_builder.h>
#include <ydb/library/yql/public/issue/yql_issue_message.h>
-
-namespace NKikimr {
-namespace NKqp {
-
-using namespace NYql;
-using namespace NYql::NDq;
-
-namespace {
-
-static constexpr TDuration MinReattachDelay = TDuration::MilliSeconds(10);
-static constexpr TDuration MaxReattachDelay = TDuration::MilliSeconds(100);
-static constexpr TDuration MaxReattachDuration = TDuration::Seconds(4);
+
+namespace NKikimr {
+namespace NKqp {
+
+using namespace NYql;
+using namespace NYql::NDq;
+
+namespace {
+
+static constexpr TDuration MinReattachDelay = TDuration::MilliSeconds(10);
+static constexpr TDuration MaxReattachDelay = TDuration::MilliSeconds(100);
+static constexpr TDuration MaxReattachDuration = TDuration::Seconds(4);
static constexpr ui32 ReplySizeLimit = 48 * 1024 * 1024; // 48 MB
-
-class TKqpDataExecuter : public TKqpExecuterBase<TKqpDataExecuter, EExecType::Data> {
- using TBase = TKqpExecuterBase<TKqpDataExecuter, EExecType::Data>;
-
- struct TEvPrivate {
- enum EEv {
- EvReattachToShard = EventSpaceBegin(TEvents::ES_PRIVATE),
- };
-
- struct TEvReattachToShard : public TEventLocal<TEvReattachToShard, EvReattachToShard> {
- const ui64 TabletId;
-
- explicit TEvReattachToShard(ui64 tabletId)
- : TabletId(tabletId) {}
- };
- };
-
- struct TReattachState {
- TDuration Delay;
- TInstant Deadline;
- ui64 Cookie = 0;
- bool Reattaching = false;
-
- bool ShouldReattach(TInstant now) {
- ++Cookie; // invalidate any previous cookie
-
- if (!Reattaching) {
- Deadline = now + MaxReattachDuration;
- Delay = TDuration::Zero();
- Reattaching = true;
- return true;
- }
-
- TDuration left = Deadline - now;
- if (!left) {
- Reattaching = false;
- return false;
- }
-
- Delay *= 2.0;
- if (Delay < MinReattachDelay) {
- Delay = MinReattachDelay;
- } else if (Delay > MaxReattachDelay) {
- Delay = MaxReattachDelay;
- }
-
- // Add ±10% jitter
- Delay *= 0.9 + 0.2 * TAppData::RandomProvider->GenRandReal4();
- if (Delay > left) {
- Delay = left;
- }
-
- return true;
- }
-
- void Reattached() {
- Reattaching = false;
- }
- };
-
- struct TShardState {
- enum class EState {
- Initial,
- Preparing, // planned tx only
- Prepared, // planned tx only
- Executing,
- Finished
- };
-
- EState State = EState::Initial;
- TSet<ui64> TaskIds;
-
- struct TDatashardState {
- ui64 ShardMinStep = 0;
- ui64 ShardMaxStep = 0;
- ui64 ReadSize = 0;
- bool ShardReadLocks = false;
+
+class TKqpDataExecuter : public TKqpExecuterBase<TKqpDataExecuter, EExecType::Data> {
+ using TBase = TKqpExecuterBase<TKqpDataExecuter, EExecType::Data>;
+
+ struct TEvPrivate {
+ enum EEv {
+ EvReattachToShard = EventSpaceBegin(TEvents::ES_PRIVATE),
+ };
+
+ struct TEvReattachToShard : public TEventLocal<TEvReattachToShard, EvReattachToShard> {
+ const ui64 TabletId;
+
+ explicit TEvReattachToShard(ui64 tabletId)
+ : TabletId(tabletId) {}
+ };
+ };
+
+ struct TReattachState {
+ TDuration Delay;
+ TInstant Deadline;
+ ui64 Cookie = 0;
+ bool Reattaching = false;
+
+ bool ShouldReattach(TInstant now) {
+ ++Cookie; // invalidate any previous cookie
+
+ if (!Reattaching) {
+ Deadline = now + MaxReattachDuration;
+ Delay = TDuration::Zero();
+ Reattaching = true;
+ return true;
+ }
+
+ TDuration left = Deadline - now;
+ if (!left) {
+ Reattaching = false;
+ return false;
+ }
+
+ Delay *= 2.0;
+ if (Delay < MinReattachDelay) {
+ Delay = MinReattachDelay;
+ } else if (Delay > MaxReattachDelay) {
+ Delay = MaxReattachDelay;
+ }
+
+ // Add ±10% jitter
+ Delay *= 0.9 + 0.2 * TAppData::RandomProvider->GenRandReal4();
+ if (Delay > left) {
+ Delay = left;
+ }
+
+ return true;
+ }
+
+ void Reattached() {
+ Reattaching = false;
+ }
+ };
+
+ struct TShardState {
+ enum class EState {
+ Initial,
+ Preparing, // planned tx only
+ Prepared, // planned tx only
+ Executing,
+ Finished
+ };
+
+ EState State = EState::Initial;
+ TSet<ui64> TaskIds;
+
+ struct TDatashardState {
+ ui64 ShardMinStep = 0;
+ ui64 ShardMaxStep = 0;
+ ui64 ReadSize = 0;
+ bool ShardReadLocks = false;
bool Follower = false;
- };
- TMaybe<TDatashardState> DatashardState;
-
- TReattachState ReattachState;
- ui32 RestartCount = 0;
- bool Restarting = false;
- };
-
-public:
- static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
- return NKikimrServices::TActivity::KQP_DATA_EXECUTER_ACTOR;
- }
-
+ };
+ TMaybe<TDatashardState> DatashardState;
+
+ TReattachState ReattachState;
+ ui32 RestartCount = 0;
+ bool Restarting = false;
+ };
+
+public:
+ static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
+ return NKikimrServices::TActivity::KQP_DATA_EXECUTER_ACTOR;
+ }
+
TKqpDataExecuter(IKqpGateway::TExecPhysicalRequest&& request, const TString& database, const TMaybe<TString>& userToken,
- TKqpRequestCounters::TPtr counters)
+ TKqpRequestCounters::TPtr counters)
: TBase(std::move(request), database, userToken, counters)
{
YQL_ENSURE(Request.IsolationLevel != NKikimrKqp::ISOLATION_LEVEL_UNDEFINED);
-
+
if (Request.AcquireLocksTxId || Request.ValidateLocks || Request.EraseLocks) {
YQL_ENSURE(Request.IsolationLevel == NKikimrKqp::ISOLATION_LEVEL_SERIALIZABLE);
}
@@ -142,794 +142,794 @@ public:
}
}
-public:
- STATEFN(WaitResolveState) {
- switch (ev->GetTypeRewrite()) {
- hFunc(TEvKqpExecuter::TEvTableResolveStatus, HandleResolve);
- hFunc(TEvKqp::TEvAbortExecution, HandleAbortExecution);
- hFunc(TEvents::TEvWakeup, HandleTimeout);
- default:
- UnexpectedEvent("WaitResolveState", ev->GetTypeRewrite());
- }
- ReportEventElapsedTime();
- }
-
- void HandleResolve(TEvKqpExecuter::TEvTableResolveStatus::TPtr& ev) {
- auto& reply = *ev->Get();
-
- auto resolveDuration = TInstant::Now() - StartResolveTime;
- Counters->TxProxyMon->TxPrepareResolveHgram->Collect(resolveDuration.MicroSeconds());
-
- KqpTableResolverId = {};
- if (Stats) {
- Stats->ExecuterCpuTime += reply.CpuTime;
- Stats->ResolveCpuTime = reply.CpuTime;
- Stats->ResolveWallTime = resolveDuration;
- }
-
- if (reply.Status != Ydb::StatusIds::SUCCESS) {
- Counters->TxProxyMon->ResolveKeySetWrongRequest->Inc();
- ReplyErrorAndDie(reply.Status, reply.Issues);
- return;
- }
-
- Execute();
- }
-
-private:
- STATEFN(PrepareState) {
- try {
- switch (ev->GetTypeRewrite()) {
- hFunc(TEvDataShard::TEvProposeTransactionResult, HandlePrepare);
- hFunc(TEvDataShard::TEvProposeTransactionRestart, HandleExecute);
- hFunc(TEvDataShard::TEvProposeTransactionAttachResult, HandlePrepare);
- hFunc(TEvPrivate::TEvReattachToShard, HandleExecute);
- hFunc(TEvDqCompute::TEvState, HandlePrepare); // from CA
- hFunc(TEvDqCompute::TEvChannelData, HandleExecute); // from CA
- hFunc(TEvPipeCache::TEvDeliveryProblem, HandlePrepare);
- hFunc(TEvKqp::TEvAbortExecution, HandlePrepare);
- hFunc(TEvents::TEvWakeup, HandlePrepare);
- default: {
- CancelProposal(0);
- UnexpectedEvent("PrepareState", ev->GetTypeRewrite());
- }
- }
- } catch (const yexception& e) {
- CancelProposal(0);
- InternalError(e.what());
- }
- ReportEventElapsedTime();
- }
-
- void HandlePrepare(TEvDataShard::TEvProposeTransactionResult::TPtr& ev) {
- TEvDataShard::TEvProposeTransactionResult* res = ev->Get();
- const ui64 shardId = res->GetOrigin();
- TShardState* shardState = ShardStates.FindPtr(shardId);
- YQL_ENSURE(shardState, "Unexpected propose result from unknown tabletId " << shardId);
-
- LOG_D("Got propose result, shard: " << shardId << ", status: "
- << NKikimrTxDataShard::TEvProposeTransactionResult_EStatus_Name(res->GetStatus())
- << ", error: " << res->GetError());
-
- if (Stats) {
- Stats->AddDatashardPrepareStats(std::move(*res->Record.MutableTxStats()));
- }
-
- switch (res->GetStatus()) {
- case NKikimrTxDataShard::TEvProposeTransactionResult::PREPARED: {
- if (!ShardPrepared(*shardState, res->Record)) {
- return CancelProposal(shardId);
- }
- return CheckPrepareCompleted();
- }
- case NKikimrTxDataShard::TEvProposeTransactionResult::COMPLETE: {
- YQL_ENSURE(false);
- }
- default: {
- CancelProposal(shardId);
- return ShardError(res->Record);
- }
- }
- }
-
- void HandlePrepare(TEvDataShard::TEvProposeTransactionAttachResult::TPtr& ev) {
- const auto& record = ev->Get()->Record;
- const ui64 tabletId = record.GetTabletId();
-
- auto* shardState = ShardStates.FindPtr(tabletId);
- YQL_ENSURE(shardState, "Unknown tablet " << tabletId);
-
- if (ev->Cookie != shardState->ReattachState.Cookie) {
- return;
- }
-
- switch (shardState->State) {
- case TShardState::EState::Preparing:
- case TShardState::EState::Prepared:
- break;
- case TShardState::EState::Initial:
- case TShardState::EState::Executing:
- case TShardState::EState::Finished:
- YQL_ENSURE(false, "Unexpected shard " << tabletId << " state " << ToString(shardState->State));
- }
-
- if (record.GetStatus() == NKikimrProto::OK) {
- // Transaction still exists at this shard
- LOG_D("Reattached to shard " << tabletId << ", state was: " << ToString(shardState->State));
- shardState->State = TShardState::EState::Prepared;
- shardState->ReattachState.Reattached();
- return CheckPrepareCompleted();
- }
-
- LOG_E("Shard " << tabletId << " transaction lost during reconnect: " << record.GetStatus());
-
- CancelProposal(tabletId);
- ReplyTxStateUnknown(tabletId);
- }
-
- void HandlePrepare(TEvDqCompute::TEvState::TPtr& ev) {
- if (ev->Get()->Record.GetState() == NDqProto::COMPUTE_STATE_FAILURE) {
- CancelProposal(0);
- }
- HandleExecute(ev);
- }
-
- void HandlePrepare(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) {
- TEvPipeCache::TEvDeliveryProblem* msg = ev->Get();
- auto* shardState = ShardStates.FindPtr(msg->TabletId);
- YQL_ENSURE(shardState, "EvDeliveryProblem from unknown tablet " << msg->TabletId);
-
- bool wasRestarting = std::exchange(shardState->Restarting, false);
-
- // We can only be sure tx was not prepared if initial propose was not delivered
- bool notPrepared = msg->NotDelivered && (shardState->RestartCount == 0);
-
- switch (shardState->State) {
- case TShardState::EState::Preparing: {
- // Disconnected while waiting for initial propose response
-
- LOG_I("Shard " << msg->TabletId << " propose error, notDelivered: " << msg->NotDelivered
- << ", notPrepared: " << notPrepared << ", wasRestart: " << wasRestarting);
-
- if (notPrepared) {
- CancelProposal(msg->TabletId);
- return ReplyUnavailable(TStringBuilder() << "Could not deliver program to shard " << msg->TabletId);
- }
-
- CancelProposal(0);
-
- if (wasRestarting) {
- // We are waiting for propose and have a restarting flag, which means shard was
- // persisting our tx. We did not receive a reply, so we cannot be sure if it
- // succeeded or not, but we know that it could not apply any side effects, since
- // we don't start transaction planning until prepare phase is complete.
- return ReplyUnavailable(TStringBuilder() << "Could not prepare program on shard " << msg->TabletId);
- }
-
- return ReplyTxStateUnknown(msg->TabletId);
- }
-
- case TShardState::EState::Prepared: {
- // Disconnected while waiting for other shards to prepare
-
- if ((wasRestarting || shardState->ReattachState.Reattaching) &&
- shardState->ReattachState.ShouldReattach(TlsActivationContext->Now()))
- {
- LOG_N("Shard " << msg->TabletId << " delivery problem (already prepared, reattaching in "
- << shardState->ReattachState.Delay << ")");
-
- Schedule(shardState->ReattachState.Delay, new TEvPrivate::TEvReattachToShard(msg->TabletId));
- ++shardState->RestartCount;
- return;
- }
-
- LOG_N("Shard " << msg->TabletId << " delivery problem (already prepared)"
- << (msg->NotDelivered ? ", last message not delivered" : ""));
-
- CancelProposal(0);
- return ReplyTxStateUnknown(msg->TabletId);
- }
-
- case TShardState::EState::Initial:
- case TShardState::EState::Executing:
- case TShardState::EState::Finished:
- YQL_ENSURE(false, "Unexpected shard " << msg->TabletId << " state " << ToString(shardState->State));
- }
- }
-
- void HandlePrepare(TEvKqp::TEvAbortExecution::TPtr& ev) {
- CancelProposal(0);
- TBase::HandleAbortExecution(ev);
- }
-
- void HandlePrepare(TEvents::TEvWakeup::TPtr& ev) {
- CancelProposal(0);
- TBase::HandleTimeout(ev);
- }
-
- void CancelProposal(ui64 exceptShardId) {
- for (auto& [shardId, state] : ShardStates) {
- if (shardId != exceptShardId &&
- (state.State == TShardState::EState::Preparing || state.State == TShardState::EState::Prepared))
- {
- state.State = TShardState::EState::Finished;
-
+public:
+ STATEFN(WaitResolveState) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvKqpExecuter::TEvTableResolveStatus, HandleResolve);
+ hFunc(TEvKqp::TEvAbortExecution, HandleAbortExecution);
+ hFunc(TEvents::TEvWakeup, HandleTimeout);
+ default:
+ UnexpectedEvent("WaitResolveState", ev->GetTypeRewrite());
+ }
+ ReportEventElapsedTime();
+ }
+
+ void HandleResolve(TEvKqpExecuter::TEvTableResolveStatus::TPtr& ev) {
+ auto& reply = *ev->Get();
+
+ auto resolveDuration = TInstant::Now() - StartResolveTime;
+ Counters->TxProxyMon->TxPrepareResolveHgram->Collect(resolveDuration.MicroSeconds());
+
+ KqpTableResolverId = {};
+ if (Stats) {
+ Stats->ExecuterCpuTime += reply.CpuTime;
+ Stats->ResolveCpuTime = reply.CpuTime;
+ Stats->ResolveWallTime = resolveDuration;
+ }
+
+ if (reply.Status != Ydb::StatusIds::SUCCESS) {
+ Counters->TxProxyMon->ResolveKeySetWrongRequest->Inc();
+ ReplyErrorAndDie(reply.Status, reply.Issues);
+ return;
+ }
+
+ Execute();
+ }
+
+private:
+ STATEFN(PrepareState) {
+ try {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvDataShard::TEvProposeTransactionResult, HandlePrepare);
+ hFunc(TEvDataShard::TEvProposeTransactionRestart, HandleExecute);
+ hFunc(TEvDataShard::TEvProposeTransactionAttachResult, HandlePrepare);
+ hFunc(TEvPrivate::TEvReattachToShard, HandleExecute);
+ hFunc(TEvDqCompute::TEvState, HandlePrepare); // from CA
+ hFunc(TEvDqCompute::TEvChannelData, HandleExecute); // from CA
+ hFunc(TEvPipeCache::TEvDeliveryProblem, HandlePrepare);
+ hFunc(TEvKqp::TEvAbortExecution, HandlePrepare);
+ hFunc(TEvents::TEvWakeup, HandlePrepare);
+ default: {
+ CancelProposal(0);
+ UnexpectedEvent("PrepareState", ev->GetTypeRewrite());
+ }
+ }
+ } catch (const yexception& e) {
+ CancelProposal(0);
+ InternalError(e.what());
+ }
+ ReportEventElapsedTime();
+ }
+
+ void HandlePrepare(TEvDataShard::TEvProposeTransactionResult::TPtr& ev) {
+ TEvDataShard::TEvProposeTransactionResult* res = ev->Get();
+ const ui64 shardId = res->GetOrigin();
+ TShardState* shardState = ShardStates.FindPtr(shardId);
+ YQL_ENSURE(shardState, "Unexpected propose result from unknown tabletId " << shardId);
+
+ LOG_D("Got propose result, shard: " << shardId << ", status: "
+ << NKikimrTxDataShard::TEvProposeTransactionResult_EStatus_Name(res->GetStatus())
+ << ", error: " << res->GetError());
+
+ if (Stats) {
+ Stats->AddDatashardPrepareStats(std::move(*res->Record.MutableTxStats()));
+ }
+
+ switch (res->GetStatus()) {
+ case NKikimrTxDataShard::TEvProposeTransactionResult::PREPARED: {
+ if (!ShardPrepared(*shardState, res->Record)) {
+ return CancelProposal(shardId);
+ }
+ return CheckPrepareCompleted();
+ }
+ case NKikimrTxDataShard::TEvProposeTransactionResult::COMPLETE: {
+ YQL_ENSURE(false);
+ }
+ default: {
+ CancelProposal(shardId);
+ return ShardError(res->Record);
+ }
+ }
+ }
+
+ void HandlePrepare(TEvDataShard::TEvProposeTransactionAttachResult::TPtr& ev) {
+ const auto& record = ev->Get()->Record;
+ const ui64 tabletId = record.GetTabletId();
+
+ auto* shardState = ShardStates.FindPtr(tabletId);
+ YQL_ENSURE(shardState, "Unknown tablet " << tabletId);
+
+ if (ev->Cookie != shardState->ReattachState.Cookie) {
+ return;
+ }
+
+ switch (shardState->State) {
+ case TShardState::EState::Preparing:
+ case TShardState::EState::Prepared:
+ break;
+ case TShardState::EState::Initial:
+ case TShardState::EState::Executing:
+ case TShardState::EState::Finished:
+ YQL_ENSURE(false, "Unexpected shard " << tabletId << " state " << ToString(shardState->State));
+ }
+
+ if (record.GetStatus() == NKikimrProto::OK) {
+ // Transaction still exists at this shard
+ LOG_D("Reattached to shard " << tabletId << ", state was: " << ToString(shardState->State));
+ shardState->State = TShardState::EState::Prepared;
+ shardState->ReattachState.Reattached();
+ return CheckPrepareCompleted();
+ }
+
+ LOG_E("Shard " << tabletId << " transaction lost during reconnect: " << record.GetStatus());
+
+ CancelProposal(tabletId);
+ ReplyTxStateUnknown(tabletId);
+ }
+
+ void HandlePrepare(TEvDqCompute::TEvState::TPtr& ev) {
+ if (ev->Get()->Record.GetState() == NDqProto::COMPUTE_STATE_FAILURE) {
+ CancelProposal(0);
+ }
+ HandleExecute(ev);
+ }
+
+ void HandlePrepare(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) {
+ TEvPipeCache::TEvDeliveryProblem* msg = ev->Get();
+ auto* shardState = ShardStates.FindPtr(msg->TabletId);
+ YQL_ENSURE(shardState, "EvDeliveryProblem from unknown tablet " << msg->TabletId);
+
+ bool wasRestarting = std::exchange(shardState->Restarting, false);
+
+ // We can only be sure tx was not prepared if initial propose was not delivered
+ bool notPrepared = msg->NotDelivered && (shardState->RestartCount == 0);
+
+ switch (shardState->State) {
+ case TShardState::EState::Preparing: {
+ // Disconnected while waiting for initial propose response
+
+ LOG_I("Shard " << msg->TabletId << " propose error, notDelivered: " << msg->NotDelivered
+ << ", notPrepared: " << notPrepared << ", wasRestart: " << wasRestarting);
+
+ if (notPrepared) {
+ CancelProposal(msg->TabletId);
+ return ReplyUnavailable(TStringBuilder() << "Could not deliver program to shard " << msg->TabletId);
+ }
+
+ CancelProposal(0);
+
+ if (wasRestarting) {
+ // We are waiting for propose and have a restarting flag, which means shard was
+ // persisting our tx. We did not receive a reply, so we cannot be sure if it
+ // succeeded or not, but we know that it could not apply any side effects, since
+ // we don't start transaction planning until prepare phase is complete.
+ return ReplyUnavailable(TStringBuilder() << "Could not prepare program on shard " << msg->TabletId);
+ }
+
+ return ReplyTxStateUnknown(msg->TabletId);
+ }
+
+ case TShardState::EState::Prepared: {
+ // Disconnected while waiting for other shards to prepare
+
+ if ((wasRestarting || shardState->ReattachState.Reattaching) &&
+ shardState->ReattachState.ShouldReattach(TlsActivationContext->Now()))
+ {
+ LOG_N("Shard " << msg->TabletId << " delivery problem (already prepared, reattaching in "
+ << shardState->ReattachState.Delay << ")");
+
+ Schedule(shardState->ReattachState.Delay, new TEvPrivate::TEvReattachToShard(msg->TabletId));
+ ++shardState->RestartCount;
+ return;
+ }
+
+ LOG_N("Shard " << msg->TabletId << " delivery problem (already prepared)"
+ << (msg->NotDelivered ? ", last message not delivered" : ""));
+
+ CancelProposal(0);
+ return ReplyTxStateUnknown(msg->TabletId);
+ }
+
+ case TShardState::EState::Initial:
+ case TShardState::EState::Executing:
+ case TShardState::EState::Finished:
+ YQL_ENSURE(false, "Unexpected shard " << msg->TabletId << " state " << ToString(shardState->State));
+ }
+ }
+
+ void HandlePrepare(TEvKqp::TEvAbortExecution::TPtr& ev) {
+ CancelProposal(0);
+ TBase::HandleAbortExecution(ev);
+ }
+
+ void HandlePrepare(TEvents::TEvWakeup::TPtr& ev) {
+ CancelProposal(0);
+ TBase::HandleTimeout(ev);
+ }
+
+ void CancelProposal(ui64 exceptShardId) {
+ for (auto& [shardId, state] : ShardStates) {
+ if (shardId != exceptShardId &&
+ (state.State == TShardState::EState::Preparing || state.State == TShardState::EState::Prepared))
+ {
+ state.State = TShardState::EState::Finished;
+
YQL_ENSURE(!state.DatashardState->Follower);
-
+
Send(MakePipePeNodeCacheID(/* allowFollowers */ false), new TEvPipeCache::TEvForward(
- new TEvDataShard::TEvCancelTransactionProposal(TxId), shardId, /* subscribe */ false));
- }
- }
- }
-
- bool ShardPrepared(TShardState& state, const NKikimrTxDataShard::TEvProposeTransactionResult& result) {
- YQL_ENSURE(state.State == TShardState::EState::Preparing);
- state.State = TShardState::EState::Prepared;
-
- state.DatashardState->ShardMinStep = result.GetMinStep();
- state.DatashardState->ShardMaxStep = result.GetMaxStep();
- state.DatashardState->ReadSize += result.GetReadSize();
-
- ui64 coordinator = 0;
- if (result.DomainCoordinatorsSize()) {
- auto domainCoordinators = TCoordinators(TVector<ui64>(result.GetDomainCoordinators().begin(),
- result.GetDomainCoordinators().end()));
- coordinator = domainCoordinators.Select(TxId);
- }
-
- if (coordinator && !TxCoordinator) {
- TxCoordinator = coordinator;
- }
-
- if (!TxCoordinator || TxCoordinator != coordinator) {
- LOG_E("Handle TEvProposeTransactionResult: unable to select coordinator. Tx canceled, actorId: " << SelfId()
- << ", previously selected coordinator: " << TxCoordinator
- << ", coordinator selected at propose result: " << coordinator);
-
- Counters->TxProxyMon->TxResultAborted->Inc();
- ReplyErrorAndDie(Ydb::StatusIds::CANCELLED, MakeIssue(
- NKikimrIssues::TIssuesIds::TX_DECLINED_IMPLICIT_COORDINATOR, "Unable to choose coordinator."));
- return false;
- }
-
- LastPrepareReply = TInstant::Now();
- if (!FirstPrepareReply) {
- FirstPrepareReply = LastPrepareReply;
- }
-
- return true;
- }
-
- void ShardError(const NKikimrTxDataShard::TEvProposeTransactionResult& result) {
- if (result.ErrorSize() != 0) {
- TStringBuilder message;
- message << NKikimrTxDataShard::TEvProposeTransactionResult_EStatus_Name(result.GetStatus()) << ": ";
- for (const auto &err : result.GetError()) {
- message << "[" << NKikimrTxDataShard::TError_EKind_Name(err.GetKind()) << "] " << err.GetReason() << "; ";
- }
- LOG_E(message);
- }
-
- switch (result.GetStatus()) {
- case NKikimrTxDataShard::TEvProposeTransactionResult::OVERLOADED: {
- Counters->TxProxyMon->TxResultShardOverloaded->Inc();
- auto issue = YqlIssue({}, TIssuesIds::KIKIMR_OVERLOADED);
- AddDataShardErrors(result, issue);
- return ReplyErrorAndDie(Ydb::StatusIds::OVERLOADED, issue);
- }
- case NKikimrTxDataShard::TEvProposeTransactionResult::ABORTED: {
- Counters->TxProxyMon->TxResultAborted->Inc();
- auto issue = YqlIssue({}, TIssuesIds::KIKIMR_OPERATION_ABORTED);
- AddDataShardErrors(result, issue);
- return ReplyErrorAndDie(Ydb::StatusIds::ABORTED, issue);
- }
- case NKikimrTxDataShard::TEvProposeTransactionResult::TRY_LATER: {
- Counters->TxProxyMon->TxResultShardTryLater->Inc();
- auto issue = YqlIssue({}, TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE);
- AddDataShardErrors(result, issue);
- return ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, issue);
- }
- case NKikimrTxDataShard::TEvProposeTransactionResult::RESULT_UNAVAILABLE: {
- Counters->TxProxyMon->TxResultResultUnavailable->Inc();
- auto issue = YqlIssue({}, TIssuesIds::KIKIMR_RESULT_UNAVAILABLE);
- AddDataShardErrors(result, issue);
- return ReplyErrorAndDie(Ydb::StatusIds::UNDETERMINED, issue);
- }
- case NKikimrTxDataShard::TEvProposeTransactionResult::CANCELLED: {
- Counters->TxProxyMon->TxResultCancelled->Inc();
- auto issue = YqlIssue({}, TIssuesIds::KIKIMR_OPERATION_CANCELLED);
- AddDataShardErrors(result, issue);
- return ReplyErrorAndDie(Ydb::StatusIds::CANCELLED, issue);
- }
- case NKikimrTxDataShard::TEvProposeTransactionResult::BAD_REQUEST: {
- Counters->TxProxyMon->TxResultCancelled->Inc();
- auto issue = YqlIssue({}, TIssuesIds::KIKIMR_BAD_REQUEST);
- AddDataShardErrors(result, issue);
- return ReplyErrorAndDie(Ydb::StatusIds::BAD_REQUEST, issue);
- }
- case NKikimrTxDataShard::TEvProposeTransactionResult::EXEC_ERROR: {
- Counters->TxProxyMon->TxResultExecError->Inc();
- for (auto& er : result.GetError()) {
- if (er.GetKind() == NKikimrTxDataShard::TError::PROGRAM_ERROR) {
- auto issue = YqlIssue({}, TIssuesIds::KIKIMR_PRECONDITION_FAILED);
- issue.AddSubIssue(new TIssue(TStringBuilder() << "Data shard error: [PROGRAM_ERROR] " << er.GetReason()));
- return ReplyErrorAndDie(Ydb::StatusIds::ABORTED, issue);
- }
- }
- auto issue = YqlIssue({}, TIssuesIds::DEFAULT_ERROR, "Error executing transaction (ExecError): Execution failed");
- AddDataShardErrors(result, issue);
- return ReplyErrorAndDie(Ydb::StatusIds::GENERIC_ERROR, issue);
- }
- case NKikimrTxDataShard::TEvProposeTransactionResult::ERROR: {
- Counters->TxProxyMon->TxResultError->Inc();
- for (auto& er : result.GetError()) {
- if (er.GetKind() == NKikimrTxDataShard::TError::SCHEME_CHANGED) {
- return ReplyErrorAndDie(Ydb::StatusIds::ABORTED, YqlIssue({}, TIssuesIds::KIKIMR_SCHEME_MISMATCH, er.GetReason()));
- }
- if (er.GetKind() == NKikimrTxDataShard::TError::SCHEME_ERROR) {
- return ReplyErrorAndDie(Ydb::StatusIds::BAD_REQUEST, YqlIssue({}, TIssuesIds::KIKIMR_SCHEME_ERROR, er.GetReason()));
- }
- }
- auto issue = YqlIssue({}, TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE);
- AddDataShardErrors(result, issue);
- return ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, issue);
- }
- default: {
- Counters->TxProxyMon->TxResultFatal->Inc();
- auto issue = YqlIssue({}, TIssuesIds::DEFAULT_ERROR, "Error executing transaction: transaction failed.");
- AddDataShardErrors(result, issue);
- return ReplyErrorAndDie(Ydb::StatusIds::GENERIC_ERROR, issue);
- }
- }
- }
-
- void CheckPrepareCompleted() {
- for (auto& [_, state] : ShardStates) {
- if (state.State != TShardState::EState::Prepared) {
- LOG_D("Not all shards are prepared, waiting...");
- return;
- }
- }
-
- Counters->TxProxyMon->TxPrepareSpreadHgram->Collect((LastPrepareReply - FirstPrepareReply).MilliSeconds());
-
- LOG_D("All shards prepared, become ExecuteState.");
- Become(&TKqpDataExecuter::ExecuteState);
- ExecutePlanned();
- }
-
- void ExecutePlanned() {
- YQL_ENSURE(TxCoordinator);
-
- auto ev = MakeHolder<TEvTxProxy::TEvProposeTransaction>();
- ev->Record.SetCoordinatorID(TxCoordinator);
-
- auto& transaction = *ev->Record.MutableTransaction();
- auto& affectedSet = *transaction.MutableAffectedSet();
- affectedSet.Reserve(static_cast<int>(ShardStates.size()));
-
- ui64 aggrMinStep = 0;
- ui64 aggrMaxStep = Max<ui64>();
- ui64 totalReadSize = 0;
-
- for (auto& [shardId, state] : ShardStates) {
- YQL_ENSURE(state.State == TShardState::EState::Prepared);
- state.State = TShardState::EState::Executing;
-
- YQL_ENSURE(state.DatashardState.Defined());
+ new TEvDataShard::TEvCancelTransactionProposal(TxId), shardId, /* subscribe */ false));
+ }
+ }
+ }
+
+ bool ShardPrepared(TShardState& state, const NKikimrTxDataShard::TEvProposeTransactionResult& result) {
+ YQL_ENSURE(state.State == TShardState::EState::Preparing);
+ state.State = TShardState::EState::Prepared;
+
+ state.DatashardState->ShardMinStep = result.GetMinStep();
+ state.DatashardState->ShardMaxStep = result.GetMaxStep();
+ state.DatashardState->ReadSize += result.GetReadSize();
+
+ ui64 coordinator = 0;
+ if (result.DomainCoordinatorsSize()) {
+ auto domainCoordinators = TCoordinators(TVector<ui64>(result.GetDomainCoordinators().begin(),
+ result.GetDomainCoordinators().end()));
+ coordinator = domainCoordinators.Select(TxId);
+ }
+
+ if (coordinator && !TxCoordinator) {
+ TxCoordinator = coordinator;
+ }
+
+ if (!TxCoordinator || TxCoordinator != coordinator) {
+ LOG_E("Handle TEvProposeTransactionResult: unable to select coordinator. Tx canceled, actorId: " << SelfId()
+ << ", previously selected coordinator: " << TxCoordinator
+ << ", coordinator selected at propose result: " << coordinator);
+
+ Counters->TxProxyMon->TxResultAborted->Inc();
+ ReplyErrorAndDie(Ydb::StatusIds::CANCELLED, MakeIssue(
+ NKikimrIssues::TIssuesIds::TX_DECLINED_IMPLICIT_COORDINATOR, "Unable to choose coordinator."));
+ return false;
+ }
+
+ LastPrepareReply = TInstant::Now();
+ if (!FirstPrepareReply) {
+ FirstPrepareReply = LastPrepareReply;
+ }
+
+ return true;
+ }
+
+ void ShardError(const NKikimrTxDataShard::TEvProposeTransactionResult& result) {
+ if (result.ErrorSize() != 0) {
+ TStringBuilder message;
+ message << NKikimrTxDataShard::TEvProposeTransactionResult_EStatus_Name(result.GetStatus()) << ": ";
+ for (const auto &err : result.GetError()) {
+ message << "[" << NKikimrTxDataShard::TError_EKind_Name(err.GetKind()) << "] " << err.GetReason() << "; ";
+ }
+ LOG_E(message);
+ }
+
+ switch (result.GetStatus()) {
+ case NKikimrTxDataShard::TEvProposeTransactionResult::OVERLOADED: {
+ Counters->TxProxyMon->TxResultShardOverloaded->Inc();
+ auto issue = YqlIssue({}, TIssuesIds::KIKIMR_OVERLOADED);
+ AddDataShardErrors(result, issue);
+ return ReplyErrorAndDie(Ydb::StatusIds::OVERLOADED, issue);
+ }
+ case NKikimrTxDataShard::TEvProposeTransactionResult::ABORTED: {
+ Counters->TxProxyMon->TxResultAborted->Inc();
+ auto issue = YqlIssue({}, TIssuesIds::KIKIMR_OPERATION_ABORTED);
+ AddDataShardErrors(result, issue);
+ return ReplyErrorAndDie(Ydb::StatusIds::ABORTED, issue);
+ }
+ case NKikimrTxDataShard::TEvProposeTransactionResult::TRY_LATER: {
+ Counters->TxProxyMon->TxResultShardTryLater->Inc();
+ auto issue = YqlIssue({}, TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE);
+ AddDataShardErrors(result, issue);
+ return ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, issue);
+ }
+ case NKikimrTxDataShard::TEvProposeTransactionResult::RESULT_UNAVAILABLE: {
+ Counters->TxProxyMon->TxResultResultUnavailable->Inc();
+ auto issue = YqlIssue({}, TIssuesIds::KIKIMR_RESULT_UNAVAILABLE);
+ AddDataShardErrors(result, issue);
+ return ReplyErrorAndDie(Ydb::StatusIds::UNDETERMINED, issue);
+ }
+ case NKikimrTxDataShard::TEvProposeTransactionResult::CANCELLED: {
+ Counters->TxProxyMon->TxResultCancelled->Inc();
+ auto issue = YqlIssue({}, TIssuesIds::KIKIMR_OPERATION_CANCELLED);
+ AddDataShardErrors(result, issue);
+ return ReplyErrorAndDie(Ydb::StatusIds::CANCELLED, issue);
+ }
+ case NKikimrTxDataShard::TEvProposeTransactionResult::BAD_REQUEST: {
+ Counters->TxProxyMon->TxResultCancelled->Inc();
+ auto issue = YqlIssue({}, TIssuesIds::KIKIMR_BAD_REQUEST);
+ AddDataShardErrors(result, issue);
+ return ReplyErrorAndDie(Ydb::StatusIds::BAD_REQUEST, issue);
+ }
+ case NKikimrTxDataShard::TEvProposeTransactionResult::EXEC_ERROR: {
+ Counters->TxProxyMon->TxResultExecError->Inc();
+ for (auto& er : result.GetError()) {
+ if (er.GetKind() == NKikimrTxDataShard::TError::PROGRAM_ERROR) {
+ auto issue = YqlIssue({}, TIssuesIds::KIKIMR_PRECONDITION_FAILED);
+ issue.AddSubIssue(new TIssue(TStringBuilder() << "Data shard error: [PROGRAM_ERROR] " << er.GetReason()));
+ return ReplyErrorAndDie(Ydb::StatusIds::ABORTED, issue);
+ }
+ }
+ auto issue = YqlIssue({}, TIssuesIds::DEFAULT_ERROR, "Error executing transaction (ExecError): Execution failed");
+ AddDataShardErrors(result, issue);
+ return ReplyErrorAndDie(Ydb::StatusIds::GENERIC_ERROR, issue);
+ }
+ case NKikimrTxDataShard::TEvProposeTransactionResult::ERROR: {
+ Counters->TxProxyMon->TxResultError->Inc();
+ for (auto& er : result.GetError()) {
+ if (er.GetKind() == NKikimrTxDataShard::TError::SCHEME_CHANGED) {
+ return ReplyErrorAndDie(Ydb::StatusIds::ABORTED, YqlIssue({}, TIssuesIds::KIKIMR_SCHEME_MISMATCH, er.GetReason()));
+ }
+ if (er.GetKind() == NKikimrTxDataShard::TError::SCHEME_ERROR) {
+ return ReplyErrorAndDie(Ydb::StatusIds::BAD_REQUEST, YqlIssue({}, TIssuesIds::KIKIMR_SCHEME_ERROR, er.GetReason()));
+ }
+ }
+ auto issue = YqlIssue({}, TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE);
+ AddDataShardErrors(result, issue);
+ return ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, issue);
+ }
+ default: {
+ Counters->TxProxyMon->TxResultFatal->Inc();
+ auto issue = YqlIssue({}, TIssuesIds::DEFAULT_ERROR, "Error executing transaction: transaction failed.");
+ AddDataShardErrors(result, issue);
+ return ReplyErrorAndDie(Ydb::StatusIds::GENERIC_ERROR, issue);
+ }
+ }
+ }
+
+ void CheckPrepareCompleted() {
+ for (auto& [_, state] : ShardStates) {
+ if (state.State != TShardState::EState::Prepared) {
+ LOG_D("Not all shards are prepared, waiting...");
+ return;
+ }
+ }
+
+ Counters->TxProxyMon->TxPrepareSpreadHgram->Collect((LastPrepareReply - FirstPrepareReply).MilliSeconds());
+
+ LOG_D("All shards prepared, become ExecuteState.");
+ Become(&TKqpDataExecuter::ExecuteState);
+ ExecutePlanned();
+ }
+
+ void ExecutePlanned() {
+ YQL_ENSURE(TxCoordinator);
+
+ auto ev = MakeHolder<TEvTxProxy::TEvProposeTransaction>();
+ ev->Record.SetCoordinatorID(TxCoordinator);
+
+ auto& transaction = *ev->Record.MutableTransaction();
+ auto& affectedSet = *transaction.MutableAffectedSet();
+ affectedSet.Reserve(static_cast<int>(ShardStates.size()));
+
+ ui64 aggrMinStep = 0;
+ ui64 aggrMaxStep = Max<ui64>();
+ ui64 totalReadSize = 0;
+
+ for (auto& [shardId, state] : ShardStates) {
+ YQL_ENSURE(state.State == TShardState::EState::Prepared);
+ state.State = TShardState::EState::Executing;
+
+ YQL_ENSURE(state.DatashardState.Defined());
YQL_ENSURE(!state.DatashardState->Follower);
-
- aggrMinStep = Max(aggrMinStep, state.DatashardState->ShardMinStep);
- aggrMaxStep = Min(aggrMaxStep, state.DatashardState->ShardMaxStep);
- totalReadSize += state.DatashardState->ReadSize;
-
- auto& item = *affectedSet.Add();
- item.SetTabletId(shardId);
-
- ui32 affectedFlags = 0;
- if (state.DatashardState->ShardReadLocks) {
- affectedFlags |= NFlatTxCoordinator::TTransactionProposal::TAffectedEntry::AffectedRead;
- }
-
- for (auto taskId : state.TaskIds) {
- auto& task = TasksGraph.GetTask(taskId);
- auto& stageInfo = TasksGraph.GetStageInfo(task.StageId);
-
- if (HasReads(stageInfo)) {
- affectedFlags |= NFlatTxCoordinator::TTransactionProposal::TAffectedEntry::AffectedRead;
- }
- if (HasWrites(stageInfo)) {
- affectedFlags |= NFlatTxCoordinator::TTransactionProposal::TAffectedEntry::AffectedWrite;
- }
- }
-
- item.SetFlags(affectedFlags);
- }
-
- ui64 sizeLimit = RequestControls.PerRequestDataSizeLimit;
- if (Request.TotalReadSizeLimitBytes > 0) {
- sizeLimit = sizeLimit
- ? std::min(sizeLimit, Request.TotalReadSizeLimitBytes)
- : Request.TotalReadSizeLimitBytes;
- }
-
- if (totalReadSize > sizeLimit) {
- auto msg = TStringBuilder() << "Transaction total read size " << totalReadSize << " exceeded limit " << sizeLimit;
- LOG_N(msg);
- ReplyErrorAndDie(Ydb::StatusIds::PRECONDITION_FAILED,
- YqlIssue({}, NYql::TIssuesIds::KIKIMR_PRECONDITION_FAILED, msg));
- return;
- }
-
- transaction.SetTxId(TxId);
- transaction.SetMinStep(aggrMinStep);
- transaction.SetMaxStep(aggrMaxStep);
-
- LOG_T("Execute planned transaction, coordinator: " << TxCoordinator);
- Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvForward(ev.Release(), TxCoordinator, /* subscribe */ true));
- }
-
-private:
- STATEFN(ExecuteState) {
- try {
- switch (ev->GetTypeRewrite()) {
- hFunc(TEvDataShard::TEvProposeTransactionResult, HandleExecute);
- hFunc(TEvDataShard::TEvProposeTransactionRestart, HandleExecute);
- hFunc(TEvDataShard::TEvProposeTransactionAttachResult, HandleExecute);
- hFunc(TEvPrivate::TEvReattachToShard, HandleExecute);
- hFunc(TEvPipeCache::TEvDeliveryProblem, HandleExecute);
- hFunc(TEvTxProxy::TEvProposeTransactionStatus, HandleExecute);
- hFunc(TEvDqCompute::TEvState, HandleExecute);
- hFunc(TEvDqCompute::TEvChannelData, HandleExecute);
- hFunc(TEvKqp::TEvAbortExecution, HandleAbortExecution);
- hFunc(TEvents::TEvWakeup, HandleTimeout);
- default:
- UnexpectedEvent("ExecuteState", ev->GetTypeRewrite());
- }
- } catch (const yexception& e) {
- InternalError(e.what());
- }
- ReportEventElapsedTime();
- }
-
- void HandleExecute(TEvDataShard::TEvProposeTransactionResult::TPtr& ev) {
- TEvDataShard::TEvProposeTransactionResult* res = ev->Get();
- const ui64 shardId = res->GetOrigin();
- TShardState* shardState = ShardStates.FindPtr(shardId);
- YQL_ENSURE(shardState);
-
- LOG_D("Got propose result, shard: " << shardId << ", status: "
- << NKikimrTxDataShard::TEvProposeTransactionResult_EStatus_Name(res->GetStatus())
- << ", error: " << res->GetError());
-
- if (Stats) {
- Stats->AddDatashardStats(std::move(*res->Record.MutableComputeActorStats()),
- std::move(*res->Record.MutableTxStats()));
- }
-
- switch (res->GetStatus()) {
- case NKikimrTxDataShard::TEvProposeTransactionResult::COMPLETE: {
- YQL_ENSURE(shardState->State == TShardState::EState::Executing);
- shardState->State = TShardState::EState::Finished;
-
- Counters->TxProxyMon->ResultsReceivedCount->Inc();
- Counters->TxProxyMon->ResultsReceivedSize->Add(res->GetTxResult().size());
-
- for (auto& lock : res->Record.GetTxLocks()) {
- LOG_D("Shard " << shardId << " completed, store lock " << lock.ShortDebugString());
- Locks.emplace_back(std::move(lock));
- }
-
- Counters->TxProxyMon->TxResultComplete->Inc();
-
- CheckExecutionComplete();
- return;
- }
- case NKikimrTxDataShard::TEvProposeTransactionResult::LOCKS_BROKEN: {
- LOG_D("Broken locks: " << res->Record.DebugString());
-
- Counters->TxProxyMon->TxResultAborted->Inc(); // TODO: dedicated counter?
-
- TMaybe<TString> tableName;
- if (!res->Record.GetTxLocks().empty()) {
- auto& lock = res->Record.GetTxLocks(0);
- auto tableId = TTableId(lock.GetSchemeShard(), lock.GetPathId());
- auto it = FindIf(TableKeys.Get(), [tableId](const auto& x){ return x.first.HasSamePath(tableId); });
- if (it != TableKeys.Get().end()) {
- tableName = it->second.Path;
- }
- }
-
- auto message = TStringBuilder() << "Transaction locks invalidated.";
- if (tableName) {
- message << " Table: " << *tableName;
- }
-
- return ReplyErrorAndDie(Ydb::StatusIds::ABORTED,
- YqlIssue({}, TIssuesIds::KIKIMR_LOCKS_INVALIDATED, message));
- }
- case NKikimrTxDataShard::TEvProposeTransactionResult::PREPARED: {
- YQL_ENSURE(false);
- }
- default: {
- return ShardError(res->Record);
- }
- }
- }
-
- void HandleExecute(TEvDataShard::TEvProposeTransactionRestart::TPtr& ev) {
- const auto& record = ev->Get()->Record;
- const ui64 shardId = record.GetTabletId();
-
- auto* shardState = ShardStates.FindPtr(shardId);
- YQL_ENSURE(shardState, "restart tx event from unknown tabletId: " << shardId << ", tx: " << TxId);
-
- LOG_D("Got transaction restart event from tabletId: " << shardId << ", state: " << ToString(shardState->State)
- << ", txPlanned: " << TxPlanned);
-
- switch (shardState->State) {
- case TShardState::EState::Preparing:
- case TShardState::EState::Prepared:
- case TShardState::EState::Executing: {
- shardState->Restarting = true;
- return;
- }
- case TShardState::EState::Finished: {
- return;
- }
- case TShardState::EState::Initial: {
- YQL_ENSURE(false);
- }
- }
- }
-
- void HandleExecute(TEvDataShard::TEvProposeTransactionAttachResult::TPtr& ev) {
- const auto& record = ev->Get()->Record;
- const ui64 tabletId = record.GetTabletId();
-
- auto* shardState = ShardStates.FindPtr(tabletId);
- YQL_ENSURE(shardState, "Unknown tablet " << tabletId);
-
- if (ev->Cookie != shardState->ReattachState.Cookie) {
- return;
- }
-
- switch (shardState->State) {
- case TShardState::EState::Executing:
- break;
- case TShardState::EState::Initial:
- case TShardState::EState::Preparing:
- case TShardState::EState::Prepared:
- case TShardState::EState::Finished:
- return;
- }
-
- if (record.GetStatus() == NKikimrProto::OK) {
- // Transaction still exists at this shard
- LOG_N("Reattached to shard " << tabletId << ", state was: " << ToString(shardState->State));
- shardState->ReattachState.Reattached();
-
- CheckExecutionComplete();
- return;
- }
-
- LOG_E("Shard " << tabletId << " transaction lost during reconnect: " << record.GetStatus());
-
- ReplyTxStateUnknown(tabletId);
- }
-
- void HandleExecute(TEvPrivate::TEvReattachToShard::TPtr& ev) {
- const ui64 tabletId = ev->Get()->TabletId;
- auto* shardState = ShardStates.FindPtr(tabletId);
- YQL_ENSURE(shardState);
-
- LOG_I("Reattach to shard " << tabletId);
-
+
+ aggrMinStep = Max(aggrMinStep, state.DatashardState->ShardMinStep);
+ aggrMaxStep = Min(aggrMaxStep, state.DatashardState->ShardMaxStep);
+ totalReadSize += state.DatashardState->ReadSize;
+
+ auto& item = *affectedSet.Add();
+ item.SetTabletId(shardId);
+
+ ui32 affectedFlags = 0;
+ if (state.DatashardState->ShardReadLocks) {
+ affectedFlags |= NFlatTxCoordinator::TTransactionProposal::TAffectedEntry::AffectedRead;
+ }
+
+ for (auto taskId : state.TaskIds) {
+ auto& task = TasksGraph.GetTask(taskId);
+ auto& stageInfo = TasksGraph.GetStageInfo(task.StageId);
+
+ if (HasReads(stageInfo)) {
+ affectedFlags |= NFlatTxCoordinator::TTransactionProposal::TAffectedEntry::AffectedRead;
+ }
+ if (HasWrites(stageInfo)) {
+ affectedFlags |= NFlatTxCoordinator::TTransactionProposal::TAffectedEntry::AffectedWrite;
+ }
+ }
+
+ item.SetFlags(affectedFlags);
+ }
+
+ ui64 sizeLimit = RequestControls.PerRequestDataSizeLimit;
+ if (Request.TotalReadSizeLimitBytes > 0) {
+ sizeLimit = sizeLimit
+ ? std::min(sizeLimit, Request.TotalReadSizeLimitBytes)
+ : Request.TotalReadSizeLimitBytes;
+ }
+
+ if (totalReadSize > sizeLimit) {
+ auto msg = TStringBuilder() << "Transaction total read size " << totalReadSize << " exceeded limit " << sizeLimit;
+ LOG_N(msg);
+ ReplyErrorAndDie(Ydb::StatusIds::PRECONDITION_FAILED,
+ YqlIssue({}, NYql::TIssuesIds::KIKIMR_PRECONDITION_FAILED, msg));
+ return;
+ }
+
+ transaction.SetTxId(TxId);
+ transaction.SetMinStep(aggrMinStep);
+ transaction.SetMaxStep(aggrMaxStep);
+
+ LOG_T("Execute planned transaction, coordinator: " << TxCoordinator);
+ Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvForward(ev.Release(), TxCoordinator, /* subscribe */ true));
+ }
+
+private:
+ STATEFN(ExecuteState) {
+ try {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvDataShard::TEvProposeTransactionResult, HandleExecute);
+ hFunc(TEvDataShard::TEvProposeTransactionRestart, HandleExecute);
+ hFunc(TEvDataShard::TEvProposeTransactionAttachResult, HandleExecute);
+ hFunc(TEvPrivate::TEvReattachToShard, HandleExecute);
+ hFunc(TEvPipeCache::TEvDeliveryProblem, HandleExecute);
+ hFunc(TEvTxProxy::TEvProposeTransactionStatus, HandleExecute);
+ hFunc(TEvDqCompute::TEvState, HandleExecute);
+ hFunc(TEvDqCompute::TEvChannelData, HandleExecute);
+ hFunc(TEvKqp::TEvAbortExecution, HandleAbortExecution);
+ hFunc(TEvents::TEvWakeup, HandleTimeout);
+ default:
+ UnexpectedEvent("ExecuteState", ev->GetTypeRewrite());
+ }
+ } catch (const yexception& e) {
+ InternalError(e.what());
+ }
+ ReportEventElapsedTime();
+ }
+
+ void HandleExecute(TEvDataShard::TEvProposeTransactionResult::TPtr& ev) {
+ TEvDataShard::TEvProposeTransactionResult* res = ev->Get();
+ const ui64 shardId = res->GetOrigin();
+ TShardState* shardState = ShardStates.FindPtr(shardId);
+ YQL_ENSURE(shardState);
+
+ LOG_D("Got propose result, shard: " << shardId << ", status: "
+ << NKikimrTxDataShard::TEvProposeTransactionResult_EStatus_Name(res->GetStatus())
+ << ", error: " << res->GetError());
+
+ if (Stats) {
+ Stats->AddDatashardStats(std::move(*res->Record.MutableComputeActorStats()),
+ std::move(*res->Record.MutableTxStats()));
+ }
+
+ switch (res->GetStatus()) {
+ case NKikimrTxDataShard::TEvProposeTransactionResult::COMPLETE: {
+ YQL_ENSURE(shardState->State == TShardState::EState::Executing);
+ shardState->State = TShardState::EState::Finished;
+
+ Counters->TxProxyMon->ResultsReceivedCount->Inc();
+ Counters->TxProxyMon->ResultsReceivedSize->Add(res->GetTxResult().size());
+
+ for (auto& lock : res->Record.GetTxLocks()) {
+ LOG_D("Shard " << shardId << " completed, store lock " << lock.ShortDebugString());
+ Locks.emplace_back(std::move(lock));
+ }
+
+ Counters->TxProxyMon->TxResultComplete->Inc();
+
+ CheckExecutionComplete();
+ return;
+ }
+ case NKikimrTxDataShard::TEvProposeTransactionResult::LOCKS_BROKEN: {
+ LOG_D("Broken locks: " << res->Record.DebugString());
+
+ Counters->TxProxyMon->TxResultAborted->Inc(); // TODO: dedicated counter?
+
+ TMaybe<TString> tableName;
+ if (!res->Record.GetTxLocks().empty()) {
+ auto& lock = res->Record.GetTxLocks(0);
+ auto tableId = TTableId(lock.GetSchemeShard(), lock.GetPathId());
+ auto it = FindIf(TableKeys.Get(), [tableId](const auto& x){ return x.first.HasSamePath(tableId); });
+ if (it != TableKeys.Get().end()) {
+ tableName = it->second.Path;
+ }
+ }
+
+ auto message = TStringBuilder() << "Transaction locks invalidated.";
+ if (tableName) {
+ message << " Table: " << *tableName;
+ }
+
+ return ReplyErrorAndDie(Ydb::StatusIds::ABORTED,
+ YqlIssue({}, TIssuesIds::KIKIMR_LOCKS_INVALIDATED, message));
+ }
+ case NKikimrTxDataShard::TEvProposeTransactionResult::PREPARED: {
+ YQL_ENSURE(false);
+ }
+ default: {
+ return ShardError(res->Record);
+ }
+ }
+ }
+
+ void HandleExecute(TEvDataShard::TEvProposeTransactionRestart::TPtr& ev) {
+ const auto& record = ev->Get()->Record;
+ const ui64 shardId = record.GetTabletId();
+
+ auto* shardState = ShardStates.FindPtr(shardId);
+ YQL_ENSURE(shardState, "restart tx event from unknown tabletId: " << shardId << ", tx: " << TxId);
+
+ LOG_D("Got transaction restart event from tabletId: " << shardId << ", state: " << ToString(shardState->State)
+ << ", txPlanned: " << TxPlanned);
+
+ switch (shardState->State) {
+ case TShardState::EState::Preparing:
+ case TShardState::EState::Prepared:
+ case TShardState::EState::Executing: {
+ shardState->Restarting = true;
+ return;
+ }
+ case TShardState::EState::Finished: {
+ return;
+ }
+ case TShardState::EState::Initial: {
+ YQL_ENSURE(false);
+ }
+ }
+ }
+
+ void HandleExecute(TEvDataShard::TEvProposeTransactionAttachResult::TPtr& ev) {
+ const auto& record = ev->Get()->Record;
+ const ui64 tabletId = record.GetTabletId();
+
+ auto* shardState = ShardStates.FindPtr(tabletId);
+ YQL_ENSURE(shardState, "Unknown tablet " << tabletId);
+
+ if (ev->Cookie != shardState->ReattachState.Cookie) {
+ return;
+ }
+
+ switch (shardState->State) {
+ case TShardState::EState::Executing:
+ break;
+ case TShardState::EState::Initial:
+ case TShardState::EState::Preparing:
+ case TShardState::EState::Prepared:
+ case TShardState::EState::Finished:
+ return;
+ }
+
+ if (record.GetStatus() == NKikimrProto::OK) {
+ // Transaction still exists at this shard
+ LOG_N("Reattached to shard " << tabletId << ", state was: " << ToString(shardState->State));
+ shardState->ReattachState.Reattached();
+
+ CheckExecutionComplete();
+ return;
+ }
+
+ LOG_E("Shard " << tabletId << " transaction lost during reconnect: " << record.GetStatus());
+
+ ReplyTxStateUnknown(tabletId);
+ }
+
+ void HandleExecute(TEvPrivate::TEvReattachToShard::TPtr& ev) {
+ const ui64 tabletId = ev->Get()->TabletId;
+ auto* shardState = ShardStates.FindPtr(tabletId);
+ YQL_ENSURE(shardState);
+
+ LOG_I("Reattach to shard " << tabletId);
+
Send(MakePipePeNodeCacheID(UseFollowers), new TEvPipeCache::TEvForward(
- new TEvDataShard::TEvProposeTransactionAttach(tabletId, TxId),
- tabletId, /* subscribe */ true), 0, ++shardState->ReattachState.Cookie);
- }
-
- void HandleExecute(TEvTxProxy::TEvProposeTransactionStatus::TPtr &ev) {
- TEvTxProxy::TEvProposeTransactionStatus* res = ev->Get();
- LOG_D("Got transaction status, status: " << res->GetStatus());
-
- switch (res->GetStatus()) {
- case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusAccepted:
- Counters->TxProxyMon->ClientTxStatusAccepted->Inc();
- break;
- case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusProcessed:
- Counters->TxProxyMon->ClientTxStatusProcessed->Inc();
- break;
- case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusConfirmed:
- Counters->TxProxyMon->ClientTxStatusConfirmed->Inc();
- break;
-
- case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusPlanned:
- Counters->TxProxyMon->ClientTxStatusPlanned->Inc();
- TxPlanned = true;
- break;
-
- case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusOutdated:
- case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusDeclined:
- case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusDeclinedNoSpace:
- case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusRestarting:
- Counters->TxProxyMon->ClientTxStatusCoordinatorDeclined->Inc();
- CancelProposal(0);
- ReplyUnavailable(TStringBuilder() << "Failed to plan transaction, status: " << res->GetStatus());
- break;
-
- case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusUnknown:
- case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusAborted:
- Counters->TxProxyMon->ClientTxStatusCoordinatorDeclined->Inc();
- InternalError(TStringBuilder() << "Unexpected TEvProposeTransactionStatus status: " << res->GetStatus());
- break;
- }
- }
-
- void HandleExecute(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) {
- TEvPipeCache::TEvDeliveryProblem* msg = ev->Get();
-
- LOG_D("DeliveryProblem to shard " << msg->TabletId << ", notDelivered: " << msg->NotDelivered
- << ", txPlanned: " << TxPlanned << ", coordinator: " << TxCoordinator);
-
- if (msg->TabletId == TxCoordinator) {
- if (msg->NotDelivered) {
- LOG_E("Not delivered to coordinator " << msg->TabletId << ", abort execution");
- CancelProposal(0);
- return ReplyUnavailable("Delivery problem: could not plan transaction.");
- }
-
- if (TxPlanned) {
- // We lost pipe to coordinator, but we already know tx is planned
- return;
- }
-
- LOG_E("Delivery problem to coordinator " << msg->TabletId << ", abort execution");
- return ReplyTxStateUnknown(msg->TabletId);
- }
-
- auto* shardState = ShardStates.FindPtr(msg->TabletId);
- YQL_ENSURE(shardState, "EvDeliveryProblem from unknown shard " << msg->TabletId);
-
- bool wasRestarting = std::exchange(shardState->Restarting, false);
-
- switch (shardState->State) {
- case TShardState::EState::Prepared: // is it correct?
- LOG_E("DeliveryProblem to shard " << msg->TabletId << ", notDelivered: " << msg->NotDelivered
- << ", txPlanned: " << TxPlanned << ", coordinator: " << TxCoordinator);
- Y_VERIFY_DEBUG(false);
+ new TEvDataShard::TEvProposeTransactionAttach(tabletId, TxId),
+ tabletId, /* subscribe */ true), 0, ++shardState->ReattachState.Cookie);
+ }
+
+ void HandleExecute(TEvTxProxy::TEvProposeTransactionStatus::TPtr &ev) {
+ TEvTxProxy::TEvProposeTransactionStatus* res = ev->Get();
+ LOG_D("Got transaction status, status: " << res->GetStatus());
+
+ switch (res->GetStatus()) {
+ case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusAccepted:
+ Counters->TxProxyMon->ClientTxStatusAccepted->Inc();
+ break;
+ case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusProcessed:
+ Counters->TxProxyMon->ClientTxStatusProcessed->Inc();
+ break;
+ case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusConfirmed:
+ Counters->TxProxyMon->ClientTxStatusConfirmed->Inc();
+ break;
+
+ case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusPlanned:
+ Counters->TxProxyMon->ClientTxStatusPlanned->Inc();
+ TxPlanned = true;
+ break;
+
+ case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusOutdated:
+ case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusDeclined:
+ case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusDeclinedNoSpace:
+ case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusRestarting:
+ Counters->TxProxyMon->ClientTxStatusCoordinatorDeclined->Inc();
+ CancelProposal(0);
+ ReplyUnavailable(TStringBuilder() << "Failed to plan transaction, status: " << res->GetStatus());
+ break;
+
+ case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusUnknown:
+ case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusAborted:
+ Counters->TxProxyMon->ClientTxStatusCoordinatorDeclined->Inc();
+ InternalError(TStringBuilder() << "Unexpected TEvProposeTransactionStatus status: " << res->GetStatus());
+ break;
+ }
+ }
+
+ void HandleExecute(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) {
+ TEvPipeCache::TEvDeliveryProblem* msg = ev->Get();
+
+ LOG_D("DeliveryProblem to shard " << msg->TabletId << ", notDelivered: " << msg->NotDelivered
+ << ", txPlanned: " << TxPlanned << ", coordinator: " << TxCoordinator);
+
+ if (msg->TabletId == TxCoordinator) {
+ if (msg->NotDelivered) {
+ LOG_E("Not delivered to coordinator " << msg->TabletId << ", abort execution");
+ CancelProposal(0);
+ return ReplyUnavailable("Delivery problem: could not plan transaction.");
+ }
+
+ if (TxPlanned) {
+ // We lost pipe to coordinator, but we already know tx is planned
+ return;
+ }
+
+ LOG_E("Delivery problem to coordinator " << msg->TabletId << ", abort execution");
+ return ReplyTxStateUnknown(msg->TabletId);
+ }
+
+ auto* shardState = ShardStates.FindPtr(msg->TabletId);
+ YQL_ENSURE(shardState, "EvDeliveryProblem from unknown shard " << msg->TabletId);
+
+ bool wasRestarting = std::exchange(shardState->Restarting, false);
+
+ switch (shardState->State) {
+ case TShardState::EState::Prepared: // is it correct?
+ LOG_E("DeliveryProblem to shard " << msg->TabletId << ", notDelivered: " << msg->NotDelivered
+ << ", txPlanned: " << TxPlanned << ", coordinator: " << TxCoordinator);
+ Y_VERIFY_DEBUG(false);
// Proceed with query processing
[[fallthrough]];
- case TShardState::EState::Executing: {
- if ((wasRestarting || shardState->ReattachState.Reattaching) &&
- shardState->ReattachState.ShouldReattach(TlsActivationContext->Now()))
- {
- LOG_N("Shard " << msg->TabletId << " lost pipe while waiting for reply (reattaching in "
- << shardState->ReattachState.Delay << ")");
-
- Schedule(shardState->ReattachState.Delay, new TEvPrivate::TEvReattachToShard(msg->TabletId));
- ++shardState->RestartCount;
- return;
- }
-
- LOG_N("Shard " << msg->TabletId << " lost pipe while waiting for reply"
- << (msg->NotDelivered ? " (last message not delivered)" : ""));
-
- return ReplyTxStateUnknown(msg->TabletId);
- }
-
- case TShardState::EState::Finished: {
- return;
- }
-
- case TShardState::EState::Initial:
- case TShardState::EState::Preparing:
- YQL_ENSURE(false, "Unexpected shard " << msg->TabletId << " state " << ToString(shardState->State));
- }
- }
-
- void HandleExecute(TEvDqCompute::TEvState::TPtr& ev) {
- TActorId computeActor = ev->Sender;
- auto& state = ev->Get()->Record;
- ui64 taskId = state.GetTaskId();
-
- LOG_D("Got execution state from compute actor: " << computeActor
- << ", task: " << taskId
- << ", state: " << NDqProto::EComputeState_Name((NDqProto::EComputeState) state.GetState()));
-
- switch (state.GetState()) {
- case NDqProto::COMPUTE_STATE_UNKNOWN: {
- YQL_ENSURE(false, "unexpected state from " << computeActor << ", task: " << taskId);
- }
-
- case NDqProto::COMPUTE_STATE_FAILURE: {
- ReplyErrorAndDie(state.GetStatus(), state.MutableIssues());
- return;
- }
-
- case NDqProto::COMPUTE_STATE_EXECUTING: {
- YQL_ENSURE(PendingComputeActors.contains(computeActor));
- YQL_ENSURE(TasksGraph.GetTask(taskId).ComputeActorId == computeActor);
- break;
- }
-
- case NDqProto::COMPUTE_STATE_FINISHED: {
- if (Stats) {
- Stats->AddComputeActorStats(computeActor.NodeId(), std::move(*state.MutableStats()));
- }
-
- if (PendingComputeActors.erase(computeActor) == 0) {
- LOG_W("Got execution state from unknown compute actor: " << computeActor << ", task: " << taskId);
- }
- }
- }
-
- CheckExecutionComplete();
- }
-
- void HandleExecute(TEvDqCompute::TEvChannelData::TPtr& ev) {
- auto& record = ev->Get()->Record;
- auto& channelData = record.GetChannelData();
-
- auto& channel = TasksGraph.GetChannel(channelData.GetChannelId());
- YQL_ENSURE(channel.DstTask == 0);
- auto shardId = TasksGraph.GetTask(channel.SrcTask).Meta.ShardId;
-
- if (Stats) {
- Stats->ResultBytes += channelData.GetData().GetRaw().size();
- Stats->ResultRows += channelData.GetData().GetRows();
- }
-
- LOG_T("Got result, channelId: " << channel.Id << ", shardId: " << shardId
- << ", inputIndex: " << channel.DstInputIndex << ", from: " << ev->Sender
- << ", finished: " << channelData.GetFinished());
-
- YQL_ENSURE(channel.DstInputIndex < Results.size());
- if (channelData.GetData().GetRows()) {
- Results[channel.DstInputIndex].Data.emplace_back(std::move(*record.MutableChannelData()->MutableData()));
- }
-
- {
- LOG_T("Send ack to channelId: " << channel.Id << ", seqNo: " << record.GetSeqNo() << ", to: " << ev->Sender);
-
- auto ackEv = MakeHolder<TEvDqCompute::TEvChannelDataAck>();
- ackEv->Record.SetSeqNo(record.GetSeqNo());
- ackEv->Record.SetChannelId(channel.Id);
- ackEv->Record.SetFreeSpace(50_MB);
- Send(ev->Sender, ackEv.Release(), /* TODO: undelivery */ 0, /* cookie */ channel.Id);
- }
- }
-
- void CheckExecutionComplete() {
- ui32 notFinished = 0;
- for (auto& x : ShardStates) {
- if (x.second.State != TShardState::EState::Finished) {
- notFinished++;
- LOG_D("Datashard " << x.first << " not finished yet: " << ToString(x.second.State));
- }
- }
- if (notFinished == 0 && PendingComputeActors.empty()) {
- Finalize();
- return;
- }
-
- if (IsDebugLogEnabled()) {
- auto sb = TStringBuilder() << "Waiting for " << PendingComputeActors.size() << " compute actor(s) and "
- << notFinished << " datashard(s): ";
- for (auto shardId : PendingComputeActors) {
+ case TShardState::EState::Executing: {
+ if ((wasRestarting || shardState->ReattachState.Reattaching) &&
+ shardState->ReattachState.ShouldReattach(TlsActivationContext->Now()))
+ {
+ LOG_N("Shard " << msg->TabletId << " lost pipe while waiting for reply (reattaching in "
+ << shardState->ReattachState.Delay << ")");
+
+ Schedule(shardState->ReattachState.Delay, new TEvPrivate::TEvReattachToShard(msg->TabletId));
+ ++shardState->RestartCount;
+ return;
+ }
+
+ LOG_N("Shard " << msg->TabletId << " lost pipe while waiting for reply"
+ << (msg->NotDelivered ? " (last message not delivered)" : ""));
+
+ return ReplyTxStateUnknown(msg->TabletId);
+ }
+
+ case TShardState::EState::Finished: {
+ return;
+ }
+
+ case TShardState::EState::Initial:
+ case TShardState::EState::Preparing:
+ YQL_ENSURE(false, "Unexpected shard " << msg->TabletId << " state " << ToString(shardState->State));
+ }
+ }
+
+ void HandleExecute(TEvDqCompute::TEvState::TPtr& ev) {
+ TActorId computeActor = ev->Sender;
+ auto& state = ev->Get()->Record;
+ ui64 taskId = state.GetTaskId();
+
+ LOG_D("Got execution state from compute actor: " << computeActor
+ << ", task: " << taskId
+ << ", state: " << NDqProto::EComputeState_Name((NDqProto::EComputeState) state.GetState()));
+
+ switch (state.GetState()) {
+ case NDqProto::COMPUTE_STATE_UNKNOWN: {
+ YQL_ENSURE(false, "unexpected state from " << computeActor << ", task: " << taskId);
+ }
+
+ case NDqProto::COMPUTE_STATE_FAILURE: {
+ ReplyErrorAndDie(state.GetStatus(), state.MutableIssues());
+ return;
+ }
+
+ case NDqProto::COMPUTE_STATE_EXECUTING: {
+ YQL_ENSURE(PendingComputeActors.contains(computeActor));
+ YQL_ENSURE(TasksGraph.GetTask(taskId).ComputeActorId == computeActor);
+ break;
+ }
+
+ case NDqProto::COMPUTE_STATE_FINISHED: {
+ if (Stats) {
+ Stats->AddComputeActorStats(computeActor.NodeId(), std::move(*state.MutableStats()));
+ }
+
+ if (PendingComputeActors.erase(computeActor) == 0) {
+ LOG_W("Got execution state from unknown compute actor: " << computeActor << ", task: " << taskId);
+ }
+ }
+ }
+
+ CheckExecutionComplete();
+ }
+
+ void HandleExecute(TEvDqCompute::TEvChannelData::TPtr& ev) {
+ auto& record = ev->Get()->Record;
+ auto& channelData = record.GetChannelData();
+
+ auto& channel = TasksGraph.GetChannel(channelData.GetChannelId());
+ YQL_ENSURE(channel.DstTask == 0);
+ auto shardId = TasksGraph.GetTask(channel.SrcTask).Meta.ShardId;
+
+ if (Stats) {
+ Stats->ResultBytes += channelData.GetData().GetRaw().size();
+ Stats->ResultRows += channelData.GetData().GetRows();
+ }
+
+ LOG_T("Got result, channelId: " << channel.Id << ", shardId: " << shardId
+ << ", inputIndex: " << channel.DstInputIndex << ", from: " << ev->Sender
+ << ", finished: " << channelData.GetFinished());
+
+ YQL_ENSURE(channel.DstInputIndex < Results.size());
+ if (channelData.GetData().GetRows()) {
+ Results[channel.DstInputIndex].Data.emplace_back(std::move(*record.MutableChannelData()->MutableData()));
+ }
+
+ {
+ LOG_T("Send ack to channelId: " << channel.Id << ", seqNo: " << record.GetSeqNo() << ", to: " << ev->Sender);
+
+ auto ackEv = MakeHolder<TEvDqCompute::TEvChannelDataAck>();
+ ackEv->Record.SetSeqNo(record.GetSeqNo());
+ ackEv->Record.SetChannelId(channel.Id);
+ ackEv->Record.SetFreeSpace(50_MB);
+ Send(ev->Sender, ackEv.Release(), /* TODO: undelivery */ 0, /* cookie */ channel.Id);
+ }
+ }
+
+ void CheckExecutionComplete() {
+ ui32 notFinished = 0;
+ for (auto& x : ShardStates) {
+ if (x.second.State != TShardState::EState::Finished) {
+ notFinished++;
+ LOG_D("Datashard " << x.first << " not finished yet: " << ToString(x.second.State));
+ }
+ }
+ if (notFinished == 0 && PendingComputeActors.empty()) {
+ Finalize();
+ return;
+ }
+
+ if (IsDebugLogEnabled()) {
+ auto sb = TStringBuilder() << "Waiting for " << PendingComputeActors.size() << " compute actor(s) and "
+ << notFinished << " datashard(s): ";
+ for (auto shardId : PendingComputeActors) {
sb << "CA " << shardId.first << ", ";
- }
- for (auto& [shardId, shardState] : ShardStates) {
- if (shardState.State != TShardState::EState::Finished) {
- sb << "DS " << shardId << " (" << ToString(shardState.State) << "), ";
- }
- }
- LOG_D(sb);
- }
- }
-
-private:
+ }
+ for (auto& [shardId, shardState] : ShardStates) {
+ if (shardState.State != TShardState::EState::Finished) {
+ sb << "DS " << shardId << " (" << ToString(shardState.State) << "), ";
+ }
+ }
+ LOG_D(sb);
+ }
+ }
+
+private:
void FillGeneralReadInfo(TTaskMeta& taskMeta, ui64 itemsLimit, bool reverse)
{
if (taskMeta.Reads && !taskMeta.Reads.GetRef().empty()) {
@@ -943,43 +943,43 @@ private:
taskMeta.ReadInfo.Reverse = reverse;
};
- void BuildDatashardTasks(TStageInfo& stageInfo, const NMiniKQL::THolderFactory& holderFactory,
- const NMiniKQL::TTypeEnvironment& typeEnv)
- {
- THashMap<ui64, ui64> shardTasks; // shardId -> taskId
-
- auto getShardTask = [&](ui64 shardId) -> TTask& {
- auto it = shardTasks.find(shardId);
- if (it != shardTasks.end()) {
- return TasksGraph.GetTask(it->second);
- }
- auto& task = TasksGraph.AddTask(stageInfo);
- task.Meta.ShardId = shardId;
- shardTasks.emplace(shardId, task.Id);
- return task;
- };
-
- auto& stage = GetStage(stageInfo);
-
- const auto& table = TableKeys.GetTable(stageInfo.Meta.TableId);
- const auto& keyTypes = table.KeyColumnTypes;;
-
- for (auto& op : stage.GetTableOps()) {
- Y_VERIFY_DEBUG(stageInfo.Meta.TablePath == op.GetTable().GetPath());
-
- auto columns = BuildKqpColumns(op, table);
+ void BuildDatashardTasks(TStageInfo& stageInfo, const NMiniKQL::THolderFactory& holderFactory,
+ const NMiniKQL::TTypeEnvironment& typeEnv)
+ {
+ THashMap<ui64, ui64> shardTasks; // shardId -> taskId
+
+ auto getShardTask = [&](ui64 shardId) -> TTask& {
+ auto it = shardTasks.find(shardId);
+ if (it != shardTasks.end()) {
+ return TasksGraph.GetTask(it->second);
+ }
+ auto& task = TasksGraph.AddTask(stageInfo);
+ task.Meta.ShardId = shardId;
+ shardTasks.emplace(shardId, task.Id);
+ return task;
+ };
+
+ auto& stage = GetStage(stageInfo);
+
+ const auto& table = TableKeys.GetTable(stageInfo.Meta.TableId);
+ const auto& keyTypes = table.KeyColumnTypes;;
+
+ for (auto& op : stage.GetTableOps()) {
+ Y_VERIFY_DEBUG(stageInfo.Meta.TablePath == op.GetTable().GetPath());
+
+ auto columns = BuildKqpColumns(op, table);
THashMap<ui64, TShardInfo> partitions;
-
- switch (op.GetTypeCase()) {
+
+ switch (op.GetTypeCase()) {
case NKqpProto::TKqpPhyTableOperation::kReadRanges:
- case NKqpProto::TKqpPhyTableOperation::kReadRange:
- case NKqpProto::TKqpPhyTableOperation::kLookup: {
- bool reverse = false;
- ui64 itemsLimit = 0;
- TString itemsLimitParamName;
- NDqProto::TData itemsLimitBytes;
+ case NKqpProto::TKqpPhyTableOperation::kReadRange:
+ case NKqpProto::TKqpPhyTableOperation::kLookup: {
+ bool reverse = false;
+ ui64 itemsLimit = 0;
+ TString itemsLimitParamName;
+ NDqProto::TData itemsLimitBytes;
NKikimr::NMiniKQL::TType* itemsLimitType = nullptr;
-
+
if (op.GetTypeCase() == NKqpProto::TKqpPhyTableOperation::kReadRanges) {
partitions = PrunePartitions(TableKeys, op.GetReadRanges(), stageInfo, holderFactory, typeEnv);
ExtractItemsLimit(stageInfo, op.GetReadRanges().GetItemsLimit(), holderFactory, typeEnv,
@@ -987,283 +987,283 @@ private:
reverse = op.GetReadRanges().GetReverse();
} else if (op.GetTypeCase() == NKqpProto::TKqpPhyTableOperation::kReadRange) {
partitions = PrunePartitions(TableKeys, op.GetReadRange(), stageInfo, holderFactory, typeEnv);
- ExtractItemsLimit(stageInfo, op.GetReadRange().GetItemsLimit(), holderFactory, typeEnv,
+ ExtractItemsLimit(stageInfo, op.GetReadRange().GetItemsLimit(), holderFactory, typeEnv,
itemsLimit, itemsLimitParamName, itemsLimitBytes, itemsLimitType);
- reverse = op.GetReadRange().GetReverse();
+ reverse = op.GetReadRange().GetReverse();
} else if (op.GetTypeCase() == NKqpProto::TKqpPhyTableOperation::kLookup) {
partitions = PrunePartitions(TableKeys, op.GetLookup(), stageInfo, holderFactory, typeEnv);
- }
-
+ }
+
for (auto& [shardId, shardInfo] : partitions) {
- YQL_ENSURE(!shardInfo.KeyWriteRanges);
-
- auto& task = getShardTask(shardId);
- for (auto& [name, value] : shardInfo.Params) {
- task.Meta.Params.emplace(name, std::move(value));
+ YQL_ENSURE(!shardInfo.KeyWriteRanges);
+
+ auto& task = getShardTask(shardId);
+ for (auto& [name, value] : shardInfo.Params) {
+ task.Meta.Params.emplace(name, std::move(value));
auto typeIterator = shardInfo.ParamTypes.find(name);
YQL_ENSURE(typeIterator != shardInfo.ParamTypes.end());
auto retType = task.Meta.ParamTypes.emplace(name, typeIterator->second);
YQL_ENSURE(retType.second);
- }
-
+ }
+
FillGeneralReadInfo(task.Meta, itemsLimit, reverse);
TTaskMeta::TShardReadInfo readInfo;
- readInfo.Ranges = std::move(*shardInfo.KeyReadRanges);
- readInfo.Columns = columns;
-
- if (itemsLimit) {
- task.Meta.Params.emplace(itemsLimitParamName, itemsLimitBytes);
+ readInfo.Ranges = std::move(*shardInfo.KeyReadRanges);
+ readInfo.Columns = columns;
+
+ if (itemsLimit) {
+ task.Meta.Params.emplace(itemsLimitParamName, itemsLimitBytes);
task.Meta.ParamTypes.emplace(itemsLimitParamName, itemsLimitType);
- }
-
- if (!task.Meta.Reads) {
- task.Meta.Reads.ConstructInPlace();
- }
- task.Meta.Reads->emplace_back(std::move(readInfo));
- }
-
- break;
- }
-
- case NKqpProto::TKqpPhyTableOperation::kUpsertRows:
- case NKqpProto::TKqpPhyTableOperation::kDeleteRows: {
- YQL_ENSURE(stage.InputsSize() <= 1, "Effect stage with multiple inputs: " << stage.GetProgramAst());
-
- if (stage.InputsSize() == 1 && stage.GetInputs(0).GetTypeCase() == NKqpProto::TKqpPhyConnection::kMapShard) {
- const auto& inputStageInfo = TasksGraph.GetStageInfo(
- TStageId(stageInfo.Id.TxId, stage.GetInputs(0).GetStageIndex()));
-
- for (ui64 inputTaskId : inputStageInfo.Tasks) {
- auto& task = getShardTask(TasksGraph.GetTask(inputTaskId).Meta.ShardId);
-
- auto& inputTask = TasksGraph.GetTask(inputTaskId);
- YQL_ENSURE(inputTask.Meta.Reads, "" << inputTask.Meta.ToString(keyTypes, *AppData()->TypeRegistry));
- for (auto& read : *inputTask.Meta.Reads) {
- if (!task.Meta.Writes) {
- task.Meta.Writes.ConstructInPlace();
- task.Meta.Writes->Ranges = read.Ranges;
- } else {
+ }
+
+ if (!task.Meta.Reads) {
+ task.Meta.Reads.ConstructInPlace();
+ }
+ task.Meta.Reads->emplace_back(std::move(readInfo));
+ }
+
+ break;
+ }
+
+ case NKqpProto::TKqpPhyTableOperation::kUpsertRows:
+ case NKqpProto::TKqpPhyTableOperation::kDeleteRows: {
+ YQL_ENSURE(stage.InputsSize() <= 1, "Effect stage with multiple inputs: " << stage.GetProgramAst());
+
+ if (stage.InputsSize() == 1 && stage.GetInputs(0).GetTypeCase() == NKqpProto::TKqpPhyConnection::kMapShard) {
+ const auto& inputStageInfo = TasksGraph.GetStageInfo(
+ TStageId(stageInfo.Id.TxId, stage.GetInputs(0).GetStageIndex()));
+
+ for (ui64 inputTaskId : inputStageInfo.Tasks) {
+ auto& task = getShardTask(TasksGraph.GetTask(inputTaskId).Meta.ShardId);
+
+ auto& inputTask = TasksGraph.GetTask(inputTaskId);
+ YQL_ENSURE(inputTask.Meta.Reads, "" << inputTask.Meta.ToString(keyTypes, *AppData()->TypeRegistry));
+ for (auto& read : *inputTask.Meta.Reads) {
+ if (!task.Meta.Writes) {
+ task.Meta.Writes.ConstructInPlace();
+ task.Meta.Writes->Ranges = read.Ranges;
+ } else {
task.Meta.Writes->Ranges.MergeWritePoints(TShardKeyRanges(read.Ranges), keyTypes);
- }
- }
- }
- } else {
- auto result = (op.GetTypeCase() == NKqpProto::TKqpPhyTableOperation::kUpsertRows)
- ? PruneEffectPartitions(TableKeys, op.GetUpsertRows(), stageInfo, holderFactory, typeEnv)
- : PruneEffectPartitions(TableKeys, op.GetDeleteRows(), stageInfo, holderFactory, typeEnv);
-
- for (auto& [shardId, shardInfo] : result) {
- YQL_ENSURE(!shardInfo.KeyReadRanges);
- YQL_ENSURE(shardInfo.KeyWriteRanges);
-
- auto& task = getShardTask(shardId);
- task.Meta.Params = std::move(shardInfo.Params);
-
- if (!task.Meta.Writes) {
- task.Meta.Writes.ConstructInPlace();
- task.Meta.Writes->Ranges = std::move(*shardInfo.KeyWriteRanges);
- } else {
+ }
+ }
+ }
+ } else {
+ auto result = (op.GetTypeCase() == NKqpProto::TKqpPhyTableOperation::kUpsertRows)
+ ? PruneEffectPartitions(TableKeys, op.GetUpsertRows(), stageInfo, holderFactory, typeEnv)
+ : PruneEffectPartitions(TableKeys, op.GetDeleteRows(), stageInfo, holderFactory, typeEnv);
+
+ for (auto& [shardId, shardInfo] : result) {
+ YQL_ENSURE(!shardInfo.KeyReadRanges);
+ YQL_ENSURE(shardInfo.KeyWriteRanges);
+
+ auto& task = getShardTask(shardId);
+ task.Meta.Params = std::move(shardInfo.Params);
+
+ if (!task.Meta.Writes) {
+ task.Meta.Writes.ConstructInPlace();
+ task.Meta.Writes->Ranges = std::move(*shardInfo.KeyWriteRanges);
+ } else {
task.Meta.Writes->Ranges.MergeWritePoints(std::move(*shardInfo.KeyWriteRanges), keyTypes);
- }
- }
- }
- break;
- }
-
- default: {
- YQL_ENSURE(false, "Unexpected table operation: " << (ui32) op.GetTypeCase());
- }
- }
- }
-
- LOG_D("Stage " << stageInfo.Id << " will be executed on " << shardTasks.size() << " shards.");
-
- for (auto& shardTask : shardTasks) {
- auto& task = TasksGraph.GetTask(shardTask.second);
- LOG_D("Stage " << stageInfo.Id << " create datashard task: " << shardTask.second
- << ", shard: " << shardTask.first
- << ", meta: " << task.Meta.ToString(keyTypes, *AppData()->TypeRegistry));
- }
- }
-
- void BuildComputeTasks(TStageInfo& stageInfo) {
- auto& stage = GetStage(stageInfo);
-
- ui32 partitionsCount = 1;
- for (ui32 inputIndex = 0; inputIndex < stage.InputsSize(); ++inputIndex) {
- const auto& input = stage.GetInputs(inputIndex);
-
- // Current assumptions:
- // 1. `Broadcast` can not be the 1st stage input unless it's a single input
- // 2. All stage's inputs, except 1st one, must be a `Broadcast` or `UnionAll`
- if (inputIndex == 0) {
- if (stage.InputsSize() > 1) {
- YQL_ENSURE(input.GetTypeCase() != NKqpProto::TKqpPhyConnection::kBroadcast);
- }
- } else {
- switch (input.GetTypeCase()) {
- case NKqpProto::TKqpPhyConnection::kBroadcast:
- case NKqpProto::TKqpPhyConnection::kHashShuffle:
- case NKqpProto::TKqpPhyConnection::kUnionAll:
+ }
+ }
+ }
+ break;
+ }
+
+ default: {
+ YQL_ENSURE(false, "Unexpected table operation: " << (ui32) op.GetTypeCase());
+ }
+ }
+ }
+
+ LOG_D("Stage " << stageInfo.Id << " will be executed on " << shardTasks.size() << " shards.");
+
+ for (auto& shardTask : shardTasks) {
+ auto& task = TasksGraph.GetTask(shardTask.second);
+ LOG_D("Stage " << stageInfo.Id << " create datashard task: " << shardTask.second
+ << ", shard: " << shardTask.first
+ << ", meta: " << task.Meta.ToString(keyTypes, *AppData()->TypeRegistry));
+ }
+ }
+
+ void BuildComputeTasks(TStageInfo& stageInfo) {
+ auto& stage = GetStage(stageInfo);
+
+ ui32 partitionsCount = 1;
+ for (ui32 inputIndex = 0; inputIndex < stage.InputsSize(); ++inputIndex) {
+ const auto& input = stage.GetInputs(inputIndex);
+
+ // Current assumptions:
+ // 1. `Broadcast` can not be the 1st stage input unless it's a single input
+ // 2. All stage's inputs, except 1st one, must be a `Broadcast` or `UnionAll`
+ if (inputIndex == 0) {
+ if (stage.InputsSize() > 1) {
+ YQL_ENSURE(input.GetTypeCase() != NKqpProto::TKqpPhyConnection::kBroadcast);
+ }
+ } else {
+ switch (input.GetTypeCase()) {
+ case NKqpProto::TKqpPhyConnection::kBroadcast:
+ case NKqpProto::TKqpPhyConnection::kHashShuffle:
+ case NKqpProto::TKqpPhyConnection::kUnionAll:
case NKqpProto::TKqpPhyConnection::kMerge:
- break;
- default:
- YQL_ENSURE(false, "Unexpected connection type: " << (ui32)input.GetTypeCase());
- }
- }
-
- auto& originStageInfo = TasksGraph.GetStageInfo(TStageId(stageInfo.Id.TxId, input.GetStageIndex()));
-
- switch (input.GetTypeCase()) {
- case NKqpProto::TKqpPhyConnection::kHashShuffle: {
- partitionsCount = std::max(partitionsCount, (ui32)originStageInfo.Tasks.size() / 2);
- partitionsCount = std::min(partitionsCount, 24u);
- break;
- }
-
- case NKqpProto::TKqpPhyConnection::kMap: {
- partitionsCount = originStageInfo.Tasks.size();
+ break;
+ default:
+ YQL_ENSURE(false, "Unexpected connection type: " << (ui32)input.GetTypeCase());
+ }
+ }
+
+ auto& originStageInfo = TasksGraph.GetStageInfo(TStageId(stageInfo.Id.TxId, input.GetStageIndex()));
+
+ switch (input.GetTypeCase()) {
+ case NKqpProto::TKqpPhyConnection::kHashShuffle: {
+ partitionsCount = std::max(partitionsCount, (ui32)originStageInfo.Tasks.size() / 2);
+ partitionsCount = std::min(partitionsCount, 24u);
break;
- }
-
- default: {
- break;
- }
- }
- }
-
- for (ui32 i = 0; i < partitionsCount; ++i) {
- auto& task = TasksGraph.AddTask(stageInfo);
- LOG_D("Stage " << stageInfo.Id << " create compute task: " << task.Id);
- }
- }
-
- void ExecuteDatashardTransaction(ui64 shardId, NKikimrTxDataShard::TKqpTransaction& kqpTx, const TMaybe<ui64> lockTxId)
- {
- TShardState shardState;
- shardState.State = ImmediateTx ? TShardState::EState::Executing : TShardState::EState::Preparing;
- shardState.DatashardState.ConstructInPlace();
+ }
+
+ case NKqpProto::TKqpPhyConnection::kMap: {
+ partitionsCount = originStageInfo.Tasks.size();
+ break;
+ }
+
+ default: {
+ break;
+ }
+ }
+ }
+
+ for (ui32 i = 0; i < partitionsCount; ++i) {
+ auto& task = TasksGraph.AddTask(stageInfo);
+ LOG_D("Stage " << stageInfo.Id << " create compute task: " << task.Id);
+ }
+ }
+
+ void ExecuteDatashardTransaction(ui64 shardId, NKikimrTxDataShard::TKqpTransaction& kqpTx, const TMaybe<ui64> lockTxId)
+ {
+ TShardState shardState;
+ shardState.State = ImmediateTx ? TShardState::EState::Executing : TShardState::EState::Preparing;
+ shardState.DatashardState.ConstructInPlace();
shardState.DatashardState->Follower = UseFollowers;
-
- if (Deadline) {
- TDuration timeout = *Deadline - TAppData::TimeProvider->Now();
- kqpTx.MutableRuntimeSettings()->SetTimeoutMs(timeout.MilliSeconds());
- }
- kqpTx.MutableRuntimeSettings()->SetExecType(NDqProto::TComputeRuntimeSettings::DATA);
- kqpTx.MutableRuntimeSettings()->SetStatsMode(Request.StatsMode);
-
- kqpTx.MutableRuntimeSettings()->SetUseLLVM(false);
- kqpTx.MutableRuntimeSettings()->SetUseSpilling(false);
-
- NKikimrTxDataShard::TDataTransaction dataTransaction;
- dataTransaction.MutableKqpTransaction()->Swap(&kqpTx);
- dataTransaction.SetImmediate(ImmediateTx);
- dataTransaction.SetReadOnly(ReadOnlyTx);
- if (CancelAt) {
- dataTransaction.SetCancelAfterMs((*CancelAt - AppData()->TimeProvider->Now()).MilliSeconds());
- }
- if (Request.PerShardKeysSizeLimitBytes) {
- YQL_ENSURE(!ReadOnlyTx);
- dataTransaction.SetPerShardKeysSizeLimitBytes(Request.PerShardKeysSizeLimitBytes);
- }
-
- if (lockTxId) {
- dataTransaction.SetLockTxId(*lockTxId);
- }
-
- for (auto& task : dataTransaction.GetKqpTransaction().GetTasks()) {
- shardState.TaskIds.insert(task.GetId());
- }
-
- auto locksCount = dataTransaction.GetKqpTransaction().GetLocks().LocksSize();
- shardState.DatashardState->ShardReadLocks = locksCount > 0;
-
- LOG_D("Executing KQP transaction on shard: " << shardId
- << ", tasks: [" << JoinStrings(shardState.TaskIds.begin(), shardState.TaskIds.end(), ",") << "]"
- << ", lockTxId: " << lockTxId
- << ", locks: " << dataTransaction.GetKqpTransaction().GetLocks().ShortDebugString());
-
+
+ if (Deadline) {
+ TDuration timeout = *Deadline - TAppData::TimeProvider->Now();
+ kqpTx.MutableRuntimeSettings()->SetTimeoutMs(timeout.MilliSeconds());
+ }
+ kqpTx.MutableRuntimeSettings()->SetExecType(NDqProto::TComputeRuntimeSettings::DATA);
+ kqpTx.MutableRuntimeSettings()->SetStatsMode(Request.StatsMode);
+
+ kqpTx.MutableRuntimeSettings()->SetUseLLVM(false);
+ kqpTx.MutableRuntimeSettings()->SetUseSpilling(false);
+
+ NKikimrTxDataShard::TDataTransaction dataTransaction;
+ dataTransaction.MutableKqpTransaction()->Swap(&kqpTx);
+ dataTransaction.SetImmediate(ImmediateTx);
+ dataTransaction.SetReadOnly(ReadOnlyTx);
+ if (CancelAt) {
+ dataTransaction.SetCancelAfterMs((*CancelAt - AppData()->TimeProvider->Now()).MilliSeconds());
+ }
+ if (Request.PerShardKeysSizeLimitBytes) {
+ YQL_ENSURE(!ReadOnlyTx);
+ dataTransaction.SetPerShardKeysSizeLimitBytes(Request.PerShardKeysSizeLimitBytes);
+ }
+
+ if (lockTxId) {
+ dataTransaction.SetLockTxId(*lockTxId);
+ }
+
+ for (auto& task : dataTransaction.GetKqpTransaction().GetTasks()) {
+ shardState.TaskIds.insert(task.GetId());
+ }
+
+ auto locksCount = dataTransaction.GetKqpTransaction().GetLocks().LocksSize();
+ shardState.DatashardState->ShardReadLocks = locksCount > 0;
+
+ LOG_D("Executing KQP transaction on shard: " << shardId
+ << ", tasks: [" << JoinStrings(shardState.TaskIds.begin(), shardState.TaskIds.end(), ",") << "]"
+ << ", lockTxId: " << lockTxId
+ << ", locks: " << dataTransaction.GetKqpTransaction().GetLocks().ShortDebugString());
+
TEvDataShard::TEvProposeTransaction* ev;
- if (Request.Snapshot.IsValid() && ReadOnlyTx) {
+ if (Request.Snapshot.IsValid() && ReadOnlyTx) {
ev = new TEvDataShard::TEvProposeTransaction(
- NKikimrTxDataShard::TX_KIND_DATA,
- SelfId(),
- TxId,
- dataTransaction.SerializeAsString(),
+ NKikimrTxDataShard::TX_KIND_DATA,
+ SelfId(),
+ TxId,
+ dataTransaction.SerializeAsString(),
Request.Snapshot.Step,
Request.Snapshot.TxId,
- ImmediateTx ? NTxDataShard::TTxFlags::Immediate : 0);
+ ImmediateTx ? NTxDataShard::TTxFlags::Immediate : 0);
} else {
ev = new TEvDataShard::TEvProposeTransaction(
NKikimrTxDataShard::TX_KIND_DATA,
SelfId(),
TxId,
dataTransaction.SerializeAsString(),
- ImmediateTx ? NTxDataShard::TTxFlags::Immediate : 0);
+ ImmediateTx ? NTxDataShard::TTxFlags::Immediate : 0);
}
-
+
Send(MakePipePeNodeCacheID(UseFollowers), new TEvPipeCache::TEvForward(ev, shardId, true));
- auto result = ShardStates.emplace(shardId, std::move(shardState));
- YQL_ENSURE(result.second);
- }
-
- void ExecuteDataComputeTask(NDqProto::TDqTask&& taskDesc) {
- auto taskId = taskDesc.GetId();
- auto& task = TasksGraph.GetTask(taskId);
-
- TComputeRuntimeSettings settings;
- if (Deadline) {
- settings.Timeout = *Deadline - TAppData::TimeProvider->Now();
- }
- //settings.ExtraMemoryAllocationPool = NRm::EKqpMemoryPool::DataQuery;
- settings.ExtraMemoryAllocationPool = NRm::EKqpMemoryPool::Unspecified;
- settings.FailOnUndelivery = true;
- settings.StatsMode = Request.StatsMode;
- settings.UseLLVM = false;
- settings.UseSpilling = false;
-
- TComputeMemoryLimits limits;
- limits.ScanBufferSize = 50_MB; // for system views only
- limits.ChannelBufferSize = 50_MB;
- limits.MkqlLightProgramMemoryLimit = Request.MkqlMemoryLimit > 0 ? std::min(500_MB, Request.MkqlMemoryLimit) : 500_MB;
- limits.MkqlHeavyProgramMemoryLimit = Request.MkqlMemoryLimit > 0 ? std::min(2_GB, Request.MkqlMemoryLimit) : 2_GB;
+ auto result = ShardStates.emplace(shardId, std::move(shardState));
+ YQL_ENSURE(result.second);
+ }
+
+ void ExecuteDataComputeTask(NDqProto::TDqTask&& taskDesc) {
+ auto taskId = taskDesc.GetId();
+ auto& task = TasksGraph.GetTask(taskId);
+
+ TComputeRuntimeSettings settings;
+ if (Deadline) {
+ settings.Timeout = *Deadline - TAppData::TimeProvider->Now();
+ }
+ //settings.ExtraMemoryAllocationPool = NRm::EKqpMemoryPool::DataQuery;
+ settings.ExtraMemoryAllocationPool = NRm::EKqpMemoryPool::Unspecified;
+ settings.FailOnUndelivery = true;
+ settings.StatsMode = Request.StatsMode;
+ settings.UseLLVM = false;
+ settings.UseSpilling = false;
+
+ TComputeMemoryLimits limits;
+ limits.ScanBufferSize = 50_MB; // for system views only
+ limits.ChannelBufferSize = 50_MB;
+ limits.MkqlLightProgramMemoryLimit = Request.MkqlMemoryLimit > 0 ? std::min(500_MB, Request.MkqlMemoryLimit) : 500_MB;
+ limits.MkqlHeavyProgramMemoryLimit = Request.MkqlMemoryLimit > 0 ? std::min(2_GB, Request.MkqlMemoryLimit) : 2_GB;
limits.AllocateMemoryFn = [TxId = TxId](auto /* txId */, ui64 taskId, ui64 memory) {
- LOG_E("Data query task cannot allocate additional memory during executing."
- << " Task: " << taskId << ", memory: " << memory);
- return false;
- };
-
+ LOG_E("Data query task cannot allocate additional memory during executing."
+ << " Task: " << taskId << ", memory: " << memory);
+ return false;
+ };
+
auto computeActor = CreateKqpComputeActor(SelfId(), TxId, std::move(taskDesc), nullptr, nullptr, settings, limits);
- auto computeActorId = Register(computeActor);
- task.ComputeActorId = computeActorId;
-
- LOG_D("Executing task: " << taskId << " on compute actor: " << task.ComputeActorId);
-
+ auto computeActorId = Register(computeActor);
+ task.ComputeActorId = computeActorId;
+
+ LOG_D("Executing task: " << taskId << " on compute actor: " << task.ComputeActorId);
+
auto result = PendingComputeActors.emplace(task.ComputeActorId, TProgressStat());
- YQL_ENSURE(result.second);
- }
-
- void Execute() {
- RequestControls.Reqister(TlsActivationContext->AsActorContext());
-
- ReadOnlyTx = true;
-
- auto& funcRegistry = *AppData()->FunctionRegistry;
- NMiniKQL::TScopedAlloc alloc(TAlignedPagePoolCounters(), funcRegistry.SupportsSizedAllocators());
- NMiniKQL::TTypeEnvironment typeEnv(alloc);
-
- NMiniKQL::TMemoryUsageInfo memInfo("PrepareTasks");
- NMiniKQL::THolderFactory holderFactory(alloc.Ref(), memInfo, &funcRegistry);
-
- for (ui32 txIdx = 0; txIdx < Request.Transactions.size(); ++txIdx) {
- auto& tx = Request.Transactions[txIdx];
-
- for (ui32 stageIdx = 0; stageIdx < tx.Body.StagesSize(); ++stageIdx) {
- auto& stage = tx.Body.GetStages(stageIdx);
- auto& stageInfo = TasksGraph.GetStageInfo(TStageId(txIdx, stageIdx));
-
+ YQL_ENSURE(result.second);
+ }
+
+ void Execute() {
+ RequestControls.Reqister(TlsActivationContext->AsActorContext());
+
+ ReadOnlyTx = true;
+
+ auto& funcRegistry = *AppData()->FunctionRegistry;
+ NMiniKQL::TScopedAlloc alloc(TAlignedPagePoolCounters(), funcRegistry.SupportsSizedAllocators());
+ NMiniKQL::TTypeEnvironment typeEnv(alloc);
+
+ NMiniKQL::TMemoryUsageInfo memInfo("PrepareTasks");
+ NMiniKQL::THolderFactory holderFactory(alloc.Ref(), memInfo, &funcRegistry);
+
+ for (ui32 txIdx = 0; txIdx < Request.Transactions.size(); ++txIdx) {
+ auto& tx = Request.Transactions[txIdx];
+
+ for (ui32 stageIdx = 0; stageIdx < tx.Body.StagesSize(); ++stageIdx) {
+ auto& stage = tx.Body.GetStages(stageIdx);
+ auto& stageInfo = TasksGraph.GetStageInfo(TStageId(txIdx, stageIdx));
+
if (stageInfo.Meta.ShardKind == NSchemeCache::TSchemeCacheRequest::KindAsyncIndexTable) {
TMaybe<TString> error;
@@ -1283,162 +1283,162 @@ private:
}
}
- LOG_D("Stage " << stageInfo.Id << " AST: " << stage.GetProgramAst());
-
- ReadOnlyTx &= !stage.GetIsEffectsStage();
-
- if (stageInfo.Meta.ShardOperations.empty()) {
- BuildComputeTasks(stageInfo);
- } else if (stageInfo.Meta.IsSysView()) {
- BuildSysViewScanTasks(stageInfo, holderFactory, typeEnv);
- } else {
- BuildDatashardTasks(stageInfo, holderFactory, typeEnv);
- }
-
- BuildKqpStageChannels(TasksGraph, TableKeys, stageInfo, TxId, /* enableSpilling */ false);
- }
-
- BuildKqpExecuterResults(tx.Body, Results);
- BuildKqpTaskGraphResultChannels(TasksGraph, tx.Body, txIdx);
- }
-
- TIssue validateIssue;
- if (!ValidateTasks(TasksGraph, EExecType::Data, /* enableSpilling */ false, validateIssue)) {
- ReplyErrorAndDie(Ydb::StatusIds::INTERNAL_ERROR, validateIssue);
- return;
- }
-
- THashMap<ui64, TVector<NDqProto::TDqTask>> datashardTasks; // shardId -> [task]
- TVector<NDqProto::TDqTask> computeTasks;
-
- for (auto& task : TasksGraph.GetTasks()) {
- auto& stageInfo = TasksGraph.GetStageInfo(task.StageId);
- auto& stage = GetStage(stageInfo);
-
- NDqProto::TDqTask taskDesc;
- taskDesc.SetId(task.Id);
- taskDesc.SetStageId(stageInfo.Id.StageId);
- ActorIdToProto(SelfId(), taskDesc.MutableExecuter()->MutableActorId());
-
- for (auto& input : task.Inputs) {
+ LOG_D("Stage " << stageInfo.Id << " AST: " << stage.GetProgramAst());
+
+ ReadOnlyTx &= !stage.GetIsEffectsStage();
+
+ if (stageInfo.Meta.ShardOperations.empty()) {
+ BuildComputeTasks(stageInfo);
+ } else if (stageInfo.Meta.IsSysView()) {
+ BuildSysViewScanTasks(stageInfo, holderFactory, typeEnv);
+ } else {
+ BuildDatashardTasks(stageInfo, holderFactory, typeEnv);
+ }
+
+ BuildKqpStageChannels(TasksGraph, TableKeys, stageInfo, TxId, /* enableSpilling */ false);
+ }
+
+ BuildKqpExecuterResults(tx.Body, Results);
+ BuildKqpTaskGraphResultChannels(TasksGraph, tx.Body, txIdx);
+ }
+
+ TIssue validateIssue;
+ if (!ValidateTasks(TasksGraph, EExecType::Data, /* enableSpilling */ false, validateIssue)) {
+ ReplyErrorAndDie(Ydb::StatusIds::INTERNAL_ERROR, validateIssue);
+ return;
+ }
+
+ THashMap<ui64, TVector<NDqProto::TDqTask>> datashardTasks; // shardId -> [task]
+ TVector<NDqProto::TDqTask> computeTasks;
+
+ for (auto& task : TasksGraph.GetTasks()) {
+ auto& stageInfo = TasksGraph.GetStageInfo(task.StageId);
+ auto& stage = GetStage(stageInfo);
+
+ NDqProto::TDqTask taskDesc;
+ taskDesc.SetId(task.Id);
+ taskDesc.SetStageId(stageInfo.Id.StageId);
+ ActorIdToProto(SelfId(), taskDesc.MutableExecuter()->MutableActorId());
+
+ for (auto& input : task.Inputs) {
FillInputDesc(*taskDesc.AddInputs(), input);
- }
-
- for (auto& output : task.Outputs) {
- FillOutputDesc(*taskDesc.AddOutputs(), output);
- }
-
- taskDesc.MutableProgram()->CopyFrom(stage.GetProgram());
-
- PrepareKqpTaskParameters(stage, stageInfo, task, taskDesc, typeEnv, holderFactory);
-
- if (task.Meta.ShardId) {
- NKikimrTxDataShard::TKqpTransaction::TDataTaskMeta protoTaskMeta;
-
- FillTableMeta(stageInfo, protoTaskMeta.MutableTable());
-
- if (task.Meta.Reads) {
- for (auto& read : *task.Meta.Reads) {
- auto* protoReadMeta = protoTaskMeta.AddReads();
- read.Ranges.SerializeTo(protoReadMeta->MutableRange());
- for (auto& column : read.Columns) {
- auto* protoColumn = protoReadMeta->AddColumns();
- protoColumn->SetId(column.Id);
- protoColumn->SetType(column.Type);
- protoColumn->SetName(column.Name);
- }
+ }
+
+ for (auto& output : task.Outputs) {
+ FillOutputDesc(*taskDesc.AddOutputs(), output);
+ }
+
+ taskDesc.MutableProgram()->CopyFrom(stage.GetProgram());
+
+ PrepareKqpTaskParameters(stage, stageInfo, task, taskDesc, typeEnv, holderFactory);
+
+ if (task.Meta.ShardId) {
+ NKikimrTxDataShard::TKqpTransaction::TDataTaskMeta protoTaskMeta;
+
+ FillTableMeta(stageInfo, protoTaskMeta.MutableTable());
+
+ if (task.Meta.Reads) {
+ for (auto& read : *task.Meta.Reads) {
+ auto* protoReadMeta = protoTaskMeta.AddReads();
+ read.Ranges.SerializeTo(protoReadMeta->MutableRange());
+ for (auto& column : read.Columns) {
+ auto* protoColumn = protoReadMeta->AddColumns();
+ protoColumn->SetId(column.Id);
+ protoColumn->SetType(column.Type);
+ protoColumn->SetName(column.Name);
+ }
protoReadMeta->SetItemsLimit(task.Meta.ReadInfo.ItemsLimit);
protoReadMeta->SetReverse(task.Meta.ReadInfo.Reverse);
- }
- }
- if (task.Meta.Writes) {
- auto* protoWrites = protoTaskMeta.MutableWrites();
- task.Meta.Writes->Ranges.SerializeTo(protoWrites->MutableRange());
- }
-
- taskDesc.MutableMeta()->PackFrom(protoTaskMeta);
- LOG_D("Task: " << task.Id << ", shard: " << task.Meta.ShardId << ", meta: " << protoTaskMeta.ShortDebugString());
-
- datashardTasks[task.Meta.ShardId].emplace_back(std::move(taskDesc));
- } else if (stageInfo.Meta.IsSysView()) {
- NKikimrTxDataShard::TKqpTransaction::TScanTaskMeta protoTaskMeta;
-
- FillTableMeta(stageInfo, protoTaskMeta.MutableTable());
-
- const auto& tableInfo = TableKeys.GetTable(stageInfo.Meta.TableId);
- for (const auto& keyColumnName : tableInfo.KeyColumns) {
- const auto& keyColumn = tableInfo.Columns.at(keyColumnName);
- protoTaskMeta.AddKeyColumnTypes(keyColumn.Type);
- }
-
- for (bool skipNullKey : stageInfo.Meta.SkipNullKeys) {
- protoTaskMeta.AddSkipNullKeys(skipNullKey);
- }
-
- YQL_ENSURE(task.Meta.Reads);
- YQL_ENSURE(!task.Meta.Writes);
-
- for (auto& column : task.Meta.Reads->front().Columns) {
- auto* protoColumn = protoTaskMeta.AddColumns();
- protoColumn->SetId(column.Id);
- protoColumn->SetType(column.Type);
- protoColumn->SetName(column.Name);
- }
-
- for (auto& read : *task.Meta.Reads) {
- auto* protoReadMeta = protoTaskMeta.AddReads();
- protoReadMeta->SetShardId(read.ShardId);
- read.Ranges.SerializeTo(protoReadMeta);
-
- YQL_ENSURE((int) read.Columns.size() == protoTaskMeta.GetColumns().size());
- for (ui64 i = 0; i < read.Columns.size(); ++i) {
- YQL_ENSURE(read.Columns[i].Id == protoTaskMeta.GetColumns()[i].GetId());
- YQL_ENSURE(read.Columns[i].Type == protoTaskMeta.GetColumns()[i].GetType());
- }
- }
-
- LOG_D("task: " << task.Id << ", node: " << task.Meta.NodeId << ", meta: " << protoTaskMeta.ShortDebugString());
-
- taskDesc.MutableMeta()->PackFrom(protoTaskMeta);
- computeTasks.emplace_back(std::move(taskDesc));
- } else {
- computeTasks.emplace_back(std::move(taskDesc));
- }
- }
-
- if (computeTasks.size() > Request.MaxComputeActors) {
- LOG_N("Too many compute actors: " << computeTasks.size());
- ReplyErrorAndDie(Ydb::StatusIds::PRECONDITION_FAILED,
- YqlIssue({}, TIssuesIds::KIKIMR_PRECONDITION_FAILED, TStringBuilder()
- << "Requested too many execution units: " << computeTasks.size()));
- return;
- }
-
- ui32 shardsLimit = Request.MaxAffectedShards;
- if (i64 msc = (i64) RequestControls.MaxShardCount; msc > 0) {
- shardsLimit = std::min(shardsLimit, (ui32) msc);
- }
- if (shardsLimit > 0 && datashardTasks.size() > shardsLimit) {
- LOG_W("Too many affected shards: datashardTasks=" << datashardTasks.size() << ", limit: " << shardsLimit);
- Counters->TxProxyMon->TxResultError->Inc();
- ReplyErrorAndDie(Ydb::StatusIds::PRECONDITION_FAILED,
- YqlIssue({}, TIssuesIds::KIKIMR_PRECONDITION_FAILED, TStringBuilder()
- << "Affected too many shards: " << datashardTasks.size()));
- return;
- }
-
- bool fitSize = AllOf(datashardTasks, [this](const auto& x){ return ValidateTaskSize(x.second); });
- if (!fitSize) {
- Counters->TxProxyMon->TxResultError->Inc();
- return;
- }
-
- auto datashardTxs = BuildDatashardTxs(datashardTasks);
-
- // Single-shard transactions are always immediate
- ImmediateTx = datashardTxs.size() <= 1;
+ }
+ }
+ if (task.Meta.Writes) {
+ auto* protoWrites = protoTaskMeta.MutableWrites();
+ task.Meta.Writes->Ranges.SerializeTo(protoWrites->MutableRange());
+ }
+
+ taskDesc.MutableMeta()->PackFrom(protoTaskMeta);
+ LOG_D("Task: " << task.Id << ", shard: " << task.Meta.ShardId << ", meta: " << protoTaskMeta.ShortDebugString());
+
+ datashardTasks[task.Meta.ShardId].emplace_back(std::move(taskDesc));
+ } else if (stageInfo.Meta.IsSysView()) {
+ NKikimrTxDataShard::TKqpTransaction::TScanTaskMeta protoTaskMeta;
+
+ FillTableMeta(stageInfo, protoTaskMeta.MutableTable());
+
+ const auto& tableInfo = TableKeys.GetTable(stageInfo.Meta.TableId);
+ for (const auto& keyColumnName : tableInfo.KeyColumns) {
+ const auto& keyColumn = tableInfo.Columns.at(keyColumnName);
+ protoTaskMeta.AddKeyColumnTypes(keyColumn.Type);
+ }
+
+ for (bool skipNullKey : stageInfo.Meta.SkipNullKeys) {
+ protoTaskMeta.AddSkipNullKeys(skipNullKey);
+ }
+
+ YQL_ENSURE(task.Meta.Reads);
+ YQL_ENSURE(!task.Meta.Writes);
+
+ for (auto& column : task.Meta.Reads->front().Columns) {
+ auto* protoColumn = protoTaskMeta.AddColumns();
+ protoColumn->SetId(column.Id);
+ protoColumn->SetType(column.Type);
+ protoColumn->SetName(column.Name);
+ }
+
+ for (auto& read : *task.Meta.Reads) {
+ auto* protoReadMeta = protoTaskMeta.AddReads();
+ protoReadMeta->SetShardId(read.ShardId);
+ read.Ranges.SerializeTo(protoReadMeta);
+
+ YQL_ENSURE((int) read.Columns.size() == protoTaskMeta.GetColumns().size());
+ for (ui64 i = 0; i < read.Columns.size(); ++i) {
+ YQL_ENSURE(read.Columns[i].Id == protoTaskMeta.GetColumns()[i].GetId());
+ YQL_ENSURE(read.Columns[i].Type == protoTaskMeta.GetColumns()[i].GetType());
+ }
+ }
+
+ LOG_D("task: " << task.Id << ", node: " << task.Meta.NodeId << ", meta: " << protoTaskMeta.ShortDebugString());
+
+ taskDesc.MutableMeta()->PackFrom(protoTaskMeta);
+ computeTasks.emplace_back(std::move(taskDesc));
+ } else {
+ computeTasks.emplace_back(std::move(taskDesc));
+ }
+ }
+
+ if (computeTasks.size() > Request.MaxComputeActors) {
+ LOG_N("Too many compute actors: " << computeTasks.size());
+ ReplyErrorAndDie(Ydb::StatusIds::PRECONDITION_FAILED,
+ YqlIssue({}, TIssuesIds::KIKIMR_PRECONDITION_FAILED, TStringBuilder()
+ << "Requested too many execution units: " << computeTasks.size()));
+ return;
+ }
+
+ ui32 shardsLimit = Request.MaxAffectedShards;
+ if (i64 msc = (i64) RequestControls.MaxShardCount; msc > 0) {
+ shardsLimit = std::min(shardsLimit, (ui32) msc);
+ }
+ if (shardsLimit > 0 && datashardTasks.size() > shardsLimit) {
+ LOG_W("Too many affected shards: datashardTasks=" << datashardTasks.size() << ", limit: " << shardsLimit);
+ Counters->TxProxyMon->TxResultError->Inc();
+ ReplyErrorAndDie(Ydb::StatusIds::PRECONDITION_FAILED,
+ YqlIssue({}, TIssuesIds::KIKIMR_PRECONDITION_FAILED, TStringBuilder()
+ << "Affected too many shards: " << datashardTasks.size()));
+ return;
+ }
+
+ bool fitSize = AllOf(datashardTasks, [this](const auto& x){ return ValidateTaskSize(x.second); });
+ if (!fitSize) {
+ Counters->TxProxyMon->TxResultError->Inc();
+ return;
+ }
+
+ auto datashardTxs = BuildDatashardTxs(datashardTasks);
+
+ // Single-shard transactions are always immediate
+ ImmediateTx = datashardTxs.size() <= 1;
switch (Request.IsolationLevel) {
- // OnlineRO with AllowInconsistentReads = true
+ // OnlineRO with AllowInconsistentReads = true
case NKikimrKqp::ISOLATION_LEVEL_READ_UNCOMMITTED:
// StaleRO transactions always execute as immediate
// (legacy behaviour, for compatibility with current execution engine)
@@ -1449,210 +1449,210 @@ private:
default:
break;
- }
+ }
+
+ if (ReadOnlyTx && Request.Snapshot.IsValid()) {
+ // Snapshot reads are always immediate
+ ImmediateTx = true;
+ }
- if (ReadOnlyTx && Request.Snapshot.IsValid()) {
- // Snapshot reads are always immediate
- ImmediateTx = true;
- }
-
UseFollowers = Request.IsolationLevel == NKikimrKqp::ISOLATION_LEVEL_READ_STALE;
- if (datashardTxs.size() > 1) {
+ if (datashardTxs.size() > 1) {
// Followers only allowed for single shard transactions.
// (legacy behaviour, for compatibility with current execution engine)
UseFollowers = false;
- }
- if (Request.Snapshot.IsValid()) {
- // TODO: KIKIMR-11912
+ }
+ if (Request.Snapshot.IsValid()) {
+ // TODO: KIKIMR-11912
UseFollowers = false;
- }
+ }
if (UseFollowers) {
- YQL_ENSURE(ReadOnlyTx);
- }
-
- if (Stats) {
- //Stats->AffectedShards = datashardTxs.size();
- Stats->DatashardStats.reserve(datashardTxs.size());
- //Stats->ComputeStats.reserve(computeTasks.size());
- }
-
- Execute(computeTasks, datashardTxs);
-
- if (ImmediateTx) {
- LOG_T("Immediate tx, become ExecuteState");
- Become(&TKqpDataExecuter::ExecuteState);
- } else {
- LOG_T("Not immediate tx, become PrepareState");
- Become(&TKqpDataExecuter::PrepareState);
- }
- }
-
- THashMap<ui64, NKikimrTxDataShard::TKqpTransaction> BuildDatashardTxs(const THashMap<ui64, TVector<NDqProto::TDqTask>>& datashardTasks) {
- THashMap<ui64, NKikimrTxDataShard::TKqpTransaction> datashardTxs;
-
- for (auto& [shardId, tasks]: datashardTasks) {
- auto& dsTxs = datashardTxs[shardId];
- for (auto& task: tasks) {
- dsTxs.AddTasks()->CopyFrom(task);
- }
- }
-
- if (auto locksMap = ExtractLocks(Request.Locks); !locksMap.empty()) {
- YQL_ENSURE(Request.ValidateLocks || Request.EraseLocks);
- auto locksOp = Request.ValidateLocks && Request.EraseLocks
- ? NKikimrTxDataShard::TKqpLocks::ValidateAndErase
- : (Request.ValidateLocks
- ? NKikimrTxDataShard::TKqpLocks::Validate
- : NKikimrTxDataShard::TKqpLocks::Erase);
-
- TSet<ui64> taskShardIds;
- if (Request.ValidateLocks) {
- for (auto& [shardId, _] : datashardTasks) {
- taskShardIds.insert(shardId);
- }
- }
-
- TSet<ui64> locksSendingShards;
- for (auto& [shardId, locksList] : locksMap) {
- auto& tx = datashardTxs[shardId];
- tx.MutableLocks()->SetOp(locksOp);
-
- for (auto& lock : locksList) {
- tx.MutableLocks()->MutableLocks()->Add()->Swap(&lock);
- }
-
- if (!locksList.empty() && Request.ValidateLocks) {
- locksSendingShards.insert(shardId);
- }
- }
-
- if (Request.ValidateLocks) {
- NProtoBuf::RepeatedField<ui64> sendingShards(locksSendingShards.begin(), locksSendingShards.end());
- NProtoBuf::RepeatedField<ui64> receivingShards(taskShardIds.begin(), taskShardIds.end());
- for (auto& [shardId, shardTx] : datashardTxs) {
- shardTx.MutableLocks()->SetOp(locksOp);
- shardTx.MutableLocks()->MutableSendingShards()->CopyFrom(sendingShards);
- shardTx.MutableLocks()->MutableReceivingShards()->CopyFrom(receivingShards);
- }
- }
- }
-
- return datashardTxs;
- }
-
- void Execute(TVector<NDqProto::TDqTask>& computeTasks, THashMap<ui64, NKikimrTxDataShard::TKqpTransaction>& datashardTxs) {
- auto lockTxId = Request.AcquireLocksTxId;
- if (lockTxId.Defined() && *lockTxId == 0) {
- lockTxId = TxId;
- }
-
- // first, start compute tasks
- TVector<ui64> computeTaskIds{Reserve(computeTasks.size())};
- for (auto&& taskDesc : computeTasks) {
- computeTaskIds.emplace_back(taskDesc.GetId());
- ExecuteDataComputeTask(std::move(taskDesc));
- }
-
- // then start data tasks with known actor ids of compute tasks
- for (auto& [shardId, shardTx] : datashardTxs) {
- shardTx.SetType(NKikimrTxDataShard::KQP_TX_TYPE_DATA);
-
- for (auto& protoTask : *shardTx.MutableTasks()) {
- ui64 taskId = protoTask.GetId();
- auto& task = TasksGraph.GetTask(taskId);
-
- for (ui64 outputIndex = 0; outputIndex < task.Outputs.size(); ++outputIndex) {
- auto& output = task.Outputs[outputIndex];
- auto* protoOutput = protoTask.MutableOutputs(outputIndex);
-
- for (ui64 outputChannelIndex = 0; outputChannelIndex < output.Channels.size(); ++outputChannelIndex) {
- ui64 outputChannelId = output.Channels[outputChannelIndex];
- auto* protoChannel = protoOutput->MutableChannels(outputChannelIndex);
-
- ui64 dstTaskId = TasksGraph.GetChannel(outputChannelId).DstTask;
- YQL_ENSURE(dstTaskId); // result channel not allowed here
-
- auto& dstTask = TasksGraph.GetTask(dstTaskId);
- if (dstTask.ComputeActorId) {
- protoChannel->MutableDstEndpoint()->Clear();
- ActorIdToProto(dstTask.ComputeActorId, protoChannel->MutableDstEndpoint()->MutableActorId());
- } else {
- if (protoChannel->HasDstEndpoint() && protoChannel->GetDstEndpoint().HasTabletId()) {
- if (protoChannel->GetDstEndpoint().GetTabletId() == shardId) {
- // inplace update
- } else {
- // TODO: send data via executer?
- // but we don't have such examples...
- YQL_ENSURE(false, "not implemented yet: " << protoTask.DebugString());
- }
- } else {
- YQL_ENSURE(!protoChannel->GetDstEndpoint().IsInitialized());
- // effects-only stage
- }
- }
- }
- }
-
- LOG_D("datashard task: " << taskId << ", proto: " << protoTask.ShortDebugString());
- }
-
- ExecuteDatashardTransaction(shardId, shardTx, lockTxId);
- }
-
+ YQL_ENSURE(ReadOnlyTx);
+ }
+
+ if (Stats) {
+ //Stats->AffectedShards = datashardTxs.size();
+ Stats->DatashardStats.reserve(datashardTxs.size());
+ //Stats->ComputeStats.reserve(computeTasks.size());
+ }
+
+ Execute(computeTasks, datashardTxs);
+
+ if (ImmediateTx) {
+ LOG_T("Immediate tx, become ExecuteState");
+ Become(&TKqpDataExecuter::ExecuteState);
+ } else {
+ LOG_T("Not immediate tx, become PrepareState");
+ Become(&TKqpDataExecuter::PrepareState);
+ }
+ }
+
+ THashMap<ui64, NKikimrTxDataShard::TKqpTransaction> BuildDatashardTxs(const THashMap<ui64, TVector<NDqProto::TDqTask>>& datashardTasks) {
+ THashMap<ui64, NKikimrTxDataShard::TKqpTransaction> datashardTxs;
+
+ for (auto& [shardId, tasks]: datashardTasks) {
+ auto& dsTxs = datashardTxs[shardId];
+ for (auto& task: tasks) {
+ dsTxs.AddTasks()->CopyFrom(task);
+ }
+ }
+
+ if (auto locksMap = ExtractLocks(Request.Locks); !locksMap.empty()) {
+ YQL_ENSURE(Request.ValidateLocks || Request.EraseLocks);
+ auto locksOp = Request.ValidateLocks && Request.EraseLocks
+ ? NKikimrTxDataShard::TKqpLocks::ValidateAndErase
+ : (Request.ValidateLocks
+ ? NKikimrTxDataShard::TKqpLocks::Validate
+ : NKikimrTxDataShard::TKqpLocks::Erase);
+
+ TSet<ui64> taskShardIds;
+ if (Request.ValidateLocks) {
+ for (auto& [shardId, _] : datashardTasks) {
+ taskShardIds.insert(shardId);
+ }
+ }
+
+ TSet<ui64> locksSendingShards;
+ for (auto& [shardId, locksList] : locksMap) {
+ auto& tx = datashardTxs[shardId];
+ tx.MutableLocks()->SetOp(locksOp);
+
+ for (auto& lock : locksList) {
+ tx.MutableLocks()->MutableLocks()->Add()->Swap(&lock);
+ }
+
+ if (!locksList.empty() && Request.ValidateLocks) {
+ locksSendingShards.insert(shardId);
+ }
+ }
+
+ if (Request.ValidateLocks) {
+ NProtoBuf::RepeatedField<ui64> sendingShards(locksSendingShards.begin(), locksSendingShards.end());
+ NProtoBuf::RepeatedField<ui64> receivingShards(taskShardIds.begin(), taskShardIds.end());
+ for (auto& [shardId, shardTx] : datashardTxs) {
+ shardTx.MutableLocks()->SetOp(locksOp);
+ shardTx.MutableLocks()->MutableSendingShards()->CopyFrom(sendingShards);
+ shardTx.MutableLocks()->MutableReceivingShards()->CopyFrom(receivingShards);
+ }
+ }
+ }
+
+ return datashardTxs;
+ }
+
+ void Execute(TVector<NDqProto::TDqTask>& computeTasks, THashMap<ui64, NKikimrTxDataShard::TKqpTransaction>& datashardTxs) {
+ auto lockTxId = Request.AcquireLocksTxId;
+ if (lockTxId.Defined() && *lockTxId == 0) {
+ lockTxId = TxId;
+ }
+
+ // first, start compute tasks
+ TVector<ui64> computeTaskIds{Reserve(computeTasks.size())};
+ for (auto&& taskDesc : computeTasks) {
+ computeTaskIds.emplace_back(taskDesc.GetId());
+ ExecuteDataComputeTask(std::move(taskDesc));
+ }
+
+ // then start data tasks with known actor ids of compute tasks
+ for (auto& [shardId, shardTx] : datashardTxs) {
+ shardTx.SetType(NKikimrTxDataShard::KQP_TX_TYPE_DATA);
+
+ for (auto& protoTask : *shardTx.MutableTasks()) {
+ ui64 taskId = protoTask.GetId();
+ auto& task = TasksGraph.GetTask(taskId);
+
+ for (ui64 outputIndex = 0; outputIndex < task.Outputs.size(); ++outputIndex) {
+ auto& output = task.Outputs[outputIndex];
+ auto* protoOutput = protoTask.MutableOutputs(outputIndex);
+
+ for (ui64 outputChannelIndex = 0; outputChannelIndex < output.Channels.size(); ++outputChannelIndex) {
+ ui64 outputChannelId = output.Channels[outputChannelIndex];
+ auto* protoChannel = protoOutput->MutableChannels(outputChannelIndex);
+
+ ui64 dstTaskId = TasksGraph.GetChannel(outputChannelId).DstTask;
+ YQL_ENSURE(dstTaskId); // result channel not allowed here
+
+ auto& dstTask = TasksGraph.GetTask(dstTaskId);
+ if (dstTask.ComputeActorId) {
+ protoChannel->MutableDstEndpoint()->Clear();
+ ActorIdToProto(dstTask.ComputeActorId, protoChannel->MutableDstEndpoint()->MutableActorId());
+ } else {
+ if (protoChannel->HasDstEndpoint() && protoChannel->GetDstEndpoint().HasTabletId()) {
+ if (protoChannel->GetDstEndpoint().GetTabletId() == shardId) {
+ // inplace update
+ } else {
+ // TODO: send data via executer?
+ // but we don't have such examples...
+ YQL_ENSURE(false, "not implemented yet: " << protoTask.DebugString());
+ }
+ } else {
+ YQL_ENSURE(!protoChannel->GetDstEndpoint().IsInitialized());
+ // effects-only stage
+ }
+ }
+ }
+ }
+
+ LOG_D("datashard task: " << taskId << ", proto: " << protoTask.ShortDebugString());
+ }
+
+ ExecuteDatashardTransaction(shardId, shardTx, lockTxId);
+ }
+
LOG_I("Total tasks: " << TasksGraph.GetTasks().size()
- << ", readonly: " << ReadOnlyTx
- << ", datashardTxs: " << datashardTxs.size()
- << ", immediate: " << ImmediateTx
+ << ", readonly: " << ReadOnlyTx
+ << ", datashardTxs: " << datashardTxs.size()
+ << ", immediate: " << ImmediateTx
<< ", useFollowers: " << UseFollowers);
- LOG_T("Updating channels after the creation of compute actors");
- THashMap<TActorId, THashSet<ui64>> updates;
- for (ui64 taskId : computeTaskIds) {
- auto& task = TasksGraph.GetTask(taskId);
- CollectTaskChannelsUpdates(task, updates);
- }
- PropagateChannelsUpdates(updates);
+ LOG_T("Updating channels after the creation of compute actors");
+ THashMap<TActorId, THashSet<ui64>> updates;
+ for (ui64 taskId : computeTaskIds) {
+ auto& task = TasksGraph.GetTask(taskId);
+ CollectTaskChannelsUpdates(task, updates);
+ }
+ PropagateChannelsUpdates(updates);
CheckExecutionComplete();
- }
-
- void Finalize() {
- auto& response = *ResponseEv->Record.MutableResponse();
-
- response.SetStatus(Ydb::StatusIds::SUCCESS);
- Counters->TxProxyMon->ReportStatusOK->Inc();
-
- TKqpProtoBuilder protoBuilder(*AppData()->FunctionRegistry);
- for (auto& result : Results) {
- auto* protoResult = response.MutableResult()->AddResults();
- if (result.IsStream) {
- protoBuilder.BuildStream(result.Data, result.ItemType, result.ResultItemType.Get(), protoResult);
- } else {
- protoBuilder.BuildValue(result.Data, result.ItemType, protoResult);
- }
- }
-
- if (!Locks.empty()) {
- BuildLocks(*response.MutableResult()->MutableLocks(), Locks);
- }
-
- if (Stats) {
- ReportEventElapsedTime();
-
- Stats->FinishTs = TInstant::Now();
- Stats->ResultRows = response.GetResult().ResultsSize();
- Stats->Finish();
-
- if (Request.StatsMode == NYql::NDqProto::DQ_STATS_MODE_PROFILE) {
- for (ui32 txId = 0; txId < Request.Transactions.size(); ++txId) {
- const auto& tx = Request.Transactions[txId].Body;
- auto planWithStats = AddExecStatsToTxPlan(tx.GetPlan(), response.GetResult().GetStats());
- response.MutableResult()->MutableStats()->AddTxPlansWithStats(planWithStats);
- }
- }
-
- Stats.reset();
- }
-
+ }
+
+ void Finalize() {
+ auto& response = *ResponseEv->Record.MutableResponse();
+
+ response.SetStatus(Ydb::StatusIds::SUCCESS);
+ Counters->TxProxyMon->ReportStatusOK->Inc();
+
+ TKqpProtoBuilder protoBuilder(*AppData()->FunctionRegistry);
+ for (auto& result : Results) {
+ auto* protoResult = response.MutableResult()->AddResults();
+ if (result.IsStream) {
+ protoBuilder.BuildStream(result.Data, result.ItemType, result.ResultItemType.Get(), protoResult);
+ } else {
+ protoBuilder.BuildValue(result.Data, result.ItemType, protoResult);
+ }
+ }
+
+ if (!Locks.empty()) {
+ BuildLocks(*response.MutableResult()->MutableLocks(), Locks);
+ }
+
+ if (Stats) {
+ ReportEventElapsedTime();
+
+ Stats->FinishTs = TInstant::Now();
+ Stats->ResultRows = response.GetResult().ResultsSize();
+ Stats->Finish();
+
+ if (Request.StatsMode == NYql::NDqProto::DQ_STATS_MODE_PROFILE) {
+ for (ui32 txId = 0; txId < Request.Transactions.size(); ++txId) {
+ const auto& tx = Request.Transactions[txId].Body;
+ auto planWithStats = AddExecStatsToTxPlan(tx.GetPlan(), response.GetResult().GetStats());
+ response.MutableResult()->MutableStats()->AddTxPlansWithStats(planWithStats);
+ }
+ }
+
+ Stats.reset();
+ }
+
auto resultSize = response.ByteSize();
if (resultSize > (int)ReplySizeLimit) {
TString message = TStringBuilder() << "Query result size limit exceeded. ("
@@ -1663,106 +1663,106 @@ private:
return;
}
- LOG_D("Sending response to: " << Target << ", results: " << Results.size());
- Send(Target, ResponseEv.release());
- PassAway();
- }
-
- void PassAway() override {
- auto totalTime = TInstant::Now() - StartTime;
- Counters->Counters->DataTxTotalTimeHistogram->Collect(totalTime.MilliSeconds());
-
- // TxProxyMon compatibility
- Counters->TxProxyMon->TxTotalTimeHgram->Collect(totalTime.MilliSeconds());
- Counters->TxProxyMon->TxExecuteTimeHgram->Collect(totalTime.MilliSeconds());
-
- Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(0));
- TBase::PassAway();
- }
-
-public:
- static void FillEndpointDesc(NDqProto::TEndpoint& endpoint, const TTask& task) {
- if (task.ComputeActorId) {
- ActorIdToProto(task.ComputeActorId, endpoint.MutableActorId());
- } else if (task.Meta.ShardId) {
- endpoint.SetTabletId(task.Meta.ShardId);
- }
- }
-
- void FillChannelDesc(NDqProto::TChannel& channelDesc, const TChannel& channel) {
- channelDesc.SetId(channel.Id);
- channelDesc.SetSrcTaskId(channel.SrcTask);
- channelDesc.SetDstTaskId(channel.DstTask);
-
- YQL_ENSURE(channel.SrcTask);
- FillEndpointDesc(*channelDesc.MutableSrcEndpoint(), TasksGraph.GetTask(channel.SrcTask));
-
- if (channel.DstTask) {
- FillEndpointDesc(*channelDesc.MutableDstEndpoint(), TasksGraph.GetTask(channel.DstTask));
- } else {
- // result channel
- ActorIdToProto(SelfId(), channelDesc.MutableDstEndpoint()->MutableActorId());
- }
-
- channelDesc.SetIsPersistent(IsCrossShardChannel(TasksGraph, channel));
- channelDesc.SetInMemory(channel.InMemory);
- }
-
-private:
- void ReplyTxStateUnknown(ui64 shardId) {
- auto message = TStringBuilder() << "Tx state unknown for shard " << shardId << ", txid " << TxId;
- if (ReadOnlyTx) {
- auto issue = YqlIssue({}, TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE);
- issue.AddSubIssue(new TIssue(message));
- issue.GetSubIssues()[0]->SetCode(NKikimrIssues::TIssuesIds::TX_STATE_UNKNOWN, TSeverityIds::S_ERROR);
- ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, issue);
- } else {
- auto issue = YqlIssue({}, TIssuesIds::KIKIMR_OPERATION_STATE_UNKNOWN);
- issue.AddSubIssue(new TIssue(message));
- issue.GetSubIssues()[0]->SetCode(NKikimrIssues::TIssuesIds::TX_STATE_UNKNOWN, TSeverityIds::S_ERROR);
- ReplyErrorAndDie(Ydb::StatusIds::UNDETERMINED, issue);
- }
- }
-
- static void AddDataShardErrors(const NKikimrTxDataShard::TEvProposeTransactionResult& result, TIssue& issue) {
- for (const auto &err : result.GetError()) {
- issue.AddSubIssue(new TIssue(TStringBuilder()
- << "[" << NKikimrTxDataShard::TError_EKind_Name(err.GetKind()) << "] " << err.GetReason()));
- }
- }
-
- static std::string_view ToString(TShardState::EState state) {
- switch (state) {
- case TShardState::EState::Initial: return "Initial"sv;
- case TShardState::EState::Preparing: return "Preparing"sv;
- case TShardState::EState::Prepared: return "Prepared"sv;
- case TShardState::EState::Executing: return "Executing"sv;
- case TShardState::EState::Finished: return "Finished"sv;
- }
- }
-
-private:
- NTxProxy::TRequestControls RequestControls;
- ui64 TxCoordinator = 0;
- THashMap<ui64, TShardState> ShardStates;
- TVector<NKikimrTxDataShard::TLock> Locks;
- TVector<TKqpExecuterTxResult> Results;
- bool ReadOnlyTx = true;
- bool ImmediateTx = false;
+ LOG_D("Sending response to: " << Target << ", results: " << Results.size());
+ Send(Target, ResponseEv.release());
+ PassAway();
+ }
+
+ void PassAway() override {
+ auto totalTime = TInstant::Now() - StartTime;
+ Counters->Counters->DataTxTotalTimeHistogram->Collect(totalTime.MilliSeconds());
+
+ // TxProxyMon compatibility
+ Counters->TxProxyMon->TxTotalTimeHgram->Collect(totalTime.MilliSeconds());
+ Counters->TxProxyMon->TxExecuteTimeHgram->Collect(totalTime.MilliSeconds());
+
+ Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(0));
+ TBase::PassAway();
+ }
+
+public:
+ static void FillEndpointDesc(NDqProto::TEndpoint& endpoint, const TTask& task) {
+ if (task.ComputeActorId) {
+ ActorIdToProto(task.ComputeActorId, endpoint.MutableActorId());
+ } else if (task.Meta.ShardId) {
+ endpoint.SetTabletId(task.Meta.ShardId);
+ }
+ }
+
+ void FillChannelDesc(NDqProto::TChannel& channelDesc, const TChannel& channel) {
+ channelDesc.SetId(channel.Id);
+ channelDesc.SetSrcTaskId(channel.SrcTask);
+ channelDesc.SetDstTaskId(channel.DstTask);
+
+ YQL_ENSURE(channel.SrcTask);
+ FillEndpointDesc(*channelDesc.MutableSrcEndpoint(), TasksGraph.GetTask(channel.SrcTask));
+
+ if (channel.DstTask) {
+ FillEndpointDesc(*channelDesc.MutableDstEndpoint(), TasksGraph.GetTask(channel.DstTask));
+ } else {
+ // result channel
+ ActorIdToProto(SelfId(), channelDesc.MutableDstEndpoint()->MutableActorId());
+ }
+
+ channelDesc.SetIsPersistent(IsCrossShardChannel(TasksGraph, channel));
+ channelDesc.SetInMemory(channel.InMemory);
+ }
+
+private:
+ void ReplyTxStateUnknown(ui64 shardId) {
+ auto message = TStringBuilder() << "Tx state unknown for shard " << shardId << ", txid " << TxId;
+ if (ReadOnlyTx) {
+ auto issue = YqlIssue({}, TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE);
+ issue.AddSubIssue(new TIssue(message));
+ issue.GetSubIssues()[0]->SetCode(NKikimrIssues::TIssuesIds::TX_STATE_UNKNOWN, TSeverityIds::S_ERROR);
+ ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, issue);
+ } else {
+ auto issue = YqlIssue({}, TIssuesIds::KIKIMR_OPERATION_STATE_UNKNOWN);
+ issue.AddSubIssue(new TIssue(message));
+ issue.GetSubIssues()[0]->SetCode(NKikimrIssues::TIssuesIds::TX_STATE_UNKNOWN, TSeverityIds::S_ERROR);
+ ReplyErrorAndDie(Ydb::StatusIds::UNDETERMINED, issue);
+ }
+ }
+
+ static void AddDataShardErrors(const NKikimrTxDataShard::TEvProposeTransactionResult& result, TIssue& issue) {
+ for (const auto &err : result.GetError()) {
+ issue.AddSubIssue(new TIssue(TStringBuilder()
+ << "[" << NKikimrTxDataShard::TError_EKind_Name(err.GetKind()) << "] " << err.GetReason()));
+ }
+ }
+
+ static std::string_view ToString(TShardState::EState state) {
+ switch (state) {
+ case TShardState::EState::Initial: return "Initial"sv;
+ case TShardState::EState::Preparing: return "Preparing"sv;
+ case TShardState::EState::Prepared: return "Prepared"sv;
+ case TShardState::EState::Executing: return "Executing"sv;
+ case TShardState::EState::Finished: return "Finished"sv;
+ }
+ }
+
+private:
+ NTxProxy::TRequestControls RequestControls;
+ ui64 TxCoordinator = 0;
+ THashMap<ui64, TShardState> ShardStates;
+ TVector<NKikimrTxDataShard::TLock> Locks;
+ TVector<TKqpExecuterTxResult> Results;
+ bool ReadOnlyTx = true;
+ bool ImmediateTx = false;
bool UseFollowers = false;
- bool TxPlanned = false;
-
- TInstant FirstPrepareReply;
- TInstant LastPrepareReply;
-};
-
-} // namespace
-
+ bool TxPlanned = false;
+
+ TInstant FirstPrepareReply;
+ TInstant LastPrepareReply;
+};
+
+} // namespace
+
IActor* CreateKqpDataExecuter(IKqpGateway::TExecPhysicalRequest&& request, const TString& database, const TMaybe<TString>& userToken,
- TKqpRequestCounters::TPtr counters)
-{
+ TKqpRequestCounters::TPtr counters)
+{
return new TKqpDataExecuter(std::move(request), database, userToken, counters);
-}
-
-} // namespace NKqp
-} // namespace NKikimr
+}
+
+} // namespace NKqp
+} // namespace NKikimr
diff --git a/ydb/core/kqp/executer/kqp_executer.h b/ydb/core/kqp/executer/kqp_executer.h
index d6e73c8d5a..13a0c10e5b 100644
--- a/ydb/core/kqp/executer/kqp_executer.h
+++ b/ydb/core/kqp/executer/kqp_executer.h
@@ -1,54 +1,54 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/kqp/common/kqp_common.h>
#include <ydb/core/kqp/common/kqp_gateway.h>
#include <ydb/core/kqp/counters/kqp_counters.h>
#include <ydb/core/protos/config.pb.h>
#include <ydb/core/protos/kqp.pb.h>
-
-namespace NKikimr {
-namespace NKqp {
-
-struct TEvKqpExecuter {
- struct TEvTxRequest : public TEventPB<TEvTxRequest, NKikimrKqp::TEvExecuterTxRequest,
- TKqpExecuterEvents::EvTxRequest> {};
-
- struct TEvTxResponse : public TEventPB<TEvTxResponse, NKikimrKqp::TEvExecuterTxResponse,
- TKqpExecuterEvents::EvTxResponse> {};
-
- struct TEvStreamData : public TEventPB<TEvStreamData, NKikimrKqp::TEvExecuterStreamData,
- TKqpExecuterEvents::EvStreamData> {};
-
- struct TEvStreamDataAck : public TEventPB<TEvStreamDataAck, NKikimrKqp::TEvExecuterStreamDataAck,
- TKqpExecuterEvents::EvStreamDataAck> {};
-
- struct TEvStreamProfile : public TEventPB<TEvStreamProfile, NKikimrKqp::TEvExecuterStreamProfile,
- TKqpExecuterEvents::EvStreamProfile> {};
-
- struct TEvExecuterProgress : public TEventPB<TEvExecuterProgress, NKikimrKqp::TEvExecuterProgress,
- TKqpExecuterEvents::EvProgress> {};
-
- struct TEvTableResolveStatus : public TEventLocal<TEvTableResolveStatus,
- TKqpExecuterEvents::EvTableResolveStatus>
- {
- Ydb::StatusIds::StatusCode Status = Ydb::StatusIds::SUCCESS;
- NYql::TIssues Issues;
- TDuration CpuTime;
- };
-
- struct TEvShardsResolveStatus : public TEventLocal<TEvShardsResolveStatus,
- TKqpExecuterEvents::EvShardsResolveStatus>
- {
- Ydb::StatusIds::StatusCode Status = Ydb::StatusIds::SUCCESS;
- NYql::TIssues Issues;
-
- TMap<ui64, ui64> ShardNodes;
- ui32 Unresolved = 0;
- };
-};
-
-IActor* CreateKqpExecuter(IKqpGateway::TExecPhysicalRequest&& request, const TString& database,
- const TMaybe<TString>& userToken, TKqpRequestCounters::TPtr counters);
-
-} // namespace NKqp
-} // namespace NKikimr
+
+namespace NKikimr {
+namespace NKqp {
+
+struct TEvKqpExecuter {
+ struct TEvTxRequest : public TEventPB<TEvTxRequest, NKikimrKqp::TEvExecuterTxRequest,
+ TKqpExecuterEvents::EvTxRequest> {};
+
+ struct TEvTxResponse : public TEventPB<TEvTxResponse, NKikimrKqp::TEvExecuterTxResponse,
+ TKqpExecuterEvents::EvTxResponse> {};
+
+ struct TEvStreamData : public TEventPB<TEvStreamData, NKikimrKqp::TEvExecuterStreamData,
+ TKqpExecuterEvents::EvStreamData> {};
+
+ struct TEvStreamDataAck : public TEventPB<TEvStreamDataAck, NKikimrKqp::TEvExecuterStreamDataAck,
+ TKqpExecuterEvents::EvStreamDataAck> {};
+
+ struct TEvStreamProfile : public TEventPB<TEvStreamProfile, NKikimrKqp::TEvExecuterStreamProfile,
+ TKqpExecuterEvents::EvStreamProfile> {};
+
+ struct TEvExecuterProgress : public TEventPB<TEvExecuterProgress, NKikimrKqp::TEvExecuterProgress,
+ TKqpExecuterEvents::EvProgress> {};
+
+ struct TEvTableResolveStatus : public TEventLocal<TEvTableResolveStatus,
+ TKqpExecuterEvents::EvTableResolveStatus>
+ {
+ Ydb::StatusIds::StatusCode Status = Ydb::StatusIds::SUCCESS;
+ NYql::TIssues Issues;
+ TDuration CpuTime;
+ };
+
+ struct TEvShardsResolveStatus : public TEventLocal<TEvShardsResolveStatus,
+ TKqpExecuterEvents::EvShardsResolveStatus>
+ {
+ Ydb::StatusIds::StatusCode Status = Ydb::StatusIds::SUCCESS;
+ NYql::TIssues Issues;
+
+ TMap<ui64, ui64> ShardNodes;
+ ui32 Unresolved = 0;
+ };
+};
+
+IActor* CreateKqpExecuter(IKqpGateway::TExecPhysicalRequest&& request, const TString& database,
+ const TMaybe<TString>& userToken, TKqpRequestCounters::TPtr counters);
+
+} // namespace NKqp
+} // namespace NKikimr
diff --git a/ydb/core/kqp/executer/kqp_executer_impl.cpp b/ydb/core/kqp/executer/kqp_executer_impl.cpp
index 34b37d4c58..0931e5d2c7 100644
--- a/ydb/core/kqp/executer/kqp_executer_impl.cpp
+++ b/ydb/core/kqp/executer/kqp_executer_impl.cpp
@@ -1,57 +1,57 @@
-#include "kqp_executer_impl.h"
-
+#include "kqp_executer_impl.h"
+
#include <ydb/core/formats/arrow_helpers.h>
#include <ydb/core/kqp/runtime/kqp_transport.h>
-
+
#include <ydb/public/api/protos/ydb_rate_limiter.pb.h>
#include <ydb/library/yql/dq/runtime/dq_transport.h>
#include <ydb/library/yql/dq/runtime/dq_arrow_helpers.h>
-
-#include <library/cpp/actors/core/log.h>
-
-namespace NKikimr {
-namespace NKqp {
-
-using namespace NYql;
-
-void BuildKqpExecuterResult(const NKqpProto::TKqpPhyResult& txResult, TKqpExecuterTxResult& result) {
- result.IsStream = txResult.GetIsStream();
- result.ItemType.CopyFrom(txResult.GetItemType());
-
- if (txResult.ColumnHintsSize() > 0) {
- TVector<TString> columnHints;
- columnHints.reserve(txResult.GetColumnHints().size());
- std::copy(txResult.GetColumnHints().begin(), txResult.GetColumnHints().end(), std::back_inserter(columnHints));
-
- result.ResultItemType.ConstructInPlace(TKqpProtoBuilder::ApplyColumnHints(result.ItemType, columnHints));
- }
-}
-
-void BuildKqpExecuterResults(const NKqpProto::TKqpPhyTx& tx, TVector<TKqpExecuterTxResult>& results) {
- ui32 i = results.size();
- results.resize(results.size() + tx.GetResults().size());
-
- for (auto& resultProto : tx.GetResults()) {
- BuildKqpExecuterResult(resultProto, results[i++]);
- }
-}
-
-void PrepareKqpTaskParameters(const NKqpProto::TKqpPhyStage& stage, const TStageInfo& stageInfo, const TTask& task,
- NDqProto::TDqTask& dqTask, const NMiniKQL::TTypeEnvironment& typeEnv, const NMiniKQL::THolderFactory& holderFactory)
-{
- for (auto& paramName : stage.GetProgramParameters()) {
- auto& dqParams = *dqTask.MutableParameters();
- if (auto* taskParam = task.Meta.Params.FindPtr(paramName)) {
- dqParams[paramName] = *taskParam;
- } else {
+
+#include <library/cpp/actors/core/log.h>
+
+namespace NKikimr {
+namespace NKqp {
+
+using namespace NYql;
+
+void BuildKqpExecuterResult(const NKqpProto::TKqpPhyResult& txResult, TKqpExecuterTxResult& result) {
+ result.IsStream = txResult.GetIsStream();
+ result.ItemType.CopyFrom(txResult.GetItemType());
+
+ if (txResult.ColumnHintsSize() > 0) {
+ TVector<TString> columnHints;
+ columnHints.reserve(txResult.GetColumnHints().size());
+ std::copy(txResult.GetColumnHints().begin(), txResult.GetColumnHints().end(), std::back_inserter(columnHints));
+
+ result.ResultItemType.ConstructInPlace(TKqpProtoBuilder::ApplyColumnHints(result.ItemType, columnHints));
+ }
+}
+
+void BuildKqpExecuterResults(const NKqpProto::TKqpPhyTx& tx, TVector<TKqpExecuterTxResult>& results) {
+ ui32 i = results.size();
+ results.resize(results.size() + tx.GetResults().size());
+
+ for (auto& resultProto : tx.GetResults()) {
+ BuildKqpExecuterResult(resultProto, results[i++]);
+ }
+}
+
+void PrepareKqpTaskParameters(const NKqpProto::TKqpPhyStage& stage, const TStageInfo& stageInfo, const TTask& task,
+ NDqProto::TDqTask& dqTask, const NMiniKQL::TTypeEnvironment& typeEnv, const NMiniKQL::THolderFactory& holderFactory)
+{
+ for (auto& paramName : stage.GetProgramParameters()) {
+ auto& dqParams = *dqTask.MutableParameters();
+ if (auto* taskParam = task.Meta.Params.FindPtr(paramName)) {
+ dqParams[paramName] = *taskParam;
+ } else {
auto* paramValue = stageInfo.Meta.Tx.Params.Values.FindPtr(paramName);
- YQL_ENSURE(paramValue);
- dqParams[paramName] = NDq::TDqDataSerializer::SerializeParam(*paramValue, typeEnv, holderFactory);
- }
- }
-}
-
+ YQL_ENSURE(paramValue);
+ dqParams[paramName] = NDq::TDqDataSerializer::SerializeParam(*paramValue, typeEnv, holderFactory);
+ }
+ }
+}
+
std::pair<TString, TString> SerializeKqpTasksParametersForOlap(const NKqpProto::TKqpPhyStage& stage,
const TStageInfo& stageInfo, const TTask& task, const NMiniKQL::THolderFactory& holderFactory,
const NMiniKQL::TTypeEnvironment& typeEnv)
@@ -123,67 +123,67 @@ TActorId ReportToRl(ui64 ru, const TString& database, const TString& userToken,
TActivationContext::AsActorContext());
}
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-IActor* CreateKqpExecuter(IKqpGateway::TExecPhysicalRequest&& request, const TString& database,
- const TMaybe<TString>& userToken, TKqpRequestCounters::TPtr counters)
-{
- if (request.Transactions.empty()) {
- // commit-only or rollback-only data transaction
- YQL_ENSURE(request.EraseLocks);
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+IActor* CreateKqpExecuter(IKqpGateway::TExecPhysicalRequest&& request, const TString& database,
+ const TMaybe<TString>& userToken, TKqpRequestCounters::TPtr counters)
+{
+ if (request.Transactions.empty()) {
+ // commit-only or rollback-only data transaction
+ YQL_ENSURE(request.EraseLocks);
return CreateKqpDataExecuter(std::move(request), database, userToken, counters);
- }
-
- if (request.Locks.empty()) {
- bool literal = true;
- for (const auto& tx : request.Transactions) {
- if (tx.Body.GetType() != NKqpProto::TKqpPhyTx::TYPE_COMPUTE) {
- literal = false;
- break;
- }
- for (const auto& stage : tx.Body.GetStages()) {
- if (stage.InputsSize() != 0) {
- literal = false; // allow only independent stages
- break;
- }
- }
- }
-
- if (literal) {
- return CreateKqpLiteralExecuter(std::move(request), counters);
- }
- }
-
- bool data = true; // `false` stands for Scan
-
- TMaybe<NKqpProto::TKqpPhyTx::EType> txsType;
- for (auto& tx : request.Transactions) {
- if (txsType) {
- YQL_ENSURE(*txsType == tx.Body.GetType(), "Mixed physical tx types in executer.");
- YQL_ENSURE(*txsType == NKqpProto::TKqpPhyTx::TYPE_DATA, "Cannot execute multiple non-data physical txs.");
- } else {
- txsType = tx.Body.GetType();
-
- switch (tx.Body.GetType()) {
- case NKqpProto::TKqpPhyTx::TYPE_COMPUTE:
- case NKqpProto::TKqpPhyTx::TYPE_DATA:
- data = true;
- break;
-
- case NKqpProto::TKqpPhyTx::TYPE_SCAN:
- data = false;
- break;
-
- default:
- YQL_ENSURE(false, "Unsupported physical tx type: " << (ui32)tx.Body.GetType());
- }
- }
- }
-
- return data
+ }
+
+ if (request.Locks.empty()) {
+ bool literal = true;
+ for (const auto& tx : request.Transactions) {
+ if (tx.Body.GetType() != NKqpProto::TKqpPhyTx::TYPE_COMPUTE) {
+ literal = false;
+ break;
+ }
+ for (const auto& stage : tx.Body.GetStages()) {
+ if (stage.InputsSize() != 0) {
+ literal = false; // allow only independent stages
+ break;
+ }
+ }
+ }
+
+ if (literal) {
+ return CreateKqpLiteralExecuter(std::move(request), counters);
+ }
+ }
+
+ bool data = true; // `false` stands for Scan
+
+ TMaybe<NKqpProto::TKqpPhyTx::EType> txsType;
+ for (auto& tx : request.Transactions) {
+ if (txsType) {
+ YQL_ENSURE(*txsType == tx.Body.GetType(), "Mixed physical tx types in executer.");
+ YQL_ENSURE(*txsType == NKqpProto::TKqpPhyTx::TYPE_DATA, "Cannot execute multiple non-data physical txs.");
+ } else {
+ txsType = tx.Body.GetType();
+
+ switch (tx.Body.GetType()) {
+ case NKqpProto::TKqpPhyTx::TYPE_COMPUTE:
+ case NKqpProto::TKqpPhyTx::TYPE_DATA:
+ data = true;
+ break;
+
+ case NKqpProto::TKqpPhyTx::TYPE_SCAN:
+ data = false;
+ break;
+
+ default:
+ YQL_ENSURE(false, "Unsupported physical tx type: " << (ui32)tx.Body.GetType());
+ }
+ }
+ }
+
+ return data
? CreateKqpDataExecuter(std::move(request), database, userToken, counters)
- : CreateKqpScanExecuter(std::move(request), database, userToken, counters);
-}
-
-} // namespace NKqp
-} // namespace NKikimr
+ : CreateKqpScanExecuter(std::move(request), database, userToken, counters);
+}
+
+} // namespace NKqp
+} // namespace NKikimr
diff --git a/ydb/core/kqp/executer/kqp_executer_impl.h b/ydb/core/kqp/executer/kqp_executer_impl.h
index 004c239181..6ecf78ab05 100644
--- a/ydb/core/kqp/executer/kqp_executer_impl.h
+++ b/ydb/core/kqp/executer/kqp_executer_impl.h
@@ -1,10 +1,10 @@
-#pragma once
-
-#include "kqp_executer.h"
-#include "kqp_executer_stats.h"
-#include "kqp_partition_helper.h"
-#include "kqp_table_resolver.h"
-
+#pragma once
+
+#include "kqp_executer.h"
+#include "kqp_executer_stats.h"
+#include "kqp_partition_helper.h"
+#include "kqp_table_resolver.h"
+
#include <ydb/core/kqp/common/kqp_ru_calc.h>
#include <ydb/core/actorlib_impl/long_timer.h>
@@ -14,246 +14,246 @@
#include <ydb/core/kqp/executer/kqp_tasks_graph.h>
#include <ydb/core/kqp/kqp.h>
#include <ydb/core/grpc_services/local_rate_limiter.h>
-
+
#include <ydb/library/mkql_proto/mkql_proto.h>
-
+
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h>
#include <ydb/library/yql/dq/proto/dq_transport.pb.h>
#include <ydb/library/yql/dq/proto/dq_tasks.pb.h>
#include <ydb/library/yql/dq/runtime/dq_transport.h>
#include <ydb/library/yql/public/issue/yql_issue.h>
#include <ydb/library/yql/public/issue/yql_issue_message.h>
-
-#include <library/cpp/actors/core/actor_bootstrapped.h>
-#include <library/cpp/actors/core/hfunc.h>
-#include <library/cpp/actors/core/log.h>
-
-#include <util/generic/size_literals.h>
-
-namespace NKikimr {
-namespace NKqp {
-
-#define LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream)
-#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream)
-#define LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream)
-#define LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream)
-#define LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream)
-#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream)
-#define LOG_C(stream) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream)
-
-enum class EExecType {
- Data,
- Scan
-};
-
-const ui64 MaxTaskSize = 48_MB;
-
-struct TKqpExecuterTxResult {
- NKikimrMiniKQL::TType ItemType;
- TMaybe<NKikimrMiniKQL::TType> ResultItemType;
- TVector<NYql::NDqProto::TData> Data; // used in KqpDataExecuter
- NMiniKQL::TUnboxedValueVector Rows; // used in KqpLiteralExecuter
- bool IsStream = true;
-};
-
-void BuildKqpExecuterResult(const NKqpProto::TKqpPhyResult& txResult, TKqpExecuterTxResult& result);
-void BuildKqpExecuterResults(const NKqpProto::TKqpPhyTx& tx, TVector<TKqpExecuterTxResult>& results);
-
-void PrepareKqpTaskParameters(const NKqpProto::TKqpPhyStage& stage, const TStageInfo& stageInfo, const TTask& task,
- NYql::NDqProto::TDqTask& dqTask, const NMiniKQL::TTypeEnvironment& typeEnv, const NMiniKQL::THolderFactory& holderFactory);
-
+
+#include <library/cpp/actors/core/actor_bootstrapped.h>
+#include <library/cpp/actors/core/hfunc.h>
+#include <library/cpp/actors/core/log.h>
+
+#include <util/generic/size_literals.h>
+
+namespace NKikimr {
+namespace NKqp {
+
+#define LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream)
+#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream)
+#define LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream)
+#define LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream)
+#define LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream)
+#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream)
+#define LOG_C(stream) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream)
+
+enum class EExecType {
+ Data,
+ Scan
+};
+
+const ui64 MaxTaskSize = 48_MB;
+
+struct TKqpExecuterTxResult {
+ NKikimrMiniKQL::TType ItemType;
+ TMaybe<NKikimrMiniKQL::TType> ResultItemType;
+ TVector<NYql::NDqProto::TData> Data; // used in KqpDataExecuter
+ NMiniKQL::TUnboxedValueVector Rows; // used in KqpLiteralExecuter
+ bool IsStream = true;
+};
+
+void BuildKqpExecuterResult(const NKqpProto::TKqpPhyResult& txResult, TKqpExecuterTxResult& result);
+void BuildKqpExecuterResults(const NKqpProto::TKqpPhyTx& tx, TVector<TKqpExecuterTxResult>& results);
+
+void PrepareKqpTaskParameters(const NKqpProto::TKqpPhyStage& stage, const TStageInfo& stageInfo, const TTask& task,
+ NYql::NDqProto::TDqTask& dqTask, const NMiniKQL::TTypeEnvironment& typeEnv, const NMiniKQL::THolderFactory& holderFactory);
+
std::pair<TString, TString> SerializeKqpTasksParametersForOlap(const NKqpProto::TKqpPhyStage& stage,
const TStageInfo& stageInfo, const TTask& task, const NMiniKQL::THolderFactory& holderFactory,
const NMiniKQL::TTypeEnvironment& typeEnv);
-inline bool IsDebugLogEnabled() {
- return TlsActivationContext->LoggerSettings() &&
- TlsActivationContext->LoggerSettings()->Satisfies(NActors::NLog::PRI_DEBUG, NKikimrServices::KQP_EXECUTER);
-}
-
+inline bool IsDebugLogEnabled() {
+ return TlsActivationContext->LoggerSettings() &&
+ TlsActivationContext->LoggerSettings()->Satisfies(NActors::NLog::PRI_DEBUG, NKikimrServices::KQP_EXECUTER);
+}
+
TActorId ReportToRl(ui64 ru, const TString& database, const TString& userToken,
const NKikimrKqp::TRlPath& path);
-template <class TDerived, EExecType ExecType>
-class TKqpExecuterBase : public TActorBootstrapped<TDerived> {
-public:
+template <class TDerived, EExecType ExecType>
+class TKqpExecuterBase : public TActorBootstrapped<TDerived> {
+public:
TKqpExecuterBase(IKqpGateway::TExecPhysicalRequest&& request, const TString& database, const TMaybe<TString>& userToken,
- TKqpRequestCounters::TPtr counters)
- : Request(std::move(request))
+ TKqpRequestCounters::TPtr counters)
+ : Request(std::move(request))
, Database(database)
- , UserToken(userToken)
- , Counters(counters)
- {
- ResponseEv = std::make_unique<TEvKqpExecuter::TEvTxResponse>();
- if (Request.StatsMode >= NYql::NDqProto::DQ_STATS_MODE_BASIC) {
- Stats = std::make_unique<TQueryExecutionStats>(Request.StatsMode, &TasksGraph,
- ResponseEv->Record.MutableResponse()->MutableResult()->MutableStats());
- }
- }
-
- void Bootstrap() {
- StartTime = TAppData::TimeProvider->Now();
- if (Request.Timeout) {
- Deadline = StartTime + Request.Timeout;
- }
- if (Request.CancelAfter) {
- CancelAt = StartTime + *Request.CancelAfter;
- }
-
- LOG_T("Bootstrap done, become ReadyState");
- this->Become(&TKqpExecuterBase::ReadyState);
- }
-
- void ReportEventElapsedTime() {
- if (Stats) {
- ui64 elapsedMicros = TlsActivationContext->GetCurrentEventTicksAsSeconds() * 1'000'000;
- Stats->ExecuterCpuTime += TDuration::MicroSeconds(elapsedMicros);
- }
- }
-
-protected:
- STATEFN(ReadyState) {
- switch (ev->GetTypeRewrite()) {
- hFunc(TEvKqpExecuter::TEvTxRequest, HandleReady);
- default: {
- UnexpectedEvent("ReadyState", ev->GetTypeRewrite());
- }
- }
- ReportEventElapsedTime();
- }
-
- void HandleReady(TEvKqpExecuter::TEvTxRequest::TPtr& ev) {
- TxId = ev->Get()->Record.GetRequest().GetTxId();
- Target = ActorIdFromProto(ev->Get()->Record.GetTarget());
-
- LOG_D("Report self actorId " << this->SelfId() << " to " << Target);
- auto progressEv = MakeHolder<TEvKqpExecuter::TEvExecuterProgress>();
- ActorIdToProto(this->SelfId(), progressEv->Record.MutableExecuterActorId());
- this->Send(Target, progressEv.Release());
-
- const auto now = TAppData::TimeProvider->Now();
- const auto& ctx = TlsActivationContext->AsActorContext();
- TMaybe<TDuration> timeout;
- if (Deadline) {
- timeout = *Deadline - now;
- DeadlineActor = CreateLongTimer(ctx, *timeout,
- new IEventHandle(this->SelfId(), this->SelfId(), new TEvents::TEvWakeup(0)));
- }
-
- TMaybe<TDuration> cancelAfter;
- if (CancelAt) {
- cancelAfter = *CancelAt - now;
- CancelAtActor = CreateLongTimer(ctx, *cancelAfter,
- new IEventHandle(this->SelfId(), this->SelfId(), new TEvents::TEvWakeup(1)));
- }
-
- LOG_I("Begin execution. Operation timeout: " << timeout << ", cancelAfter: " << cancelAfter
- << ", txs: " << Request.Transactions.size());
-
- if (IsDebugLogEnabled()) {
- for (auto& tx : Request.Transactions) {
- LOG_D("Executing physical tx, type: " << (ui32) tx.Body.GetType() << ", stages: " << tx.Body.StagesSize());
- }
- }
-
- auto kqpTableResolver = CreateKqpTableResolver(this->SelfId(), TxId, UserToken, Request.Transactions,
- TableKeys, TasksGraph);
- KqpTableResolverId = this->RegisterWithSameMailbox(kqpTableResolver);
-
- LOG_T("Got request, become WaitResolveState");
- this->Become(&TDerived::WaitResolveState);
- StartResolveTime = now;
-
- if (Stats) {
- Stats->StartTs = now;
- }
- }
-
-protected:
- void HandleAbortExecution(TEvKqp::TEvAbortExecution::TPtr& ev) {
- auto& msg = ev->Get()->Record;
- LOG_D("Got EvAbortExecution, status: " << Ydb::StatusIds_StatusCode_Name(msg.GetStatusCode())
- << ", message: " << msg.GetMessage());
- if (msg.GetStatusCode() == Ydb::StatusIds::INTERNAL_ERROR) {
- InternalError(msg.GetMessage());
- } else if (msg.GetStatusCode() == Ydb::StatusIds::TIMEOUT) {
- auto abortEv = MakeHolder<TEvKqp::TEvAbortExecution>(Ydb::StatusIds::TIMEOUT, "Request timeout exceeded");
- this->Send(Target, abortEv.Release());
-
- TerminateComputeActors(Ydb::StatusIds::TIMEOUT, "timeout");
- this->PassAway();
- } else {
- RuntimeError(msg.GetStatusCode(), msg.GetMessage());
- }
- }
-
- void HandleTimeout(TEvents::TEvWakeup::TPtr& ev) {
- bool cancel = ev->Get()->Tag == 1;
- LOG_I((cancel ? "CancelAt" : "Timeout") << " exceeded. Send timeout event to the rpc actor " << Target);
-
- if (cancel) {
- auto abortEv = MakeHolder<TEvKqp::TEvAbortExecution>(Ydb::StatusIds::CANCELLED, "Request timeout exceeded");
- this->Send(Target, abortEv.Release());
- CancelAtActor = {};
- } else {
- auto abortEv = MakeHolder<TEvKqp::TEvAbortExecution>(Ydb::StatusIds::TIMEOUT, "Request timeout exceeded");
- this->Send(Target, abortEv.Release());
- DeadlineActor = {};
- }
-
- TerminateComputeActors(Ydb::StatusIds::TIMEOUT, "timeout");
- this->PassAway();
- }
-
-protected:
- void CollectTaskChannelsUpdates(const TKqpTasksGraph::TTaskType& task, THashMap<TActorId, THashSet<ui64>>& updates) {
- YQL_ENSURE(task.ComputeActorId);
-
- LOG_T("Collect channels updates for task: " << task.Id << " at actor " << task.ComputeActorId);
-
- // N.B. update only output channels
-
- for (auto& input : task.Inputs) {
- for (auto channelId : input.Channels) {
- auto& channel = TasksGraph.GetChannel(channelId);
- YQL_ENSURE(channel.DstTask == task.Id);
- YQL_ENSURE(channel.SrcTask);
-
- auto& srcTask = TasksGraph.GetTask(channel.SrcTask);
- if (srcTask.ComputeActorId) {
- updates[srcTask.ComputeActorId].emplace(channelId);
- }
-
- LOG_T("Task: " << task.Id << ", input channelId: " << channelId << ", src task: " << channel.SrcTask
- << ", at actor " << srcTask.ComputeActorId);
- }
- }
-
- for (auto& output : task.Outputs) {
- for (auto channelId : output.Channels) {
- updates[task.ComputeActorId].emplace(channelId);
- }
- }
- }
-
- void PropagateChannelsUpdates(const THashMap<TActorId, THashSet<ui64>>& updates) {
- for (auto& pair : updates) {
- auto computeActorId = pair.first;
- auto& channelIds = pair.second;
-
- auto channelsInfoEv = MakeHolder<NYql::NDq::TEvDqCompute::TEvChannelsInfo>();
- auto& record = channelsInfoEv->Record;
-
- for (auto& channelId : channelIds) {
- static_cast<TDerived*>(this)->FillChannelDesc(*record.AddUpdate(), TasksGraph.GetChannel(channelId));
- }
-
- LOG_T("Sending channels info to compute actor: " << computeActorId << ", channels: " << channelIds.size());
- bool sent = this->Send(computeActorId, channelsInfoEv.Release());
- YQL_ENSURE(sent, "Failed to send event to " << computeActorId.ToString());
- }
- }
-
+ , UserToken(userToken)
+ , Counters(counters)
+ {
+ ResponseEv = std::make_unique<TEvKqpExecuter::TEvTxResponse>();
+ if (Request.StatsMode >= NYql::NDqProto::DQ_STATS_MODE_BASIC) {
+ Stats = std::make_unique<TQueryExecutionStats>(Request.StatsMode, &TasksGraph,
+ ResponseEv->Record.MutableResponse()->MutableResult()->MutableStats());
+ }
+ }
+
+ void Bootstrap() {
+ StartTime = TAppData::TimeProvider->Now();
+ if (Request.Timeout) {
+ Deadline = StartTime + Request.Timeout;
+ }
+ if (Request.CancelAfter) {
+ CancelAt = StartTime + *Request.CancelAfter;
+ }
+
+ LOG_T("Bootstrap done, become ReadyState");
+ this->Become(&TKqpExecuterBase::ReadyState);
+ }
+
+ void ReportEventElapsedTime() {
+ if (Stats) {
+ ui64 elapsedMicros = TlsActivationContext->GetCurrentEventTicksAsSeconds() * 1'000'000;
+ Stats->ExecuterCpuTime += TDuration::MicroSeconds(elapsedMicros);
+ }
+ }
+
+protected:
+ STATEFN(ReadyState) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvKqpExecuter::TEvTxRequest, HandleReady);
+ default: {
+ UnexpectedEvent("ReadyState", ev->GetTypeRewrite());
+ }
+ }
+ ReportEventElapsedTime();
+ }
+
+ void HandleReady(TEvKqpExecuter::TEvTxRequest::TPtr& ev) {
+ TxId = ev->Get()->Record.GetRequest().GetTxId();
+ Target = ActorIdFromProto(ev->Get()->Record.GetTarget());
+
+ LOG_D("Report self actorId " << this->SelfId() << " to " << Target);
+ auto progressEv = MakeHolder<TEvKqpExecuter::TEvExecuterProgress>();
+ ActorIdToProto(this->SelfId(), progressEv->Record.MutableExecuterActorId());
+ this->Send(Target, progressEv.Release());
+
+ const auto now = TAppData::TimeProvider->Now();
+ const auto& ctx = TlsActivationContext->AsActorContext();
+ TMaybe<TDuration> timeout;
+ if (Deadline) {
+ timeout = *Deadline - now;
+ DeadlineActor = CreateLongTimer(ctx, *timeout,
+ new IEventHandle(this->SelfId(), this->SelfId(), new TEvents::TEvWakeup(0)));
+ }
+
+ TMaybe<TDuration> cancelAfter;
+ if (CancelAt) {
+ cancelAfter = *CancelAt - now;
+ CancelAtActor = CreateLongTimer(ctx, *cancelAfter,
+ new IEventHandle(this->SelfId(), this->SelfId(), new TEvents::TEvWakeup(1)));
+ }
+
+ LOG_I("Begin execution. Operation timeout: " << timeout << ", cancelAfter: " << cancelAfter
+ << ", txs: " << Request.Transactions.size());
+
+ if (IsDebugLogEnabled()) {
+ for (auto& tx : Request.Transactions) {
+ LOG_D("Executing physical tx, type: " << (ui32) tx.Body.GetType() << ", stages: " << tx.Body.StagesSize());
+ }
+ }
+
+ auto kqpTableResolver = CreateKqpTableResolver(this->SelfId(), TxId, UserToken, Request.Transactions,
+ TableKeys, TasksGraph);
+ KqpTableResolverId = this->RegisterWithSameMailbox(kqpTableResolver);
+
+ LOG_T("Got request, become WaitResolveState");
+ this->Become(&TDerived::WaitResolveState);
+ StartResolveTime = now;
+
+ if (Stats) {
+ Stats->StartTs = now;
+ }
+ }
+
+protected:
+ void HandleAbortExecution(TEvKqp::TEvAbortExecution::TPtr& ev) {
+ auto& msg = ev->Get()->Record;
+ LOG_D("Got EvAbortExecution, status: " << Ydb::StatusIds_StatusCode_Name(msg.GetStatusCode())
+ << ", message: " << msg.GetMessage());
+ if (msg.GetStatusCode() == Ydb::StatusIds::INTERNAL_ERROR) {
+ InternalError(msg.GetMessage());
+ } else if (msg.GetStatusCode() == Ydb::StatusIds::TIMEOUT) {
+ auto abortEv = MakeHolder<TEvKqp::TEvAbortExecution>(Ydb::StatusIds::TIMEOUT, "Request timeout exceeded");
+ this->Send(Target, abortEv.Release());
+
+ TerminateComputeActors(Ydb::StatusIds::TIMEOUT, "timeout");
+ this->PassAway();
+ } else {
+ RuntimeError(msg.GetStatusCode(), msg.GetMessage());
+ }
+ }
+
+ void HandleTimeout(TEvents::TEvWakeup::TPtr& ev) {
+ bool cancel = ev->Get()->Tag == 1;
+ LOG_I((cancel ? "CancelAt" : "Timeout") << " exceeded. Send timeout event to the rpc actor " << Target);
+
+ if (cancel) {
+ auto abortEv = MakeHolder<TEvKqp::TEvAbortExecution>(Ydb::StatusIds::CANCELLED, "Request timeout exceeded");
+ this->Send(Target, abortEv.Release());
+ CancelAtActor = {};
+ } else {
+ auto abortEv = MakeHolder<TEvKqp::TEvAbortExecution>(Ydb::StatusIds::TIMEOUT, "Request timeout exceeded");
+ this->Send(Target, abortEv.Release());
+ DeadlineActor = {};
+ }
+
+ TerminateComputeActors(Ydb::StatusIds::TIMEOUT, "timeout");
+ this->PassAway();
+ }
+
+protected:
+ void CollectTaskChannelsUpdates(const TKqpTasksGraph::TTaskType& task, THashMap<TActorId, THashSet<ui64>>& updates) {
+ YQL_ENSURE(task.ComputeActorId);
+
+ LOG_T("Collect channels updates for task: " << task.Id << " at actor " << task.ComputeActorId);
+
+ // N.B. update only output channels
+
+ for (auto& input : task.Inputs) {
+ for (auto channelId : input.Channels) {
+ auto& channel = TasksGraph.GetChannel(channelId);
+ YQL_ENSURE(channel.DstTask == task.Id);
+ YQL_ENSURE(channel.SrcTask);
+
+ auto& srcTask = TasksGraph.GetTask(channel.SrcTask);
+ if (srcTask.ComputeActorId) {
+ updates[srcTask.ComputeActorId].emplace(channelId);
+ }
+
+ LOG_T("Task: " << task.Id << ", input channelId: " << channelId << ", src task: " << channel.SrcTask
+ << ", at actor " << srcTask.ComputeActorId);
+ }
+ }
+
+ for (auto& output : task.Outputs) {
+ for (auto channelId : output.Channels) {
+ updates[task.ComputeActorId].emplace(channelId);
+ }
+ }
+ }
+
+ void PropagateChannelsUpdates(const THashMap<TActorId, THashSet<ui64>>& updates) {
+ for (auto& pair : updates) {
+ auto computeActorId = pair.first;
+ auto& channelIds = pair.second;
+
+ auto channelsInfoEv = MakeHolder<NYql::NDq::TEvDqCompute::TEvChannelsInfo>();
+ auto& record = channelsInfoEv->Record;
+
+ for (auto& channelId : channelIds) {
+ static_cast<TDerived*>(this)->FillChannelDesc(*record.AddUpdate(), TasksGraph.GetChannel(channelId));
+ }
+
+ LOG_T("Sending channels info to compute actor: " << computeActorId << ", channels: " << channelIds.size());
+ bool sent = this->Send(computeActorId, channelsInfoEv.Release());
+ YQL_ENSURE(sent, "Failed to send event to " << computeActorId.ToString());
+ }
+ }
+
void UpdateResourcesUsage(bool force) {
TInstant now = TActivationContext::Now();
if ((now - LastResourceUsageUpdate < ResourceUsageUpdateInterval) && !force)
@@ -303,23 +303,23 @@ protected:
-protected:
- void BuildSysViewScanTasks(TStageInfo& stageInfo, const NMiniKQL::THolderFactory& holderFactory,
- const NMiniKQL::TTypeEnvironment& typeEnv)
- {
- Y_VERIFY_DEBUG(stageInfo.Meta.IsSysView());
-
- auto& stage = GetStage(stageInfo);
-
- const auto& table = TableKeys.GetTable(stageInfo.Meta.TableId);
- const auto& keyTypes = table.KeyColumnTypes;
-
- for (auto& op : stage.GetTableOps()) {
- Y_VERIFY_DEBUG(stageInfo.Meta.TablePath == op.GetTable().GetPath());
-
- auto& task = TasksGraph.AddTask(stageInfo);
+protected:
+ void BuildSysViewScanTasks(TStageInfo& stageInfo, const NMiniKQL::THolderFactory& holderFactory,
+ const NMiniKQL::TTypeEnvironment& typeEnv)
+ {
+ Y_VERIFY_DEBUG(stageInfo.Meta.IsSysView());
+
+ auto& stage = GetStage(stageInfo);
+
+ const auto& table = TableKeys.GetTable(stageInfo.Meta.TableId);
+ const auto& keyTypes = table.KeyColumnTypes;
+
+ for (auto& op : stage.GetTableOps()) {
+ Y_VERIFY_DEBUG(stageInfo.Meta.TablePath == op.GetTable().GetPath());
+
+ auto& task = TasksGraph.AddTask(stageInfo);
TShardKeyRanges keyRanges;
-
+
switch (op.GetTypeCase()) {
case NKqpProto::TKqpPhyTableOperation::kReadRange:
stageInfo.Meta.SkipNullKeys.assign(
@@ -337,25 +337,25 @@ protected:
default:
YQL_ENSURE(false, "Unexpected table scan operation: " << (ui32) op.GetTypeCase());
}
-
+
TTaskMeta::TShardReadInfo readInfo = {
.Ranges = std::move(keyRanges),
.Columns = BuildKqpColumns(op, table),
};
-
- task.Meta.Reads.ConstructInPlace();
- task.Meta.Reads->emplace_back(std::move(readInfo));
+
+ task.Meta.Reads.ConstructInPlace();
+ task.Meta.Reads->emplace_back(std::move(readInfo));
task.Meta.ReadInfo.Reverse = op.GetReadRange().GetReverse();
-
- LOG_D("Stage " << stageInfo.Id << " create sysview scan task: " << task.Id);
- }
- }
-
-protected:
- // in derived classes
- // void FillEndpointDesc(NYql::NDqProto::TEndpoint& endpoint, const TTask& task);
- // void FillChannelDesc(NYql::NDqProto::TChannel& channelDesc, const TChannel& channel);
-
+
+ LOG_D("Stage " << stageInfo.Id << " create sysview scan task: " << task.Id);
+ }
+ }
+
+protected:
+ // in derived classes
+ // void FillEndpointDesc(NYql::NDqProto::TEndpoint& endpoint, const TTask& task);
+ // void FillChannelDesc(NYql::NDqProto::TChannel& channelDesc, const TChannel& channel);
+
void FillInputDesc(NYql::NDqProto::TTaskInput& inputDesc, const TTaskInput& input) {
switch (input.Type()) {
case NYql::NDq::TTaskInputType::UnionAll: {
@@ -383,85 +383,85 @@ protected:
}
}
- void FillOutputDesc(NYql::NDqProto::TTaskOutput& outputDesc, const TTaskOutput& output) {
- switch (output.Type) {
- case TTaskOutputType::Map:
- YQL_ENSURE(output.Channels.size() == 1);
- outputDesc.MutableMap();
- break;
-
- case TTaskOutputType::HashPartition: {
- auto& hashPartitionDesc = *outputDesc.MutableHashPartition();
- for (auto& column : output.KeyColumns) {
- hashPartitionDesc.AddKeyColumns(column);
- }
- hashPartitionDesc.SetPartitionsCount(output.PartitionsCount);
- break;
- }
-
- case TKqpTaskOutputType::ShardRangePartition: {
- auto& rangePartitionDesc = *outputDesc.MutableRangePartition();
- auto& columns = *rangePartitionDesc.MutableKeyColumns();
- for (auto& column : output.KeyColumns) {
- *columns.Add() = column;
- }
-
- auto& partitionsDesc = *rangePartitionDesc.MutablePartitions();
- for (auto& pair : output.Meta.ShardPartitions) {
- auto& range = *pair.second->Range;
- auto& partitionDesc = *partitionsDesc.Add();
- partitionDesc.SetEndKeyPrefix(range.EndKeyPrefix.GetBuffer());
- partitionDesc.SetIsInclusive(range.IsInclusive);
- partitionDesc.SetIsPoint(range.IsPoint);
- partitionDesc.SetChannelId(pair.first);
- }
- break;
- }
-
- case TTaskOutputType::Broadcast: {
- outputDesc.MutableBroadcast();
- break;
- }
-
- case TTaskOutputType::Effects: {
- outputDesc.MutableEffects();
- break;
- }
-
- default: {
- YQL_ENSURE(false, "Unexpected task output type " << output.Type);
- }
- }
-
- for (auto& channel : output.Channels) {
- auto& channelDesc = *outputDesc.AddChannels();
- static_cast<TDerived*>(this)->FillChannelDesc(channelDesc, TasksGraph.GetChannel(channel));
- }
- }
-
- void FillTableMeta(const TStageInfo& stageInfo, NKikimrTxDataShard::TKqpTransaction_TTableMeta* meta) {
- meta->SetTablePath(stageInfo.Meta.TablePath);
- meta->MutableTableId()->SetTableId(stageInfo.Meta.TableId.PathId.LocalPathId);
- meta->MutableTableId()->SetOwnerId(stageInfo.Meta.TableId.PathId.OwnerId);
- meta->SetSchemaVersion(stageInfo.Meta.TableId.SchemaVersion);
- meta->SetSysViewInfo(stageInfo.Meta.TableId.SysViewInfo);
+ void FillOutputDesc(NYql::NDqProto::TTaskOutput& outputDesc, const TTaskOutput& output) {
+ switch (output.Type) {
+ case TTaskOutputType::Map:
+ YQL_ENSURE(output.Channels.size() == 1);
+ outputDesc.MutableMap();
+ break;
+
+ case TTaskOutputType::HashPartition: {
+ auto& hashPartitionDesc = *outputDesc.MutableHashPartition();
+ for (auto& column : output.KeyColumns) {
+ hashPartitionDesc.AddKeyColumns(column);
+ }
+ hashPartitionDesc.SetPartitionsCount(output.PartitionsCount);
+ break;
+ }
+
+ case TKqpTaskOutputType::ShardRangePartition: {
+ auto& rangePartitionDesc = *outputDesc.MutableRangePartition();
+ auto& columns = *rangePartitionDesc.MutableKeyColumns();
+ for (auto& column : output.KeyColumns) {
+ *columns.Add() = column;
+ }
+
+ auto& partitionsDesc = *rangePartitionDesc.MutablePartitions();
+ for (auto& pair : output.Meta.ShardPartitions) {
+ auto& range = *pair.second->Range;
+ auto& partitionDesc = *partitionsDesc.Add();
+ partitionDesc.SetEndKeyPrefix(range.EndKeyPrefix.GetBuffer());
+ partitionDesc.SetIsInclusive(range.IsInclusive);
+ partitionDesc.SetIsPoint(range.IsPoint);
+ partitionDesc.SetChannelId(pair.first);
+ }
+ break;
+ }
+
+ case TTaskOutputType::Broadcast: {
+ outputDesc.MutableBroadcast();
+ break;
+ }
+
+ case TTaskOutputType::Effects: {
+ outputDesc.MutableEffects();
+ break;
+ }
+
+ default: {
+ YQL_ENSURE(false, "Unexpected task output type " << output.Type);
+ }
+ }
+
+ for (auto& channel : output.Channels) {
+ auto& channelDesc = *outputDesc.AddChannels();
+ static_cast<TDerived*>(this)->FillChannelDesc(channelDesc, TasksGraph.GetChannel(channel));
+ }
+ }
+
+ void FillTableMeta(const TStageInfo& stageInfo, NKikimrTxDataShard::TKqpTransaction_TTableMeta* meta) {
+ meta->SetTablePath(stageInfo.Meta.TablePath);
+ meta->MutableTableId()->SetTableId(stageInfo.Meta.TableId.PathId.LocalPathId);
+ meta->MutableTableId()->SetOwnerId(stageInfo.Meta.TableId.PathId.OwnerId);
+ meta->SetSchemaVersion(stageInfo.Meta.TableId.SchemaVersion);
+ meta->SetSysViewInfo(stageInfo.Meta.TableId.SysViewInfo);
meta->SetTableKind((ui32)stageInfo.Meta.TableKind);
- }
-
- void ExtractItemsLimit(const TStageInfo& stageInfo, const NKqpProto::TKqpPhyParamValue& paramValue,
- const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv,
+ }
+
+ void ExtractItemsLimit(const TStageInfo& stageInfo, const NKqpProto::TKqpPhyParamValue& paramValue,
+ const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv,
ui64& itemsLimit, TString& itemsLimitParamName, NYql::NDqProto::TData& itemsLimitBytes,
NKikimr::NMiniKQL::TType*& itemsLimitType)
- {
- itemsLimitParamName = paramValue.GetParamName();
-
+ {
+ itemsLimitParamName = paramValue.GetParamName();
+
if (!itemsLimitParamName) {
return;
}
-
+
auto* itemsLimitParam = stageInfo.Meta.Tx.Params.Values.FindPtr(itemsLimitParamName);
YQL_ENSURE(itemsLimitParam);
-
+
auto [type, value] = NMiniKQL::ImportValueFromProto(itemsLimitParam->GetType(), itemsLimitParam->GetValue(), typeEnv, holderFactory);
YQL_ENSURE(type->GetKind() == NMiniKQL::TType::EKind::Data);
@@ -470,171 +470,171 @@ protected:
NYql::NDq::TDqDataSerializer dataSerializer(typeEnv, holderFactory, NYql::NDqProto::DATA_TRANSPORT_UV_PICKLE_1_0);
itemsLimitBytes = dataSerializer.Serialize(value, type);
itemsLimitType = type;
- }
-
-protected:
- void TerminateComputeActors(Ydb::StatusIds::StatusCode code, const TString& message) {
- for (const auto& task : this->TasksGraph.GetTasks()) {
- if (task.ComputeActorId) {
- LOG_I("aborting compute actor execution, message: " << message
- << ", compute actor: " << task.ComputeActorId << ", task: " << task.Id);
-
- auto ev = MakeHolder<TEvKqp::TEvAbortExecution>(code, message);
- this->Send(task.ComputeActorId, ev.Release());
- } else {
- LOG_I("task: " << task.Id << ", does not have Compute ActorId yet");
- }
- }
- }
-
-protected:
- void UnexpectedEvent(const TString& state, ui32 eventType) {
- LOG_C("TKqpExecuter, unexpected event: " << eventType << ", at state:" << state << ", selfID: " << this->SelfId());
- InternalError(TStringBuilder() << "Unexpected event at TKqpScanExecuter, state: " << state
- << ", event: " << eventType);
- }
-
- void InternalError(const TString& message) {
- LOG_E(message);
- TerminateComputeActors(Ydb::StatusIds::INTERNAL_ERROR, message);
+ }
+
+protected:
+ void TerminateComputeActors(Ydb::StatusIds::StatusCode code, const TString& message) {
+ for (const auto& task : this->TasksGraph.GetTasks()) {
+ if (task.ComputeActorId) {
+ LOG_I("aborting compute actor execution, message: " << message
+ << ", compute actor: " << task.ComputeActorId << ", task: " << task.Id);
+
+ auto ev = MakeHolder<TEvKqp::TEvAbortExecution>(code, message);
+ this->Send(task.ComputeActorId, ev.Release());
+ } else {
+ LOG_I("task: " << task.Id << ", does not have Compute ActorId yet");
+ }
+ }
+ }
+
+protected:
+ void UnexpectedEvent(const TString& state, ui32 eventType) {
+ LOG_C("TKqpExecuter, unexpected event: " << eventType << ", at state:" << state << ", selfID: " << this->SelfId());
+ InternalError(TStringBuilder() << "Unexpected event at TKqpScanExecuter, state: " << state
+ << ", event: " << eventType);
+ }
+
+ void InternalError(const TString& message) {
+ LOG_E(message);
+ TerminateComputeActors(Ydb::StatusIds::INTERNAL_ERROR, message);
auto issue = NYql::YqlIssue({}, NYql::TIssuesIds::UNEXPECTED, "Internal error while executing transaction.");
- issue.AddSubIssue(MakeIntrusive<NYql::TIssue>(message));
- ReplyErrorAndDie(Ydb::StatusIds::INTERNAL_ERROR, issue);
- }
-
- void ReplyUnavailable(const TString& message) {
- LOG_E("UNAVAILABLE: " << message);
- TerminateComputeActors(Ydb::StatusIds::UNAVAILABLE, message);
- auto issue = NYql::YqlIssue({}, NYql::TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE);
- issue.AddSubIssue(new NYql::TIssue(message));
- ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, issue);
- }
-
- void RuntimeError(Ydb::StatusIds::StatusCode code, const TString& message) {
- LOG_E(Ydb::StatusIds_StatusCode_Name(code) << ": " << message);
- TerminateComputeActors(code, message);
- ReplyErrorAndDie(code, NYql::TIssue(message));
- }
-
- void ReplyErrorAndDie(Ydb::StatusIds::StatusCode status, const NYql::TIssues& issues) {
- google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage> protoIssues;
- IssuesToMessage(issues, &protoIssues);
- ReplyErrorAndDie(status, &protoIssues);
- }
-
- void ReplyErrorAndDie(Ydb::StatusIds::StatusCode status, const NYql::TIssue& issue) {
- google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage> issues;
- IssueToMessage(issue, issues.Add());
- ReplyErrorAndDie(status, &issues);
- }
-
- virtual void ReplyErrorAndDie(Ydb::StatusIds::StatusCode status,
- google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage>* issues)
- {
- for (auto computeActor : PendingComputeActors) {
+ issue.AddSubIssue(MakeIntrusive<NYql::TIssue>(message));
+ ReplyErrorAndDie(Ydb::StatusIds::INTERNAL_ERROR, issue);
+ }
+
+ void ReplyUnavailable(const TString& message) {
+ LOG_E("UNAVAILABLE: " << message);
+ TerminateComputeActors(Ydb::StatusIds::UNAVAILABLE, message);
+ auto issue = NYql::YqlIssue({}, NYql::TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE);
+ issue.AddSubIssue(new NYql::TIssue(message));
+ ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, issue);
+ }
+
+ void RuntimeError(Ydb::StatusIds::StatusCode code, const TString& message) {
+ LOG_E(Ydb::StatusIds_StatusCode_Name(code) << ": " << message);
+ TerminateComputeActors(code, message);
+ ReplyErrorAndDie(code, NYql::TIssue(message));
+ }
+
+ void ReplyErrorAndDie(Ydb::StatusIds::StatusCode status, const NYql::TIssues& issues) {
+ google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage> protoIssues;
+ IssuesToMessage(issues, &protoIssues);
+ ReplyErrorAndDie(status, &protoIssues);
+ }
+
+ void ReplyErrorAndDie(Ydb::StatusIds::StatusCode status, const NYql::TIssue& issue) {
+ google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage> issues;
+ IssueToMessage(issue, issues.Add());
+ ReplyErrorAndDie(status, &issues);
+ }
+
+ virtual void ReplyErrorAndDie(Ydb::StatusIds::StatusCode status,
+ google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage>* issues)
+ {
+ for (auto computeActor : PendingComputeActors) {
LOG_D("terminate compute actor " << computeActor.first);
-
- auto ev = MakeHolder<TEvKqp::TEvAbortExecution>(status, "Terminate execution");
+
+ auto ev = MakeHolder<TEvKqp::TEvAbortExecution>(status, "Terminate execution");
this->Send(computeActor.first, ev.Release());
- }
-
- auto& response = *ResponseEv->Record.MutableResponse();
-
- response.SetStatus(status);
- response.MutableIssues()->Swap(issues);
-
- LOG_T("ReplyErrorAndDie. " << response.DebugString());
-
- if constexpr (ExecType == EExecType::Data) {
- if (status != Ydb::StatusIds::SUCCESS) {
- Counters->TxProxyMon->ReportStatusNotOK->Inc();
- } else {
- Counters->TxProxyMon->ReportStatusOK->Inc();
- }
- }
-
- this->Send(Target, ResponseEv.release());
- this->PassAway();
- }
-
-protected:
- template <class TCollection>
- bool ValidateTaskSize(const TCollection& tasks) {
- for (const auto& task : tasks) {
- if (ui32 size = task.ByteSize(); size > MaxTaskSize) {
- LOG_E("Abort execution. Task #" << task.GetId() << " size is too big: " << size << " > " << MaxTaskSize);
- ReplyErrorAndDie(Ydb::StatusIds::ABORTED,
- MakeIssue(NKikimrIssues::TIssuesIds::SHARD_PROGRAM_SIZE_EXCEEDED, TStringBuilder() <<
- "Datashard program size limit exceeded (" << size << " > " << MaxTaskSize << ")"));
- return false;
- }
- }
- return true;
- }
-
-protected:
- void PassAway() override {
- LOG_D("terminate execution.");
- if (DeadlineActor) {
- this->Send(DeadlineActor, new TEvents::TEvPoison);
- }
- if (CancelAtActor) {
- this->Send(CancelAtActor, new TEvents::TEvPoison);
- }
- if (KqpTableResolverId) {
- this->Send(KqpTableResolverId, new TEvents::TEvPoison);
- this->Send(this->SelfId(), new TEvents::TEvPoison);
- LOG_T("Terminate, become ZombieState");
- this->Become(&TKqpExecuterBase::ZombieState);
- } else {
- IActor::PassAway();
- }
- }
-
- STATEFN(ZombieState) {
- if (ev->GetTypeRewrite() == TEvents::TEvPoison::EventType) {
- IActor::PassAway();
- }
- }
-
-protected:
- IKqpGateway::TExecPhysicalRequest Request;
+ }
+
+ auto& response = *ResponseEv->Record.MutableResponse();
+
+ response.SetStatus(status);
+ response.MutableIssues()->Swap(issues);
+
+ LOG_T("ReplyErrorAndDie. " << response.DebugString());
+
+ if constexpr (ExecType == EExecType::Data) {
+ if (status != Ydb::StatusIds::SUCCESS) {
+ Counters->TxProxyMon->ReportStatusNotOK->Inc();
+ } else {
+ Counters->TxProxyMon->ReportStatusOK->Inc();
+ }
+ }
+
+ this->Send(Target, ResponseEv.release());
+ this->PassAway();
+ }
+
+protected:
+ template <class TCollection>
+ bool ValidateTaskSize(const TCollection& tasks) {
+ for (const auto& task : tasks) {
+ if (ui32 size = task.ByteSize(); size > MaxTaskSize) {
+ LOG_E("Abort execution. Task #" << task.GetId() << " size is too big: " << size << " > " << MaxTaskSize);
+ ReplyErrorAndDie(Ydb::StatusIds::ABORTED,
+ MakeIssue(NKikimrIssues::TIssuesIds::SHARD_PROGRAM_SIZE_EXCEEDED, TStringBuilder() <<
+ "Datashard program size limit exceeded (" << size << " > " << MaxTaskSize << ")"));
+ return false;
+ }
+ }
+ return true;
+ }
+
+protected:
+ void PassAway() override {
+ LOG_D("terminate execution.");
+ if (DeadlineActor) {
+ this->Send(DeadlineActor, new TEvents::TEvPoison);
+ }
+ if (CancelAtActor) {
+ this->Send(CancelAtActor, new TEvents::TEvPoison);
+ }
+ if (KqpTableResolverId) {
+ this->Send(KqpTableResolverId, new TEvents::TEvPoison);
+ this->Send(this->SelfId(), new TEvents::TEvPoison);
+ LOG_T("Terminate, become ZombieState");
+ this->Become(&TKqpExecuterBase::ZombieState);
+ } else {
+ IActor::PassAway();
+ }
+ }
+
+ STATEFN(ZombieState) {
+ if (ev->GetTypeRewrite() == TEvents::TEvPoison::EventType) {
+ IActor::PassAway();
+ }
+ }
+
+protected:
+ IKqpGateway::TExecPhysicalRequest Request;
const TString Database;
const TMaybe<TString> UserToken;
- TKqpRequestCounters::TPtr Counters;
- std::unique_ptr<TQueryExecutionStats> Stats;
- TInstant StartTime;
- TMaybe<TInstant> Deadline;
- TActorId DeadlineActor;
- TMaybe<TInstant> CancelAt;
- TActorId CancelAtActor;
- TActorId Target;
- ui64 TxId = 0;
-
- TKqpTasksGraph TasksGraph;
- TKqpTableKeys TableKeys;
-
- TActorId KqpTableResolverId;
+ TKqpRequestCounters::TPtr Counters;
+ std::unique_ptr<TQueryExecutionStats> Stats;
+ TInstant StartTime;
+ TMaybe<TInstant> Deadline;
+ TActorId DeadlineActor;
+ TMaybe<TInstant> CancelAt;
+ TActorId CancelAtActor;
+ TActorId Target;
+ ui64 TxId = 0;
+
+ TKqpTasksGraph TasksGraph;
+ TKqpTableKeys TableKeys;
+
+ TActorId KqpTableResolverId;
THashMap<TActorId, TProgressStat> PendingComputeActors; // Running compute actors (pure and DS)
TVector<TProgressStat> LastStats;
-
- TInstant StartResolveTime;
+
+ TInstant StartResolveTime;
TInstant LastResourceUsageUpdate;
-
- std::unique_ptr<TEvKqpExecuter::TEvTxResponse> ResponseEv;
+
+ std::unique_ptr<TEvKqpExecuter::TEvTxResponse> ResponseEv;
private:
static constexpr TDuration ResourceUsageUpdateInterval = TDuration::MilliSeconds(100);
-};
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-IActor* CreateKqpLiteralExecuter(IKqpGateway::TExecPhysicalRequest&& request, TKqpRequestCounters::TPtr counters);
-
+};
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+IActor* CreateKqpLiteralExecuter(IKqpGateway::TExecPhysicalRequest&& request, TKqpRequestCounters::TPtr counters);
+
IActor* CreateKqpDataExecuter(IKqpGateway::TExecPhysicalRequest&& request, const TString& database,
const TMaybe<TString>& userToken, TKqpRequestCounters::TPtr counters);
-
-IActor* CreateKqpScanExecuter(IKqpGateway::TExecPhysicalRequest&& request, const TString& database,
- const TMaybe<TString>& userToken, TKqpRequestCounters::TPtr counters);
-
-} // namespace NKqp
-} // namespace NKikimr
+
+IActor* CreateKqpScanExecuter(IKqpGateway::TExecPhysicalRequest&& request, const TString& database,
+ const TMaybe<TString>& userToken, TKqpRequestCounters::TPtr counters);
+
+} // namespace NKqp
+} // namespace NKikimr
diff --git a/ydb/core/kqp/executer/kqp_executer_stats.cpp b/ydb/core/kqp/executer/kqp_executer_stats.cpp
index 699f4bebac..8c3605fd91 100644
--- a/ydb/core/kqp/executer/kqp_executer_stats.cpp
+++ b/ydb/core/kqp/executer/kqp_executer_stats.cpp
@@ -1,257 +1,257 @@
-#include "kqp_executer_stats.h"
-
+#include "kqp_executer_stats.h"
+
#include <ydb/core/protos/kqp.pb.h>
-
-namespace NKikimr::NKqp {
-
-using namespace NYql;
+
+namespace NKikimr::NKqp {
+
+using namespace NYql;
using namespace NYql::NDq;
-
-namespace {
-
-TTableStat operator - (const TTableStat& l, const TTableStat& r) {
- return {.Rows = l.Rows - r.Rows, .Bytes = l.Bytes - r.Bytes};
+
+namespace {
+
+TTableStat operator - (const TTableStat& l, const TTableStat& r) {
+ return {.Rows = l.Rows - r.Rows, .Bytes = l.Bytes - r.Bytes};
}
TProgressStatEntry operator - (const TProgressStatEntry& l, const TProgressStatEntry& r) {
return TProgressStatEntry {
- .ComputeTime = l.ComputeTime - r.ComputeTime,
- .ReadIOStat = l.ReadIOStat - r.ReadIOStat
+ .ComputeTime = l.ComputeTime - r.ComputeTime,
+ .ReadIOStat = l.ReadIOStat - r.ReadIOStat
};
}
-void UpdateAggr(NDqProto::TDqStatsAggr* aggr, ui64 value) noexcept {
- if (aggr->GetMin() == 0) {
- aggr->SetMin(value);
- } else {
- aggr->SetMin(std::min(aggr->GetMin(), value));
- }
- aggr->SetMax(std::max(aggr->GetMax(), value));
- aggr->SetSum(aggr->GetSum() + value);
- aggr->SetCnt(aggr->GetCnt() + 1);
-}
-
-void UpdateMinMax(NDqProto::TDqStatsMinMax* minMax, ui64 value) noexcept {
- if (minMax->GetMin() == 0) {
- minMax->SetMin(value);
- } else {
- minMax->SetMin(std::min(minMax->GetMin(), value));
- }
+void UpdateAggr(NDqProto::TDqStatsAggr* aggr, ui64 value) noexcept {
+ if (aggr->GetMin() == 0) {
+ aggr->SetMin(value);
+ } else {
+ aggr->SetMin(std::min(aggr->GetMin(), value));
+ }
+ aggr->SetMax(std::max(aggr->GetMax(), value));
+ aggr->SetSum(aggr->GetSum() + value);
+ aggr->SetCnt(aggr->GetCnt() + 1);
+}
+
+void UpdateMinMax(NDqProto::TDqStatsMinMax* minMax, ui64 value) noexcept {
+ if (minMax->GetMin() == 0) {
+ minMax->SetMin(value);
+ } else {
+ minMax->SetMin(std::min(minMax->GetMin(), value));
+ }
minMax->SetMax(std::max(minMax->GetMax(), value));
-}
-
-NDqProto::TDqStageStats* GetOrCreateStageStats(const NYql::NDqProto::TDqTaskStats& taskStats,
- const TKqpTasksGraph& tasksGraph, NDqProto::TDqExecutionStats& execStats)
-{
- auto& task = tasksGraph.GetTask(taskStats.GetTaskId());
- auto& stageId = task.StageId;
- auto& stageInfo = tasksGraph.GetStageInfo(stageId);
- auto& stageProto = stageInfo.Meta.Tx.Body.GetStages(stageId.StageId);
-
-
- for (auto& stage : *execStats.MutableStages()) {
- if (stage.GetStageGuid() == stageProto.GetStageGuid()) {
- return &stage;
- }
- }
-
- auto* newStage = execStats.AddStages();
- newStage->SetStageId(stageId.StageId);
- newStage->SetStageGuid(stageProto.GetStageGuid());
- newStage->SetProgram(stageProto.GetProgramAst());
- return newStage;
}
-
-NDqProto::TDqTableAggrStats* GetOrCreateTableAggrStats(NDqProto::TDqStageStats* stage, const TString& tablePath) {
- for(auto& table : *stage->MutableTables()) {
- if (table.GetTablePath() == tablePath) {
- return &table;
- }
- }
- auto table = stage->AddTables();
- table->SetTablePath(tablePath);
- return table;
+
+NDqProto::TDqStageStats* GetOrCreateStageStats(const NYql::NDqProto::TDqTaskStats& taskStats,
+ const TKqpTasksGraph& tasksGraph, NDqProto::TDqExecutionStats& execStats)
+{
+ auto& task = tasksGraph.GetTask(taskStats.GetTaskId());
+ auto& stageId = task.StageId;
+ auto& stageInfo = tasksGraph.GetStageInfo(stageId);
+ auto& stageProto = stageInfo.Meta.Tx.Body.GetStages(stageId.StageId);
+
+
+ for (auto& stage : *execStats.MutableStages()) {
+ if (stage.GetStageGuid() == stageProto.GetStageGuid()) {
+ return &stage;
+ }
+ }
+
+ auto* newStage = execStats.AddStages();
+ newStage->SetStageId(stageId.StageId);
+ newStage->SetStageGuid(stageProto.GetStageGuid());
+ newStage->SetProgram(stageProto.GetProgramAst());
+ return newStage;
+}
+
+NDqProto::TDqTableAggrStats* GetOrCreateTableAggrStats(NDqProto::TDqStageStats* stage, const TString& tablePath) {
+ for(auto& table : *stage->MutableTables()) {
+ if (table.GetTablePath() == tablePath) {
+ return &table;
+ }
+ }
+ auto table = stage->AddTables();
+ table->SetTablePath(tablePath);
+ return table;
}
-
+
} // anonymous namespace
-
-void TQueryExecutionStats::AddComputeActorStats(ui32 /* nodeId */, NYql::NDqProto::TDqComputeActorStats&& stats) {
-// Cerr << (TStringBuilder() << "::AddComputeActorStats " << stats.DebugString() << Endl);
-
- Result->SetCpuTimeUs(Result->GetCpuTimeUs() + stats.GetCpuTimeUs());
- TotalTasks += stats.GetTasks().size();
-
- UpdateAggr(ExtraStats.MutableComputeCpuTimeUs(), stats.GetCpuTimeUs());
-
- for (auto& task : stats.GetTasks()) {
- for (auto& table : task.GetTables()) {
- NYql::NDqProto::TDqTableStats* tableAggr = nullptr;
- if (auto it = TableStats.find(table.GetTablePath()); it != TableStats.end()) {
- tableAggr = it->second;
- } else {
- tableAggr = Result->AddTables();
- tableAggr->SetTablePath(table.GetTablePath());
- TableStats.emplace(table.GetTablePath(), tableAggr);
- }
-
- tableAggr->SetReadRows(tableAggr->GetReadRows() + table.GetReadRows());
- tableAggr->SetReadBytes(tableAggr->GetReadBytes() + table.GetReadBytes());
- tableAggr->SetWriteRows(tableAggr->GetWriteRows() + table.GetWriteRows());
- tableAggr->SetWriteBytes(tableAggr->GetWriteBytes() + table.GetWriteBytes());
- tableAggr->SetEraseRows(tableAggr->GetEraseRows() + table.GetEraseRows());
- tableAggr->SetAffectedPartitions(tableAggr->GetAffectedPartitions() + table.GetAffectedPartitions());
- }
- }
-
- if (StatsMode >= NDqProto::DQ_STATS_MODE_PROFILE) {
- for (auto& task : stats.GetTasks()) {
- auto* stageStats = GetOrCreateStageStats(task, *TasksGraph, *Result);
-
- stageStats->SetTotalTasksCount(stageStats->GetTotalTasksCount() + 1);
- UpdateAggr(stageStats->MutableCpuTimeUs(), task.GetCpuTimeUs());
- UpdateAggr(stageStats->MutableInputRows(), task.GetInputRows());
- UpdateAggr(stageStats->MutableInputBytes(), task.GetInputBytes());
- UpdateAggr(stageStats->MutableOutputRows(), task.GetOutputRows());
- UpdateAggr(stageStats->MutableOutputBytes(), task.GetOutputBytes());
-
- UpdateMinMax(stageStats->MutableFirstRowTimeMs(), task.GetFirstRowTimeMs());
- UpdateMinMax(stageStats->MutableFinishTimeMs(), task.GetFinishTimeMs());
-
- stageStats->SetDurationUs((stageStats->GetFinishTimeMs().GetMax() - stageStats->GetFirstRowTimeMs().GetMin()) * 1'000);
-
- stageStats->AddComputeActors()->CopyFrom(stats);
- }
- }
-}
-
-void TQueryExecutionStats::AddDatashardPrepareStats(NKikimrQueryStats::TTxStats&& txStats) {
-// Cerr << (TStringBuilder() << "::AddDatashardPrepareStats " << txStats.DebugString() << Endl);
-
- ui64 cpuUs = txStats.GetComputeCpuTimeUsec();
- for (const auto& perShard : txStats.GetPerShardStats()) {
- AffectedShards.emplace(perShard.GetShardId());
- cpuUs += perShard.GetCpuTimeUsec();
- }
-
- Result->SetCpuTimeUs(Result->GetCpuTimeUs() + cpuUs);
-}
-
-void TQueryExecutionStats::AddDatashardStats(NYql::NDqProto::TDqComputeActorStats&& stats,
- NKikimrQueryStats::TTxStats&& txStats)
-{
-// Cerr << (TStringBuilder() << "::AddDatashardStats " << stats.DebugString() << ", " << txStats.DebugString() << Endl);
-
- ui64 datashardCpuTimeUs = 0;
- for (const auto& perShard : txStats.GetPerShardStats()) {
- AffectedShards.emplace(perShard.GetShardId());
-
- datashardCpuTimeUs += perShard.GetCpuTimeUsec();
- UpdateAggr(ExtraStats.MutableShardsCpuTimeUs(), perShard.GetCpuTimeUsec());
- }
-
- Result->SetCpuTimeUs(Result->GetCpuTimeUs() + datashardCpuTimeUs);
- TotalTasks += stats.GetTasks().size();
-
- for (auto& task : stats.GetTasks()) {
- for (auto& table : task.GetTables()) {
- NYql::NDqProto::TDqTableStats* tableAggr = nullptr;
- if (auto it = TableStats.find(table.GetTablePath()); it != TableStats.end()) {
- tableAggr = it->second;
- } else {
- tableAggr = Result->AddTables();
- tableAggr->SetTablePath(table.GetTablePath());
- TableStats.emplace(table.GetTablePath(), tableAggr);
- }
-
- tableAggr->SetReadRows(tableAggr->GetReadRows() + table.GetReadRows());
- tableAggr->SetReadBytes(tableAggr->GetReadBytes() + table.GetReadBytes());
- tableAggr->SetWriteRows(tableAggr->GetWriteRows() + table.GetWriteRows());
- tableAggr->SetWriteBytes(tableAggr->GetWriteBytes() + table.GetWriteBytes());
- tableAggr->SetEraseRows(tableAggr->GetEraseRows() + table.GetEraseRows());
-
- auto& shards = TableShards[table.GetTablePath()];
- for (const auto& perShard : txStats.GetPerShardStats()) {
- shards.insert(perShard.GetShardId());
- }
- tableAggr->SetAffectedPartitions(shards.size());
- }
- }
-
- if (StatsMode == NDqProto::DQ_STATS_MODE_PROFILE) {
- for (auto& task : *stats.MutableTasks()) {
- auto* stageStats = GetOrCreateStageStats(task, *TasksGraph, *Result);
-
- stageStats->SetTotalTasksCount(stageStats->GetTotalTasksCount() + 1);
- UpdateAggr(stageStats->MutableCpuTimeUs(), task.GetCpuTimeUs());
- UpdateAggr(stageStats->MutableInputRows(), task.GetInputRows());
- UpdateAggr(stageStats->MutableInputBytes(), task.GetInputBytes());
- UpdateAggr(stageStats->MutableOutputRows(), task.GetOutputRows());
- UpdateAggr(stageStats->MutableOutputBytes(), task.GetOutputBytes());
-
- UpdateMinMax(stageStats->MutableFirstRowTimeMs(), task.GetFirstRowTimeMs());
- UpdateMinMax(stageStats->MutableFinishTimeMs(), task.GetFinishTimeMs());
-
- stageStats->SetDurationUs((stageStats->GetFinishTimeMs().GetMax() - stageStats->GetFirstRowTimeMs().GetMin()) * 1'000);
-
- for (auto& tableStats: task.GetTables()) {
- auto* tableAggrStats = GetOrCreateTableAggrStats(stageStats, tableStats.GetTablePath());
-
- UpdateAggr(tableAggrStats->MutableReadRows(), tableStats.GetReadRows());
- UpdateAggr(tableAggrStats->MutableReadBytes(), tableStats.GetReadBytes());
- UpdateAggr(tableAggrStats->MutableWriteRows(), tableStats.GetWriteRows());
- UpdateAggr(tableAggrStats->MutableWriteBytes(), tableStats.GetWriteBytes());
- UpdateAggr(tableAggrStats->MutableEraseRows(), tableStats.GetEraseRows());
-
- NKqpProto::TKqpShardTableExtraStats tableExtraStats;
- if (tableStats.GetExtra().UnpackTo(&tableExtraStats)) {
- NKqpProto::TKqpShardTableAggrExtraStats tableAggrExtraStats;
- if (tableAggrStats->HasExtra()) {
- bool ok = tableAggrStats->MutableExtra()->UnpackTo(&tableAggrExtraStats);
- YQL_ENSURE(ok);
- }
-
- tableAggrExtraStats.SetAffectedShards(TableShards[tableStats.GetTablePath()].size());
- UpdateAggr(tableAggrExtraStats.MutableShardCpuTimeUs(), datashardCpuTimeUs);
-
- tableAggrStats->MutableExtra()->PackFrom(tableAggrExtraStats);
- }
- }
-
- NKqpProto::TKqpStageExtraStats stageExtraStats;
- if (stageStats->HasExtra()) {
- bool ok = stageStats->GetExtra().UnpackTo(&stageExtraStats);
- YQL_ENSURE(ok);
- }
- stageExtraStats.AddDatashardTasks()->Swap(&task);
- stageStats->MutableExtra()->PackFrom(stageExtraStats);
- }
-
- DatashardStats.emplace_back(std::move(txStats));
- }
-}
-
-void TQueryExecutionStats::Finish() {
-// Cerr << (TStringBuilder() << "-- finish: executerTime: " << ExecuterCpuTime.MicroSeconds() << Endl);
-
- Result->SetCpuTimeUs(Result->GetCpuTimeUs() + ExecuterCpuTime.MicroSeconds());
- Result->SetDurationUs(FinishTs.MicroSeconds() - StartTs.MicroSeconds());
-
- Result->SetResultBytes(ResultBytes);
- Result->SetResultRows(ResultRows);
-
- ExtraStats.SetAffectedShards(AffectedShards.size());
- Result->MutableExtra()->PackFrom(ExtraStats);
-
- if (StatsMode >= NYql::NDqProto::DQ_STATS_MODE_PROFILE) {
- Result->SetExecuterCpuTimeUs(ExecuterCpuTime.MicroSeconds());
-
- Result->SetStartTimeMs(StartTs.MilliSeconds());
- Result->SetFinishTimeMs(FinishTs.MilliSeconds());
- }
-
- // Cerr << (TStringBuilder() << "TQueryExecutionStats::Finish" << Endl << Result->DebugString() << Endl);
-}
-
+
+void TQueryExecutionStats::AddComputeActorStats(ui32 /* nodeId */, NYql::NDqProto::TDqComputeActorStats&& stats) {
+// Cerr << (TStringBuilder() << "::AddComputeActorStats " << stats.DebugString() << Endl);
+
+ Result->SetCpuTimeUs(Result->GetCpuTimeUs() + stats.GetCpuTimeUs());
+ TotalTasks += stats.GetTasks().size();
+
+ UpdateAggr(ExtraStats.MutableComputeCpuTimeUs(), stats.GetCpuTimeUs());
+
+ for (auto& task : stats.GetTasks()) {
+ for (auto& table : task.GetTables()) {
+ NYql::NDqProto::TDqTableStats* tableAggr = nullptr;
+ if (auto it = TableStats.find(table.GetTablePath()); it != TableStats.end()) {
+ tableAggr = it->second;
+ } else {
+ tableAggr = Result->AddTables();
+ tableAggr->SetTablePath(table.GetTablePath());
+ TableStats.emplace(table.GetTablePath(), tableAggr);
+ }
+
+ tableAggr->SetReadRows(tableAggr->GetReadRows() + table.GetReadRows());
+ tableAggr->SetReadBytes(tableAggr->GetReadBytes() + table.GetReadBytes());
+ tableAggr->SetWriteRows(tableAggr->GetWriteRows() + table.GetWriteRows());
+ tableAggr->SetWriteBytes(tableAggr->GetWriteBytes() + table.GetWriteBytes());
+ tableAggr->SetEraseRows(tableAggr->GetEraseRows() + table.GetEraseRows());
+ tableAggr->SetAffectedPartitions(tableAggr->GetAffectedPartitions() + table.GetAffectedPartitions());
+ }
+ }
+
+ if (StatsMode >= NDqProto::DQ_STATS_MODE_PROFILE) {
+ for (auto& task : stats.GetTasks()) {
+ auto* stageStats = GetOrCreateStageStats(task, *TasksGraph, *Result);
+
+ stageStats->SetTotalTasksCount(stageStats->GetTotalTasksCount() + 1);
+ UpdateAggr(stageStats->MutableCpuTimeUs(), task.GetCpuTimeUs());
+ UpdateAggr(stageStats->MutableInputRows(), task.GetInputRows());
+ UpdateAggr(stageStats->MutableInputBytes(), task.GetInputBytes());
+ UpdateAggr(stageStats->MutableOutputRows(), task.GetOutputRows());
+ UpdateAggr(stageStats->MutableOutputBytes(), task.GetOutputBytes());
+
+ UpdateMinMax(stageStats->MutableFirstRowTimeMs(), task.GetFirstRowTimeMs());
+ UpdateMinMax(stageStats->MutableFinishTimeMs(), task.GetFinishTimeMs());
+
+ stageStats->SetDurationUs((stageStats->GetFinishTimeMs().GetMax() - stageStats->GetFirstRowTimeMs().GetMin()) * 1'000);
+
+ stageStats->AddComputeActors()->CopyFrom(stats);
+ }
+ }
+}
+
+void TQueryExecutionStats::AddDatashardPrepareStats(NKikimrQueryStats::TTxStats&& txStats) {
+// Cerr << (TStringBuilder() << "::AddDatashardPrepareStats " << txStats.DebugString() << Endl);
+
+ ui64 cpuUs = txStats.GetComputeCpuTimeUsec();
+ for (const auto& perShard : txStats.GetPerShardStats()) {
+ AffectedShards.emplace(perShard.GetShardId());
+ cpuUs += perShard.GetCpuTimeUsec();
+ }
+
+ Result->SetCpuTimeUs(Result->GetCpuTimeUs() + cpuUs);
+}
+
+void TQueryExecutionStats::AddDatashardStats(NYql::NDqProto::TDqComputeActorStats&& stats,
+ NKikimrQueryStats::TTxStats&& txStats)
+{
+// Cerr << (TStringBuilder() << "::AddDatashardStats " << stats.DebugString() << ", " << txStats.DebugString() << Endl);
+
+ ui64 datashardCpuTimeUs = 0;
+ for (const auto& perShard : txStats.GetPerShardStats()) {
+ AffectedShards.emplace(perShard.GetShardId());
+
+ datashardCpuTimeUs += perShard.GetCpuTimeUsec();
+ UpdateAggr(ExtraStats.MutableShardsCpuTimeUs(), perShard.GetCpuTimeUsec());
+ }
+
+ Result->SetCpuTimeUs(Result->GetCpuTimeUs() + datashardCpuTimeUs);
+ TotalTasks += stats.GetTasks().size();
+
+ for (auto& task : stats.GetTasks()) {
+ for (auto& table : task.GetTables()) {
+ NYql::NDqProto::TDqTableStats* tableAggr = nullptr;
+ if (auto it = TableStats.find(table.GetTablePath()); it != TableStats.end()) {
+ tableAggr = it->second;
+ } else {
+ tableAggr = Result->AddTables();
+ tableAggr->SetTablePath(table.GetTablePath());
+ TableStats.emplace(table.GetTablePath(), tableAggr);
+ }
+
+ tableAggr->SetReadRows(tableAggr->GetReadRows() + table.GetReadRows());
+ tableAggr->SetReadBytes(tableAggr->GetReadBytes() + table.GetReadBytes());
+ tableAggr->SetWriteRows(tableAggr->GetWriteRows() + table.GetWriteRows());
+ tableAggr->SetWriteBytes(tableAggr->GetWriteBytes() + table.GetWriteBytes());
+ tableAggr->SetEraseRows(tableAggr->GetEraseRows() + table.GetEraseRows());
+
+ auto& shards = TableShards[table.GetTablePath()];
+ for (const auto& perShard : txStats.GetPerShardStats()) {
+ shards.insert(perShard.GetShardId());
+ }
+ tableAggr->SetAffectedPartitions(shards.size());
+ }
+ }
+
+ if (StatsMode == NDqProto::DQ_STATS_MODE_PROFILE) {
+ for (auto& task : *stats.MutableTasks()) {
+ auto* stageStats = GetOrCreateStageStats(task, *TasksGraph, *Result);
+
+ stageStats->SetTotalTasksCount(stageStats->GetTotalTasksCount() + 1);
+ UpdateAggr(stageStats->MutableCpuTimeUs(), task.GetCpuTimeUs());
+ UpdateAggr(stageStats->MutableInputRows(), task.GetInputRows());
+ UpdateAggr(stageStats->MutableInputBytes(), task.GetInputBytes());
+ UpdateAggr(stageStats->MutableOutputRows(), task.GetOutputRows());
+ UpdateAggr(stageStats->MutableOutputBytes(), task.GetOutputBytes());
+
+ UpdateMinMax(stageStats->MutableFirstRowTimeMs(), task.GetFirstRowTimeMs());
+ UpdateMinMax(stageStats->MutableFinishTimeMs(), task.GetFinishTimeMs());
+
+ stageStats->SetDurationUs((stageStats->GetFinishTimeMs().GetMax() - stageStats->GetFirstRowTimeMs().GetMin()) * 1'000);
+
+ for (auto& tableStats: task.GetTables()) {
+ auto* tableAggrStats = GetOrCreateTableAggrStats(stageStats, tableStats.GetTablePath());
+
+ UpdateAggr(tableAggrStats->MutableReadRows(), tableStats.GetReadRows());
+ UpdateAggr(tableAggrStats->MutableReadBytes(), tableStats.GetReadBytes());
+ UpdateAggr(tableAggrStats->MutableWriteRows(), tableStats.GetWriteRows());
+ UpdateAggr(tableAggrStats->MutableWriteBytes(), tableStats.GetWriteBytes());
+ UpdateAggr(tableAggrStats->MutableEraseRows(), tableStats.GetEraseRows());
+
+ NKqpProto::TKqpShardTableExtraStats tableExtraStats;
+ if (tableStats.GetExtra().UnpackTo(&tableExtraStats)) {
+ NKqpProto::TKqpShardTableAggrExtraStats tableAggrExtraStats;
+ if (tableAggrStats->HasExtra()) {
+ bool ok = tableAggrStats->MutableExtra()->UnpackTo(&tableAggrExtraStats);
+ YQL_ENSURE(ok);
+ }
+
+ tableAggrExtraStats.SetAffectedShards(TableShards[tableStats.GetTablePath()].size());
+ UpdateAggr(tableAggrExtraStats.MutableShardCpuTimeUs(), datashardCpuTimeUs);
+
+ tableAggrStats->MutableExtra()->PackFrom(tableAggrExtraStats);
+ }
+ }
+
+ NKqpProto::TKqpStageExtraStats stageExtraStats;
+ if (stageStats->HasExtra()) {
+ bool ok = stageStats->GetExtra().UnpackTo(&stageExtraStats);
+ YQL_ENSURE(ok);
+ }
+ stageExtraStats.AddDatashardTasks()->Swap(&task);
+ stageStats->MutableExtra()->PackFrom(stageExtraStats);
+ }
+
+ DatashardStats.emplace_back(std::move(txStats));
+ }
+}
+
+void TQueryExecutionStats::Finish() {
+// Cerr << (TStringBuilder() << "-- finish: executerTime: " << ExecuterCpuTime.MicroSeconds() << Endl);
+
+ Result->SetCpuTimeUs(Result->GetCpuTimeUs() + ExecuterCpuTime.MicroSeconds());
+ Result->SetDurationUs(FinishTs.MicroSeconds() - StartTs.MicroSeconds());
+
+ Result->SetResultBytes(ResultBytes);
+ Result->SetResultRows(ResultRows);
+
+ ExtraStats.SetAffectedShards(AffectedShards.size());
+ Result->MutableExtra()->PackFrom(ExtraStats);
+
+ if (StatsMode >= NYql::NDqProto::DQ_STATS_MODE_PROFILE) {
+ Result->SetExecuterCpuTimeUs(ExecuterCpuTime.MicroSeconds());
+
+ Result->SetStartTimeMs(StartTs.MilliSeconds());
+ Result->SetFinishTimeMs(FinishTs.MilliSeconds());
+ }
+
+ // Cerr << (TStringBuilder() << "TQueryExecutionStats::Finish" << Endl << Result->DebugString() << Endl);
+}
+
TTableStat& TTableStat::operator +=(const TTableStat& rhs) {
Rows += rhs.Rows;
Bytes += rhs.Bytes;
@@ -265,31 +265,31 @@ TTableStat& TTableStat::operator -=(const TTableStat& rhs) {
}
TProgressStatEntry& TProgressStatEntry::operator +=(const TProgressStatEntry& rhs) {
- ComputeTime += rhs.ComputeTime;
- ReadIOStat += rhs.ReadIOStat;
+ ComputeTime += rhs.ComputeTime;
+ ReadIOStat += rhs.ReadIOStat;
return *this;
}
-TTableStat CalcSumTableReadStat(const TProgressStatEntry& entry) {
- return entry.ReadIOStat;
+TTableStat CalcSumTableReadStat(const TProgressStatEntry& entry) {
+ return entry.ReadIOStat;
}
TDuration CalcCumComputeTime(const TProgressStatEntry& entry) {
- return entry.ComputeTime;
+ return entry.ComputeTime;
}
void TProgressStatEntry::Out(IOutputStream& o) const {
- o << "ComputeTime: " << ComputeTime << " ReadRows: " << ReadIOStat.Rows << " ReadBytes: " << ReadIOStat.Bytes;
+ o << "ComputeTime: " << ComputeTime << " ReadRows: " << ReadIOStat.Rows << " ReadBytes: " << ReadIOStat.Bytes;
}
-void TProgressStat::Set(const NDqProto::TDqComputeActorStats& stats) {
- Cur.ComputeTime += TDuration::MicroSeconds(stats.GetCpuTimeUs());
- for (auto& task : stats.GetTasks()) {
- for (auto& table: task.GetTables()) {
- Cur.ReadIOStat.Rows += table.GetReadRows();
- Cur.ReadIOStat.Bytes += table.GetReadBytes();
- }
+void TProgressStat::Set(const NDqProto::TDqComputeActorStats& stats) {
+ Cur.ComputeTime += TDuration::MicroSeconds(stats.GetCpuTimeUs());
+ for (auto& task : stats.GetTasks()) {
+ for (auto& table: task.GetTables()) {
+ Cur.ReadIOStat.Rows += table.GetReadRows();
+ Cur.ReadIOStat.Bytes += table.GetReadBytes();
+ }
}
}
@@ -302,4 +302,4 @@ void TProgressStat::Update() {
Cur = TEntry();
}
-} // namespace NKikimr::NKqp
+} // namespace NKikimr::NKqp
diff --git a/ydb/core/kqp/executer/kqp_executer_stats.h b/ydb/core/kqp/executer/kqp_executer_stats.h
index 915e21c7eb..56c7921318 100644
--- a/ydb/core/kqp/executer/kqp_executer_stats.h
+++ b/ydb/core/kqp/executer/kqp_executer_stats.h
@@ -1,93 +1,93 @@
-#pragma once
-
+#pragma once
+
#include "kqp_tasks_graph.h"
#include <ydb/library/yql/dq/actors/protos/dq_stats.pb.h>
-#include <util/generic/vector.h>
-
-namespace NKikimr {
-namespace NKqp {
-
-struct TQueryExecutionStats {
- const NYql::NDqProto::EDqStatsMode StatsMode;
- const TKqpTasksGraph* const TasksGraph = nullptr;
- NYql::NDqProto::TDqExecutionStats* const Result;
-
- // basic stats
- std::unordered_set<ui64> AffectedShards;
- ui32 TotalTasks = 0;
- ui64 ResultBytes = 0;
- ui64 ResultRows = 0;
- TDuration ExecuterCpuTime;
-
- TInstant StartTs;
- TInstant FinishTs;
-
- std::unordered_map<TString, NYql::NDqProto::TDqTableStats*> TableStats;
- std::unordered_map<TString, std::unordered_set<ui64>> TableShards;
-
- NKqpProto::TKqpExecutionExtraStats ExtraStats;
-
- // profile stats
- TDuration ResolveCpuTime;
- TDuration ResolveWallTime;
- TVector<NKikimrQueryStats::TTxStats> DatashardStats;
-
- TQueryExecutionStats(NYql::NDqProto::EDqStatsMode statsMode, const TKqpTasksGraph* const tasksGraph,
- NYql::NDqProto::TDqExecutionStats* const result)
- : StatsMode(statsMode)
- , TasksGraph(tasksGraph)
- , Result(result)
- {
- YQL_ENSURE(StatsMode >= NYql::NDqProto::DQ_STATS_MODE_BASIC);
- }
-
- void AddComputeActorStats(ui32 nodeId, NYql::NDqProto::TDqComputeActorStats&& stats);
-
- void AddDatashardPrepareStats(NKikimrQueryStats::TTxStats&& txStats);
- void AddDatashardStats(NYql::NDqProto::TDqComputeActorStats&& stats, NKikimrQueryStats::TTxStats&& txStats);
-
- void Finish();
-};
-
+#include <util/generic/vector.h>
+
+namespace NKikimr {
+namespace NKqp {
+
+struct TQueryExecutionStats {
+ const NYql::NDqProto::EDqStatsMode StatsMode;
+ const TKqpTasksGraph* const TasksGraph = nullptr;
+ NYql::NDqProto::TDqExecutionStats* const Result;
+
+ // basic stats
+ std::unordered_set<ui64> AffectedShards;
+ ui32 TotalTasks = 0;
+ ui64 ResultBytes = 0;
+ ui64 ResultRows = 0;
+ TDuration ExecuterCpuTime;
+
+ TInstant StartTs;
+ TInstant FinishTs;
+
+ std::unordered_map<TString, NYql::NDqProto::TDqTableStats*> TableStats;
+ std::unordered_map<TString, std::unordered_set<ui64>> TableShards;
+
+ NKqpProto::TKqpExecutionExtraStats ExtraStats;
+
+ // profile stats
+ TDuration ResolveCpuTime;
+ TDuration ResolveWallTime;
+ TVector<NKikimrQueryStats::TTxStats> DatashardStats;
+
+ TQueryExecutionStats(NYql::NDqProto::EDqStatsMode statsMode, const TKqpTasksGraph* const tasksGraph,
+ NYql::NDqProto::TDqExecutionStats* const result)
+ : StatsMode(statsMode)
+ , TasksGraph(tasksGraph)
+ , Result(result)
+ {
+ YQL_ENSURE(StatsMode >= NYql::NDqProto::DQ_STATS_MODE_BASIC);
+ }
+
+ void AddComputeActorStats(ui32 nodeId, NYql::NDqProto::TDqComputeActorStats&& stats);
+
+ void AddDatashardPrepareStats(NKikimrQueryStats::TTxStats&& txStats);
+ void AddDatashardStats(NYql::NDqProto::TDqComputeActorStats&& stats, NKikimrQueryStats::TTxStats&& txStats);
+
+ void Finish();
+};
+
struct TTableStat {
ui64 Rows = 0;
ui64 Bytes = 0;
-
- TTableStat& operator+=(const TTableStat& rhs);
- TTableStat& operator-=(const TTableStat& rhs);
+
+ TTableStat& operator+=(const TTableStat& rhs);
+ TTableStat& operator-=(const TTableStat& rhs);
};
struct TProgressStatEntry {
- TDuration ComputeTime;
- TTableStat ReadIOStat;
-
- TProgressStatEntry& operator+=(const TProgressStatEntry& rhs);
-
+ TDuration ComputeTime;
+ TTableStat ReadIOStat;
+
+ TProgressStatEntry& operator+=(const TProgressStatEntry& rhs);
+
void Out(IOutputStream& o) const;
};
-TTableStat CalcSumTableReadStat(const TProgressStatEntry& entry);
+TTableStat CalcSumTableReadStat(const TProgressStatEntry& entry);
TDuration CalcCumComputeTime(const TProgressStatEntry& entry);
class TProgressStat {
public:
using TEntry = TProgressStatEntry;
-
+
TProgressStat() = default;
-
- void Set(const NYql::NDqProto::TDqComputeActorStats& stats);
-
+
+ void Set(const NYql::NDqProto::TDqComputeActorStats& stats);
+
void Update();
-
+
TEntry GetLastUsage() const;
-
+
private:
TEntry Total;
TEntry Cur;
};
-} // namespace NKqp
-} // namespace NKikimr
+} // namespace NKqp
+} // namespace NKikimr
template<>
inline void Out<NKikimr::NKqp::TProgressStatEntry>(IOutputStream& o, const NKikimr::NKqp::TProgressStatEntry& x) {
diff --git a/ydb/core/kqp/executer/kqp_literal_executer.cpp b/ydb/core/kqp/executer/kqp_literal_executer.cpp
index ec56b548a2..3aabd41548 100644
--- a/ydb/core/kqp/executer/kqp_literal_executer.cpp
+++ b/ydb/core/kqp/executer/kqp_literal_executer.cpp
@@ -1,416 +1,416 @@
-#include "kqp_executer.h"
-#include "kqp_executer_impl.h"
-
+#include "kqp_executer.h"
+#include "kqp_executer_impl.h"
+
#include <ydb/core/kqp/common/kqp_yql.h>
#include <ydb/core/kqp/rm/kqp_rm.h>
#include <ydb/core/kqp/runtime/kqp_compute.h>
#include <ydb/core/kqp/runtime/kqp_tasks_runner.h>
#include <ydb/core/kqp/runtime/kqp_transport.h>
#include <ydb/core/kqp/prepare/kqp_query_plan.h>
-
-namespace NKikimr {
-namespace NKqp {
-
-using namespace NYql;
-using namespace NYql::NDq;
-
-namespace {
-
-TDqTaskRunnerContext CreateTaskRunnerContext(NMiniKQL::TKqpComputeContextBase* computeCtx, NMiniKQL::TScopedAlloc* alloc,
- NMiniKQL::TTypeEnvironment* typeEnv)
-{
- TDqTaskRunnerContext context;
- context.FuncRegistry = AppData()->FunctionRegistry;
- context.RandomProvider = TAppData::RandomProvider.Get();
- context.TimeProvider = TAppData::TimeProvider.Get();
- context.ComputeCtx = computeCtx;
- context.ComputationFactory = NMiniKQL::GetKqpBaseComputeFactory(computeCtx);
- context.Alloc = alloc;
- context.TypeEnv = typeEnv;
- context.ApplyCtx = nullptr;
- return context;
-}
-
-TDqTaskRunnerSettings CreateTaskRunnerSettings(NDqProto::EDqStatsMode statsMode) {
- TDqTaskRunnerSettings settings;
- settings.CollectBasicStats = statsMode >= NDqProto::DQ_STATS_MODE_BASIC;
- settings.CollectProfileStats = statsMode >= NDqProto::DQ_STATS_MODE_PROFILE;
- settings.OptLLVM = "OFF";
- settings.TerminateOnError = false;
- settings.AllowGeneratorsInUnboxedValues = false;
- return settings;
-}
-
-TDqTaskRunnerMemoryLimits CreateTaskRunnerMemoryLimits() {
- TDqTaskRunnerMemoryLimits memoryLimits;
- memoryLimits.ChannelBufferSize = std::numeric_limits<ui32>::max();
- memoryLimits.OutputChunkMaxSize = std::numeric_limits<ui32>::max();
- return memoryLimits;
-}
-
-TDqTaskRunnerExecutionContext CreateTaskRunnerExecutionContext() {
- return {};
-}
-
-class TKqpLiteralExecuter : public TActorBootstrapped<TKqpLiteralExecuter> {
- using TBase = TActorBootstrapped<TKqpLiteralExecuter>;
-
-public:
- static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
- return NKikimrServices::TActivity::KQP_LITERAL_EXECUTER_ACTOR;
- }
-
-public:
- TKqpLiteralExecuter(IKqpGateway::TExecPhysicalRequest&& request, TKqpRequestCounters::TPtr counters)
- : Request(std::move(request))
- , Counters(counters)
- {
- ResponseEv = std::make_unique<TEvKqpExecuter::TEvTxResponse>();
- if (Request.StatsMode >= NDqProto::DQ_STATS_MODE_BASIC) {
- Stats = std::make_unique<TQueryExecutionStats>(Request.StatsMode, &TasksGraph,
- ResponseEv->Record.MutableResponse()->MutableResult()->MutableStats());
- }
- }
-
- void Bootstrap() {
- StartTime = TAppData::TimeProvider->Now();
- if (Request.Timeout) {
- Deadline = StartTime + Request.Timeout;
- }
- if (Request.CancelAfter) {
- CancelAt = StartTime + *Request.CancelAfter;
- }
-
- LOG_D("Begin literal execution. Operation timeout: " << Request.Timeout << ", cancelAfter: " << Request.CancelAfter);
-
- Become(&TKqpLiteralExecuter::WorkState);
- }
-
-private:
- STATEFN(WorkState) {
- try {
- switch (ev->GetTypeRewrite()) {
- hFunc(TEvKqpExecuter::TEvTxRequest, Handle);
- default: {
- LOG_C("TKqpLiteralExecuter, unexpected event: " << ev->GetTypeRewrite() << ", selfID: " << SelfId());
- InternalError("Unexpected event");
- }
- }
- } catch (const TMemoryLimitExceededException& e) {
+
+namespace NKikimr {
+namespace NKqp {
+
+using namespace NYql;
+using namespace NYql::NDq;
+
+namespace {
+
+TDqTaskRunnerContext CreateTaskRunnerContext(NMiniKQL::TKqpComputeContextBase* computeCtx, NMiniKQL::TScopedAlloc* alloc,
+ NMiniKQL::TTypeEnvironment* typeEnv)
+{
+ TDqTaskRunnerContext context;
+ context.FuncRegistry = AppData()->FunctionRegistry;
+ context.RandomProvider = TAppData::RandomProvider.Get();
+ context.TimeProvider = TAppData::TimeProvider.Get();
+ context.ComputeCtx = computeCtx;
+ context.ComputationFactory = NMiniKQL::GetKqpBaseComputeFactory(computeCtx);
+ context.Alloc = alloc;
+ context.TypeEnv = typeEnv;
+ context.ApplyCtx = nullptr;
+ return context;
+}
+
+TDqTaskRunnerSettings CreateTaskRunnerSettings(NDqProto::EDqStatsMode statsMode) {
+ TDqTaskRunnerSettings settings;
+ settings.CollectBasicStats = statsMode >= NDqProto::DQ_STATS_MODE_BASIC;
+ settings.CollectProfileStats = statsMode >= NDqProto::DQ_STATS_MODE_PROFILE;
+ settings.OptLLVM = "OFF";
+ settings.TerminateOnError = false;
+ settings.AllowGeneratorsInUnboxedValues = false;
+ return settings;
+}
+
+TDqTaskRunnerMemoryLimits CreateTaskRunnerMemoryLimits() {
+ TDqTaskRunnerMemoryLimits memoryLimits;
+ memoryLimits.ChannelBufferSize = std::numeric_limits<ui32>::max();
+ memoryLimits.OutputChunkMaxSize = std::numeric_limits<ui32>::max();
+ return memoryLimits;
+}
+
+TDqTaskRunnerExecutionContext CreateTaskRunnerExecutionContext() {
+ return {};
+}
+
+class TKqpLiteralExecuter : public TActorBootstrapped<TKqpLiteralExecuter> {
+ using TBase = TActorBootstrapped<TKqpLiteralExecuter>;
+
+public:
+ static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
+ return NKikimrServices::TActivity::KQP_LITERAL_EXECUTER_ACTOR;
+ }
+
+public:
+ TKqpLiteralExecuter(IKqpGateway::TExecPhysicalRequest&& request, TKqpRequestCounters::TPtr counters)
+ : Request(std::move(request))
+ , Counters(counters)
+ {
+ ResponseEv = std::make_unique<TEvKqpExecuter::TEvTxResponse>();
+ if (Request.StatsMode >= NDqProto::DQ_STATS_MODE_BASIC) {
+ Stats = std::make_unique<TQueryExecutionStats>(Request.StatsMode, &TasksGraph,
+ ResponseEv->Record.MutableResponse()->MutableResult()->MutableStats());
+ }
+ }
+
+ void Bootstrap() {
+ StartTime = TAppData::TimeProvider->Now();
+ if (Request.Timeout) {
+ Deadline = StartTime + Request.Timeout;
+ }
+ if (Request.CancelAfter) {
+ CancelAt = StartTime + *Request.CancelAfter;
+ }
+
+ LOG_D("Begin literal execution. Operation timeout: " << Request.Timeout << ", cancelAfter: " << Request.CancelAfter);
+
+ Become(&TKqpLiteralExecuter::WorkState);
+ }
+
+private:
+ STATEFN(WorkState) {
+ try {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvKqpExecuter::TEvTxRequest, Handle);
+ default: {
+ LOG_C("TKqpLiteralExecuter, unexpected event: " << ev->GetTypeRewrite() << ", selfID: " << SelfId());
+ InternalError("Unexpected event");
+ }
+ }
+ } catch (const TMemoryLimitExceededException& e) {
LOG_W("TKqpLiteralExecuter, memory limit exceeded.");
ReplyErrorAndDie(Ydb::StatusIds::PRECONDITION_FAILED,
YqlIssue({}, TIssuesIds::KIKIMR_PRECONDITION_FAILED, "Memory limit exceeded"));
- } catch (...) {
- auto msg = CurrentExceptionMessage();
- LOG_C("TKqpLiteralExecuter, unexpected exception caught: " << msg);
- InternalError(TStringBuilder() << "Unexpected exception: " << msg);
- }
- }
-
- void Handle(TEvKqpExecuter::TEvTxRequest::TPtr& ev) {
- if (Stats) {
- Stats->StartTs = TInstant::Now();
- }
-
- TxId = ev->Get()->Record.GetRequest().GetTxId();
- Target = ActorIdFromProto(ev->Get()->Record.GetTarget());
-
- {
- LOG_D("Report self actorId " << SelfId() << " to " << Target);
- auto progressEv = MakeHolder<TEvKqpExecuter::TEvExecuterProgress>();
- ActorIdToProto(SelfId(), progressEv->Record.MutableExecuterActorId());
- Send(Target, progressEv.Release());
- }
-
- LOG_D("Begin literal execution, txs: " << Request.Transactions.size());
-
- FillKqpTasksGraphStages(TasksGraph, Request.Transactions);
-
- for (ui32 txIdx = 0; txIdx < Request.Transactions.size(); ++txIdx) {
- auto& tx = Request.Transactions[txIdx];
-
- for (ui32 stageIdx = 0; stageIdx < tx.Body.StagesSize(); ++stageIdx) {
- auto& stage = tx.Body.GetStages(stageIdx);
- auto& stageInfo = TasksGraph.GetStageInfo(TStageId(txIdx, stageIdx));
- LOG_D("Stage " << stageInfo.Id << " AST: " << stage.GetProgramAst());
-
- YQL_ENSURE(stageInfo.Meta.ShardOperations.empty());
- YQL_ENSURE(stageInfo.InputsCount == 0);
-
- TasksGraph.AddTask(stageInfo);
- }
-
- BuildKqpExecuterResults(tx.Body, Results);
- BuildKqpTaskGraphResultChannels(TasksGraph, tx.Body, txIdx);
- }
-
- if (TerminateIfTimeout()) {
- return;
- }
-
- auto funcRegistry = AppData()->FunctionRegistry;
- NMiniKQL::TScopedAlloc alloc(TAlignedPagePoolCounters(), funcRegistry->SupportsSizedAllocators());
- NMiniKQL::TTypeEnvironment typeEnv(alloc);
- NMiniKQL::TMemoryUsageInfo memInfo("KqpLocalExecuter");
- NMiniKQL::THolderFactory holderFactory(alloc.Ref(), memInfo, funcRegistry);
-
- auto rmConfig = GetKqpResourceManager()->GetConfig();
- ui64 limit = Request.MkqlMemoryLimit > 0
- ? std::min(Request.MkqlMemoryLimit, rmConfig.GetMkqlLightProgramMemoryLimit())
- : rmConfig.GetMkqlLightProgramMemoryLimit();
- alloc.SetLimit(limit);
-
- // task runner settings
- NMiniKQL::TKqpComputeContextBase computeCtx;
- TDqTaskRunnerContext context = CreateTaskRunnerContext(&computeCtx, &alloc, &typeEnv);
- TDqTaskRunnerSettings settings = CreateTaskRunnerSettings(Request.StatsMode);
-
- Y_DEFER {
- // clear allocator state
- Results.crop(0);
- TaskRunners.crop(0);
- };
-
- for (auto& task : TasksGraph.GetTasks()) {
- RunTask(task, context, settings);
-
- if (TerminateIfTimeout()) {
- return;
- }
- }
-
- Finalize(context, holderFactory);
- PassAway();
- }
-
- void RunTask(TTask& task, const TDqTaskRunnerContext& context, const TDqTaskRunnerSettings& settings) {
- auto& stageInfo = TasksGraph.GetStageInfo(task.StageId);
- auto& stage = GetStage(stageInfo);
-
- NDqProto::TDqTask protoTask;
- protoTask.SetId(task.Id);
- protoTask.SetStageId(task.StageId.StageId);
- protoTask.MutableProgram()->CopyFrom(stage.GetProgram()); // it's not good...
-
- TaskId2StageId[task.Id] = task.StageId.StageId;
-
- for (auto& output : task.Outputs) {
- YQL_ENSURE(output.Type == TTaskOutputType::Map, "" << output.Type);
- YQL_ENSURE(output.Channels.size() == 1);
-
- auto* protoOutput = protoTask.AddOutputs();
- protoOutput->MutableMap();
-
- auto& resultChannel = TasksGraph.GetChannel(output.Channels[0]);
- auto* protoResultChannel = protoOutput->AddChannels();
-
- protoResultChannel->SetId(resultChannel.Id);
- protoResultChannel->SetSrcTaskId(resultChannel.SrcTask);
- protoResultChannel->SetDstTaskId(resultChannel.DstTask);
- protoResultChannel->SetInMemory(true);
-
- YQL_ENSURE(resultChannel.SrcTask != 0);
- YQL_ENSURE(resultChannel.DstTask == 0);
- }
-
- auto parameterProvider = [&task, &stageInfo](std::string_view name, NMiniKQL::TType* type,
- const NMiniKQL::TTypeEnvironment& typeEnv, const NMiniKQL::THolderFactory& holderFactory,
- NUdf::TUnboxedValue& value)
- {
- if (auto* data = task.Meta.Params.FindPtr(name)) {
- TDqDataSerializer::DeserializeParam(*data, type, holderFactory, value);
- return true;
- }
-
- if (auto* param = stageInfo.Meta.Tx.Params.Values.FindPtr(name)) {
+ } catch (...) {
+ auto msg = CurrentExceptionMessage();
+ LOG_C("TKqpLiteralExecuter, unexpected exception caught: " << msg);
+ InternalError(TStringBuilder() << "Unexpected exception: " << msg);
+ }
+ }
+
+ void Handle(TEvKqpExecuter::TEvTxRequest::TPtr& ev) {
+ if (Stats) {
+ Stats->StartTs = TInstant::Now();
+ }
+
+ TxId = ev->Get()->Record.GetRequest().GetTxId();
+ Target = ActorIdFromProto(ev->Get()->Record.GetTarget());
+
+ {
+ LOG_D("Report self actorId " << SelfId() << " to " << Target);
+ auto progressEv = MakeHolder<TEvKqpExecuter::TEvExecuterProgress>();
+ ActorIdToProto(SelfId(), progressEv->Record.MutableExecuterActorId());
+ Send(Target, progressEv.Release());
+ }
+
+ LOG_D("Begin literal execution, txs: " << Request.Transactions.size());
+
+ FillKqpTasksGraphStages(TasksGraph, Request.Transactions);
+
+ for (ui32 txIdx = 0; txIdx < Request.Transactions.size(); ++txIdx) {
+ auto& tx = Request.Transactions[txIdx];
+
+ for (ui32 stageIdx = 0; stageIdx < tx.Body.StagesSize(); ++stageIdx) {
+ auto& stage = tx.Body.GetStages(stageIdx);
+ auto& stageInfo = TasksGraph.GetStageInfo(TStageId(txIdx, stageIdx));
+ LOG_D("Stage " << stageInfo.Id << " AST: " << stage.GetProgramAst());
+
+ YQL_ENSURE(stageInfo.Meta.ShardOperations.empty());
+ YQL_ENSURE(stageInfo.InputsCount == 0);
+
+ TasksGraph.AddTask(stageInfo);
+ }
+
+ BuildKqpExecuterResults(tx.Body, Results);
+ BuildKqpTaskGraphResultChannels(TasksGraph, tx.Body, txIdx);
+ }
+
+ if (TerminateIfTimeout()) {
+ return;
+ }
+
+ auto funcRegistry = AppData()->FunctionRegistry;
+ NMiniKQL::TScopedAlloc alloc(TAlignedPagePoolCounters(), funcRegistry->SupportsSizedAllocators());
+ NMiniKQL::TTypeEnvironment typeEnv(alloc);
+ NMiniKQL::TMemoryUsageInfo memInfo("KqpLocalExecuter");
+ NMiniKQL::THolderFactory holderFactory(alloc.Ref(), memInfo, funcRegistry);
+
+ auto rmConfig = GetKqpResourceManager()->GetConfig();
+ ui64 limit = Request.MkqlMemoryLimit > 0
+ ? std::min(Request.MkqlMemoryLimit, rmConfig.GetMkqlLightProgramMemoryLimit())
+ : rmConfig.GetMkqlLightProgramMemoryLimit();
+ alloc.SetLimit(limit);
+
+ // task runner settings
+ NMiniKQL::TKqpComputeContextBase computeCtx;
+ TDqTaskRunnerContext context = CreateTaskRunnerContext(&computeCtx, &alloc, &typeEnv);
+ TDqTaskRunnerSettings settings = CreateTaskRunnerSettings(Request.StatsMode);
+
+ Y_DEFER {
+ // clear allocator state
+ Results.crop(0);
+ TaskRunners.crop(0);
+ };
+
+ for (auto& task : TasksGraph.GetTasks()) {
+ RunTask(task, context, settings);
+
+ if (TerminateIfTimeout()) {
+ return;
+ }
+ }
+
+ Finalize(context, holderFactory);
+ PassAway();
+ }
+
+ void RunTask(TTask& task, const TDqTaskRunnerContext& context, const TDqTaskRunnerSettings& settings) {
+ auto& stageInfo = TasksGraph.GetStageInfo(task.StageId);
+ auto& stage = GetStage(stageInfo);
+
+ NDqProto::TDqTask protoTask;
+ protoTask.SetId(task.Id);
+ protoTask.SetStageId(task.StageId.StageId);
+ protoTask.MutableProgram()->CopyFrom(stage.GetProgram()); // it's not good...
+
+ TaskId2StageId[task.Id] = task.StageId.StageId;
+
+ for (auto& output : task.Outputs) {
+ YQL_ENSURE(output.Type == TTaskOutputType::Map, "" << output.Type);
+ YQL_ENSURE(output.Channels.size() == 1);
+
+ auto* protoOutput = protoTask.AddOutputs();
+ protoOutput->MutableMap();
+
+ auto& resultChannel = TasksGraph.GetChannel(output.Channels[0]);
+ auto* protoResultChannel = protoOutput->AddChannels();
+
+ protoResultChannel->SetId(resultChannel.Id);
+ protoResultChannel->SetSrcTaskId(resultChannel.SrcTask);
+ protoResultChannel->SetDstTaskId(resultChannel.DstTask);
+ protoResultChannel->SetInMemory(true);
+
+ YQL_ENSURE(resultChannel.SrcTask != 0);
+ YQL_ENSURE(resultChannel.DstTask == 0);
+ }
+
+ auto parameterProvider = [&task, &stageInfo](std::string_view name, NMiniKQL::TType* type,
+ const NMiniKQL::TTypeEnvironment& typeEnv, const NMiniKQL::THolderFactory& holderFactory,
+ NUdf::TUnboxedValue& value)
+ {
+ if (auto* data = task.Meta.Params.FindPtr(name)) {
+ TDqDataSerializer::DeserializeParam(*data, type, holderFactory, value);
+ return true;
+ }
+
+ if (auto* param = stageInfo.Meta.Tx.Params.Values.FindPtr(name)) {
NMiniKQL::TType* typeFromProto;
std::tie(typeFromProto, value) = ImportValueFromProto(param->GetType(), param->GetValue(), typeEnv, holderFactory);
-#ifndef NDEBUG
- YQL_ENSURE(ToString(*type) == ToString(*typeFromProto), "" << *type << " != " << *typeFromProto);
-#else
+#ifndef NDEBUG
+ YQL_ENSURE(ToString(*type) == ToString(*typeFromProto), "" << *type << " != " << *typeFromProto);
+#else
Y_UNUSED(typeFromProto);
-#endif
- return true;
- }
-
- return false;
- };
-
- auto log = [as = TlsActivationContext->ActorSystem(), txId = TxId, taskId = task.Id](const TString& message) {
- LOG_DEBUG_S(*as, NKikimrServices::KQP_TASKS_RUNNER, "TxId: " << txId << ", task: " << taskId << ". "
- << message);
- };
-
- auto taskRunner = CreateKqpTaskRunner(context, settings, log);
- TaskRunners.emplace_back(taskRunner);
- taskRunner->Prepare(protoTask, CreateTaskRunnerMemoryLimits(), CreateTaskRunnerExecutionContext(),
- parameterProvider);
-
- auto status = taskRunner->Run();
- YQL_ENSURE(status == ERunStatus::Finished);
-
- for (auto& taskOutput : task.Outputs) {
- for (ui64 outputChannelId : taskOutput.Channels) {
- auto outputChannel = taskRunner->GetOutputChannel(outputChannelId);
- auto& channelDesc = TasksGraph.GetChannel(outputChannelId);
-
- outputChannel->PopAll(Results[channelDesc.DstInputIndex].Rows);
- YQL_ENSURE(outputChannel->IsFinished());
- }
- }
- }
-
- void Finalize(const TDqTaskRunnerContext& context, NMiniKQL::THolderFactory& holderFactory) {
- auto& response = *ResponseEv->Record.MutableResponse();
-
- response.SetStatus(Ydb::StatusIds::SUCCESS);
- Counters->TxProxyMon->ReportStatusOK->Inc();
-
- ui64 rows = 0;
- ui64 bytes = 0;
-
- TKqpProtoBuilder protoBuilder(context.Alloc, context.TypeEnv, &holderFactory);
- for (auto& result : Results) {
- rows += result.Rows.size();
- auto* protoResult = response.MutableResult()->AddResults();
- if (result.IsStream) {
- protoBuilder.BuildStream(result.Rows, result.ItemType, result.ResultItemType.Get(), protoResult);
- } else {
- protoBuilder.BuildValue(result.Rows, result.ItemType, protoResult);
- }
- bytes += protoResult->ByteSizeLong();
- }
-
- if (Stats) {
- ui64 elapsedMicros = TlsActivationContext->GetCurrentEventTicksAsSeconds() * 1'000'000;
- TDuration executerCpuTime = TDuration::MicroSeconds(elapsedMicros);
-
- NYql::NDqProto::TDqComputeActorStats fakeComputeActorStats;
-
- for (auto& taskRunner : TaskRunners) {
- auto* stats = taskRunner->GetStats();
- auto taskCpuTime = stats->BuildCpuTime + stats->ComputeCpuTime;
- executerCpuTime -= taskCpuTime;
- NYql::NDq::FillTaskRunnerStats(taskRunner->GetTaskId(), TaskId2StageId[taskRunner->GetTaskId()],
- *stats, fakeComputeActorStats.AddTasks(), Request.StatsMode >= NYql::NDqProto::DQ_STATS_MODE_PROFILE);
- fakeComputeActorStats.SetCpuTimeUs(fakeComputeActorStats.GetCpuTimeUs() + taskCpuTime.MicroSeconds());
- }
-
- fakeComputeActorStats.SetDurationUs(elapsedMicros);
-
- Stats->AddComputeActorStats(SelfId().NodeId(), std::move(fakeComputeActorStats));
-
- Stats->ExecuterCpuTime = executerCpuTime;
- Stats->FinishTs = Stats->StartTs + TDuration::MicroSeconds(elapsedMicros);
- Stats->ResultRows = rows;
- Stats->ResultBytes = bytes;
-
- Stats->Finish();
-
- if (Y_UNLIKELY(Request.StatsMode >= NDqProto::DQ_STATS_MODE_PROFILE)) {
- for (ui32 txId = 0; txId < Request.Transactions.size(); ++txId) {
- const auto& tx = Request.Transactions[txId].Body;
- auto planWithStats = AddExecStatsToTxPlan(tx.GetPlan(), response.GetResult().GetStats());
- response.MutableResult()->MutableStats()->AddTxPlansWithStats(planWithStats);
- }
- }
- }
-
- LOG_D("Sending response to: " << Target << ", results: " << Results.size());
- Send(Target, ResponseEv.release());
- }
-
-private:
- bool TerminateIfTimeout() {
- auto now = AppData()->TimeProvider->Now();
-
- if (Deadline && *Deadline <= now) {
- LOG_I("Timeout exceeded. Send timeout event to the rpc actor " << Target);
-
- ReplyErrorAndDie(Ydb::StatusIds::TIMEOUT,
- YqlIssue({}, TIssuesIds::KIKIMR_TIMEOUT, "Request timeout exceeded."));
- return true;
- }
-
- if (CancelAt && *CancelAt <= now) {
- LOG_I("CancelAt exceeded. Send cancel event to the rpc actor " << Target);
-
- ReplyErrorAndDie(Ydb::StatusIds::CANCELLED,
- YqlIssue({}, TIssuesIds::KIKIMR_OPERATION_CANCELLED, "Request timeout exceeded."));
- return true;
- }
-
- return false;
- }
-
-private:
- void InternalError(const TString& message) {
- LOG_E(message);
+#endif
+ return true;
+ }
+
+ return false;
+ };
+
+ auto log = [as = TlsActivationContext->ActorSystem(), txId = TxId, taskId = task.Id](const TString& message) {
+ LOG_DEBUG_S(*as, NKikimrServices::KQP_TASKS_RUNNER, "TxId: " << txId << ", task: " << taskId << ". "
+ << message);
+ };
+
+ auto taskRunner = CreateKqpTaskRunner(context, settings, log);
+ TaskRunners.emplace_back(taskRunner);
+ taskRunner->Prepare(protoTask, CreateTaskRunnerMemoryLimits(), CreateTaskRunnerExecutionContext(),
+ parameterProvider);
+
+ auto status = taskRunner->Run();
+ YQL_ENSURE(status == ERunStatus::Finished);
+
+ for (auto& taskOutput : task.Outputs) {
+ for (ui64 outputChannelId : taskOutput.Channels) {
+ auto outputChannel = taskRunner->GetOutputChannel(outputChannelId);
+ auto& channelDesc = TasksGraph.GetChannel(outputChannelId);
+
+ outputChannel->PopAll(Results[channelDesc.DstInputIndex].Rows);
+ YQL_ENSURE(outputChannel->IsFinished());
+ }
+ }
+ }
+
+ void Finalize(const TDqTaskRunnerContext& context, NMiniKQL::THolderFactory& holderFactory) {
+ auto& response = *ResponseEv->Record.MutableResponse();
+
+ response.SetStatus(Ydb::StatusIds::SUCCESS);
+ Counters->TxProxyMon->ReportStatusOK->Inc();
+
+ ui64 rows = 0;
+ ui64 bytes = 0;
+
+ TKqpProtoBuilder protoBuilder(context.Alloc, context.TypeEnv, &holderFactory);
+ for (auto& result : Results) {
+ rows += result.Rows.size();
+ auto* protoResult = response.MutableResult()->AddResults();
+ if (result.IsStream) {
+ protoBuilder.BuildStream(result.Rows, result.ItemType, result.ResultItemType.Get(), protoResult);
+ } else {
+ protoBuilder.BuildValue(result.Rows, result.ItemType, protoResult);
+ }
+ bytes += protoResult->ByteSizeLong();
+ }
+
+ if (Stats) {
+ ui64 elapsedMicros = TlsActivationContext->GetCurrentEventTicksAsSeconds() * 1'000'000;
+ TDuration executerCpuTime = TDuration::MicroSeconds(elapsedMicros);
+
+ NYql::NDqProto::TDqComputeActorStats fakeComputeActorStats;
+
+ for (auto& taskRunner : TaskRunners) {
+ auto* stats = taskRunner->GetStats();
+ auto taskCpuTime = stats->BuildCpuTime + stats->ComputeCpuTime;
+ executerCpuTime -= taskCpuTime;
+ NYql::NDq::FillTaskRunnerStats(taskRunner->GetTaskId(), TaskId2StageId[taskRunner->GetTaskId()],
+ *stats, fakeComputeActorStats.AddTasks(), Request.StatsMode >= NYql::NDqProto::DQ_STATS_MODE_PROFILE);
+ fakeComputeActorStats.SetCpuTimeUs(fakeComputeActorStats.GetCpuTimeUs() + taskCpuTime.MicroSeconds());
+ }
+
+ fakeComputeActorStats.SetDurationUs(elapsedMicros);
+
+ Stats->AddComputeActorStats(SelfId().NodeId(), std::move(fakeComputeActorStats));
+
+ Stats->ExecuterCpuTime = executerCpuTime;
+ Stats->FinishTs = Stats->StartTs + TDuration::MicroSeconds(elapsedMicros);
+ Stats->ResultRows = rows;
+ Stats->ResultBytes = bytes;
+
+ Stats->Finish();
+
+ if (Y_UNLIKELY(Request.StatsMode >= NDqProto::DQ_STATS_MODE_PROFILE)) {
+ for (ui32 txId = 0; txId < Request.Transactions.size(); ++txId) {
+ const auto& tx = Request.Transactions[txId].Body;
+ auto planWithStats = AddExecStatsToTxPlan(tx.GetPlan(), response.GetResult().GetStats());
+ response.MutableResult()->MutableStats()->AddTxPlansWithStats(planWithStats);
+ }
+ }
+ }
+
+ LOG_D("Sending response to: " << Target << ", results: " << Results.size());
+ Send(Target, ResponseEv.release());
+ }
+
+private:
+ bool TerminateIfTimeout() {
+ auto now = AppData()->TimeProvider->Now();
+
+ if (Deadline && *Deadline <= now) {
+ LOG_I("Timeout exceeded. Send timeout event to the rpc actor " << Target);
+
+ ReplyErrorAndDie(Ydb::StatusIds::TIMEOUT,
+ YqlIssue({}, TIssuesIds::KIKIMR_TIMEOUT, "Request timeout exceeded."));
+ return true;
+ }
+
+ if (CancelAt && *CancelAt <= now) {
+ LOG_I("CancelAt exceeded. Send cancel event to the rpc actor " << Target);
+
+ ReplyErrorAndDie(Ydb::StatusIds::CANCELLED,
+ YqlIssue({}, TIssuesIds::KIKIMR_OPERATION_CANCELLED, "Request timeout exceeded."));
+ return true;
+ }
+
+ return false;
+ }
+
+private:
+ void InternalError(const TString& message) {
+ LOG_E(message);
auto issue = NYql::YqlIssue({}, NYql::TIssuesIds::UNEXPECTED, "Internal error while executing transaction.");
- issue.AddSubIssue(MakeIntrusive<TIssue>(message));
- ReplyErrorAndDie(Ydb::StatusIds::INTERNAL_ERROR, issue);
- }
-
- void ReplyErrorAndDie(Ydb::StatusIds::StatusCode status, const TIssue& issue) {
- google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage> issues;
- IssueToMessage(issue, issues.Add());
- ReplyErrorAndDie(status, &issues);
- }
-
- void ReplyErrorAndDie(Ydb::StatusIds::StatusCode status,
- google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage>* issues)
- {
- if (status != Ydb::StatusIds::SUCCESS) {
- Counters->TxProxyMon->ReportStatusNotOK->Inc();
- } else {
- Counters->TxProxyMon->ReportStatusOK->Inc();
- }
-
- if (Stats) {
- ui64 elapsedMicros = TlsActivationContext->GetCurrentEventTicksAsSeconds() * 1'000'000;
- Stats->ExecuterCpuTime += TDuration::MicroSeconds(elapsedMicros);
- }
-
- // TODO: fill stats
-
- auto& response = *ResponseEv->Record.MutableResponse();
-
- response.SetStatus(status);
- response.MutableIssues()->Swap(issues);
-
- Send(Target, ResponseEv.release());
- PassAway();
- }
-
- void PassAway() override {
- auto totalTime = TInstant::Now() - StartTime;
- Counters->Counters->LiteralTxTotalTimeHistogram->Collect(totalTime.MilliSeconds());
-
- TBase::PassAway();
- }
-
-private:
- IKqpGateway::TExecPhysicalRequest Request;
- TKqpRequestCounters::TPtr Counters;
- TInstant StartTime;
- std::unique_ptr<TQueryExecutionStats> Stats;
- TMaybe<TInstant> Deadline;
- TMaybe<TInstant> CancelAt;
- TActorId Target;
- ui64 TxId = 0;
- TKqpTasksGraph TasksGraph;
- TVector<TKqpExecuterTxResult> Results;
- TVector<TIntrusivePtr<IDqTaskRunner>> TaskRunners;
- std::unordered_map<ui64, ui32> TaskId2StageId;
- std::unique_ptr<TEvKqpExecuter::TEvTxResponse> ResponseEv;
-};
-
-} // anonymous namespace
-
-IActor* CreateKqpLiteralExecuter(IKqpGateway::TExecPhysicalRequest&& request, TKqpRequestCounters::TPtr counters) {
- return new TKqpLiteralExecuter(std::move(request), counters);
-}
-
-} // namespace NKqp
-} // namespace NKikimr
-
+ issue.AddSubIssue(MakeIntrusive<TIssue>(message));
+ ReplyErrorAndDie(Ydb::StatusIds::INTERNAL_ERROR, issue);
+ }
+
+ void ReplyErrorAndDie(Ydb::StatusIds::StatusCode status, const TIssue& issue) {
+ google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage> issues;
+ IssueToMessage(issue, issues.Add());
+ ReplyErrorAndDie(status, &issues);
+ }
+
+ void ReplyErrorAndDie(Ydb::StatusIds::StatusCode status,
+ google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage>* issues)
+ {
+ if (status != Ydb::StatusIds::SUCCESS) {
+ Counters->TxProxyMon->ReportStatusNotOK->Inc();
+ } else {
+ Counters->TxProxyMon->ReportStatusOK->Inc();
+ }
+
+ if (Stats) {
+ ui64 elapsedMicros = TlsActivationContext->GetCurrentEventTicksAsSeconds() * 1'000'000;
+ Stats->ExecuterCpuTime += TDuration::MicroSeconds(elapsedMicros);
+ }
+
+ // TODO: fill stats
+
+ auto& response = *ResponseEv->Record.MutableResponse();
+
+ response.SetStatus(status);
+ response.MutableIssues()->Swap(issues);
+
+ Send(Target, ResponseEv.release());
+ PassAway();
+ }
+
+ void PassAway() override {
+ auto totalTime = TInstant::Now() - StartTime;
+ Counters->Counters->LiteralTxTotalTimeHistogram->Collect(totalTime.MilliSeconds());
+
+ TBase::PassAway();
+ }
+
+private:
+ IKqpGateway::TExecPhysicalRequest Request;
+ TKqpRequestCounters::TPtr Counters;
+ TInstant StartTime;
+ std::unique_ptr<TQueryExecutionStats> Stats;
+ TMaybe<TInstant> Deadline;
+ TMaybe<TInstant> CancelAt;
+ TActorId Target;
+ ui64 TxId = 0;
+ TKqpTasksGraph TasksGraph;
+ TVector<TKqpExecuterTxResult> Results;
+ TVector<TIntrusivePtr<IDqTaskRunner>> TaskRunners;
+ std::unordered_map<ui64, ui32> TaskId2StageId;
+ std::unique_ptr<TEvKqpExecuter::TEvTxResponse> ResponseEv;
+};
+
+} // anonymous namespace
+
+IActor* CreateKqpLiteralExecuter(IKqpGateway::TExecPhysicalRequest&& request, TKqpRequestCounters::TPtr counters) {
+ return new TKqpLiteralExecuter(std::move(request), counters);
+}
+
+} // namespace NKqp
+} // namespace NKikimr
+
diff --git a/ydb/core/kqp/executer/kqp_locks_helper.cpp b/ydb/core/kqp/executer/kqp_locks_helper.cpp
index 17b00c85ba..cb1178b74f 100644
--- a/ydb/core/kqp/executer/kqp_locks_helper.cpp
+++ b/ydb/core/kqp/executer/kqp_locks_helper.cpp
@@ -1,77 +1,77 @@
-#include "kqp_locks_helper.h"
-
+#include "kqp_locks_helper.h"
+
#include <ydb/library/yql/minikql/mkql_node.h>
#include <ydb/library/yql/utils/yql_panic.h>
-
-
-namespace NKikimr::NKqp {
-
-void BuildLocks(NKikimrMiniKQL::TResult& result, const TVector<NKikimrTxDataShard::TLock>& locks) {
- auto setMemberDataType = [] (NKikimrMiniKQL::TMember& member, const TString& name, ui32 scheme) {
- member.SetName(name);
- member.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data);
- member.MutableType()->MutableData()->SetScheme(scheme);
- };
-
- auto& type = *result.MutableType();
- type.SetKind(NKikimrMiniKQL::ETypeKind::List);
- auto& itemType = *type.MutableList()->MutableItem();
- itemType.SetKind(NKikimrMiniKQL::ETypeKind::Struct);
- auto& structType = *itemType.MutableStruct();
- setMemberDataType(*structType.AddMember(), "Counter", NKikimr::NUdf::TDataType<ui64>::Id);
- setMemberDataType(*structType.AddMember(), "DataShard", NKikimr::NUdf::TDataType<ui64>::Id);
- setMemberDataType(*structType.AddMember(), "Generation", NKikimr::NUdf::TDataType<ui32>::Id);
- setMemberDataType(*structType.AddMember(), "LockId", NKikimr::NUdf::TDataType<ui64>::Id);
- setMemberDataType(*structType.AddMember(), "PathId", NKikimr::NUdf::TDataType<ui64>::Id);
- setMemberDataType(*structType.AddMember(), "SchemeShard", NKikimr::NUdf::TDataType<ui64>::Id);
-
- auto& value = *result.MutableValue();
- for (auto& lock : locks) {
- auto& item = *value.AddList();
- item.AddStruct()->SetUint64(lock.GetCounter());
- item.AddStruct()->SetUint64(lock.GetDataShard());
- item.AddStruct()->SetUint32(lock.GetGeneration());
- item.AddStruct()->SetUint64(lock.GetLockId());
- item.AddStruct()->SetUint64(lock.GetPathId());
- item.AddStruct()->SetUint64(lock.GetSchemeShard());
- }
-}
-
-TMap<ui64, TVector<NKikimrTxDataShard::TLock>> ExtractLocks(const TVector<NYql::NDq::TMkqlValueRef>& locks) {
- auto ensureMemberDataType = [] (const NKikimrMiniKQL::TMember& member, const TString& name, ui32 scheme) {
- YQL_ENSURE(member.GetName() == name);
- YQL_ENSURE(member.GetType().GetKind() == NKikimrMiniKQL::ETypeKind::Data);
- YQL_ENSURE(member.GetType().GetData().GetScheme() == scheme);
- };
-
- TMap<ui64, TVector<NKikimrTxDataShard::TLock>> locksMap;
- for (auto& lock : locks) {
- const auto& type = lock.GetType();
- const auto& value = lock.GetValue();
-
- YQL_ENSURE(type.GetKind() == NKikimrMiniKQL::ETypeKind::Struct);
- auto& structType = type.GetStruct();
-
- YQL_ENSURE(structType.MemberSize() == 6);
- ensureMemberDataType(structType.GetMember(0), "Counter", NKikimr::NUdf::TDataType<ui64>::Id);
- ensureMemberDataType(structType.GetMember(1), "DataShard", NKikimr::NUdf::TDataType<ui64>::Id);
- ensureMemberDataType(structType.GetMember(2), "Generation", NKikimr::NUdf::TDataType<ui32>::Id);
- ensureMemberDataType(structType.GetMember(3), "LockId", NKikimr::NUdf::TDataType<ui64>::Id);
- ensureMemberDataType(structType.GetMember(4), "PathId", NKikimr::NUdf::TDataType<ui64>::Id);
- ensureMemberDataType(structType.GetMember(5), "SchemeShard", NKikimr::NUdf::TDataType<ui64>::Id);
-
- NKikimrTxDataShard::TLock dsLock;
- dsLock.SetCounter(value.GetStruct(0).GetUint64());
- dsLock.SetDataShard(value.GetStruct(1).GetUint64());
- dsLock.SetGeneration(value.GetStruct(2).GetUint32());
- dsLock.SetLockId(value.GetStruct(3).GetUint64());
- dsLock.SetPathId(value.GetStruct(4).GetUint64());
- dsLock.SetSchemeShard(value.GetStruct(5).GetUint64());
-
- locksMap[dsLock.GetDataShard()].emplace_back(std::move(dsLock));
- }
-
- return locksMap;
-}
-
-} // namespace NKikimr::NKqp
+
+
+namespace NKikimr::NKqp {
+
+void BuildLocks(NKikimrMiniKQL::TResult& result, const TVector<NKikimrTxDataShard::TLock>& locks) {
+ auto setMemberDataType = [] (NKikimrMiniKQL::TMember& member, const TString& name, ui32 scheme) {
+ member.SetName(name);
+ member.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data);
+ member.MutableType()->MutableData()->SetScheme(scheme);
+ };
+
+ auto& type = *result.MutableType();
+ type.SetKind(NKikimrMiniKQL::ETypeKind::List);
+ auto& itemType = *type.MutableList()->MutableItem();
+ itemType.SetKind(NKikimrMiniKQL::ETypeKind::Struct);
+ auto& structType = *itemType.MutableStruct();
+ setMemberDataType(*structType.AddMember(), "Counter", NKikimr::NUdf::TDataType<ui64>::Id);
+ setMemberDataType(*structType.AddMember(), "DataShard", NKikimr::NUdf::TDataType<ui64>::Id);
+ setMemberDataType(*structType.AddMember(), "Generation", NKikimr::NUdf::TDataType<ui32>::Id);
+ setMemberDataType(*structType.AddMember(), "LockId", NKikimr::NUdf::TDataType<ui64>::Id);
+ setMemberDataType(*structType.AddMember(), "PathId", NKikimr::NUdf::TDataType<ui64>::Id);
+ setMemberDataType(*structType.AddMember(), "SchemeShard", NKikimr::NUdf::TDataType<ui64>::Id);
+
+ auto& value = *result.MutableValue();
+ for (auto& lock : locks) {
+ auto& item = *value.AddList();
+ item.AddStruct()->SetUint64(lock.GetCounter());
+ item.AddStruct()->SetUint64(lock.GetDataShard());
+ item.AddStruct()->SetUint32(lock.GetGeneration());
+ item.AddStruct()->SetUint64(lock.GetLockId());
+ item.AddStruct()->SetUint64(lock.GetPathId());
+ item.AddStruct()->SetUint64(lock.GetSchemeShard());
+ }
+}
+
+TMap<ui64, TVector<NKikimrTxDataShard::TLock>> ExtractLocks(const TVector<NYql::NDq::TMkqlValueRef>& locks) {
+ auto ensureMemberDataType = [] (const NKikimrMiniKQL::TMember& member, const TString& name, ui32 scheme) {
+ YQL_ENSURE(member.GetName() == name);
+ YQL_ENSURE(member.GetType().GetKind() == NKikimrMiniKQL::ETypeKind::Data);
+ YQL_ENSURE(member.GetType().GetData().GetScheme() == scheme);
+ };
+
+ TMap<ui64, TVector<NKikimrTxDataShard::TLock>> locksMap;
+ for (auto& lock : locks) {
+ const auto& type = lock.GetType();
+ const auto& value = lock.GetValue();
+
+ YQL_ENSURE(type.GetKind() == NKikimrMiniKQL::ETypeKind::Struct);
+ auto& structType = type.GetStruct();
+
+ YQL_ENSURE(structType.MemberSize() == 6);
+ ensureMemberDataType(structType.GetMember(0), "Counter", NKikimr::NUdf::TDataType<ui64>::Id);
+ ensureMemberDataType(structType.GetMember(1), "DataShard", NKikimr::NUdf::TDataType<ui64>::Id);
+ ensureMemberDataType(structType.GetMember(2), "Generation", NKikimr::NUdf::TDataType<ui32>::Id);
+ ensureMemberDataType(structType.GetMember(3), "LockId", NKikimr::NUdf::TDataType<ui64>::Id);
+ ensureMemberDataType(structType.GetMember(4), "PathId", NKikimr::NUdf::TDataType<ui64>::Id);
+ ensureMemberDataType(structType.GetMember(5), "SchemeShard", NKikimr::NUdf::TDataType<ui64>::Id);
+
+ NKikimrTxDataShard::TLock dsLock;
+ dsLock.SetCounter(value.GetStruct(0).GetUint64());
+ dsLock.SetDataShard(value.GetStruct(1).GetUint64());
+ dsLock.SetGeneration(value.GetStruct(2).GetUint32());
+ dsLock.SetLockId(value.GetStruct(3).GetUint64());
+ dsLock.SetPathId(value.GetStruct(4).GetUint64());
+ dsLock.SetSchemeShard(value.GetStruct(5).GetUint64());
+
+ locksMap[dsLock.GetDataShard()].emplace_back(std::move(dsLock));
+ }
+
+ return locksMap;
+}
+
+} // namespace NKikimr::NKqp
diff --git a/ydb/core/kqp/executer/kqp_locks_helper.h b/ydb/core/kqp/executer/kqp_locks_helper.h
index 6ea8299c59..1e8b175219 100644
--- a/ydb/core/kqp/executer/kqp_locks_helper.h
+++ b/ydb/core/kqp/executer/kqp_locks_helper.h
@@ -1,15 +1,15 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/mkql_proto/protos/minikql.pb.h>
#include <ydb/core/protos/tx_datashard.pb.h>
-
+
#include <ydb/library/yql/dq/common/dq_value.h>
-
-
-namespace NKikimr::NKqp {
-
-void BuildLocks(NKikimrMiniKQL::TResult& result, const TVector<NKikimrTxDataShard::TLock>& locks);
-
-TMap<ui64, TVector<NKikimrTxDataShard::TLock>> ExtractLocks(const TVector<NYql::NDq::TMkqlValueRef>& locks);
-
-} // namespace NKikimr::NKqp
+
+
+namespace NKikimr::NKqp {
+
+void BuildLocks(NKikimrMiniKQL::TResult& result, const TVector<NKikimrTxDataShard::TLock>& locks);
+
+TMap<ui64, TVector<NKikimrTxDataShard::TLock>> ExtractLocks(const TVector<NYql::NDq::TMkqlValueRef>& locks);
+
+} // namespace NKikimr::NKqp
diff --git a/ydb/core/kqp/executer/kqp_partition_helper.cpp b/ydb/core/kqp/executer/kqp_partition_helper.cpp
index 5018c39de4..ac786f6c24 100644
--- a/ydb/core/kqp/executer/kqp_partition_helper.cpp
+++ b/ydb/core/kqp/executer/kqp_partition_helper.cpp
@@ -1,222 +1,222 @@
-#include "kqp_partition_helper.h"
-#include "kqp_table_resolver.h"
-
+#include "kqp_partition_helper.h"
+#include "kqp_table_resolver.h"
+
#include <ydb/core/base/appdata.h>
#include <ydb/core/tx/datashard/range_ops.h>
#include <ydb/library/mkql_proto/mkql_proto.h>
-
+
#include <ydb/library/yql/dq/runtime/dq_columns_resolve.h>
#include <ydb/library/yql/dq/runtime/dq_transport.h>
#include <ydb/library/yql/utils/log/log.h>
-
-namespace NKikimr::NKqp {
-
-namespace {
-
-using namespace NYql;
-
-struct TShardParamValuesAndRanges {
- NDqProto::TData ParamValues;
+
+namespace NKikimr::NKqp {
+
+namespace {
+
+using namespace NYql;
+
+struct TShardParamValuesAndRanges {
+ NDqProto::TData ParamValues;
NKikimr::NMiniKQL::TType* ParamType;
- // either FullRange or Ranges are set
- TVector<TSerializedPointOrRange> Ranges;
- std::optional<TSerializedTableRange> FullRange;
-};
-
-THashMap<ui64, TShardParamValuesAndRanges> PartitionParamByKey(const NDq::TMkqlValueRef& param, const TTableId& tableId,
- const TKqpTableKeys& tableKeys, const TKeyDesc& key, const NMiniKQL::THolderFactory& holderFactory,
- const NMiniKQL::TTypeEnvironment& typeEnv)
-{
- YQL_ENSURE(tableId.HasSamePath(key.TableId));
- auto& table = tableKeys.GetTable(tableId);
-
- THashMap<ui64, TShardParamValuesAndRanges> ret;
- THashMap<ui64, NMiniKQL::TUnboxedValueVector> shardParamValues;
-
+ // either FullRange or Ranges are set
+ TVector<TSerializedPointOrRange> Ranges;
+ std::optional<TSerializedTableRange> FullRange;
+};
+
+THashMap<ui64, TShardParamValuesAndRanges> PartitionParamByKey(const NDq::TMkqlValueRef& param, const TTableId& tableId,
+ const TKqpTableKeys& tableKeys, const TKeyDesc& key, const NMiniKQL::THolderFactory& holderFactory,
+ const NMiniKQL::TTypeEnvironment& typeEnv)
+{
+ YQL_ENSURE(tableId.HasSamePath(key.TableId));
+ auto& table = tableKeys.GetTable(tableId);
+
+ THashMap<ui64, TShardParamValuesAndRanges> ret;
+ THashMap<ui64, NMiniKQL::TUnboxedValueVector> shardParamValues;
+
auto [type, value] = ImportValueFromProto(param.GetType(), param.GetValue(), typeEnv, holderFactory);
-
- YQL_ENSURE(type->GetKind() == NMiniKQL::TType::EKind::List);
- auto* itemType = static_cast<NMiniKQL::TListType*>(type)->GetItemType();
- YQL_ENSURE(itemType->GetKind() == NMiniKQL::TType::EKind::Struct);
- auto* structType = static_cast<NMiniKQL::TStructType*>(itemType);
-
- const ui64 keyLen = table.KeyColumns.size();
-
- TVector<ui32> keyColumnIndices;
- keyColumnIndices.reserve(keyLen);
- for (auto& keyColumn : table.KeyColumns) {
- keyColumnIndices.push_back(structType->GetMemberIndex(keyColumn));
- }
-
- NUdf::TUnboxedValue paramValue;
- auto it = value.GetListIterator();
- while (it.Next(paramValue)) {
- auto keyValue = MakeKeyCells(paramValue, table.KeyColumnTypes, keyColumnIndices, typeEnv, /* copyValues */ true);
- Y_VERIFY_DEBUG(keyValue.size() == keyLen);
-
- ui32 partitionIndex = FindKeyPartitionIndex(keyValue, key.Partitions, table.KeyColumnTypes,
- [] (const auto& partition) { return *partition.Range; });
-
- ui64 shardId = key.Partitions[partitionIndex].ShardId;
-
- shardParamValues[shardId].emplace_back(std::move(paramValue));
-
- auto point = TSerializedCellVec(TSerializedCellVec::Serialize(keyValue));
-
- auto& shardData = ret[shardId];
- if (key.Partitions[partitionIndex].Range->IsPoint) {
- // singular case when partition is just a point
- shardData.FullRange.emplace(TSerializedTableRange(point.GetBuffer(), "", true, true));
- shardData.FullRange->Point = true;
- shardData.Ranges.clear();
- } else {
- shardData.Ranges.emplace_back(std::move(point));
- }
+
+ YQL_ENSURE(type->GetKind() == NMiniKQL::TType::EKind::List);
+ auto* itemType = static_cast<NMiniKQL::TListType*>(type)->GetItemType();
+ YQL_ENSURE(itemType->GetKind() == NMiniKQL::TType::EKind::Struct);
+ auto* structType = static_cast<NMiniKQL::TStructType*>(itemType);
+
+ const ui64 keyLen = table.KeyColumns.size();
+
+ TVector<ui32> keyColumnIndices;
+ keyColumnIndices.reserve(keyLen);
+ for (auto& keyColumn : table.KeyColumns) {
+ keyColumnIndices.push_back(structType->GetMemberIndex(keyColumn));
+ }
+
+ NUdf::TUnboxedValue paramValue;
+ auto it = value.GetListIterator();
+ while (it.Next(paramValue)) {
+ auto keyValue = MakeKeyCells(paramValue, table.KeyColumnTypes, keyColumnIndices, typeEnv, /* copyValues */ true);
+ Y_VERIFY_DEBUG(keyValue.size() == keyLen);
+
+ ui32 partitionIndex = FindKeyPartitionIndex(keyValue, key.Partitions, table.KeyColumnTypes,
+ [] (const auto& partition) { return *partition.Range; });
+
+ ui64 shardId = key.Partitions[partitionIndex].ShardId;
+
+ shardParamValues[shardId].emplace_back(std::move(paramValue));
+
+ auto point = TSerializedCellVec(TSerializedCellVec::Serialize(keyValue));
+
+ auto& shardData = ret[shardId];
+ if (key.Partitions[partitionIndex].Range->IsPoint) {
+ // singular case when partition is just a point
+ shardData.FullRange.emplace(TSerializedTableRange(point.GetBuffer(), "", true, true));
+ shardData.FullRange->Point = true;
+ shardData.Ranges.clear();
+ } else {
+ shardData.Ranges.emplace_back(std::move(point));
+ }
shardData.ParamType = itemType;
- }
-
+ }
+
NDq::TDqDataSerializer dataSerializer{typeEnv, holderFactory, NDqProto::EDataTransportVersion::DATA_TRANSPORT_UV_PICKLE_1_0};
- for (auto& [shardId, data] : ret) {
- ret[shardId].ParamValues = dataSerializer.Serialize(shardParamValues[shardId], itemType);
- }
-
- return ret;
-}
-
-THashMap<ui64, TShardParamValuesAndRanges> PartitionParamByKeyPrefix(const NDq::TMkqlValueRef& param,
- const TTableId& tableId, const TKqpTableKeys& tableKeys, const TKeyDesc& key,
- const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv)
-{
- YQL_ENSURE(tableId.HasSamePath(key.TableId));
- auto& table = tableKeys.GetTable(tableId);
-
- THashMap<ui64, TShardParamValuesAndRanges> ret;
- THashMap<ui64, NMiniKQL::TUnboxedValueVector> shardParamValues;
-
+ for (auto& [shardId, data] : ret) {
+ ret[shardId].ParamValues = dataSerializer.Serialize(shardParamValues[shardId], itemType);
+ }
+
+ return ret;
+}
+
+THashMap<ui64, TShardParamValuesAndRanges> PartitionParamByKeyPrefix(const NDq::TMkqlValueRef& param,
+ const TTableId& tableId, const TKqpTableKeys& tableKeys, const TKeyDesc& key,
+ const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv)
+{
+ YQL_ENSURE(tableId.HasSamePath(key.TableId));
+ auto& table = tableKeys.GetTable(tableId);
+
+ THashMap<ui64, TShardParamValuesAndRanges> ret;
+ THashMap<ui64, NMiniKQL::TUnboxedValueVector> shardParamValues;
+
auto [type, value] = ImportValueFromProto(param.GetType(), param.GetValue(), typeEnv, holderFactory);
-
- YQL_ENSURE(type->GetKind() == NMiniKQL::TType::EKind::List);
- auto itemType = static_cast<NMiniKQL::TListType&>(*type).GetItemType();
- YQL_ENSURE(itemType->GetKind() == NMiniKQL::TType::EKind::Struct);
- auto& structType = static_cast<NMiniKQL::TStructType&>(*itemType);
-
- const ui64 keyLen = table.KeyColumns.size();
-
- TVector<NUdf::TDataTypeId> keyFullType{Reserve(keyLen)};
- TVector<NUdf::TDataTypeId> keyPrefixType{Reserve(keyLen)};
- TVector<ui32> keyPrefixIndices{Reserve(keyLen)};
-
- for (const auto& keyColumn : table.KeyColumns) {
- auto columnInfo = NDq::FindColumnInfo(&structType, keyColumn);
- if (!columnInfo) {
- break;
- }
-
- keyFullType.push_back(columnInfo->TypeId);
- keyPrefixType.push_back(columnInfo->TypeId);
- keyPrefixIndices.push_back(columnInfo->Index);
- }
-
- YQL_ENSURE(!keyPrefixType.empty());
-
- for (ui64 i = keyFullType.size(); i < keyLen; ++i) {
- keyFullType.push_back(table.Columns.at(table.KeyColumns[i]).Type);
- }
-
- NUdf::TUnboxedValue paramValue;
- auto it = value.GetListIterator();
- while (it.Next(paramValue)) {
- auto fromValues = MakeKeyCells(paramValue, keyPrefixType, keyPrefixIndices, typeEnv, /* copyValues */ false);
- auto toValuesPrefix = fromValues;
-
- // append `from key` with nulls
- for (ui32 i = keyPrefixIndices.size(); i < keyLen; ++i) {
- fromValues.push_back(TCell()); // null
- // skip or toValuesPrefix.push_back(+inf);
- }
- Y_VERIFY_DEBUG(fromValues.size() == keyLen);
-
- const bool point = toValuesPrefix.size() == keyLen;
-
- auto range = TTableRange(fromValues, /* inclusiveFrom */ true,
- point ? TConstArrayRef<TCell>() : toValuesPrefix, /* inclusiveTo */ true,
- /* point */ point);
- TVector<TPartitionWithRange> rangePartitions = GetKeyRangePartitions(range, key.Partitions, keyFullType);
-
- for (TPartitionWithRange& partitionWithRange : rangePartitions) {
- ui64 shardId = partitionWithRange.PartitionInfo->ShardId;
-
+
+ YQL_ENSURE(type->GetKind() == NMiniKQL::TType::EKind::List);
+ auto itemType = static_cast<NMiniKQL::TListType&>(*type).GetItemType();
+ YQL_ENSURE(itemType->GetKind() == NMiniKQL::TType::EKind::Struct);
+ auto& structType = static_cast<NMiniKQL::TStructType&>(*itemType);
+
+ const ui64 keyLen = table.KeyColumns.size();
+
+ TVector<NUdf::TDataTypeId> keyFullType{Reserve(keyLen)};
+ TVector<NUdf::TDataTypeId> keyPrefixType{Reserve(keyLen)};
+ TVector<ui32> keyPrefixIndices{Reserve(keyLen)};
+
+ for (const auto& keyColumn : table.KeyColumns) {
+ auto columnInfo = NDq::FindColumnInfo(&structType, keyColumn);
+ if (!columnInfo) {
+ break;
+ }
+
+ keyFullType.push_back(columnInfo->TypeId);
+ keyPrefixType.push_back(columnInfo->TypeId);
+ keyPrefixIndices.push_back(columnInfo->Index);
+ }
+
+ YQL_ENSURE(!keyPrefixType.empty());
+
+ for (ui64 i = keyFullType.size(); i < keyLen; ++i) {
+ keyFullType.push_back(table.Columns.at(table.KeyColumns[i]).Type);
+ }
+
+ NUdf::TUnboxedValue paramValue;
+ auto it = value.GetListIterator();
+ while (it.Next(paramValue)) {
+ auto fromValues = MakeKeyCells(paramValue, keyPrefixType, keyPrefixIndices, typeEnv, /* copyValues */ false);
+ auto toValuesPrefix = fromValues;
+
+ // append `from key` with nulls
+ for (ui32 i = keyPrefixIndices.size(); i < keyLen; ++i) {
+ fromValues.push_back(TCell()); // null
+ // skip or toValuesPrefix.push_back(+inf);
+ }
+ Y_VERIFY_DEBUG(fromValues.size() == keyLen);
+
+ const bool point = toValuesPrefix.size() == keyLen;
+
+ auto range = TTableRange(fromValues, /* inclusiveFrom */ true,
+ point ? TConstArrayRef<TCell>() : toValuesPrefix, /* inclusiveTo */ true,
+ /* point */ point);
+ TVector<TPartitionWithRange> rangePartitions = GetKeyRangePartitions(range, key.Partitions, keyFullType);
+
+ for (TPartitionWithRange& partitionWithRange : rangePartitions) {
+ ui64 shardId = partitionWithRange.PartitionInfo->ShardId;
+
shardParamValues[shardId].emplace_back(paramValue);
-
- auto& shardData = ret[shardId];
- if (partitionWithRange.FullRange) {
- shardData.FullRange = std::move(partitionWithRange.FullRange);
- shardData.Ranges.clear();
- } else if (!shardData.FullRange) {
- shardData.Ranges.emplace_back(std::move(partitionWithRange.PointOrRange));
- }
+
+ auto& shardData = ret[shardId];
+ if (partitionWithRange.FullRange) {
+ shardData.FullRange = std::move(partitionWithRange.FullRange);
+ shardData.Ranges.clear();
+ } else if (!shardData.FullRange) {
+ shardData.Ranges.emplace_back(std::move(partitionWithRange.PointOrRange));
+ }
shardData.ParamType = itemType;
- }
- }
-
+ }
+ }
+
NDq::TDqDataSerializer dataSerializer(typeEnv, holderFactory, NDqProto::EDataTransportVersion::DATA_TRANSPORT_UV_PICKLE_1_0);
- for (auto& [shardId, data] : ret) {
- data.ParamValues = dataSerializer.Serialize(shardParamValues[shardId], itemType);
- }
-
- return ret;
-}
-
-TVector<TCell> FillKeyValues(const TVector<NUdf::TDataTypeId>& keyColumnTypes, const NKqpProto::TKqpPhyKeyBound& bound,
- const TStageInfo& stageInfo, const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv)
-{
- YQL_ENSURE(bound.ValuesSize() <= keyColumnTypes.size());
-
- TVector<TCell> keyValues;
- keyValues.reserve(bound.ValuesSize());
-
- for (ui32 i = 0; i < bound.ValuesSize(); ++i) {
- auto& tupleValue = bound.GetValues(i);
-
- TString paramName;
- TMaybe<ui32> paramIndex;
- switch (tupleValue.GetKindCase()) {
- case NKqpProto::TKqpPhyValue::kParamValue:
- paramName = tupleValue.GetParamValue().GetParamName();
- break;
- case NKqpProto::TKqpPhyValue::kParamElementValue:
- paramName = tupleValue.GetParamElementValue().GetParamName();
- paramIndex = tupleValue.GetParamElementValue().GetElementIndex();
- break;
- default:
- YQL_ENSURE(false, "Unexpected type case " << (int) tupleValue.GetKindCase());
- }
-
+ for (auto& [shardId, data] : ret) {
+ data.ParamValues = dataSerializer.Serialize(shardParamValues[shardId], itemType);
+ }
+
+ return ret;
+}
+
+TVector<TCell> FillKeyValues(const TVector<NUdf::TDataTypeId>& keyColumnTypes, const NKqpProto::TKqpPhyKeyBound& bound,
+ const TStageInfo& stageInfo, const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv)
+{
+ YQL_ENSURE(bound.ValuesSize() <= keyColumnTypes.size());
+
+ TVector<TCell> keyValues;
+ keyValues.reserve(bound.ValuesSize());
+
+ for (ui32 i = 0; i < bound.ValuesSize(); ++i) {
+ auto& tupleValue = bound.GetValues(i);
+
+ TString paramName;
+ TMaybe<ui32> paramIndex;
+ switch (tupleValue.GetKindCase()) {
+ case NKqpProto::TKqpPhyValue::kParamValue:
+ paramName = tupleValue.GetParamValue().GetParamName();
+ break;
+ case NKqpProto::TKqpPhyValue::kParamElementValue:
+ paramName = tupleValue.GetParamElementValue().GetParamName();
+ paramIndex = tupleValue.GetParamElementValue().GetElementIndex();
+ break;
+ default:
+ YQL_ENSURE(false, "Unexpected type case " << (int) tupleValue.GetKindCase());
+ }
+
auto param = stageInfo.Meta.Tx.Params.Values.FindPtr(paramName);
- YQL_ENSURE(param, "Param not found: " << paramName);
-
- const auto* protoType = &param->GetType();
- const auto* protoValue = &param->GetValue();
- if (paramIndex) {
- YQL_ENSURE(protoType->GetKind() == NKikimrMiniKQL::Tuple);
- YQL_ENSURE(*paramIndex < protoType->GetTuple().ElementSize());
- YQL_ENSURE(*paramIndex < protoValue->TupleSize());
- protoType = &protoType->GetTuple().GetElement(*paramIndex);
- protoValue = &protoValue->GetTuple(*paramIndex);
- }
-
+ YQL_ENSURE(param, "Param not found: " << paramName);
+
+ const auto* protoType = &param->GetType();
+ const auto* protoValue = &param->GetValue();
+ if (paramIndex) {
+ YQL_ENSURE(protoType->GetKind() == NKikimrMiniKQL::Tuple);
+ YQL_ENSURE(*paramIndex < protoType->GetTuple().ElementSize());
+ YQL_ENSURE(*paramIndex < protoValue->TupleSize());
+ protoType = &protoType->GetTuple().GetElement(*paramIndex);
+ protoValue = &protoValue->GetTuple(*paramIndex);
+ }
+
auto [type, value] = ImportValueFromProto(*protoType, *protoValue, typeEnv, holderFactory);
-
- keyValues.emplace_back(NMiniKQL::MakeCell(keyColumnTypes[i], value, typeEnv, /* copy */ true));
- }
-
- return keyValues;
-}
-
-TSerializedPointOrRange FillOneRange(NUdf::TUnboxedValue& begin, NUdf::TUnboxedValue& end,
+
+ keyValues.emplace_back(NMiniKQL::MakeCell(keyColumnTypes[i], value, typeEnv, /* copy */ true));
+ }
+
+ return keyValues;
+}
+
+TSerializedPointOrRange FillOneRange(NUdf::TUnboxedValue& begin, NUdf::TUnboxedValue& end,
const TVector<NUdf::TDataTypeId>& keyColumnTypes, const NMiniKQL::TTypeEnvironment& typeEnv)
{
- const ui32 keyColumnsSize = keyColumnTypes.size();
+ const ui32 keyColumnsSize = keyColumnTypes.size();
// Range tuple contains ranges over all key colums + inclusive sign
YQL_ENSURE((keyColumnsSize + 1) == begin.GetListLength());
@@ -240,7 +240,7 @@ TSerializedPointOrRange FillOneRange(NUdf::TUnboxedValue& begin, NUdf::TUnboxedV
}
auto cell = NMiniKQL::MakeCell(keyColumnTypes[i], element, typeEnv, /* copy */ true);
- keyValues.emplace_back(std::move(cell));
+ keyValues.emplace_back(std::move(cell));
}
return keyValues;
@@ -292,22 +292,22 @@ TSerializedPointOrRange FillOneRange(NUdf::TUnboxedValue& begin, NUdf::TUnboxedV
}
}
- bool point = false;
- if (fromInclusive && toInclusive && fromKeyValues.size() == keyColumnsSize) {
- if (toKeyValues.empty()) {
- point = true;
- } else if (toKeyValues.size() == keyColumnsSize) {
- point = CompareTypedCellVectors(fromKeyValues.data(), toKeyValues.data(), keyColumnTypes.data(), keyColumnTypes.size()) == 0;
- }
- }
-
- if (point) {
- YQL_CLOG(DEBUG, ProviderKqp) << "Formed point [extract predicate]: "
- << DebugPrintPoint(keyColumnTypes, fromKeyValues, *AppData()->TypeRegistry);
-
- return TSerializedCellVec(TSerializedCellVec::Serialize(fromKeyValues));
- }
-
+ bool point = false;
+ if (fromInclusive && toInclusive && fromKeyValues.size() == keyColumnsSize) {
+ if (toKeyValues.empty()) {
+ point = true;
+ } else if (toKeyValues.size() == keyColumnsSize) {
+ point = CompareTypedCellVectors(fromKeyValues.data(), toKeyValues.data(), keyColumnTypes.data(), keyColumnTypes.size()) == 0;
+ }
+ }
+
+ if (point) {
+ YQL_CLOG(DEBUG, ProviderKqp) << "Formed point [extract predicate]: "
+ << DebugPrintPoint(keyColumnTypes, fromKeyValues, *AppData()->TypeRegistry);
+
+ return TSerializedCellVec(TSerializedCellVec::Serialize(fromKeyValues));
+ }
+
auto range = TSerializedTableRange(fromKeyValues, fromInclusive, toKeyValues, toInclusive);
YQL_CLOG(DEBUG, ProviderKqp) << "Formed range [extract predicate]: "
@@ -316,7 +316,7 @@ TSerializedPointOrRange FillOneRange(NUdf::TUnboxedValue& begin, NUdf::TUnboxedV
return range;
}
-TVector<TSerializedPointOrRange> BuildFullRange(const TVector<NUdf::TDataTypeId>& keyColumnTypes) {
+TVector<TSerializedPointOrRange> BuildFullRange(const TVector<NUdf::TDataTypeId>& keyColumnTypes) {
// Build range from NULL, NULL ... NULL to +inf, +inf ... +inf
TVector<TCell> fromKeyValues(keyColumnTypes.size());
@@ -325,10 +325,10 @@ TVector<TSerializedPointOrRange> BuildFullRange(const TVector<NUdf::TDataTypeId>
YQL_CLOG(DEBUG, ProviderKqp) << "Formed full range [extract predicate]: "
<< DebugPrintRange(keyColumnTypes, range.ToTableRange(), *AppData()->TypeRegistry);
- return {std::move(range)};
+ return {std::move(range)};
}
-TVector<TSerializedPointOrRange> FillRangesFromParameter(const TVector<NUdf::TDataTypeId>& keyColumnTypes,
+TVector<TSerializedPointOrRange> FillRangesFromParameter(const TVector<NUdf::TDataTypeId>& keyColumnTypes,
const NKqpProto::TKqpPhyParamValue& rangesParam, const TStageInfo& stageInfo,
const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv)
{
@@ -349,7 +349,7 @@ TVector<TSerializedPointOrRange> FillRangesFromParameter(const TVector<NUdf::TDa
auto rangesList = value.GetElement(0);
YQL_ENSURE(rangesList.IsBoxed());
- TVector<TSerializedPointOrRange> out;
+ TVector<TSerializedPointOrRange> out;
out.reserve(rangesList.GetListLength());
const auto it = rangesList.GetListIterator();
@@ -368,7 +368,7 @@ TVector<TSerializedPointOrRange> FillRangesFromParameter(const TVector<NUdf::TDa
}
template <typename PhyOpReadRanges>
-TVector<TSerializedPointOrRange> FillReadRangesInternal(const TVector<NUdf::TDataTypeId>& keyColumnTypes,
+TVector<TSerializedPointOrRange> FillReadRangesInternal(const TVector<NUdf::TDataTypeId>& keyColumnTypes,
const PhyOpReadRanges& readRanges, const TStageInfo& stageInfo,
const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv)
{
@@ -383,131 +383,131 @@ TVector<TSerializedPointOrRange> FillReadRangesInternal(const TVector<NUdf::TDat
} // anonymous namespace
-TVector<TSerializedPointOrRange> FillReadRanges(const TVector<NUdf::TDataTypeId>& keyColumnTypes,
+TVector<TSerializedPointOrRange> FillReadRanges(const TVector<NUdf::TDataTypeId>& keyColumnTypes,
const NKqpProto::TKqpPhyOpReadOlapRanges& readRange, const TStageInfo& stageInfo,
const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv)
{
return FillReadRangesInternal(keyColumnTypes, readRange, stageInfo, holderFactory, typeEnv);
}
-TVector<TSerializedPointOrRange> FillReadRanges(const TVector<NUdf::TDataTypeId>& keyColumnTypes,
+TVector<TSerializedPointOrRange> FillReadRanges(const TVector<NUdf::TDataTypeId>& keyColumnTypes,
const NKqpProto::TKqpPhyOpReadRanges& readRange, const TStageInfo& stageInfo,
const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv)
{
return FillReadRangesInternal(keyColumnTypes, readRange, stageInfo, holderFactory, typeEnv);
}
-TSerializedTableRange MakeKeyRange(const TVector<NUdf::TDataTypeId>& keyColumnTypes,
- const NKqpProto::TKqpPhyKeyRange& range, const TStageInfo& stageInfo, const NMiniKQL::THolderFactory& holderFactory,
- const NMiniKQL::TTypeEnvironment& typeEnv)
-{
- YQL_ENSURE(range.HasFrom());
- YQL_ENSURE(range.HasTo());
-
- auto fromValues = FillKeyValues(keyColumnTypes, range.GetFrom(), stageInfo, holderFactory, typeEnv);
- if (range.GetFrom().GetIsInclusive()) {
- for (ui32 i = fromValues.size(); i < keyColumnTypes.size(); ++i) {
- fromValues.emplace_back(TCell());
- }
- }
-
- auto toValues = FillKeyValues(keyColumnTypes, range.GetTo(), stageInfo, holderFactory, typeEnv);
- if (!range.GetTo().GetIsInclusive()) {
- for (ui32 i = toValues.size(); i < keyColumnTypes.size(); ++i) {
- toValues.emplace_back(TCell());
- }
- }
-
+TSerializedTableRange MakeKeyRange(const TVector<NUdf::TDataTypeId>& keyColumnTypes,
+ const NKqpProto::TKqpPhyKeyRange& range, const TStageInfo& stageInfo, const NMiniKQL::THolderFactory& holderFactory,
+ const NMiniKQL::TTypeEnvironment& typeEnv)
+{
+ YQL_ENSURE(range.HasFrom());
+ YQL_ENSURE(range.HasTo());
+
+ auto fromValues = FillKeyValues(keyColumnTypes, range.GetFrom(), stageInfo, holderFactory, typeEnv);
+ if (range.GetFrom().GetIsInclusive()) {
+ for (ui32 i = fromValues.size(); i < keyColumnTypes.size(); ++i) {
+ fromValues.emplace_back(TCell());
+ }
+ }
+
+ auto toValues = FillKeyValues(keyColumnTypes, range.GetTo(), stageInfo, holderFactory, typeEnv);
+ if (!range.GetTo().GetIsInclusive()) {
+ for (ui32 i = toValues.size(); i < keyColumnTypes.size(); ++i) {
+ toValues.emplace_back(TCell());
+ }
+ }
+
auto serialized = TSerializedTableRange(fromValues, range.GetFrom().GetIsInclusive(), toValues, range.GetTo().GetIsInclusive());
YQL_CLOG(DEBUG, ProviderKqp) << "Formed range: "
<< DebugPrintRange(keyColumnTypes, serialized.ToTableRange(), *AppData()->TypeRegistry);
return serialized;
-}
-
+}
+
namespace {
-void FillFullRange(const TStageInfo& stageInfo, THashMap<ui64, TShardInfo>& shardInfoMap, bool read) {
- for (ui64 i = 0; i < stageInfo.Meta.ShardKey->Partitions.size(); ++i) {
- auto& partition = stageInfo.Meta.ShardKey->Partitions[i];
- auto& partitionRange = *partition.Range;
- auto& shardInfo = shardInfoMap[partition.ShardId];
-
- auto& ranges = read ? shardInfo.KeyReadRanges : shardInfo.KeyWriteRanges;
-
- ranges.ConstructInPlace();
-
- if (partitionRange.IsPoint) {
- YQL_ENSURE(partitionRange.IsInclusive);
- auto point = TSerializedTableRange(partitionRange.EndKeyPrefix.GetCells(), true, {}, true);
- point.Point = true;
-
- ranges->MakeFullRange(std::move(point));
- continue;
- }
-
- if (i != 0) {
- auto& prevPartition = stageInfo.Meta.ShardKey->Partitions[i - 1];
-
- ranges->MakeFull(TSerializedTableRange(prevPartition.Range->EndKeyPrefix.GetCells(), !prevPartition.Range->IsInclusive,
- partitionRange.EndKeyPrefix.GetCells(), partitionRange.IsInclusive));
- } else {
- TVector<TCell> fromValues;
- for (auto x : partitionRange.EndKeyPrefix.GetCells()) {
- Y_UNUSED(x);
- fromValues.emplace_back(TCell());
- }
-
- ranges->MakeFullRange(TSerializedTableRange(fromValues, true,
- partitionRange.EndKeyPrefix.GetCells(), partitionRange.IsInclusive));
- }
- }
-}
-} // anonymous namespace
-
-TString TShardInfo::ToString(const TVector<NScheme::TTypeId>& keyTypes, const NScheme::TTypeRegistry& typeRegistry) const
-{
- TStringBuilder sb;
- sb << "TShardInfo{ ";
- sb << "ReadRanges: " << (KeyReadRanges ? KeyReadRanges->ToString(keyTypes, typeRegistry) : "<none>");
- sb << ", WriteRanges: " << (KeyWriteRanges ? KeyWriteRanges->ToString(keyTypes, typeRegistry) : "<none>");
- sb << ", Parameters: {";
- for (auto& param: Params) {
- sb << param.first << ", ";
- }
- sb << "} }";
- return sb;
-}
-
-THashMap<ui64, TShardInfo> PrunePartitions(const TKqpTableKeys& tableKeys,
- const NKqpProto::TKqpPhyOpReadRange& readRange, const TStageInfo& stageInfo,
- const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv)
-{
- const auto* table = tableKeys.FindTablePtr(stageInfo.Meta.TableId);
- YQL_ENSURE(table);
-
- const auto& keyColumnTypes = table->KeyColumnTypes;
- YQL_ENSURE(readRange.HasKeyRange());
-
- auto range = MakeKeyRange(keyColumnTypes, readRange.GetKeyRange(), stageInfo, holderFactory, typeEnv);
- auto readPartitions = GetKeyRangePartitions(range.ToTableRange(), stageInfo.Meta.ShardKey->Partitions, keyColumnTypes);
-
- THashMap<ui64, TShardInfo> shardInfoMap;
- for (TPartitionWithRange& partitionWithRange : readPartitions) {
- auto& shardInfo = shardInfoMap[partitionWithRange.PartitionInfo->ShardId];
-
- YQL_ENSURE(!shardInfo.KeyReadRanges);
- shardInfo.KeyReadRanges.ConstructInPlace();
-
- if (partitionWithRange.FullRange) {
- shardInfo.KeyReadRanges->MakeFullRange(std::move(*partitionWithRange.FullRange));
- } else if (!shardInfo.KeyReadRanges->IsFullRange()) {
- shardInfo.KeyReadRanges->Add(std::move(partitionWithRange.PointOrRange));
- }
- }
-
- return shardInfoMap;
-}
+void FillFullRange(const TStageInfo& stageInfo, THashMap<ui64, TShardInfo>& shardInfoMap, bool read) {
+ for (ui64 i = 0; i < stageInfo.Meta.ShardKey->Partitions.size(); ++i) {
+ auto& partition = stageInfo.Meta.ShardKey->Partitions[i];
+ auto& partitionRange = *partition.Range;
+ auto& shardInfo = shardInfoMap[partition.ShardId];
+
+ auto& ranges = read ? shardInfo.KeyReadRanges : shardInfo.KeyWriteRanges;
+
+ ranges.ConstructInPlace();
+
+ if (partitionRange.IsPoint) {
+ YQL_ENSURE(partitionRange.IsInclusive);
+ auto point = TSerializedTableRange(partitionRange.EndKeyPrefix.GetCells(), true, {}, true);
+ point.Point = true;
+
+ ranges->MakeFullRange(std::move(point));
+ continue;
+ }
+
+ if (i != 0) {
+ auto& prevPartition = stageInfo.Meta.ShardKey->Partitions[i - 1];
+
+ ranges->MakeFull(TSerializedTableRange(prevPartition.Range->EndKeyPrefix.GetCells(), !prevPartition.Range->IsInclusive,
+ partitionRange.EndKeyPrefix.GetCells(), partitionRange.IsInclusive));
+ } else {
+ TVector<TCell> fromValues;
+ for (auto x : partitionRange.EndKeyPrefix.GetCells()) {
+ Y_UNUSED(x);
+ fromValues.emplace_back(TCell());
+ }
+
+ ranges->MakeFullRange(TSerializedTableRange(fromValues, true,
+ partitionRange.EndKeyPrefix.GetCells(), partitionRange.IsInclusive));
+ }
+ }
+}
+} // anonymous namespace
+
+TString TShardInfo::ToString(const TVector<NScheme::TTypeId>& keyTypes, const NScheme::TTypeRegistry& typeRegistry) const
+{
+ TStringBuilder sb;
+ sb << "TShardInfo{ ";
+ sb << "ReadRanges: " << (KeyReadRanges ? KeyReadRanges->ToString(keyTypes, typeRegistry) : "<none>");
+ sb << ", WriteRanges: " << (KeyWriteRanges ? KeyWriteRanges->ToString(keyTypes, typeRegistry) : "<none>");
+ sb << ", Parameters: {";
+ for (auto& param: Params) {
+ sb << param.first << ", ";
+ }
+ sb << "} }";
+ return sb;
+}
+
+THashMap<ui64, TShardInfo> PrunePartitions(const TKqpTableKeys& tableKeys,
+ const NKqpProto::TKqpPhyOpReadRange& readRange, const TStageInfo& stageInfo,
+ const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv)
+{
+ const auto* table = tableKeys.FindTablePtr(stageInfo.Meta.TableId);
+ YQL_ENSURE(table);
+
+ const auto& keyColumnTypes = table->KeyColumnTypes;
+ YQL_ENSURE(readRange.HasKeyRange());
+
+ auto range = MakeKeyRange(keyColumnTypes, readRange.GetKeyRange(), stageInfo, holderFactory, typeEnv);
+ auto readPartitions = GetKeyRangePartitions(range.ToTableRange(), stageInfo.Meta.ShardKey->Partitions, keyColumnTypes);
+
+ THashMap<ui64, TShardInfo> shardInfoMap;
+ for (TPartitionWithRange& partitionWithRange : readPartitions) {
+ auto& shardInfo = shardInfoMap[partitionWithRange.PartitionInfo->ShardId];
+
+ YQL_ENSURE(!shardInfo.KeyReadRanges);
+ shardInfo.KeyReadRanges.ConstructInPlace();
+
+ if (partitionWithRange.FullRange) {
+ shardInfo.KeyReadRanges->MakeFullRange(std::move(*partitionWithRange.FullRange));
+ } else if (!shardInfo.KeyReadRanges->IsFullRange()) {
+ shardInfo.KeyReadRanges->Add(std::move(partitionWithRange.PointOrRange));
+ }
+ }
+
+ return shardInfoMap;
+}
THashMap<ui64, TShardInfo> PrunePartitions(const TKqpTableKeys& tableKeys,
const NKqpProto::TKqpPhyOpReadRanges& readRanges, const TStageInfo& stageInfo,
@@ -516,19 +516,19 @@ THashMap<ui64, TShardInfo> PrunePartitions(const TKqpTableKeys& tableKeys,
const auto* table = tableKeys.FindTablePtr(stageInfo.Meta.TableId);
YQL_ENSURE(table);
- const auto& keyColumnTypes = table->KeyColumnTypes;
+ const auto& keyColumnTypes = table->KeyColumnTypes;
auto ranges = FillReadRangesInternal(keyColumnTypes, readRanges, stageInfo, holderFactory, typeEnv);
THashMap<ui64, TShardInfo> shardInfoMap;
// KeyReadRanges must be sorted & non-intersecting, they came in such condition from predicate extraction.
for (auto& range: ranges) {
- TTableRange tableRange = std::holds_alternative<TSerializedCellVec>(range)
- ? TTableRange(std::get<TSerializedCellVec>(range).GetCells(), true, std::get<TSerializedCellVec>(range).GetCells(), true, true)
- : TTableRange(std::get<TSerializedTableRange>(range).ToTableRange());
+ TTableRange tableRange = std::holds_alternative<TSerializedCellVec>(range)
+ ? TTableRange(std::get<TSerializedCellVec>(range).GetCells(), true, std::get<TSerializedCellVec>(range).GetCells(), true, true)
+ : TTableRange(std::get<TSerializedTableRange>(range).ToTableRange());
+
+ auto readPartitions = GetKeyRangePartitions(tableRange, stageInfo.Meta.ShardKey->Partitions, keyColumnTypes);
- auto readPartitions = GetKeyRangePartitions(tableRange, stageInfo.Meta.ShardKey->Partitions, keyColumnTypes);
-
for (TPartitionWithRange& partitionWithRange : readPartitions) {
auto& shardInfo = shardInfoMap[partitionWithRange.PartitionInfo->ShardId];
@@ -537,268 +537,268 @@ THashMap<ui64, TShardInfo> PrunePartitions(const TKqpTableKeys& tableKeys,
}
if (partitionWithRange.FullRange) {
- shardInfo.KeyReadRanges->MakeFullRange(std::move(*partitionWithRange.FullRange));
+ shardInfo.KeyReadRanges->MakeFullRange(std::move(*partitionWithRange.FullRange));
continue;
}
- shardInfo.KeyReadRanges->Add(std::move(partitionWithRange.PointOrRange));
+ shardInfo.KeyReadRanges->Add(std::move(partitionWithRange.PointOrRange));
+ }
+ }
+
+ return shardInfoMap;
+}
+
+namespace {
+
+using namespace NMiniKQL;
+
+THashMap<ui64, TShardInfo> PartitionLookupByParameterValue(const NKqpProto::TKqpPhyParamValue& proto,
+ const TKqpTableKeys& tableKeys, const TStageInfo& stageInfo, const THolderFactory& holderFactory,
+ const TTypeEnvironment& typeEnv)
+{
+ const auto& name = proto.GetParamName();
+ auto param = stageInfo.Meta.Tx.Params.Values.FindPtr(name);
+ YQL_ENSURE(param);
+
+ auto shardsMap = PartitionParamByKeyPrefix(*param, stageInfo.Meta.TableId, tableKeys, *stageInfo.Meta.ShardKey,
+ holderFactory, typeEnv);
+
+ THashMap<ui64, TShardInfo> shardInfoMap;
+
+ for (auto& [shardId, shardData] : shardsMap) {
+ auto& shardInfo = shardInfoMap[shardId];
+
+ if (!shardInfo.KeyReadRanges) {
+ shardInfo.KeyReadRanges.ConstructInPlace();
+ }
+
+ auto ret = shardInfo.Params.emplace(name, std::move(shardData.ParamValues));
+ Y_VERIFY_DEBUG(ret.second);
+ auto retType = shardInfo.ParamTypes.emplace(name, std::move(shardData.ParamType));
+ Y_VERIFY_DEBUG(retType.second);
+
+ if (shardData.FullRange) {
+ shardInfo.KeyReadRanges->MakeFullRange(std::move(*shardData.FullRange));
+ } else {
+ for (auto& range : shardData.Ranges) {
+ shardInfo.KeyReadRanges->Add(std::move(range));
+ }
}
}
return shardInfoMap;
}
-namespace {
-
-using namespace NMiniKQL;
-
-THashMap<ui64, TShardInfo> PartitionLookupByParameterValue(const NKqpProto::TKqpPhyParamValue& proto,
- const TKqpTableKeys& tableKeys, const TStageInfo& stageInfo, const THolderFactory& holderFactory,
- const TTypeEnvironment& typeEnv)
-{
- const auto& name = proto.GetParamName();
- auto param = stageInfo.Meta.Tx.Params.Values.FindPtr(name);
- YQL_ENSURE(param);
-
- auto shardsMap = PartitionParamByKeyPrefix(*param, stageInfo.Meta.TableId, tableKeys, *stageInfo.Meta.ShardKey,
- holderFactory, typeEnv);
-
- THashMap<ui64, TShardInfo> shardInfoMap;
-
- for (auto& [shardId, shardData] : shardsMap) {
- auto& shardInfo = shardInfoMap[shardId];
-
- if (!shardInfo.KeyReadRanges) {
- shardInfo.KeyReadRanges.ConstructInPlace();
- }
-
- auto ret = shardInfo.Params.emplace(name, std::move(shardData.ParamValues));
- Y_VERIFY_DEBUG(ret.second);
- auto retType = shardInfo.ParamTypes.emplace(name, std::move(shardData.ParamType));
- Y_VERIFY_DEBUG(retType.second);
-
- if (shardData.FullRange) {
- shardInfo.KeyReadRanges->MakeFullRange(std::move(*shardData.FullRange));
- } else {
- for (auto& range : shardData.Ranges) {
- shardInfo.KeyReadRanges->Add(std::move(range));
- }
- }
- }
-
- return shardInfoMap;
-}
-
-THashMap<ui64, TShardInfo> PartitionLookupByRowsList(const NKqpProto::TKqpPhyRowsList& proto,
- const TKqpTableKeys& tableKeys, const TStageInfo& stageInfo, const THolderFactory& holderFactory,
- const TTypeEnvironment& typeEnv)
-{
- const auto& table = tableKeys.GetTable(stageInfo.Meta.ShardKey->TableId);
-
- struct TParamDesc {
- const NDq::TMkqlValueRef* MkqlValueRef = nullptr;
- NMiniKQL::TType* MkqlType = nullptr;
- NUdf::TUnboxedValue MkqlValue;
- };
-
- std::unordered_map<std::string_view, TParamDesc> params; // already known parameters
- std::unordered_map<ui64, THashSet<TString>> shardParams; // shardId -> paramNames
- std::unordered_map<ui64, TShardParamValuesAndRanges> ret;
-
- THashMap<ui64, TShardInfo> shardInfoMap;
-
- for (const auto& row : proto.GetRows()) {
- TVector<TCell> keyFrom, keyTo;
- keyFrom.resize(table.KeyColumns.size());
- keyTo.resize(row.GetColumns().size());
-
- NMiniKQL::TType* mkqlType = nullptr;
- NUdf::TUnboxedValue mkqlValue;
-
- for (const auto& [columnName, columnValue]: row.GetColumns()) {
- switch (columnValue.GetKindCase()) {
- case NKqpProto::TKqpPhyRowsList_TValue::kParamValue: {
- const auto& paramName = columnValue.GetParamValue().GetParamName();
-
- TParamDesc* paramDesc;
- if (auto it = params.find(paramName); it != params.end()) {
- paramDesc = &it->second;
- } else {
- auto param = stageInfo.Meta.Tx.Params.Values.FindPtr(paramName);
- YQL_ENSURE(param);
-
- auto iter = params.emplace(paramName, TParamDesc());
- paramDesc = &iter.first->second;
-
- paramDesc->MkqlValueRef = param;
+THashMap<ui64, TShardInfo> PartitionLookupByRowsList(const NKqpProto::TKqpPhyRowsList& proto,
+ const TKqpTableKeys& tableKeys, const TStageInfo& stageInfo, const THolderFactory& holderFactory,
+ const TTypeEnvironment& typeEnv)
+{
+ const auto& table = tableKeys.GetTable(stageInfo.Meta.ShardKey->TableId);
+
+ struct TParamDesc {
+ const NDq::TMkqlValueRef* MkqlValueRef = nullptr;
+ NMiniKQL::TType* MkqlType = nullptr;
+ NUdf::TUnboxedValue MkqlValue;
+ };
+
+ std::unordered_map<std::string_view, TParamDesc> params; // already known parameters
+ std::unordered_map<ui64, THashSet<TString>> shardParams; // shardId -> paramNames
+ std::unordered_map<ui64, TShardParamValuesAndRanges> ret;
+
+ THashMap<ui64, TShardInfo> shardInfoMap;
+
+ for (const auto& row : proto.GetRows()) {
+ TVector<TCell> keyFrom, keyTo;
+ keyFrom.resize(table.KeyColumns.size());
+ keyTo.resize(row.GetColumns().size());
+
+ NMiniKQL::TType* mkqlType = nullptr;
+ NUdf::TUnboxedValue mkqlValue;
+
+ for (const auto& [columnName, columnValue]: row.GetColumns()) {
+ switch (columnValue.GetKindCase()) {
+ case NKqpProto::TKqpPhyRowsList_TValue::kParamValue: {
+ const auto& paramName = columnValue.GetParamValue().GetParamName();
+
+ TParamDesc* paramDesc;
+ if (auto it = params.find(paramName); it != params.end()) {
+ paramDesc = &it->second;
+ } else {
+ auto param = stageInfo.Meta.Tx.Params.Values.FindPtr(paramName);
+ YQL_ENSURE(param);
+
+ auto iter = params.emplace(paramName, TParamDesc());
+ paramDesc = &iter.first->second;
+
+ paramDesc->MkqlValueRef = param;
std::tie(paramDesc->MkqlType, paramDesc->MkqlValue) = ImportValueFromProto(param->GetType(), param->GetValue(), typeEnv, holderFactory);
- }
-
- mkqlType = paramDesc->MkqlType;
- mkqlValue = paramDesc->MkqlValue;
-
- break;
- }
-
- case NKqpProto::TKqpPhyRowsList_TValue::kLiteralValue: {
- const auto& literal = columnValue.GetLiteralValue();
-
+ }
+
+ mkqlType = paramDesc->MkqlType;
+ mkqlValue = paramDesc->MkqlValue;
+
+ break;
+ }
+
+ case NKqpProto::TKqpPhyRowsList_TValue::kLiteralValue: {
+ const auto& literal = columnValue.GetLiteralValue();
+
std::tie(mkqlType, mkqlValue) = ImportValueFromProto(literal.GetType(), literal.GetValue(), typeEnv, holderFactory);
-
- break;
- }
-
- case NKqpProto::TKqpPhyRowsList_TValue::KIND_NOT_SET: {
- YQL_ENSURE(false);
- }
- }
-
- for (ui64 i = 0; i < table.KeyColumns.size(); ++i) {
- if (table.KeyColumns[i] == columnName) {
- keyFrom[i] = keyTo[i] = NMiniKQL::MakeCell(
- table.KeyColumnTypes[i], mkqlValue, typeEnv, /* copyValue */ false);
- break;
- }
- }
- }
-
- auto range = TTableRange(keyFrom, true, keyTo, true, /* point */ false);
- auto partitions = GetKeyRangePartitions(range, stageInfo.Meta.ShardKey->Partitions, table.KeyColumnTypes);
-
- for (auto& partitionWithRange: partitions) {
- ui64 shardId = partitionWithRange.PartitionInfo->ShardId;
-
- for (const auto& [columnName, columnValue] : row.GetColumns()) {
- if (columnValue.GetKindCase() == NKqpProto::TKqpPhyRowsList_TValue::kParamValue) {
- shardParams[shardId].emplace(columnValue.GetParamValue().GetParamName());
- }
- }
-
- auto& shardData = ret[shardId];
- if (partitionWithRange.FullRange) {
- shardData.FullRange = std::move(partitionWithRange.FullRange);
- shardData.Ranges.clear();
- } else {
- shardData.Ranges.emplace_back(std::move(partitionWithRange.PointOrRange));
- }
- }
- }
-
- for (auto& [shardId, shardData] : ret) {
- auto& shardInfo = shardInfoMap[shardId];
-
- if (!shardInfo.KeyReadRanges) {
- shardInfo.KeyReadRanges.ConstructInPlace();
- }
-
- for (const auto& paramName : shardParams[shardId]) {
- auto it = params.find(paramName);
- YQL_ENSURE(it != params.end());
- shardInfo.Params.emplace(
- paramName,
- NDq::TDqDataSerializer::SerializeParamValue(it->second.MkqlType, it->second.MkqlValue)
- );
- shardInfo.ParamTypes.emplace(paramName, it->second.MkqlType);
- }
-
- if (shardData.FullRange) {
- shardInfo.KeyReadRanges->MakeFullRange(std::move(*shardData.FullRange));
- } else {
- for (auto& range : shardData.Ranges) {
- shardInfo.KeyReadRanges->Add(std::move(range));
- }
- }
- }
-
- return shardInfoMap;
-}
-
-} // namespace
-
-THashMap<ui64, TShardInfo> PrunePartitions(const TKqpTableKeys& tableKeys, const NKqpProto::TKqpPhyOpLookup& lookup,
- const TStageInfo& stageInfo, const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv)
-{
- YQL_CLOG(TRACE, ProviderKqp) << "PrunePartitions: " << lookup.DebugString();
-
- if (!lookup.HasKeysValue()) {
- THashMap<ui64, TShardInfo> shardInfoMap;
- FillFullRange(stageInfo, shardInfoMap, /* read */ true);
- return shardInfoMap;
- }
-
- switch (auto kind = lookup.GetKeysValue().GetKindCase()) {
- case NKqpProto::TKqpPhyValue::kParamValue: {
- return PartitionLookupByParameterValue(lookup.GetKeysValue().GetParamValue(), tableKeys, stageInfo,
- holderFactory, typeEnv);
- }
-
- case NKqpProto::TKqpPhyValue::kRowsList: {
- return PartitionLookupByRowsList(lookup.GetKeysValue().GetRowsList(), tableKeys, stageInfo,
- holderFactory, typeEnv);
- }
-
- case NKqpProto::TKqpPhyValue::kParamElementValue:
- case NKqpProto::TKqpPhyValue::kLiteralValue:
- case NKqpProto::TKqpPhyValue::KIND_NOT_SET:
- YQL_ENSURE(false, "Unexpected lookup kind " << (int) kind);
- return {};
- }
-}
-
-template <typename TEffect>
-THashMap<ui64, TShardInfo> PruneEffectPartitionsImpl(const TKqpTableKeys& tableKeys, const TEffect& effect,
- const TStageInfo& stageInfo, const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv)
-{
- THashMap<ui64, TShardInfo> shardInfoMap;
- if (effect.HasRowsValue() &&
- effect.GetRowsValue().GetKindCase() == NKqpProto::TKqpPhyValue::kParamValue)
- {
- const auto& name = effect.GetRowsValue().GetParamValue().GetParamName();
+
+ break;
+ }
+
+ case NKqpProto::TKqpPhyRowsList_TValue::KIND_NOT_SET: {
+ YQL_ENSURE(false);
+ }
+ }
+
+ for (ui64 i = 0; i < table.KeyColumns.size(); ++i) {
+ if (table.KeyColumns[i] == columnName) {
+ keyFrom[i] = keyTo[i] = NMiniKQL::MakeCell(
+ table.KeyColumnTypes[i], mkqlValue, typeEnv, /* copyValue */ false);
+ break;
+ }
+ }
+ }
+
+ auto range = TTableRange(keyFrom, true, keyTo, true, /* point */ false);
+ auto partitions = GetKeyRangePartitions(range, stageInfo.Meta.ShardKey->Partitions, table.KeyColumnTypes);
+
+ for (auto& partitionWithRange: partitions) {
+ ui64 shardId = partitionWithRange.PartitionInfo->ShardId;
+
+ for (const auto& [columnName, columnValue] : row.GetColumns()) {
+ if (columnValue.GetKindCase() == NKqpProto::TKqpPhyRowsList_TValue::kParamValue) {
+ shardParams[shardId].emplace(columnValue.GetParamValue().GetParamName());
+ }
+ }
+
+ auto& shardData = ret[shardId];
+ if (partitionWithRange.FullRange) {
+ shardData.FullRange = std::move(partitionWithRange.FullRange);
+ shardData.Ranges.clear();
+ } else {
+ shardData.Ranges.emplace_back(std::move(partitionWithRange.PointOrRange));
+ }
+ }
+ }
+
+ for (auto& [shardId, shardData] : ret) {
+ auto& shardInfo = shardInfoMap[shardId];
+
+ if (!shardInfo.KeyReadRanges) {
+ shardInfo.KeyReadRanges.ConstructInPlace();
+ }
+
+ for (const auto& paramName : shardParams[shardId]) {
+ auto it = params.find(paramName);
+ YQL_ENSURE(it != params.end());
+ shardInfo.Params.emplace(
+ paramName,
+ NDq::TDqDataSerializer::SerializeParamValue(it->second.MkqlType, it->second.MkqlValue)
+ );
+ shardInfo.ParamTypes.emplace(paramName, it->second.MkqlType);
+ }
+
+ if (shardData.FullRange) {
+ shardInfo.KeyReadRanges->MakeFullRange(std::move(*shardData.FullRange));
+ } else {
+ for (auto& range : shardData.Ranges) {
+ shardInfo.KeyReadRanges->Add(std::move(range));
+ }
+ }
+ }
+
+ return shardInfoMap;
+}
+
+} // namespace
+
+THashMap<ui64, TShardInfo> PrunePartitions(const TKqpTableKeys& tableKeys, const NKqpProto::TKqpPhyOpLookup& lookup,
+ const TStageInfo& stageInfo, const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv)
+{
+ YQL_CLOG(TRACE, ProviderKqp) << "PrunePartitions: " << lookup.DebugString();
+
+ if (!lookup.HasKeysValue()) {
+ THashMap<ui64, TShardInfo> shardInfoMap;
+ FillFullRange(stageInfo, shardInfoMap, /* read */ true);
+ return shardInfoMap;
+ }
+
+ switch (auto kind = lookup.GetKeysValue().GetKindCase()) {
+ case NKqpProto::TKqpPhyValue::kParamValue: {
+ return PartitionLookupByParameterValue(lookup.GetKeysValue().GetParamValue(), tableKeys, stageInfo,
+ holderFactory, typeEnv);
+ }
+
+ case NKqpProto::TKqpPhyValue::kRowsList: {
+ return PartitionLookupByRowsList(lookup.GetKeysValue().GetRowsList(), tableKeys, stageInfo,
+ holderFactory, typeEnv);
+ }
+
+ case NKqpProto::TKqpPhyValue::kParamElementValue:
+ case NKqpProto::TKqpPhyValue::kLiteralValue:
+ case NKqpProto::TKqpPhyValue::KIND_NOT_SET:
+ YQL_ENSURE(false, "Unexpected lookup kind " << (int) kind);
+ return {};
+ }
+}
+
+template <typename TEffect>
+THashMap<ui64, TShardInfo> PruneEffectPartitionsImpl(const TKqpTableKeys& tableKeys, const TEffect& effect,
+ const TStageInfo& stageInfo, const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv)
+{
+ THashMap<ui64, TShardInfo> shardInfoMap;
+ if (effect.HasRowsValue() &&
+ effect.GetRowsValue().GetKindCase() == NKqpProto::TKqpPhyValue::kParamValue)
+ {
+ const auto& name = effect.GetRowsValue().GetParamValue().GetParamName();
auto param = stageInfo.Meta.Tx.Params.Values.FindPtr(name);
- YQL_ENSURE(param);
-
- auto shardsMap = PartitionParamByKey(*param, stageInfo.Meta.TableId, tableKeys, *stageInfo.Meta.ShardKey,
- holderFactory, typeEnv);
-
- for (auto& [shardId, shardData] : shardsMap) {
- auto& shardInfo = shardInfoMap[shardId];
-
- auto ret = shardInfo.Params.emplace(name, std::move(shardData.ParamValues));
- YQL_ENSURE(ret.second);
+ YQL_ENSURE(param);
+
+ auto shardsMap = PartitionParamByKey(*param, stageInfo.Meta.TableId, tableKeys, *stageInfo.Meta.ShardKey,
+ holderFactory, typeEnv);
+
+ for (auto& [shardId, shardData] : shardsMap) {
+ auto& shardInfo = shardInfoMap[shardId];
+
+ auto ret = shardInfo.Params.emplace(name, std::move(shardData.ParamValues));
+ YQL_ENSURE(ret.second);
auto retType = shardInfo.ParamTypes.emplace(name, std::move(shardData.ParamType));
YQL_ENSURE(retType.second);
-
- if (!shardInfo.KeyWriteRanges) {
- shardInfo.KeyWriteRanges.ConstructInPlace();
- }
-
- if (shardData.FullRange) {
- shardInfo.KeyWriteRanges->MakeFullRange(std::move(*shardData.FullRange));
- } else {
- for (auto& range : shardData.Ranges) {
- shardInfo.KeyWriteRanges->Add(std::move(range));
- }
- }
- }
- } else {
- FillFullRange(stageInfo, shardInfoMap, /* read */ false);
- }
-
- return shardInfoMap;
-}
-
-THashMap<ui64, TShardInfo> PruneEffectPartitions(const TKqpTableKeys& tableKeys,
- const NKqpProto::TKqpPhyOpUpsertRows& effect, const TStageInfo& stageInfo,
- const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv)
-{
- return PruneEffectPartitionsImpl(tableKeys, effect, stageInfo, holderFactory, typeEnv);
-}
-
-THashMap<ui64, TShardInfo> PruneEffectPartitions(const TKqpTableKeys& tableKeys,
- const NKqpProto::TKqpPhyOpDeleteRows& effect, const TStageInfo& stageInfo,
- const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv)
-{
- return PruneEffectPartitionsImpl(tableKeys, effect, stageInfo, holderFactory, typeEnv);
-}
-
-} // namespace NKikimr::NKqp
+
+ if (!shardInfo.KeyWriteRanges) {
+ shardInfo.KeyWriteRanges.ConstructInPlace();
+ }
+
+ if (shardData.FullRange) {
+ shardInfo.KeyWriteRanges->MakeFullRange(std::move(*shardData.FullRange));
+ } else {
+ for (auto& range : shardData.Ranges) {
+ shardInfo.KeyWriteRanges->Add(std::move(range));
+ }
+ }
+ }
+ } else {
+ FillFullRange(stageInfo, shardInfoMap, /* read */ false);
+ }
+
+ return shardInfoMap;
+}
+
+THashMap<ui64, TShardInfo> PruneEffectPartitions(const TKqpTableKeys& tableKeys,
+ const NKqpProto::TKqpPhyOpUpsertRows& effect, const TStageInfo& stageInfo,
+ const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv)
+{
+ return PruneEffectPartitionsImpl(tableKeys, effect, stageInfo, holderFactory, typeEnv);
+}
+
+THashMap<ui64, TShardInfo> PruneEffectPartitions(const TKqpTableKeys& tableKeys,
+ const NKqpProto::TKqpPhyOpDeleteRows& effect, const TStageInfo& stageInfo,
+ const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv)
+{
+ return PruneEffectPartitionsImpl(tableKeys, effect, stageInfo, holderFactory, typeEnv);
+}
+
+} // namespace NKikimr::NKqp
diff --git a/ydb/core/kqp/executer/kqp_partition_helper.h b/ydb/core/kqp/executer/kqp_partition_helper.h
index 4f0dc20f72..f01f25ab4a 100644
--- a/ydb/core/kqp/executer/kqp_partition_helper.h
+++ b/ydb/core/kqp/executer/kqp_partition_helper.h
@@ -1,57 +1,57 @@
-#pragma once
-
-#include "kqp_tasks_graph.h"
-
+#pragma once
+
+#include "kqp_tasks_graph.h"
+
#include <ydb/library/yql/dq/common/dq_value.h>
#include <ydb/library/yql/dq/proto/dq_tasks.pb.h>
#include <ydb/library/yql/dq/proto/dq_transport.pb.h>
-
+
#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
-
-#include <util/generic/variant.h>
-
-namespace NKikimr::NKqp {
-
-struct TShardInfo {
- TMap<TString, NYql::NDqProto::TData> Params;
+
+#include <util/generic/variant.h>
+
+namespace NKikimr::NKqp {
+
+struct TShardInfo {
+ TMap<TString, NYql::NDqProto::TData> Params;
TMap<TString, NKikimr::NMiniKQL::TType*> ParamTypes;
-
- TMaybe<TShardKeyRanges> KeyReadRanges; // empty -> no reads
- TMaybe<TShardKeyRanges> KeyWriteRanges; // empty -> no writes
-
- TString ToString(const TVector<NScheme::TTypeId>& keyTypes, const NScheme::TTypeRegistry& typeRegistry) const;
-};
-
+
+ TMaybe<TShardKeyRanges> KeyReadRanges; // empty -> no reads
+ TMaybe<TShardKeyRanges> KeyWriteRanges; // empty -> no writes
+
+ TString ToString(const TVector<NScheme::TTypeId>& keyTypes, const NScheme::TTypeRegistry& typeRegistry) const;
+};
+
TSerializedTableRange MakeKeyRange(const TVector<NUdf::TDataTypeId>& keyColumnTypes,
const NKqpProto::TKqpPhyKeyRange& range, const TStageInfo& stageInfo, const NMiniKQL::THolderFactory& holderFactory,
const NMiniKQL::TTypeEnvironment& typeEnv);
-TVector<TSerializedPointOrRange> FillReadRanges(const TVector<NUdf::TDataTypeId>& keyColumnTypes,
+TVector<TSerializedPointOrRange> FillReadRanges(const TVector<NUdf::TDataTypeId>& keyColumnTypes,
const NKqpProto::TKqpPhyOpReadOlapRanges& readRange, const TStageInfo& stageInfo,
const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv);
-TVector<TSerializedPointOrRange> FillReadRanges(const TVector<NUdf::TDataTypeId>& keyColumnTypes,
+TVector<TSerializedPointOrRange> FillReadRanges(const TVector<NUdf::TDataTypeId>& keyColumnTypes,
const NKqpProto::TKqpPhyOpReadRanges& readRange, const TStageInfo& stageInfo,
const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv);
-THashMap<ui64, TShardInfo> PrunePartitions(const TKqpTableKeys& tableKeys,
- const NKqpProto::TKqpPhyOpReadRange& readRange, const TStageInfo& stageInfo,
- const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv);
-
-THashMap<ui64, TShardInfo> PrunePartitions(const TKqpTableKeys& tableKeys,
+THashMap<ui64, TShardInfo> PrunePartitions(const TKqpTableKeys& tableKeys,
+ const NKqpProto::TKqpPhyOpReadRange& readRange, const TStageInfo& stageInfo,
+ const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv);
+
+THashMap<ui64, TShardInfo> PrunePartitions(const TKqpTableKeys& tableKeys,
const NKqpProto::TKqpPhyOpReadRanges& readRanges, const TStageInfo& stageInfo,
const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv);
THashMap<ui64, TShardInfo> PrunePartitions(const TKqpTableKeys& tableKeys,
- const NKqpProto::TKqpPhyOpLookup& lookup, const TStageInfo& stageInfo,
- const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv);
-
-THashMap<ui64, TShardInfo> PruneEffectPartitions(const TKqpTableKeys& tableKeys,
- const NKqpProto::TKqpPhyOpUpsertRows& effect, const TStageInfo& stageInfo,
- const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv);
-
-THashMap<ui64, TShardInfo> PruneEffectPartitions(const TKqpTableKeys& tableKeys,
- const NKqpProto::TKqpPhyOpDeleteRows& effect, const TStageInfo& stageInfo,
- const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv);
-
-} // namespace NKikimr::NKqp
+ const NKqpProto::TKqpPhyOpLookup& lookup, const TStageInfo& stageInfo,
+ const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv);
+
+THashMap<ui64, TShardInfo> PruneEffectPartitions(const TKqpTableKeys& tableKeys,
+ const NKqpProto::TKqpPhyOpUpsertRows& effect, const TStageInfo& stageInfo,
+ const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv);
+
+THashMap<ui64, TShardInfo> PruneEffectPartitions(const TKqpTableKeys& tableKeys,
+ const NKqpProto::TKqpPhyOpDeleteRows& effect, const TStageInfo& stageInfo,
+ const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv);
+
+} // namespace NKikimr::NKqp
diff --git a/ydb/core/kqp/executer/kqp_planner.cpp b/ydb/core/kqp/executer/kqp_planner.cpp
index 46cae15016..6bcfa85c66 100644
--- a/ydb/core/kqp/executer/kqp_planner.cpp
+++ b/ydb/core/kqp/executer/kqp_planner.cpp
@@ -1,271 +1,271 @@
-#include "kqp_planner.h"
-#include "kqp_planner_strategy.h"
+#include "kqp_planner.h"
+#include "kqp_planner_strategy.h"
#include "kqp_shards_resolver.h"
-
+
#include <ydb/core/base/appdata.h>
#include <ydb/core/kqp/rm/kqp_rm.h>
#include <ydb/core/kqp/rm/kqp_resource_estimation.h>
-
+
#include <util/generic/set.h>
-
-namespace NKikimr::NKqp {
-
-#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream)
-#define LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream)
-#define LOG_C(stream) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream)
-#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream)
-
-using namespace NYql;
-
-// Task can allocate extra memory during execution.
-// So, we estimate total memory amount required for task as apriori task size multiplied by this constant.
-constexpr ui32 MEMORY_ESTIMATION_OVERFLOW = 2;
-
-TKqpPlanner::TKqpPlanner(ui64 txId, const TActorId& executer, TVector<NDqProto::TDqTask>&& tasks,
- THashMap<ui64, TVector<NDqProto::TDqTask>>&& scanTasks, const IKqpGateway::TKqpSnapshot& snapshot,
- const TString& database, const TMaybe<TString>& userToken, TInstant deadline, const NYql::NDqProto::EDqStatsMode& statsMode,
+
+namespace NKikimr::NKqp {
+
+#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream)
+#define LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream)
+#define LOG_C(stream) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream)
+#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream)
+
+using namespace NYql;
+
+// Task can allocate extra memory during execution.
+// So, we estimate total memory amount required for task as apriori task size multiplied by this constant.
+constexpr ui32 MEMORY_ESTIMATION_OVERFLOW = 2;
+
+TKqpPlanner::TKqpPlanner(ui64 txId, const TActorId& executer, TVector<NDqProto::TDqTask>&& tasks,
+ THashMap<ui64, TVector<NDqProto::TDqTask>>&& scanTasks, const IKqpGateway::TKqpSnapshot& snapshot,
+ const TString& database, const TMaybe<TString>& userToken, TInstant deadline, const NYql::NDqProto::EDqStatsMode& statsMode,
bool disableLlvmForUdfStages, bool enableLlvm, bool withSpilling, const TMaybe<NKikimrKqp::TRlPath>& rlPath)
- : TxId(txId)
- , ExecuterId(executer)
- , Tasks(std::move(tasks))
+ : TxId(txId)
+ , ExecuterId(executer)
+ , Tasks(std::move(tasks))
, ScanTasks(std::move(scanTasks))
, Snapshot(snapshot)
- , Database(database)
+ , Database(database)
, UserToken(userToken)
- , Deadline(deadline)
+ , Deadline(deadline)
, StatsMode(statsMode)
- , DisableLlvmForUdfStages(disableLlvmForUdfStages)
+ , DisableLlvmForUdfStages(disableLlvmForUdfStages)
, EnableLlvm(enableLlvm)
- , WithSpilling(withSpilling)
+ , WithSpilling(withSpilling)
, RlPath(rlPath)
-{
- if (!Database) {
- // a piece of magic for tests
- for (auto& x : AppData()->DomainsInfo->DomainByName) {
- Database = TStringBuilder() << '/' << x.first;
- LOG_E("Database not set, use " << Database);
- }
- }
-}
-
-void TKqpPlanner::Bootstrap(const TActorContext&) {
- GetKqpResourceManager()->RequestClusterResourcesInfo(
- [as = TlsActivationContext->ActorSystem(), self = SelfId()](TVector<NKikimrKqp::TKqpNodeResources>&& resources) {
- TAutoPtr<IEventHandle> eh = new IEventHandle(self, self, new TEvPrivate::TEvResourcesSnapshot(std::move(resources)));
- as->Send(eh);
- });
-
- Become(&TKqpPlanner::WaitState);
-}
-
-void TKqpPlanner::WaitState(TAutoPtr<IEventHandle>& ev, const TActorContext&) {
- switch (ev->GetTypeRewrite()) {
- hFunc(TEvPrivate::TEvResourcesSnapshot, HandleWait);
- hFunc(TEvKqp::TEvAbortExecution, HandleWait);
- default:
- LOG_C("Unexpected event type: " << ev->GetTypeRewrite() << " at Wait state"
- << ", event: " << (ev->HasEvent() ? ev->GetBase()->ToString().data() : "<serialized>"));
- }
-}
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-/// Wait State
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-void TKqpPlanner::HandleWait(TEvPrivate::TEvResourcesSnapshot::TPtr& ev) {
- if (ev->Get()->Snapshot.empty()) {
- LOG_E("Can not find default state storage group for database " << Database);
- RunLocal(ev->Get()->Snapshot);
- return;
- }
-
- Process(ev->Get()->Snapshot);
-}
-
-void TKqpPlanner::HandleWait(TEvKqp::TEvAbortExecution::TPtr& ev) {
- LOG_E("Terminate KqpPlanner, reason: " << ev->Get()->Record.GetMessage());
- PassAway();
-}
-
-void TKqpPlanner::Process(const TVector<NKikimrKqp::TKqpNodeResources>& snapshot) {
- auto rmConfig = GetKqpResourceManager()->GetConfig();
-
- ui32 tasksCount = Tasks.size();
- for (auto& [shardId, tasks] : ScanTasks) {
- tasksCount += tasks.size();
- }
-
- TVector<TTaskResourceEstimation> est;
- est.resize(tasksCount);
-
- ui64 localRunMemoryEst = 0;
-
- ui64 i = 0;
- for (auto& task : Tasks) {
- EstimateTaskResources(task, 0, 0, rmConfig, est[i]);
- localRunMemoryEst += est[i].TotalMemoryLimit;
- i++;
- }
- if (auto it = ScanTasks.find(SelfId().NodeId()); it != ScanTasks.end()) {
- for (auto& task : it->second) {
- EstimateTaskResources(task, 0, 0, rmConfig, est[i]);
- localRunMemoryEst += est[i].TotalMemoryLimit;
- i++;
- }
- }
-
- auto localResources = GetKqpResourceManager()->GetLocalResources();
- if (localRunMemoryEst * MEMORY_ESTIMATION_OVERFLOW <= localResources.Memory[NRm::EKqpMemoryPool::ScanQuery] &&
- tasksCount <= localResources.ExecutionUnits)
- {
- RunLocal(snapshot);
- return;
- }
-
- if (snapshot.empty() || (snapshot.size() == 1 && snapshot[0].GetNodeId() == SelfId().NodeId())) {
- // try to run without memory overflow settings
- if (localRunMemoryEst <= localResources.Memory[NRm::EKqpMemoryPool::ScanQuery] &&
- tasksCount <= localResources.ExecutionUnits)
- {
- RunLocal(snapshot);
- return;
- }
-
- LOG_E("Not enough resources to execute query locally and no information about other nodes");
- auto ev = MakeHolder<TEvKqp::TEvAbortExecution>(Ydb::StatusIds::PRECONDITION_FAILED,
- "Not enough resources to execute query locally and no information about other nodes");
- Send(ExecuterId, ev.Release());
- PassAway();
- return;
- }
-
- auto planner = CreateKqpGreedyPlanner();
-
- auto ctx = TlsActivationContext->AsActorContext();
- if (ctx.LoggerSettings() && ctx.LoggerSettings()->Satisfies(NActors::NLog::PRI_DEBUG, NKikimrServices::KQP_EXECUTER)) {
- planner->SetLogFunc([TxId = TxId](TStringBuf msg) { LOG_D(msg); });
- }
-
+{
+ if (!Database) {
+ // a piece of magic for tests
+ for (auto& x : AppData()->DomainsInfo->DomainByName) {
+ Database = TStringBuilder() << '/' << x.first;
+ LOG_E("Database not set, use " << Database);
+ }
+ }
+}
+
+void TKqpPlanner::Bootstrap(const TActorContext&) {
+ GetKqpResourceManager()->RequestClusterResourcesInfo(
+ [as = TlsActivationContext->ActorSystem(), self = SelfId()](TVector<NKikimrKqp::TKqpNodeResources>&& resources) {
+ TAutoPtr<IEventHandle> eh = new IEventHandle(self, self, new TEvPrivate::TEvResourcesSnapshot(std::move(resources)));
+ as->Send(eh);
+ });
+
+ Become(&TKqpPlanner::WaitState);
+}
+
+void TKqpPlanner::WaitState(TAutoPtr<IEventHandle>& ev, const TActorContext&) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvPrivate::TEvResourcesSnapshot, HandleWait);
+ hFunc(TEvKqp::TEvAbortExecution, HandleWait);
+ default:
+ LOG_C("Unexpected event type: " << ev->GetTypeRewrite() << " at Wait state"
+ << ", event: " << (ev->HasEvent() ? ev->GetBase()->ToString().data() : "<serialized>"));
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+/// Wait State
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+void TKqpPlanner::HandleWait(TEvPrivate::TEvResourcesSnapshot::TPtr& ev) {
+ if (ev->Get()->Snapshot.empty()) {
+ LOG_E("Can not find default state storage group for database " << Database);
+ RunLocal(ev->Get()->Snapshot);
+ return;
+ }
+
+ Process(ev->Get()->Snapshot);
+}
+
+void TKqpPlanner::HandleWait(TEvKqp::TEvAbortExecution::TPtr& ev) {
+ LOG_E("Terminate KqpPlanner, reason: " << ev->Get()->Record.GetMessage());
+ PassAway();
+}
+
+void TKqpPlanner::Process(const TVector<NKikimrKqp::TKqpNodeResources>& snapshot) {
+ auto rmConfig = GetKqpResourceManager()->GetConfig();
+
+ ui32 tasksCount = Tasks.size();
+ for (auto& [shardId, tasks] : ScanTasks) {
+ tasksCount += tasks.size();
+ }
+
+ TVector<TTaskResourceEstimation> est;
+ est.resize(tasksCount);
+
+ ui64 localRunMemoryEst = 0;
+
+ ui64 i = 0;
+ for (auto& task : Tasks) {
+ EstimateTaskResources(task, 0, 0, rmConfig, est[i]);
+ localRunMemoryEst += est[i].TotalMemoryLimit;
+ i++;
+ }
+ if (auto it = ScanTasks.find(SelfId().NodeId()); it != ScanTasks.end()) {
+ for (auto& task : it->second) {
+ EstimateTaskResources(task, 0, 0, rmConfig, est[i]);
+ localRunMemoryEst += est[i].TotalMemoryLimit;
+ i++;
+ }
+ }
+
+ auto localResources = GetKqpResourceManager()->GetLocalResources();
+ if (localRunMemoryEst * MEMORY_ESTIMATION_OVERFLOW <= localResources.Memory[NRm::EKqpMemoryPool::ScanQuery] &&
+ tasksCount <= localResources.ExecutionUnits)
+ {
+ RunLocal(snapshot);
+ return;
+ }
+
+ if (snapshot.empty() || (snapshot.size() == 1 && snapshot[0].GetNodeId() == SelfId().NodeId())) {
+ // try to run without memory overflow settings
+ if (localRunMemoryEst <= localResources.Memory[NRm::EKqpMemoryPool::ScanQuery] &&
+ tasksCount <= localResources.ExecutionUnits)
+ {
+ RunLocal(snapshot);
+ return;
+ }
+
+ LOG_E("Not enough resources to execute query locally and no information about other nodes");
+ auto ev = MakeHolder<TEvKqp::TEvAbortExecution>(Ydb::StatusIds::PRECONDITION_FAILED,
+ "Not enough resources to execute query locally and no information about other nodes");
+ Send(ExecuterId, ev.Release());
+ PassAway();
+ return;
+ }
+
+ auto planner = CreateKqpGreedyPlanner();
+
+ auto ctx = TlsActivationContext->AsActorContext();
+ if (ctx.LoggerSettings() && ctx.LoggerSettings()->Satisfies(NActors::NLog::PRI_DEBUG, NKikimrServices::KQP_EXECUTER)) {
+ planner->SetLogFunc([TxId = TxId](TStringBuf msg) { LOG_D(msg); });
+ }
+
THashMap<ui64, size_t> nodeIdtoIdx;
for (size_t idx = 0; idx < snapshot.size(); ++idx) {
nodeIdtoIdx[snapshot[idx].nodeid()] = idx;
}
- auto plan = planner->Plan(snapshot, std::move(est));
-
- if (!plan.empty()) {
- for (auto& group : plan) {
- auto ev = PrepareKqpNodeRequest(group.TaskIds);
- AddScansToKqpNodeRequest(ev, group.NodeId);
-
- auto target = MakeKqpNodeServiceID(group.NodeId);
- TlsActivationContext->Send(new IEventHandle(target, ExecuterId, ev.Release(),
- CalcSendMessageFlagsForNode(target.NodeId())));
- }
+ auto plan = planner->Plan(snapshot, std::move(est));
+
+ if (!plan.empty()) {
+ for (auto& group : plan) {
+ auto ev = PrepareKqpNodeRequest(group.TaskIds);
+ AddScansToKqpNodeRequest(ev, group.NodeId);
+
+ auto target = MakeKqpNodeServiceID(group.NodeId);
+ TlsActivationContext->Send(new IEventHandle(target, ExecuterId, ev.Release(),
+ CalcSendMessageFlagsForNode(target.NodeId())));
+ }
TVector<ui64> nodes;
- nodes.reserve(ScanTasks.size());
- for (auto& [nodeId, _]: ScanTasks) {
- nodes.push_back(nodeId);
+ nodes.reserve(ScanTasks.size());
+ for (auto& [nodeId, _]: ScanTasks) {
+ nodes.push_back(nodeId);
}
-
- for (ui64 nodeId: nodes) {
- auto ev = PrepareKqpNodeRequest({});
- AddScansToKqpNodeRequest(ev, nodeId);
-
- auto target = MakeKqpNodeServiceID(nodeId);
- TlsActivationContext->Send(new IEventHandle(target, ExecuterId, ev.Release(),
- CalcSendMessageFlagsForNode(target.NodeId())));
+
+ for (ui64 nodeId: nodes) {
+ auto ev = PrepareKqpNodeRequest({});
+ AddScansToKqpNodeRequest(ev, nodeId);
+
+ auto target = MakeKqpNodeServiceID(nodeId);
+ TlsActivationContext->Send(new IEventHandle(target, ExecuterId, ev.Release(),
+ CalcSendMessageFlagsForNode(target.NodeId())));
}
- Y_VERIFY(ScanTasks.empty());
- } else {
- auto ev = MakeHolder<TEvKqp::TEvAbortExecution>(Ydb::StatusIds::PRECONDITION_FAILED,
- "Not enough resources to execute query");
- Send(ExecuterId, ev.Release());
- }
-
- PassAway();
-}
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-/// Local Execution
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-void TKqpPlanner::RunLocal(const TVector<NKikimrKqp::TKqpNodeResources>& snapshot) {
- LOG_D("Execute query locally");
-
- auto ev = PrepareKqpNodeRequest({});
- AddScansToKqpNodeRequest(ev, SelfId().NodeId());
-
- auto target = MakeKqpNodeServiceID(SelfId().NodeId());
- TlsActivationContext->Send(new IEventHandle(target, ExecuterId, ev.Release(), IEventHandle::FlagTrackDelivery));
-
+ Y_VERIFY(ScanTasks.empty());
+ } else {
+ auto ev = MakeHolder<TEvKqp::TEvAbortExecution>(Ydb::StatusIds::PRECONDITION_FAILED,
+ "Not enough resources to execute query");
+ Send(ExecuterId, ev.Release());
+ }
+
+ PassAway();
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+/// Local Execution
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+void TKqpPlanner::RunLocal(const TVector<NKikimrKqp::TKqpNodeResources>& snapshot) {
+ LOG_D("Execute query locally");
+
+ auto ev = PrepareKqpNodeRequest({});
+ AddScansToKqpNodeRequest(ev, SelfId().NodeId());
+
+ auto target = MakeKqpNodeServiceID(SelfId().NodeId());
+ TlsActivationContext->Send(new IEventHandle(target, ExecuterId, ev.Release(), IEventHandle::FlagTrackDelivery));
+
TVector<ui64> nodes;
for (const auto& pair: ScanTasks) {
nodes.push_back(pair.first);
- YQL_ENSURE(pair.first != SelfId().NodeId());
+ YQL_ENSURE(pair.first != SelfId().NodeId());
}
-
- THashMap<ui64, size_t> nodeIdToIdx;
+
+ THashMap<ui64, size_t> nodeIdToIdx;
for (size_t idx = 0; idx < snapshot.size(); ++idx) {
- nodeIdToIdx[snapshot[idx].nodeid()] = idx;
- LOG_D("snapshot #" << idx << ": " << snapshot[idx].ShortDebugString());
+ nodeIdToIdx[snapshot[idx].nodeid()] = idx;
+ LOG_D("snapshot #" << idx << ": " << snapshot[idx].ShortDebugString());
}
for (auto nodeId: nodes) {
- auto ev = PrepareKqpNodeRequest({});
- AddScansToKqpNodeRequest(ev, nodeId);
- auto target = MakeKqpNodeServiceID(nodeId);
- TlsActivationContext->Send(new IEventHandle(target, ExecuterId, ev.Release(),
- CalcSendMessageFlagsForNode(target.NodeId())));
+ auto ev = PrepareKqpNodeRequest({});
+ AddScansToKqpNodeRequest(ev, nodeId);
+ auto target = MakeKqpNodeServiceID(nodeId);
+ TlsActivationContext->Send(new IEventHandle(target, ExecuterId, ev.Release(),
+ CalcSendMessageFlagsForNode(target.NodeId())));
}
Y_VERIFY(ScanTasks.size() == 0);
- PassAway();
-}
-
-void TKqpPlanner::PassAway() {
- TBase::PassAway();
-}
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-THolder<TEvKqpNode::TEvStartKqpTasksRequest> TKqpPlanner::PrepareKqpNodeRequest(const TVector<ui64>& taskIds) {
- auto ev = MakeHolder<TEvKqpNode::TEvStartKqpTasksRequest>();
-
- ev->Record.SetTxId(TxId);
- ActorIdToProto(ExecuterId, ev->Record.MutableExecuterActorId());
-
+ PassAway();
+}
+
+void TKqpPlanner::PassAway() {
+ TBase::PassAway();
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+THolder<TEvKqpNode::TEvStartKqpTasksRequest> TKqpPlanner::PrepareKqpNodeRequest(const TVector<ui64>& taskIds) {
+ auto ev = MakeHolder<TEvKqpNode::TEvStartKqpTasksRequest>();
+
+ ev->Record.SetTxId(TxId);
+ ActorIdToProto(ExecuterId, ev->Record.MutableExecuterActorId());
+
bool withLLVM = EnableLlvm;
-
- if (taskIds.empty()) {
- for (auto& taskDesc : Tasks) {
- if (taskDesc.GetId()) {
- if (DisableLlvmForUdfStages && taskDesc.GetProgram().GetSettings().GetHasUdf()) {
- withLLVM = false;
- }
- ev->Record.AddTasks()->Swap(&taskDesc);
- }
- }
- } else {
- for (auto& taskDesc : Tasks) {
- if (taskDesc.GetId() && Find(taskIds, taskDesc.GetId()) != taskIds.end()) {
- if (DisableLlvmForUdfStages && taskDesc.GetProgram().GetSettings().GetHasUdf()) {
- withLLVM = false;
- }
- ev->Record.AddTasks()->Swap(&taskDesc);
- }
- }
- }
-
- if (Deadline) {
- TDuration timeout = Deadline - TAppData::TimeProvider->Now();
- ev->Record.MutableRuntimeSettings()->SetTimeoutMs(timeout.MilliSeconds());
- }
-
- ev->Record.MutableRuntimeSettings()->SetExecType(NDqProto::TComputeRuntimeSettings::SCAN);
- ev->Record.MutableRuntimeSettings()->SetStatsMode(StatsMode);
- ev->Record.MutableRuntimeSettings()->SetUseLLVM(withLLVM);
- ev->Record.MutableRuntimeSettings()->SetUseSpilling(WithSpilling);
-
+
+ if (taskIds.empty()) {
+ for (auto& taskDesc : Tasks) {
+ if (taskDesc.GetId()) {
+ if (DisableLlvmForUdfStages && taskDesc.GetProgram().GetSettings().GetHasUdf()) {
+ withLLVM = false;
+ }
+ ev->Record.AddTasks()->Swap(&taskDesc);
+ }
+ }
+ } else {
+ for (auto& taskDesc : Tasks) {
+ if (taskDesc.GetId() && Find(taskIds, taskDesc.GetId()) != taskIds.end()) {
+ if (DisableLlvmForUdfStages && taskDesc.GetProgram().GetSettings().GetHasUdf()) {
+ withLLVM = false;
+ }
+ ev->Record.AddTasks()->Swap(&taskDesc);
+ }
+ }
+ }
+
+ if (Deadline) {
+ TDuration timeout = Deadline - TAppData::TimeProvider->Now();
+ ev->Record.MutableRuntimeSettings()->SetTimeoutMs(timeout.MilliSeconds());
+ }
+
+ ev->Record.MutableRuntimeSettings()->SetExecType(NDqProto::TComputeRuntimeSettings::SCAN);
+ ev->Record.MutableRuntimeSettings()->SetStatsMode(StatsMode);
+ ev->Record.MutableRuntimeSettings()->SetUseLLVM(withLLVM);
+ ev->Record.MutableRuntimeSettings()->SetUseSpilling(WithSpilling);
+
if (RlPath) {
auto rlPath = ev->Record.MutableRuntimeSettings()->MutableRlPath();
rlPath->SetCoordinationNode(RlPath->GetCoordinationNode());
@@ -275,55 +275,55 @@ THolder<TEvKqpNode::TEvStartKqpTasksRequest> TKqpPlanner::PrepareKqpNodeRequest(
rlPath->SetToken(UserToken.GetRef());
}
- ev->Record.SetStartAllOrFail(true);
-
- return ev;
-}
-
-void TKqpPlanner::AddScansToKqpNodeRequest(THolder<TEvKqpNode::TEvStartKqpTasksRequest>& ev, ui64 nodeId) {
- if (!Snapshot.IsValid()) {
- Y_ASSERT(ScanTasks.size() == 0);
- return;
- }
-
- bool withLLVM = true;
- if (auto nodeTasks = ScanTasks.FindPtr(nodeId)) {
- LOG_D("Adding " << nodeTasks->size() << " scans to KqpNode request");
-
- ev->Record.MutableSnapshot()->SetTxId(Snapshot.TxId);
- ev->Record.MutableSnapshot()->SetStep(Snapshot.Step);
-
- for (auto& task: *nodeTasks) {
- if (DisableLlvmForUdfStages && task.GetProgram().GetSettings().GetHasUdf()) {
- withLLVM = false;
- }
- ev->Record.AddTasks()->Swap(&task);
- }
- ScanTasks.erase(nodeId);
- }
-
- if (ev->Record.GetRuntimeSettings().GetUseLLVM()) {
- ev->Record.MutableRuntimeSettings()->SetUseLLVM(withLLVM);
- }
-}
-
-ui32 TKqpPlanner::CalcSendMessageFlagsForNode(ui32 nodeId) {
- ui32 flags = IEventHandle::FlagTrackDelivery;
- if (TrackingNodes.insert(nodeId).second) {
- flags |= IEventHandle::FlagSubscribeOnSession;
- }
- return flags;
-}
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-IActor* CreateKqpPlanner(ui64 txId, const TActorId& executer, TVector<NDqProto::TDqTask>&& tasks,
- THashMap<ui64, TVector<NDqProto::TDqTask>>&& scanTasks, const IKqpGateway::TKqpSnapshot& snapshot,
- const TString& database, const TMaybe<TString>& token, TInstant deadline, const NYql::NDqProto::EDqStatsMode& statsMode,
+ ev->Record.SetStartAllOrFail(true);
+
+ return ev;
+}
+
+void TKqpPlanner::AddScansToKqpNodeRequest(THolder<TEvKqpNode::TEvStartKqpTasksRequest>& ev, ui64 nodeId) {
+ if (!Snapshot.IsValid()) {
+ Y_ASSERT(ScanTasks.size() == 0);
+ return;
+ }
+
+ bool withLLVM = true;
+ if (auto nodeTasks = ScanTasks.FindPtr(nodeId)) {
+ LOG_D("Adding " << nodeTasks->size() << " scans to KqpNode request");
+
+ ev->Record.MutableSnapshot()->SetTxId(Snapshot.TxId);
+ ev->Record.MutableSnapshot()->SetStep(Snapshot.Step);
+
+ for (auto& task: *nodeTasks) {
+ if (DisableLlvmForUdfStages && task.GetProgram().GetSettings().GetHasUdf()) {
+ withLLVM = false;
+ }
+ ev->Record.AddTasks()->Swap(&task);
+ }
+ ScanTasks.erase(nodeId);
+ }
+
+ if (ev->Record.GetRuntimeSettings().GetUseLLVM()) {
+ ev->Record.MutableRuntimeSettings()->SetUseLLVM(withLLVM);
+ }
+}
+
+ui32 TKqpPlanner::CalcSendMessageFlagsForNode(ui32 nodeId) {
+ ui32 flags = IEventHandle::FlagTrackDelivery;
+ if (TrackingNodes.insert(nodeId).second) {
+ flags |= IEventHandle::FlagSubscribeOnSession;
+ }
+ return flags;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+IActor* CreateKqpPlanner(ui64 txId, const TActorId& executer, TVector<NDqProto::TDqTask>&& tasks,
+ THashMap<ui64, TVector<NDqProto::TDqTask>>&& scanTasks, const IKqpGateway::TKqpSnapshot& snapshot,
+ const TString& database, const TMaybe<TString>& token, TInstant deadline, const NYql::NDqProto::EDqStatsMode& statsMode,
bool disableLlvmForUdfStages, bool enableLlvm, bool withSpilling, const TMaybe<NKikimrKqp::TRlPath>& rlPath)
-{
+{
return new TKqpPlanner(txId, executer, std::move(tasks), std::move(scanTasks), snapshot,
database, token, deadline, statsMode, disableLlvmForUdfStages, enableLlvm, withSpilling, rlPath);
-}
-
-} // namespace NKikimr::NKqp
-
+}
+
+} // namespace NKikimr::NKqp
+
diff --git a/ydb/core/kqp/executer/kqp_planner.h b/ydb/core/kqp/executer/kqp_planner.h
index 9f22f94929..dc5e993a74 100644
--- a/ydb/core/kqp/executer/kqp_planner.h
+++ b/ydb/core/kqp/executer/kqp_planner.h
@@ -1,79 +1,79 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/kqp/common/kqp_gateway.h>
#include <ydb/core/kqp/kqp.h>
#include <ydb/core/kqp/node/kqp_node.h>
#include <ydb/core/kqp/rm/kqp_rm.h>
-
-#include <library/cpp/actors/core/actor_bootstrapped.h>
-#include <library/cpp/actors/core/hfunc.h>
-#include <library/cpp/actors/core/log.h>
-
-#include <util/string/vector.h>
-
-
-namespace NKikimr::NKqp {
-
-class TKqpPlanner : public TActorBootstrapped<TKqpPlanner> {
- using TBase = TActorBootstrapped<TKqpPlanner>;
-
- struct TEvPrivate {
- enum EEv {
- EvResourcesSnapshot = EventSpaceBegin(TEvents::ES_PRIVATE)
- };
-
- struct TEvResourcesSnapshot : public TEventLocal<TEvResourcesSnapshot, EEv::EvResourcesSnapshot> {
- TVector<NKikimrKqp::TKqpNodeResources> Snapshot;
-
- TEvResourcesSnapshot(TVector<NKikimrKqp::TKqpNodeResources>&& snapshot)
- : Snapshot(std::move(snapshot)) {}
- };
- };
-
-public:
+
+#include <library/cpp/actors/core/actor_bootstrapped.h>
+#include <library/cpp/actors/core/hfunc.h>
+#include <library/cpp/actors/core/log.h>
+
+#include <util/string/vector.h>
+
+
+namespace NKikimr::NKqp {
+
+class TKqpPlanner : public TActorBootstrapped<TKqpPlanner> {
+ using TBase = TActorBootstrapped<TKqpPlanner>;
+
+ struct TEvPrivate {
+ enum EEv {
+ EvResourcesSnapshot = EventSpaceBegin(TEvents::ES_PRIVATE)
+ };
+
+ struct TEvResourcesSnapshot : public TEventLocal<TEvResourcesSnapshot, EEv::EvResourcesSnapshot> {
+ TVector<NKikimrKqp::TKqpNodeResources> Snapshot;
+
+ TEvResourcesSnapshot(TVector<NKikimrKqp::TKqpNodeResources>&& snapshot)
+ : Snapshot(std::move(snapshot)) {}
+ };
+ };
+
+public:
TKqpPlanner(ui64 txId, const TActorId& executer, TVector<NYql::NDqProto::TDqTask>&& tasks,
THashMap<ui64, TVector<NYql::NDqProto::TDqTask>>&& scanTasks, const IKqpGateway::TKqpSnapshot& snapshot,
- const TString& database, const TMaybe<TString>& userToken, TInstant deadline, const NYql::NDqProto::EDqStatsMode& statsMode,
+ const TString& database, const TMaybe<TString>& userToken, TInstant deadline, const NYql::NDqProto::EDqStatsMode& statsMode,
bool disableLlvmForUdfStages, bool enableLlvm, bool withSpilling, const TMaybe<NKikimrKqp::TRlPath>& rlPath);
-
- void Bootstrap(const TActorContext& ctx);
-
-private:
- STATEFN(WaitState);
-
- void HandleWait(TEvPrivate::TEvResourcesSnapshot::TPtr& ev);
- void HandleWait(TEvKqp::TEvAbortExecution::TPtr& ev);
-
- void Process(const TVector<NKikimrKqp::TKqpNodeResources>& snapshot);
- void RunLocal(const TVector<NKikimrKqp::TKqpNodeResources>& snapshot);
-
- void PassAway() override;
-
- THolder<TEvKqpNode::TEvStartKqpTasksRequest> PrepareKqpNodeRequest(const TVector<ui64>& taskIds);
- void AddScansToKqpNodeRequest(THolder<TEvKqpNode::TEvStartKqpTasksRequest>& ev, ui64 nodeId);
-
- ui32 CalcSendMessageFlagsForNode(ui32 nodeId);
-
-private:
- const ui64 TxId;
+
+ void Bootstrap(const TActorContext& ctx);
+
+private:
+ STATEFN(WaitState);
+
+ void HandleWait(TEvPrivate::TEvResourcesSnapshot::TPtr& ev);
+ void HandleWait(TEvKqp::TEvAbortExecution::TPtr& ev);
+
+ void Process(const TVector<NKikimrKqp::TKqpNodeResources>& snapshot);
+ void RunLocal(const TVector<NKikimrKqp::TKqpNodeResources>& snapshot);
+
+ void PassAway() override;
+
+ THolder<TEvKqpNode::TEvStartKqpTasksRequest> PrepareKqpNodeRequest(const TVector<ui64>& taskIds);
+ void AddScansToKqpNodeRequest(THolder<TEvKqpNode::TEvStartKqpTasksRequest>& ev, ui64 nodeId);
+
+ ui32 CalcSendMessageFlagsForNode(ui32 nodeId);
+
+private:
+ const ui64 TxId;
const TActorId ExecuterId;
- TVector<NYql::NDqProto::TDqTask> Tasks;
+ TVector<NYql::NDqProto::TDqTask> Tasks;
THashMap<ui64, TVector<NYql::NDqProto::TDqTask>> ScanTasks;
const IKqpGateway::TKqpSnapshot Snapshot;
- TString Database;
+ TString Database;
const TMaybe<TString> UserToken;
- const TInstant Deadline;
- const NYql::NDqProto::EDqStatsMode StatsMode;
- const bool DisableLlvmForUdfStages;
+ const TInstant Deadline;
+ const NYql::NDqProto::EDqStatsMode StatsMode;
+ const bool DisableLlvmForUdfStages;
const bool EnableLlvm;
- const bool WithSpilling;
+ const bool WithSpilling;
const TMaybe<NKikimrKqp::TRlPath> RlPath;
- THashSet<ui32> TrackingNodes;
-};
-
+ THashSet<ui32> TrackingNodes;
+};
+
IActor* CreateKqpPlanner(ui64 txId, const TActorId& executer, TVector<NYql::NDqProto::TDqTask>&& tasks,
THashMap<ui64, TVector<NYql::NDqProto::TDqTask>>&& scanTasks, const IKqpGateway::TKqpSnapshot& snapshot,
- const TString& database, const TMaybe<TString>& userToken, TInstant deadline, const NYql::NDqProto::EDqStatsMode& statsMode,
+ const TString& database, const TMaybe<TString>& userToken, TInstant deadline, const NYql::NDqProto::EDqStatsMode& statsMode,
bool disableLlvmForUdfStages, bool enableLlvm, bool withSpilling, const TMaybe<NKikimrKqp::TRlPath>& rlPath);
-
-} // namespace NKikimr::NKqp
+
+} // namespace NKikimr::NKqp
diff --git a/ydb/core/kqp/executer/kqp_planner_strategy.cpp b/ydb/core/kqp/executer/kqp_planner_strategy.cpp
index bb375dabec..b0f323dbc2 100644
--- a/ydb/core/kqp/executer/kqp_planner_strategy.cpp
+++ b/ydb/core/kqp/executer/kqp_planner_strategy.cpp
@@ -1,134 +1,134 @@
-#include "kqp_planner_strategy.h"
-
-#include <library/cpp/actors/core/event_pb.h>
-
-#include <util/generic/queue.h>
-#include <util/string/builder.h>
-#include <util/string/join.h>
-
-
-namespace NKikimr::NKqp {
-
-namespace {
-
-using namespace NActors;
-
-// Task can allocate extra memory during execution.
-// So, we estimate total memory amount required for task as apriori task size multiplied by this constant.
-constexpr float TASK_MEMORY_ESTIMATION_OVERFLOW = 1.2f;
-
-class TKqpGreedyPlanner : public IKqpPlannerStrategy {
-public:
- ~TKqpGreedyPlanner() override {}
-
- TVector<TResult> Plan(const TVector<NKikimrKqp::TKqpNodeResources>& nodeResources,
- TVector<TTaskResourceEstimation>&& tasks) override
- {
- TVector<TResult> result;
-
- struct TNodeDesc {
- ui32 NodeId = std::numeric_limits<ui32>::max();
+#include "kqp_planner_strategy.h"
+
+#include <library/cpp/actors/core/event_pb.h>
+
+#include <util/generic/queue.h>
+#include <util/string/builder.h>
+#include <util/string/join.h>
+
+
+namespace NKikimr::NKqp {
+
+namespace {
+
+using namespace NActors;
+
+// Task can allocate extra memory during execution.
+// So, we estimate total memory amount required for task as apriori task size multiplied by this constant.
+constexpr float TASK_MEMORY_ESTIMATION_OVERFLOW = 1.2f;
+
+class TKqpGreedyPlanner : public IKqpPlannerStrategy {
+public:
+ ~TKqpGreedyPlanner() override {}
+
+ TVector<TResult> Plan(const TVector<NKikimrKqp::TKqpNodeResources>& nodeResources,
+ TVector<TTaskResourceEstimation>&& tasks) override
+ {
+ TVector<TResult> result;
+
+ struct TNodeDesc {
+ ui32 NodeId = std::numeric_limits<ui32>::max();
TActorId ResourceManagerId;
- ui64 RemainsMemory = 0;
- ui32 RemainsComputeActors = 0;
- TVector<ui64> Tasks;
- };
-
- struct TComp {
- bool operator ()(const TNodeDesc& l, const TNodeDesc& r) {
+ ui64 RemainsMemory = 0;
+ ui32 RemainsComputeActors = 0;
+ TVector<ui64> Tasks;
+ };
+
+ struct TComp {
+ bool operator ()(const TNodeDesc& l, const TNodeDesc& r) {
return r.RemainsMemory > l.RemainsMemory && l.RemainsComputeActors > 0;
- }
- };
-
- TPriorityQueue<TNodeDesc, TVector<TNodeDesc>, TComp> nodes{TComp()};
-
- for (auto& node : nodeResources) {
- nodes.emplace(TNodeDesc{
- node.GetNodeId(),
+ }
+ };
+
+ TPriorityQueue<TNodeDesc, TVector<TNodeDesc>, TComp> nodes{TComp()};
+
+ for (auto& node : nodeResources) {
+ nodes.emplace(TNodeDesc{
+ node.GetNodeId(),
ActorIdFromProto(node.GetResourceManagerActorId()),
- node.GetTotalMemory() - node.GetUsedMemory(),
- node.GetAvailableComputeActors(),
- {}
- });
-
- if (LogFunc) {
- LogFunc(TStringBuilder() << "[AvailableResources] node #" << node.GetNodeId()
- << " memory: " << (node.GetTotalMemory() - node.GetUsedMemory())
- << ", ca: " << node.GetAvailableComputeActors());
- }
- }
-
- Sort(tasks, [](const auto& l, const auto& r) { return l.TotalMemoryLimit > r.TotalMemoryLimit; });
-
- if (LogFunc) {
- for (auto& task : tasks) {
- LogFunc(TStringBuilder() << "[TaskResources] task: " << task.TaskId << ", memory: " << task.TotalMemoryLimit);
- }
- }
-
- for (auto& taskEstimation : tasks) {
- TNodeDesc node = nodes.top();
-
- if (node.RemainsComputeActors > 0 &&
- node.RemainsMemory > taskEstimation.TotalMemoryLimit * TASK_MEMORY_ESTIMATION_OVERFLOW)
- {
- nodes.pop();
- node.RemainsComputeActors--;
- node.RemainsMemory -= taskEstimation.TotalMemoryLimit * TASK_MEMORY_ESTIMATION_OVERFLOW;
- node.Tasks.push_back(taskEstimation.TaskId);
-
- if (LogFunc) {
- LogFunc(TStringBuilder() << "Schedule task: " << taskEstimation.TaskId
- << " (" << taskEstimation.TotalMemoryLimit << " bytes) "
- << "to node #" << node.NodeId << ". "
- << "Remains memory: " << node.RemainsMemory << ", ca: " << node.RemainsComputeActors);
- }
-
- nodes.emplace(std::move(node));
- } else {
- if (LogFunc) {
- TStringBuilder err;
- err << "Not enough resources to execute query. Task " << taskEstimation.TaskId
- << " (" << taskEstimation.TotalMemoryLimit << " bytes) "
- << "Node: " << node.NodeId << ", remains memory: " << node.RemainsMemory
- << ", ca: " << node.RemainsComputeActors;
-
- LogFunc(err);
- }
-
- return result;
- }
- }
-
- while (!nodes.empty()) {
- TNodeDesc node = nodes.top();
- nodes.pop();
-
- if (node.Tasks.empty()) {
- continue;
- }
-
- if (LogFunc) {
- LogFunc(TStringBuilder() << "About to execute tasks [" << JoinSeq(", ", node.Tasks) << "]"
- << " on node " << node.NodeId);
- }
-
- result.push_back({});
- TResult& item = result.back();
- item.NodeId = node.NodeId;
- item.ResourceManager = node.ResourceManagerId;
- item.TaskIds.swap(node.Tasks);
- }
-
- return result;
- }
-};
-
-} // anonymous namespace
-
-THolder<IKqpPlannerStrategy> CreateKqpGreedyPlanner() {
+ node.GetTotalMemory() - node.GetUsedMemory(),
+ node.GetAvailableComputeActors(),
+ {}
+ });
+
+ if (LogFunc) {
+ LogFunc(TStringBuilder() << "[AvailableResources] node #" << node.GetNodeId()
+ << " memory: " << (node.GetTotalMemory() - node.GetUsedMemory())
+ << ", ca: " << node.GetAvailableComputeActors());
+ }
+ }
+
+ Sort(tasks, [](const auto& l, const auto& r) { return l.TotalMemoryLimit > r.TotalMemoryLimit; });
+
+ if (LogFunc) {
+ for (auto& task : tasks) {
+ LogFunc(TStringBuilder() << "[TaskResources] task: " << task.TaskId << ", memory: " << task.TotalMemoryLimit);
+ }
+ }
+
+ for (auto& taskEstimation : tasks) {
+ TNodeDesc node = nodes.top();
+
+ if (node.RemainsComputeActors > 0 &&
+ node.RemainsMemory > taskEstimation.TotalMemoryLimit * TASK_MEMORY_ESTIMATION_OVERFLOW)
+ {
+ nodes.pop();
+ node.RemainsComputeActors--;
+ node.RemainsMemory -= taskEstimation.TotalMemoryLimit * TASK_MEMORY_ESTIMATION_OVERFLOW;
+ node.Tasks.push_back(taskEstimation.TaskId);
+
+ if (LogFunc) {
+ LogFunc(TStringBuilder() << "Schedule task: " << taskEstimation.TaskId
+ << " (" << taskEstimation.TotalMemoryLimit << " bytes) "
+ << "to node #" << node.NodeId << ". "
+ << "Remains memory: " << node.RemainsMemory << ", ca: " << node.RemainsComputeActors);
+ }
+
+ nodes.emplace(std::move(node));
+ } else {
+ if (LogFunc) {
+ TStringBuilder err;
+ err << "Not enough resources to execute query. Task " << taskEstimation.TaskId
+ << " (" << taskEstimation.TotalMemoryLimit << " bytes) "
+ << "Node: " << node.NodeId << ", remains memory: " << node.RemainsMemory
+ << ", ca: " << node.RemainsComputeActors;
+
+ LogFunc(err);
+ }
+
+ return result;
+ }
+ }
+
+ while (!nodes.empty()) {
+ TNodeDesc node = nodes.top();
+ nodes.pop();
+
+ if (node.Tasks.empty()) {
+ continue;
+ }
+
+ if (LogFunc) {
+ LogFunc(TStringBuilder() << "About to execute tasks [" << JoinSeq(", ", node.Tasks) << "]"
+ << " on node " << node.NodeId);
+ }
+
+ result.push_back({});
+ TResult& item = result.back();
+ item.NodeId = node.NodeId;
+ item.ResourceManager = node.ResourceManagerId;
+ item.TaskIds.swap(node.Tasks);
+ }
+
+ return result;
+ }
+};
+
+} // anonymous namespace
+
+THolder<IKqpPlannerStrategy> CreateKqpGreedyPlanner() {
return MakeHolder<TKqpGreedyPlanner>();
-}
-
-} // namespace NKikimr::NKqp
-
+}
+
+} // namespace NKikimr::NKqp
+
diff --git a/ydb/core/kqp/executer/kqp_planner_strategy.h b/ydb/core/kqp/executer/kqp_planner_strategy.h
index 29db8cf3f3..42df6703ab 100644
--- a/ydb/core/kqp/executer/kqp_planner_strategy.h
+++ b/ydb/core/kqp/executer/kqp_planner_strategy.h
@@ -1,35 +1,35 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/protos/kqp.pb.h>
#include <ydb/core/kqp/rm/kqp_resource_estimation.h>
-
-#include <library/cpp/actors/core/actorid.h>
-
-namespace NKikimr::NKqp {
-
-class IKqpPlannerStrategy {
-public:
- virtual ~IKqpPlannerStrategy() = default;
-
- using TLogFunc = std::function<void(TStringBuf message)>;
-
- void SetLogFunc(TLogFunc&& logFunc) {
- LogFunc = std::move(logFunc);
- }
-
- struct TResult {
- ui32 NodeId;
+
+#include <library/cpp/actors/core/actorid.h>
+
+namespace NKikimr::NKqp {
+
+class IKqpPlannerStrategy {
+public:
+ virtual ~IKqpPlannerStrategy() = default;
+
+ using TLogFunc = std::function<void(TStringBuf message)>;
+
+ void SetLogFunc(TLogFunc&& logFunc) {
+ LogFunc = std::move(logFunc);
+ }
+
+ struct TResult {
+ ui32 NodeId;
NActors::TActorId ResourceManager;
- TVector<ui64> TaskIds;
- };
-
- virtual TVector<TResult> Plan(const TVector<NKikimrKqp::TKqpNodeResources>& nodeResources,
- TVector<TTaskResourceEstimation>&& estimatedResources) = 0;
-
-protected:
- TLogFunc LogFunc;
-};
-
-THolder<IKqpPlannerStrategy> CreateKqpGreedyPlanner();
-
-} // namespace NKikimr::NKqp
+ TVector<ui64> TaskIds;
+ };
+
+ virtual TVector<TResult> Plan(const TVector<NKikimrKqp::TKqpNodeResources>& nodeResources,
+ TVector<TTaskResourceEstimation>&& estimatedResources) = 0;
+
+protected:
+ TLogFunc LogFunc;
+};
+
+THolder<IKqpPlannerStrategy> CreateKqpGreedyPlanner();
+
+} // namespace NKikimr::NKqp
diff --git a/ydb/core/kqp/executer/kqp_result_channel.cpp b/ydb/core/kqp/executer/kqp_result_channel.cpp
index 2472036d7e..0fe235323e 100644
--- a/ydb/core/kqp/executer/kqp_result_channel.cpp
+++ b/ydb/core/kqp/executer/kqp_result_channel.cpp
@@ -1,57 +1,57 @@
-#include "kqp_result_channel.h"
-
-#include "kqp_executer.h"
+#include "kqp_result_channel.h"
+
+#include "kqp_executer.h"
#include "kqp_executer_impl.h"
-#include "kqp_executer_stats.h"
-
+#include "kqp_executer_stats.h"
+
#include <ydb/core/base/appdata.h>
#include <ydb/core/kqp/kqp.h>
#include <ydb/core/kqp/runtime/kqp_transport.h>
-
+
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h>
-
-namespace NKikimr {
-namespace NKqp {
+
+namespace NKikimr {
+namespace NKqp {
namespace {
-
+
class TResultCommonChannelProxy : public NActors::TActor<TResultCommonChannelProxy> {
-public:
- static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
+public:
+ static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
return NKikimrServices::TActivity::KQP_RESULT_CHANNEL_PROXY;
- }
-
- TResultCommonChannelProxy(ui64 txId, ui64 channelId, TQueryExecutionStats* stats, TActorId executer)
+ }
+
+ TResultCommonChannelProxy(ui64 txId, ui64 channelId, TQueryExecutionStats* stats, TActorId executer)
: TActor(&TResultCommonChannelProxy::WorkState)
- , TxId(txId)
- , ChannelId(channelId)
- , Stats(stats)
+ , TxId(txId)
+ , ChannelId(channelId)
+ , Stats(stats)
, Executer(executer) {}
-
+
protected:
virtual void SendResults(NYql::NDqProto::TEvComputeChannelData& computeData, TActorId sender) = 0;
-private:
- STATEFN(WorkState) {
- switch (ev->GetTypeRewrite()) {
- hFunc(NYql::NDq::TEvDqCompute::TEvChannelData, HandleWork);
- hFunc(TEvKqpExecuter::TEvStreamDataAck, HandleWork);
- hFunc(TEvents::TEvPoison, HandlePoison);
- default: {
- InternalError(TStringBuilder() << "TxId: " << TxId << ", channelId: " << ChannelId
- << "Handle unexpected event " << ev->GetTypeRewrite());
- }
- }
- }
-
+private:
+ STATEFN(WorkState) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(NYql::NDq::TEvDqCompute::TEvChannelData, HandleWork);
+ hFunc(TEvKqpExecuter::TEvStreamDataAck, HandleWork);
+ hFunc(TEvents::TEvPoison, HandlePoison);
+ default: {
+ InternalError(TStringBuilder() << "TxId: " << TxId << ", channelId: " << ChannelId
+ << "Handle unexpected event " << ev->GetTypeRewrite());
+ }
+ }
+ }
+
void UpdateStatistics(const ::NYql::NDqProto::TData& data) {
if (!Stats) {
return;
}
-
+
Stats->ResultBytes += data.GetRaw().size();
Stats->ResultRows += data.GetRows();
}
-
+
void HandleWork(NYql::NDq::TEvDqCompute::TEvChannelData::TPtr& ev) {
NYql::NDqProto::TEvComputeChannelData& record = ev->Get()->Record;
auto& channelData = record.GetChannelData();
@@ -67,61 +67,61 @@ private:
SendResults(record, ev->Sender);
}
- void HandleWork(TEvKqpExecuter::TEvStreamDataAck::TPtr& ev) {
- ui64 seqNo = ev->Get()->Record.GetSeqNo();
- ui64 freeSpace = ev->Get()->Record.GetFreeSpace();
-
- LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId
- << ", send ack to channelId: " << ChannelId
- << ", seqNo: " << seqNo
+ void HandleWork(TEvKqpExecuter::TEvStreamDataAck::TPtr& ev) {
+ ui64 seqNo = ev->Get()->Record.GetSeqNo();
+ ui64 freeSpace = ev->Get()->Record.GetFreeSpace();
+
+ LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId
+ << ", send ack to channelId: " << ChannelId
+ << ", seqNo: " << seqNo
<< ", enough: " << ev->Get()->Record.GetEnough()
- << ", freeSpace: " << freeSpace
- << ", to: " << ComputeActor);
-
- auto ackEv = MakeHolder<NYql::NDq::TEvDqCompute::TEvChannelDataAck>();
- ackEv->Record.SetSeqNo(seqNo);
- ackEv->Record.SetChannelId(ChannelId);
- ackEv->Record.SetFreeSpace(freeSpace);
+ << ", freeSpace: " << freeSpace
+ << ", to: " << ComputeActor);
+
+ auto ackEv = MakeHolder<NYql::NDq::TEvDqCompute::TEvChannelDataAck>();
+ ackEv->Record.SetSeqNo(seqNo);
+ ackEv->Record.SetChannelId(ChannelId);
+ ackEv->Record.SetFreeSpace(freeSpace);
ackEv->Record.SetFinish(ev->Get()->Record.GetEnough());
- Send(ComputeActor, ackEv.Release(), /* TODO: undelivery */ 0, /* cookie */ ChannelId);
- }
-
- void InternalError(const TString& msg) {
- LOG_CRIT_S(*NActors::TlsActivationContext, NKikimrServices::KQP_EXECUTER, msg);
-
- auto evAbort = MakeHolder<TEvKqp::TEvAbortExecution>(Ydb::StatusIds::INTERNAL_ERROR, msg);
- Send(Executer, evAbort.Release());
-
+ Send(ComputeActor, ackEv.Release(), /* TODO: undelivery */ 0, /* cookie */ ChannelId);
+ }
+
+ void InternalError(const TString& msg) {
+ LOG_CRIT_S(*NActors::TlsActivationContext, NKikimrServices::KQP_EXECUTER, msg);
+
+ auto evAbort = MakeHolder<TEvKqp::TEvAbortExecution>(Ydb::StatusIds::INTERNAL_ERROR, msg);
+ Send(Executer, evAbort.Release());
+
Become(&TResultCommonChannelProxy::DeadState);
- }
-
-private:
- STATEFN(DeadState) {
- switch (ev->GetTypeRewrite()) {
- hFunc(TEvents::TEvPoison, HandlePoison);
- }
- }
-
-private:
- void HandlePoison(TEvents::TEvPoison::TPtr&) {
- LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId
- << ", result channelId: " << ChannelId << ", pass away");
- PassAway();
- }
-
-private:
- const ui64 TxId;
- const ui64 ChannelId;
- TQueryExecutionStats* Stats; // owned by KqpExecuter
+ }
+
+private:
+ STATEFN(DeadState) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvents::TEvPoison, HandlePoison);
+ }
+ }
+
+private:
+ void HandlePoison(TEvents::TEvPoison::TPtr&) {
+ LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId
+ << ", result channelId: " << ChannelId << ", pass away");
+ PassAway();
+ }
+
+private:
+ const ui64 TxId;
+ const ui64 ChannelId;
+ TQueryExecutionStats* Stats; // owned by KqpExecuter
const NActors::TActorId Executer;
NActors::TActorId ComputeActor;
-};
-
+};
+
class TResultStreamChannelProxy : public TResultCommonChannelProxy {
public:
TResultStreamChannelProxy(ui64 txId, ui64 channelId, const NKikimrMiniKQL::TType& itemType,
const NKikimrMiniKQL::TType* resultItemType, TActorId target, TQueryExecutionStats* stats,
- TActorId executer)
+ TActorId executer)
: TResultCommonChannelProxy(txId, channelId, stats, executer)
, ResultItemType(resultItemType)
, ItemType(itemType)
@@ -154,8 +154,8 @@ private:
class TResultDataChannelProxy : public TResultCommonChannelProxy {
public:
- TResultDataChannelProxy(ui64 txId, ui64 channelId, TQueryExecutionStats* stats, TActorId executer,
- TVector<NYql::NDqProto::TData>* resultReceiver)
+ TResultDataChannelProxy(ui64 txId, ui64 channelId, TQueryExecutionStats* stats, TActorId executer,
+ TVector<NYql::NDqProto::TData>* resultReceiver)
: TResultCommonChannelProxy(txId, channelId, stats, executer)
, ResultReceiver(resultReceiver) {}
@@ -187,7 +187,7 @@ private:
NActors::IActor* CreateResultStreamChannelProxy(ui64 txId, ui64 channelId, const NKikimrMiniKQL::TType& itemType,
const NKikimrMiniKQL::TType* resultItemType, TActorId target, TQueryExecutionStats* stats, TActorId executer)
-{
+{
LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_EXECUTER,
"CreateResultStreamChannelProxy: TxId: " << txId <<
", channelId: " << channelId
@@ -206,7 +206,7 @@ NActors::IActor* CreateResultDataChannelProxy(ui64 txId, ui64 channelId,
);
return new TResultDataChannelProxy(txId, channelId, stats, executer, resultsReceiver);
-}
-
-} // namespace NKqp
-} // namespace NKikimr
+}
+
+} // namespace NKqp
+} // namespace NKikimr
diff --git a/ydb/core/kqp/executer/kqp_result_channel.h b/ydb/core/kqp/executer/kqp_result_channel.h
index 627991e4e1..299b0f686b 100644
--- a/ydb/core/kqp/executer/kqp_result_channel.h
+++ b/ydb/core/kqp/executer/kqp_result_channel.h
@@ -1,15 +1,15 @@
-#pragma once
-
+#pragma once
+
#include "kqp_tasks_graph.h"
#include <library/cpp/actors/core/actor.h>
-
-namespace NYql {
-class TTypeAnnotationNode;
+namespace NYql {
+
+class TTypeAnnotationNode;
namespace NDqProto {
-
+
class TData;
} // namespace NDqProto
@@ -19,16 +19,16 @@ namespace NKikimrMiniKQL {
class TType;
} // namespace NKikimrMiniKQL
-namespace NKikimr::NKqp {
-
-struct TQueryExecutionStats;
+namespace NKikimr::NKqp {
+
+struct TQueryExecutionStats;
struct TKqpExecuterTxResult;
-
+
NActors::IActor* CreateResultStreamChannelProxy(ui64 txId, ui64 channelId, const NKikimrMiniKQL::TType& itemType,
const NKikimrMiniKQL::TType* resultItemType, NActors::TActorId target, TQueryExecutionStats* stats,
NActors::TActorId executer);
-
+
NActors::IActor* CreateResultDataChannelProxy(ui64 txId, ui64 channelId, TQueryExecutionStats* stats,
NActors::TActorId executer, TVector<NYql::NDqProto::TData>* resultReceiver);
-} // namespace NKikimr::NKqp
+} // namespace NKikimr::NKqp
diff --git a/ydb/core/kqp/executer/kqp_scan_executer.cpp b/ydb/core/kqp/executer/kqp_scan_executer.cpp
index 3ef6cd2b98..07c29f64e2 100644
--- a/ydb/core/kqp/executer/kqp_scan_executer.cpp
+++ b/ydb/core/kqp/executer/kqp_scan_executer.cpp
@@ -1,11 +1,11 @@
#include "kqp_executer.h"
-#include "kqp_executer_impl.h"
-#include "kqp_partition_helper.h"
-#include "kqp_planner.h"
-#include "kqp_result_channel.h"
+#include "kqp_executer_impl.h"
+#include "kqp_partition_helper.h"
+#include "kqp_planner.h"
+#include "kqp_result_channel.h"
#include "kqp_tasks_graph.h"
#include "kqp_tasks_validate.h"
-#include "kqp_shards_resolver.h"
+#include "kqp_shards_resolver.h"
#include <ydb/core/base/appdata.h>
#include <ydb/core/base/tablet_pipecache.h>
@@ -23,7 +23,7 @@
#include <library/cpp/actors/core/actor_bootstrapped.h>
#include <library/cpp/actors/core/hfunc.h>
-#include <library/cpp/actors/core/interconnect.h>
+#include <library/cpp/actors/core/interconnect.h>
#include <library/cpp/actors/core/log.h>
namespace NKikimr {
@@ -34,24 +34,24 @@ using namespace NYql::NDq;
namespace {
-class TKqpScanExecuter : public TKqpExecuterBase<TKqpScanExecuter, EExecType::Scan> {
- using TBase = TKqpExecuterBase<TKqpScanExecuter, EExecType::Scan>;
-
+class TKqpScanExecuter : public TKqpExecuterBase<TKqpScanExecuter, EExecType::Scan> {
+ using TBase = TKqpExecuterBase<TKqpScanExecuter, EExecType::Scan>;
+
public:
static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
return NKikimrServices::TActivity::KQP_EXECUTER_ACTOR;
}
- TKqpScanExecuter(IKqpGateway::TExecPhysicalRequest&& request, const TString& database,
- const TMaybe<TString>& userToken, TKqpRequestCounters::TPtr counters)
+ TKqpScanExecuter(IKqpGateway::TExecPhysicalRequest&& request, const TString& database,
+ const TMaybe<TString>& userToken, TKqpRequestCounters::TPtr counters)
: TBase(std::move(request), database, userToken, counters)
{
- YQL_ENSURE(Request.Transactions.size() == 1);
- YQL_ENSURE(Request.Locks.empty());
- YQL_ENSURE(!Request.ValidateLocks);
- YQL_ENSURE(!Request.EraseLocks);
+ YQL_ENSURE(Request.Transactions.size() == 1);
+ YQL_ENSURE(Request.Locks.empty());
+ YQL_ENSURE(!Request.ValidateLocks);
+ YQL_ENSURE(!Request.EraseLocks);
YQL_ENSURE(Request.IsolationLevel == NKikimrKqp::ISOLATION_LEVEL_UNDEFINED);
- YQL_ENSURE(Request.Snapshot.IsValid());
+ YQL_ENSURE(Request.Snapshot.IsValid());
size_t resultsSize = Request.Transactions[0].Body.ResultsSize();
YQL_ENSURE(resultsSize != 0);
@@ -67,200 +67,200 @@ public:
}
}
-public:
+public:
TActorId KqpShardsResolverId;
STATEFN(WaitResolveState) {
- switch (ev->GetTypeRewrite()) {
+ switch (ev->GetTypeRewrite()) {
hFunc(TEvKqpExecuter::TEvTableResolveStatus, HandleResolve);
hFunc(TEvKqpExecuter::TEvShardsResolveStatus, HandleResolve);
- hFunc(TEvKqp::TEvAbortExecution, HandleAbortExecution);
- hFunc(TEvents::TEvWakeup, HandleTimeout);
+ hFunc(TEvKqp::TEvAbortExecution, HandleAbortExecution);
+ hFunc(TEvents::TEvWakeup, HandleTimeout);
default:
- UnexpectedEvent("WaitResolveState", ev->GetTypeRewrite());
+ UnexpectedEvent("WaitResolveState", ev->GetTypeRewrite());
}
- ReportEventElapsedTime();
+ ReportEventElapsedTime();
}
void HandleResolve(TEvKqpExecuter::TEvTableResolveStatus::TPtr& ev) {
auto& reply = *ev->Get();
-
+
KqpTableResolverId = {};
if (reply.Status != Ydb::StatusIds::SUCCESS) {
- TBase::ReplyErrorAndDie(reply.Status, reply.Issues);
+ TBase::ReplyErrorAndDie(reply.Status, reply.Issues);
return;
}
- TSet<ui64> shardIds;
- for (auto& [stageId, stageInfo] : TasksGraph.GetStagesInfo()) {
- if (stageInfo.Meta.ShardKey) {
- for (auto& partition : stageInfo.Meta.ShardKey->Partitions) {
- shardIds.insert(partition.ShardId);
- }
- }
- }
-
- if (shardIds.size() > 0) {
- LOG_D("Start resolving tablets nodes... (" << shardIds.size() << ")");
- auto kqpShardsResolver = CreateKqpShardsResolver(SelfId(), TxId, std::move(shardIds), 0);
- KqpShardsResolverId = RegisterWithSameMailbox(kqpShardsResolver);
- } else {
- Execute();
- }
- }
-
+ TSet<ui64> shardIds;
+ for (auto& [stageId, stageInfo] : TasksGraph.GetStagesInfo()) {
+ if (stageInfo.Meta.ShardKey) {
+ for (auto& partition : stageInfo.Meta.ShardKey->Partitions) {
+ shardIds.insert(partition.ShardId);
+ }
+ }
+ }
+
+ if (shardIds.size() > 0) {
+ LOG_D("Start resolving tablets nodes... (" << shardIds.size() << ")");
+ auto kqpShardsResolver = CreateKqpShardsResolver(SelfId(), TxId, std::move(shardIds), 0);
+ KqpShardsResolverId = RegisterWithSameMailbox(kqpShardsResolver);
+ } else {
+ Execute();
+ }
+ }
+
void HandleResolve(TEvKqpExecuter::TEvShardsResolveStatus::TPtr& ev) {
auto& reply = *ev->Get();
-
+
KqpShardsResolverId = {};
- // TODO: count resolve time in CpuTime
-
+ // TODO: count resolve time in CpuTime
+
if (reply.Status != Ydb::StatusIds::SUCCESS) {
- LOG_W("Shards nodes resolve failed, status: " << Ydb::StatusIds_StatusCode_Name(reply.Status)
- << ", issues: " << reply.Issues.ToString());
- TBase::ReplyErrorAndDie(reply.Status, reply.Issues);
+ LOG_W("Shards nodes resolve failed, status: " << Ydb::StatusIds_StatusCode_Name(reply.Status)
+ << ", issues: " << reply.Issues.ToString());
+ TBase::ReplyErrorAndDie(reply.Status, reply.Issues);
return;
}
LOG_D("Shards nodes resolved, success: " << reply.ShardNodes.size() << ", failed: " << reply.Unresolved);
ShardIdToNodeId = std::move(reply.ShardNodes);
- for (auto& [shardId, nodeId] : ShardIdToNodeId) {
- ShardsOnNode[nodeId].push_back(shardId);
- }
-
- if (IsDebugLogEnabled()) {
- TStringBuilder sb;
- sb << "Shards on nodes: " << Endl;
- for (auto& pair : ShardsOnNode) {
- sb << " node " << pair.first << ": [";
- if (pair.second.size() <= 20) {
- sb << JoinSeq(", ", pair.second) << "]" << Endl;
- } else {
- sb << JoinRange(", ", pair.second.begin(), std::next(pair.second.begin(), 20)) << ", ...] "
- << "(total " << pair.second.size() << ") " << Endl;
- }
- }
- LOG_D(sb);
- }
-
- Execute();
+ for (auto& [shardId, nodeId] : ShardIdToNodeId) {
+ ShardsOnNode[nodeId].push_back(shardId);
+ }
+
+ if (IsDebugLogEnabled()) {
+ TStringBuilder sb;
+ sb << "Shards on nodes: " << Endl;
+ for (auto& pair : ShardsOnNode) {
+ sb << " node " << pair.first << ": [";
+ if (pair.second.size() <= 20) {
+ sb << JoinSeq(", ", pair.second) << "]" << Endl;
+ } else {
+ sb << JoinRange(", ", pair.second.begin(), std::next(pair.second.begin(), 20)) << ", ...] "
+ << "(total " << pair.second.size() << ") " << Endl;
+ }
+ }
+ LOG_D(sb);
+ }
+
+ Execute();
}
-private:
- STATEFN(ExecuteState) {
+private:
+ STATEFN(ExecuteState) {
try {
switch (ev->GetTypeRewrite()) {
- hFunc(TEvDqCompute::TEvState, HandleExecute);
- hFunc(TEvKqpExecuter::TEvStreamDataAck, HandleExecute);
- hFunc(TEvKqp::TEvAbortExecution, HandleAbortExecution);
- hFunc(TEvents::TEvWakeup, HandleTimeout);
- hFunc(TEvents::TEvUndelivered, HandleExecute);
- hFunc(TEvKqpNode::TEvStartKqpTasksResponse, HandleExecute);
- IgnoreFunc(TEvKqpNode::TEvCancelKqpTasksResponse);
- hFunc(TEvInterconnect::TEvNodeDisconnected, HandleExecute);
- IgnoreFunc(TEvInterconnect::TEvNodeConnected);
- default:
- UnexpectedEvent("ExecuteState", ev->GetTypeRewrite());
+ hFunc(TEvDqCompute::TEvState, HandleExecute);
+ hFunc(TEvKqpExecuter::TEvStreamDataAck, HandleExecute);
+ hFunc(TEvKqp::TEvAbortExecution, HandleAbortExecution);
+ hFunc(TEvents::TEvWakeup, HandleTimeout);
+ hFunc(TEvents::TEvUndelivered, HandleExecute);
+ hFunc(TEvKqpNode::TEvStartKqpTasksResponse, HandleExecute);
+ IgnoreFunc(TEvKqpNode::TEvCancelKqpTasksResponse);
+ hFunc(TEvInterconnect::TEvNodeDisconnected, HandleExecute);
+ IgnoreFunc(TEvInterconnect::TEvNodeConnected);
+ default:
+ UnexpectedEvent("ExecuteState", ev->GetTypeRewrite());
}
} catch (const yexception& e) {
- InternalError(e.what());
+ InternalError(e.what());
}
- ReportEventElapsedTime();
+ ReportEventElapsedTime();
}
- void HandleExecute(TEvDqCompute::TEvState::TPtr& ev) {
+ void HandleExecute(TEvDqCompute::TEvState::TPtr& ev) {
TActorId computeActor = ev->Sender;
auto& state = ev->Get()->Record;
- ui64 taskId = state.GetTaskId();
+ ui64 taskId = state.GetTaskId();
- LOG_D("Got execution state from compute actor: " << computeActor
- << ", task: " << taskId
+ LOG_D("Got execution state from compute actor: " << computeActor
+ << ", task: " << taskId
<< ", state: " << NDqProto::EComputeState_Name((NDqProto::EComputeState) state.GetState())
<< ", stats: " << state.GetStats());
- switch (state.GetState()) {
- case NDqProto::COMPUTE_STATE_UNKNOWN: {
- YQL_ENSURE(false, "unexpected state from " << computeActor << ", task: " << taskId);
- return;
- }
-
- case NDqProto::COMPUTE_STATE_FAILURE: {
- ReplyErrorAndDie(state.GetStatus(), state.MutableIssues());
- return;
- }
-
- case NDqProto::COMPUTE_STATE_EXECUTING: {
- // initial TEvState event from Compute Actor
- // there can be race with RM answer
- if (PendingComputeTasks.erase(taskId)) {
+ switch (state.GetState()) {
+ case NDqProto::COMPUTE_STATE_UNKNOWN: {
+ YQL_ENSURE(false, "unexpected state from " << computeActor << ", task: " << taskId);
+ return;
+ }
+
+ case NDqProto::COMPUTE_STATE_FAILURE: {
+ ReplyErrorAndDie(state.GetStatus(), state.MutableIssues());
+ return;
+ }
+
+ case NDqProto::COMPUTE_STATE_EXECUTING: {
+ // initial TEvState event from Compute Actor
+ // there can be race with RM answer
+ if (PendingComputeTasks.erase(taskId)) {
auto it = PendingComputeActors.emplace(computeActor, TProgressStat());
- YQL_ENSURE(it.second);
-
- if (state.HasStats()) {
- it.first->second.Set(state.GetStats());
+ YQL_ENSURE(it.second);
+
+ if (state.HasStats()) {
+ it.first->second.Set(state.GetStats());
}
- auto& task = TasksGraph.GetTask(taskId);
- task.ComputeActorId = computeActor;
-
+ auto& task = TasksGraph.GetTask(taskId);
+ task.ComputeActorId = computeActor;
+
THashMap<TActorId, THashSet<ui64>> updates;
- CollectTaskChannelsUpdates(task, updates);
- PropagateChannelsUpdates(updates);
+ CollectTaskChannelsUpdates(task, updates);
+ PropagateChannelsUpdates(updates);
} else {
auto it = PendingComputeActors.find(computeActor);
if (it != PendingComputeActors.end()) {
- if (state.HasStats()) {
- it->second.Set(state.GetStats());
+ if (state.HasStats()) {
+ it->second.Set(state.GetStats());
}
}
- }
- break;
- }
-
- case NDqProto::COMPUTE_STATE_FINISHED: {
- if (Stats) {
- Stats->AddComputeActorStats(computeActor.NodeId(), std::move(*state.MutableStats()));
- }
-
+ }
+ break;
+ }
+
+ case NDqProto::COMPUTE_STATE_FINISHED: {
+ if (Stats) {
+ Stats->AddComputeActorStats(computeActor.NodeId(), std::move(*state.MutableStats()));
+ }
+
auto it = PendingComputeActors.find(computeActor);
if (it == PendingComputeActors.end()) {
- LOG_W("Got execution state for compute actor: " << computeActor
- << ", task: " << taskId
- << ", state: " << NDqProto::EComputeState_Name((NDqProto::EComputeState) state.GetState())
- << ", too early (waiting reply from RM)");
-
- if (PendingComputeTasks.erase(taskId)) {
- LOG_E("Got execution state for compute actor: " << computeActor
- << ", for unknown task: " << state.GetTaskId()
- << ", state: " << NDqProto::EComputeState_Name((NDqProto::EComputeState) state.GetState()));
- return;
- }
+ LOG_W("Got execution state for compute actor: " << computeActor
+ << ", task: " << taskId
+ << ", state: " << NDqProto::EComputeState_Name((NDqProto::EComputeState) state.GetState())
+ << ", too early (waiting reply from RM)");
+
+ if (PendingComputeTasks.erase(taskId)) {
+ LOG_E("Got execution state for compute actor: " << computeActor
+ << ", for unknown task: " << state.GetTaskId()
+ << ", state: " << NDqProto::EComputeState_Name((NDqProto::EComputeState) state.GetState()));
+ return;
+ }
} else {
- if (state.HasStats()) {
- it->second.Set(state.GetStats());
+ if (state.HasStats()) {
+ it->second.Set(state.GetStats());
}
LastStats.emplace_back(std::move(it->second));
PendingComputeActors.erase(it);
YQL_ENSURE(PendingComputeTasks.find(taskId) == PendingComputeTasks.end());
- }
- }
- }
-
- CheckExecutionComplete();
+ }
+ }
+ }
+
+ CheckExecutionComplete();
}
- void HandleExecute(TEvKqpExecuter::TEvStreamDataAck::TPtr& ev) {
- LOG_T("Recv stream data ack, seqNo: " << ev->Get()->Record.GetSeqNo()
- << ", freeSpace: " << ev->Get()->Record.GetFreeSpace()
+ void HandleExecute(TEvKqpExecuter::TEvStreamDataAck::TPtr& ev) {
+ LOG_T("Recv stream data ack, seqNo: " << ev->Get()->Record.GetSeqNo()
+ << ", freeSpace: " << ev->Get()->Record.GetFreeSpace()
<< ", enough: " << ev->Get()->Record.GetEnough()
- << ", from: " << ev->Sender);
-
+ << ", from: " << ev->Sender);
+
if (ResultChannelProxies.empty()) {
return;
- }
+ }
// Forward only for stream results, data results acks event theirselves.
YQL_ENSURE(!Results.empty() && Results[0].IsStream);
@@ -268,107 +268,107 @@ private:
auto channelIt = ResultChannelProxies.begin();
auto handle = ev->Forward(channelIt->second->SelfId());
channelIt->second->Receive(handle, TlsActivationContext->AsActorContext());
- }
-
- void HandleExecute(TEvKqpNode::TEvStartKqpTasksResponse::TPtr& ev) {
- auto& record = ev->Get()->Record;
- YQL_ENSURE(record.GetTxId() == TxId);
-
- if (record.NotStartedTasksSize() != 0) {
- auto reason = record.GetNotStartedTasks()[0].GetReason();
- auto& message = record.GetNotStartedTasks()[0].GetMessage();
-
- LOG_E("Stop executing, reason: " << NKikimrKqp::TEvStartKqpTasksResponse_ENotStartedTaskReason_Name(reason)
- << ", message: " << message);
-
- switch (reason) {
- case NKikimrKqp::TEvStartKqpTasksResponse::NOT_ENOUGH_MEMORY: {
- TBase::ReplyErrorAndDie(Ydb::StatusIds::OVERLOADED,
- YqlIssue({}, TIssuesIds::KIKIMR_OVERLOADED, "Not enough memory to execute query"));
- break;
- }
-
- case NKikimrKqp::TEvStartKqpTasksResponse::NOT_ENOUGH_EXECUTION_UNITS: {
- TBase::ReplyErrorAndDie(Ydb::StatusIds::OVERLOADED,
- YqlIssue({}, TIssuesIds::KIKIMR_OVERLOADED, "Not enough computation units to execute query"));
- break;
- }
-
- case NKikimrKqp::TEvStartKqpTasksResponse::QUERY_MEMORY_LIMIT_EXCEEDED: {
- TBase::ReplyErrorAndDie(Ydb::StatusIds::PRECONDITION_FAILED,
- YqlIssue({}, TIssuesIds::KIKIMR_PRECONDITION_FAILED, "Memory limit exceeded"));
- break;
- }
-
- case NKikimrKqp::TEvStartKqpTasksResponse::QUERY_EXECUTION_UNITS_LIMIT_EXCEEDED: {
- TBase::ReplyErrorAndDie(Ydb::StatusIds::OVERLOADED,
- YqlIssue({}, TIssuesIds::KIKIMR_OVERLOADED, "Not enough computation units to execute query"));
- break;
- }
-
- case NKikimrKqp::TEvStartKqpTasksResponse::INTERNAL_ERROR: {
- InternalError("KqpNode internal error");
- break;
- }
- }
-
- return;
- }
-
- THashMap<TActorId, THashSet<ui64>> channelsUpdates;
-
- for (auto& startedTask : record.GetStartedTasks()) {
- auto taskId = startedTask.GetTaskId();
- auto& task = TasksGraph.GetTask(taskId);
-
- task.ComputeActorId = ActorIdFromProto(startedTask.GetActorId());
-
- LOG_D("Executing task: " << taskId << " on compute actor: " << task.ComputeActorId);
-
- if (PendingComputeTasks.erase(taskId) == 0) {
- LOG_D("Executing task: " << taskId << ", compute actor: " << task.ComputeActorId << ", already finished");
- } else {
+ }
+
+ void HandleExecute(TEvKqpNode::TEvStartKqpTasksResponse::TPtr& ev) {
+ auto& record = ev->Get()->Record;
+ YQL_ENSURE(record.GetTxId() == TxId);
+
+ if (record.NotStartedTasksSize() != 0) {
+ auto reason = record.GetNotStartedTasks()[0].GetReason();
+ auto& message = record.GetNotStartedTasks()[0].GetMessage();
+
+ LOG_E("Stop executing, reason: " << NKikimrKqp::TEvStartKqpTasksResponse_ENotStartedTaskReason_Name(reason)
+ << ", message: " << message);
+
+ switch (reason) {
+ case NKikimrKqp::TEvStartKqpTasksResponse::NOT_ENOUGH_MEMORY: {
+ TBase::ReplyErrorAndDie(Ydb::StatusIds::OVERLOADED,
+ YqlIssue({}, TIssuesIds::KIKIMR_OVERLOADED, "Not enough memory to execute query"));
+ break;
+ }
+
+ case NKikimrKqp::TEvStartKqpTasksResponse::NOT_ENOUGH_EXECUTION_UNITS: {
+ TBase::ReplyErrorAndDie(Ydb::StatusIds::OVERLOADED,
+ YqlIssue({}, TIssuesIds::KIKIMR_OVERLOADED, "Not enough computation units to execute query"));
+ break;
+ }
+
+ case NKikimrKqp::TEvStartKqpTasksResponse::QUERY_MEMORY_LIMIT_EXCEEDED: {
+ TBase::ReplyErrorAndDie(Ydb::StatusIds::PRECONDITION_FAILED,
+ YqlIssue({}, TIssuesIds::KIKIMR_PRECONDITION_FAILED, "Memory limit exceeded"));
+ break;
+ }
+
+ case NKikimrKqp::TEvStartKqpTasksResponse::QUERY_EXECUTION_UNITS_LIMIT_EXCEEDED: {
+ TBase::ReplyErrorAndDie(Ydb::StatusIds::OVERLOADED,
+ YqlIssue({}, TIssuesIds::KIKIMR_OVERLOADED, "Not enough computation units to execute query"));
+ break;
+ }
+
+ case NKikimrKqp::TEvStartKqpTasksResponse::INTERNAL_ERROR: {
+ InternalError("KqpNode internal error");
+ break;
+ }
+ }
+
+ return;
+ }
+
+ THashMap<TActorId, THashSet<ui64>> channelsUpdates;
+
+ for (auto& startedTask : record.GetStartedTasks()) {
+ auto taskId = startedTask.GetTaskId();
+ auto& task = TasksGraph.GetTask(taskId);
+
+ task.ComputeActorId = ActorIdFromProto(startedTask.GetActorId());
+
+ LOG_D("Executing task: " << taskId << " on compute actor: " << task.ComputeActorId);
+
+ if (PendingComputeTasks.erase(taskId) == 0) {
+ LOG_D("Executing task: " << taskId << ", compute actor: " << task.ComputeActorId << ", already finished");
+ } else {
auto result = PendingComputeActors.emplace(std::make_pair(task.ComputeActorId, TProgressStat()));
- YQL_ENSURE(result.second);
-
- CollectTaskChannelsUpdates(task, channelsUpdates);
- }
- }
-
- PropagateChannelsUpdates(channelsUpdates);
- }
-
- void HandleExecute(TEvents::TEvUndelivered::TPtr& ev) {
- ui32 eventType = ev->Get()->SourceType;
- auto reason = ev->Get()->Reason;
- switch (eventType) {
- case TEvKqpNode::TEvStartKqpTasksRequest::EventType: {
- return InternalError(TStringBuilder()
- << "TEvKqpNode::TEvStartKqpTasksRequest lost: " << reason);
- }
- default: {
- LOG_E("Event lost, type: " << eventType << ", reason: " << reason);
- }
- }
- }
-
- void HandleExecute(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) {
- auto nodeId = ev->Get()->NodeId;
- LOG_N("Disconnected node " << nodeId);
-
- for (auto computeActor : PendingComputeActors) {
+ YQL_ENSURE(result.second);
+
+ CollectTaskChannelsUpdates(task, channelsUpdates);
+ }
+ }
+
+ PropagateChannelsUpdates(channelsUpdates);
+ }
+
+ void HandleExecute(TEvents::TEvUndelivered::TPtr& ev) {
+ ui32 eventType = ev->Get()->SourceType;
+ auto reason = ev->Get()->Reason;
+ switch (eventType) {
+ case TEvKqpNode::TEvStartKqpTasksRequest::EventType: {
+ return InternalError(TStringBuilder()
+ << "TEvKqpNode::TEvStartKqpTasksRequest lost: " << reason);
+ }
+ default: {
+ LOG_E("Event lost, type: " << eventType << ", reason: " << reason);
+ }
+ }
+ }
+
+ void HandleExecute(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) {
+ auto nodeId = ev->Get()->NodeId;
+ LOG_N("Disconnected node " << nodeId);
+
+ for (auto computeActor : PendingComputeActors) {
if (computeActor.first.NodeId() == nodeId) {
- return ReplyUnavailable(TStringBuilder() << "Connection with node " << nodeId << " lost.");
- }
- }
-
- for (auto& task : TasksGraph.GetTasks()) {
- if (task.Meta.NodeId == nodeId && PendingComputeTasks.contains(task.Id)) {
- return ReplyUnavailable(TStringBuilder() << "Connection with node " << nodeId << " lost.");
- }
- }
- }
-
+ return ReplyUnavailable(TStringBuilder() << "Connection with node " << nodeId << " lost.");
+ }
+ }
+
+ for (auto& task : TasksGraph.GetTasks()) {
+ if (task.Meta.NodeId == nodeId && PendingComputeTasks.contains(task.Id)) {
+ return ReplyUnavailable(TStringBuilder() << "Connection with node " << nodeId << " lost.");
+ }
+ }
+ }
+
private:
void FillReadInfo(TTaskMeta& taskMeta, ui64 itemsLimit, bool reverse,
const TMaybe<::NKqpProto::TKqpPhyOpReadOlapRanges>& readOlapRange)
@@ -404,9 +404,9 @@ private:
void BuildDatashardScanTasks(TStageInfo& stageInfo, const NMiniKQL::THolderFactory& holderFactory,
const NMiniKQL::TTypeEnvironment& typeEnv)
{
- Y_VERIFY_DEBUG(stageInfo.Meta.IsDatashard());
-
- THashMap<ui64, TVector<ui64>> nodeTasks; // nodeId -> [taskId]
+ Y_VERIFY_DEBUG(stageInfo.Meta.IsDatashard());
+
+ THashMap<ui64, TVector<ui64>> nodeTasks; // nodeId -> [taskId]
auto getNodeTask = [&](ui64 nodeId, ui32 taskIdx) -> TTask& {
auto& tasks = nodeTasks[nodeId];
@@ -425,12 +425,12 @@ private:
auto& stage = GetStage(stageInfo);
const auto& table = TableKeys.GetTable(stageInfo.Meta.TableId);
- const auto& keyTypes = table.KeyColumnTypes;
+ const auto& keyTypes = table.KeyColumnTypes;
for (auto& op : stage.GetTableOps()) {
Y_VERIFY_DEBUG(stageInfo.Meta.TablePath == op.GetTable().GetPath());
- auto columns = BuildKqpColumns(op, table);
+ auto columns = BuildKqpColumns(op, table);
THashMap<ui64, TShardInfo> partitions;
switch (op.GetTypeCase()) {
@@ -453,15 +453,15 @@ private:
TString itemsLimitParamName;
NDqProto::TData itemsLimitBytes;
NKikimr::NMiniKQL::TType* itemsLimitType = nullptr;
-
+
// TODO: Support reverse, skipnull and limit for kReadRanges
if (op.GetTypeCase() == NKqpProto::TKqpPhyTableOperation::kReadRange) {
ExtractItemsLimit(stageInfo, op.GetReadRange().GetItemsLimit(), holderFactory,
typeEnv, itemsLimit, itemsLimitParamName, itemsLimitBytes, itemsLimitType);
reverse = op.GetReadRange().GetReverse();
-
+
YQL_ENSURE(!reverse); // TODO: not supported yet
-
+
stageInfo.Meta.SkipNullKeys.assign(op.GetReadRange().GetSkipNullKeys().begin(),
op.GetReadRange().GetSkipNullKeys().end());
}
@@ -521,19 +521,19 @@ private:
* Sort read ranges so that sequential scan of that ranges produce sorted result.
*
* Partition pruner feed us with set of non-intersecting ranges with filled right boundary.
- * So we may sort ranges based solely on the their rightmost point.
+ * So we may sort ranges based solely on the their rightmost point.
*/
std::sort(taskReads.begin(), taskReads.end(), [&](const auto& lhs, const auto& rhs){
- YQL_ENSURE(lhs.ShardId != rhs.ShardId);
+ YQL_ENSURE(lhs.ShardId != rhs.ShardId);
+
+ const std::pair<const TSerializedCellVec*, bool> k1 = lhs.Ranges.GetRightBorder();
+ const std::pair<const TSerializedCellVec*, bool> k2 = lhs.Ranges.GetRightBorder();
- const std::pair<const TSerializedCellVec*, bool> k1 = lhs.Ranges.GetRightBorder();
- const std::pair<const TSerializedCellVec*, bool> k2 = lhs.Ranges.GetRightBorder();
-
const int cmp = CompareBorders<false, false>(
- k1.first->GetCells(),
- k2.first->GetCells(),
- k1.second,
- k2.second,
+ k1.first->GetCells(),
+ k2.first->GetCells(),
+ k1.second,
+ k2.second,
keyTypes);
return (cmp < 0);
@@ -558,7 +558,7 @@ private:
YQL_ENSURE(table->TableKind == ETableKind::Olap);
YQL_ENSURE(stageInfo.Meta.TableKind == ETableKind::Olap);
- const auto& keyColumnTypes = table->KeyColumnTypes;
+ const auto& keyColumnTypes = table->KeyColumnTypes;
auto ranges = FillReadRanges(keyColumnTypes, readRanges, stageInfo, holderFactory, typeEnv);
THashMap<ui64, TShardInfo> shardInfoMap;
@@ -581,7 +581,7 @@ private:
auto& stage = GetStage(stageInfo);
const auto& table = TableKeys.GetTable(stageInfo.Meta.TableId);
- const auto& keyTypes = table.KeyColumnTypes;
+ const auto& keyTypes = table.KeyColumnTypes;
TMap<ui64, TKeyDesc::TPartitionRangeInfo> shard2range;
for (const auto& part: stageInfo.Meta.ShardKey->Partitions) {
@@ -593,7 +593,7 @@ private:
for (auto& op : stage.GetTableOps()) {
Y_VERIFY_DEBUG(stageInfo.Meta.TablePath == op.GetTable().GetPath());
- auto columns = BuildKqpColumns(op, table);
+ auto columns = BuildKqpColumns(op, table);
YQL_ENSURE(
op.GetTypeCase() == NKqpProto::TKqpPhyTableOperation::kReadOlapRange,
@@ -675,7 +675,7 @@ private:
} else {
switch (input.GetTypeCase()) {
case NKqpProto::TKqpPhyConnection::kBroadcast:
- case NKqpProto::TKqpPhyConnection::kHashShuffle:
+ case NKqpProto::TKqpPhyConnection::kHashShuffle:
case NKqpProto::TKqpPhyConnection::kUnionAll:
case NKqpProto::TKqpPhyConnection::kMerge:
break;
@@ -687,15 +687,15 @@ private:
auto& originStageInfo = TasksGraph.GetStageInfo(TStageId(stageInfo.Id.TxId, input.GetStageIndex()));
switch (input.GetTypeCase()) {
- case NKqpProto::TKqpPhyConnection::kHashShuffle: {
+ case NKqpProto::TKqpPhyConnection::kHashShuffle: {
partitionsCount = std::max(partitionsCount, (ui32)originStageInfo.Tasks.size() / 2);
- ui32 nodes = ShardsOnNode.size();
- if (nodes) {
- // <= 2 tasks on node
- partitionsCount = std::min(partitionsCount, std::min(24u, nodes * 2));
- } else {
- partitionsCount = std::min(partitionsCount, 24u);
- }
+ ui32 nodes = ShardsOnNode.size();
+ if (nodes) {
+ // <= 2 tasks on node
+ partitionsCount = std::min(partitionsCount, std::min(24u, nodes * 2));
+ } else {
+ partitionsCount = std::min(partitionsCount, 24u);
+ }
break;
}
@@ -709,57 +709,57 @@ private:
}
for (ui32 i = 0; i < partitionsCount; ++i) {
- auto& task = TasksGraph.AddTask(stageInfo);
- LOG_D("Stage " << stageInfo.Id << " create compute task: " << task.Id);
+ auto& task = TasksGraph.AddTask(stageInfo);
+ LOG_D("Stage " << stageInfo.Id << " create compute task: " << task.Id);
}
}
- void Execute() {
- auto& funcRegistry = *AppData()->FunctionRegistry;
+ void Execute() {
+ auto& funcRegistry = *AppData()->FunctionRegistry;
NMiniKQL::TScopedAlloc alloc(TAlignedPagePoolCounters(), funcRegistry.SupportsSizedAllocators());
NMiniKQL::TTypeEnvironment typeEnv(alloc);
NMiniKQL::TMemoryUsageInfo memInfo("PrepareTasks");
NMiniKQL::THolderFactory holderFactory(alloc.Ref(), memInfo, &funcRegistry);
- auto& tx = Request.Transactions[0];
- for (ui32 stageIdx = 0; stageIdx < tx.Body.StagesSize(); ++stageIdx) {
- auto& stage = tx.Body.GetStages(stageIdx);
- auto& stageInfo = TasksGraph.GetStageInfo(TStageId(0, stageIdx));
+ auto& tx = Request.Transactions[0];
+ for (ui32 stageIdx = 0; stageIdx < tx.Body.StagesSize(); ++stageIdx) {
+ auto& stage = tx.Body.GetStages(stageIdx);
+ auto& stageInfo = TasksGraph.GetStageInfo(TStageId(0, stageIdx));
- LOG_D("Stage " << stageInfo.Id << " AST: " << stage.GetProgramAst());
+ LOG_D("Stage " << stageInfo.Id << " AST: " << stage.GetProgramAst());
- Y_VERIFY_DEBUG(!stage.GetIsEffectsStage());
-
- if (stageInfo.Meta.ShardOperations.empty()) {
- BuildComputeTasks(stageInfo);
- } else if (stageInfo.Meta.IsSysView()) {
- BuildSysViewScanTasks(stageInfo, holderFactory, typeEnv);
+ Y_VERIFY_DEBUG(!stage.GetIsEffectsStage());
+
+ if (stageInfo.Meta.ShardOperations.empty()) {
+ BuildComputeTasks(stageInfo);
+ } else if (stageInfo.Meta.IsSysView()) {
+ BuildSysViewScanTasks(stageInfo, holderFactory, typeEnv);
} else if (stageInfo.Meta.IsOlap()) {
BuildColumnshardScanTasks(stageInfo, holderFactory, typeEnv);
- } else if (stageInfo.Meta.IsDatashard()) {
- BuildDatashardScanTasks(stageInfo, holderFactory, typeEnv);
- } else {
- YQL_ENSURE(false, "Unexpected stage type " << (int) stageInfo.Meta.TableKind);
+ } else if (stageInfo.Meta.IsDatashard()) {
+ BuildDatashardScanTasks(stageInfo, holderFactory, typeEnv);
+ } else {
+ YQL_ENSURE(false, "Unexpected stage type " << (int) stageInfo.Meta.TableKind);
}
- BuildKqpStageChannels(TasksGraph, TableKeys, stageInfo, TxId, AppData()->EnableKqpSpilling);
+ BuildKqpStageChannels(TasksGraph, TableKeys, stageInfo, TxId, AppData()->EnableKqpSpilling);
}
BuildKqpExecuterResults(tx.Body, Results);
- BuildKqpTaskGraphResultChannels(TasksGraph, tx.Body, 0);
-
+ BuildKqpTaskGraphResultChannels(TasksGraph, tx.Body, 0);
+
TIssue validateIssue;
- if (!ValidateTasks(TasksGraph, EExecType::Scan, AppData()->EnableKqpSpilling, validateIssue)) {
- TBase::ReplyErrorAndDie(Ydb::StatusIds::INTERNAL_ERROR, validateIssue);
+ if (!ValidateTasks(TasksGraph, EExecType::Scan, AppData()->EnableKqpSpilling, validateIssue)) {
+ TBase::ReplyErrorAndDie(Ydb::StatusIds::INTERNAL_ERROR, validateIssue);
return;
}
-
+
// NodeId -> {Tasks}
THashMap<ui64, TVector<NYql::NDqProto::TDqTask>> scanTasks;
ui32 nShardScans = 0;
ui32 nScanTasks = 0;
-
+
TVector<NYql::NDqProto::TDqTask> computeTasks;
for (auto& task : TasksGraph.GetTasks()) {
@@ -779,21 +779,21 @@ private:
}
taskDesc.MutableProgram()->CopyFrom(stage.GetProgram());
- taskDesc.SetStageId(task.StageId.StageId);
-
- PrepareKqpTaskParameters(stage, stageInfo, task, taskDesc, typeEnv, holderFactory);
-
- if (task.Meta.NodeId || stageInfo.Meta.IsSysView()) {
- NKikimrTxDataShard::TKqpTransaction::TScanTaskMeta protoTaskMeta;
-
- FillTableMeta(stageInfo, protoTaskMeta.MutableTable());
-
- const auto& tableInfo = TableKeys.GetTable(stageInfo.Meta.TableId);
- for (const auto& keyColumnName : tableInfo.KeyColumns) {
- const auto& keyColumn = tableInfo.Columns.at(keyColumnName);
- protoTaskMeta.AddKeyColumnTypes(keyColumn.Type);
- }
-
+ taskDesc.SetStageId(task.StageId.StageId);
+
+ PrepareKqpTaskParameters(stage, stageInfo, task, taskDesc, typeEnv, holderFactory);
+
+ if (task.Meta.NodeId || stageInfo.Meta.IsSysView()) {
+ NKikimrTxDataShard::TKqpTransaction::TScanTaskMeta protoTaskMeta;
+
+ FillTableMeta(stageInfo, protoTaskMeta.MutableTable());
+
+ const auto& tableInfo = TableKeys.GetTable(stageInfo.Meta.TableId);
+ for (const auto& keyColumnName : tableInfo.KeyColumns) {
+ const auto& keyColumn = tableInfo.Columns.at(keyColumnName);
+ protoTaskMeta.AddKeyColumnTypes(keyColumn.Type);
+ }
+
switch (tableInfo.TableKind) {
case ETableKind::Unknown:
case ETableKind::SysView: {
@@ -814,106 +814,106 @@ private:
}
}
- for (bool skipNullKey : stageInfo.Meta.SkipNullKeys) {
- protoTaskMeta.AddSkipNullKeys(skipNullKey);
- }
-
- YQL_ENSURE(task.Meta.Reads);
- YQL_ENSURE(!task.Meta.Writes);
-
- for (auto& column : task.Meta.Reads->front().Columns) {
- auto* protoColumn = protoTaskMeta.AddColumns();
- protoColumn->SetId(column.Id);
- protoColumn->SetType(column.Type);
- protoColumn->SetName(column.Name);
- }
-
+ for (bool skipNullKey : stageInfo.Meta.SkipNullKeys) {
+ protoTaskMeta.AddSkipNullKeys(skipNullKey);
+ }
+
+ YQL_ENSURE(task.Meta.Reads);
+ YQL_ENSURE(!task.Meta.Writes);
+
+ for (auto& column : task.Meta.Reads->front().Columns) {
+ auto* protoColumn = protoTaskMeta.AddColumns();
+ protoColumn->SetId(column.Id);
+ protoColumn->SetType(column.Type);
+ protoColumn->SetName(column.Name);
+ }
+
if (!task.Meta.Reads->empty()) {
protoTaskMeta.SetReverse(task.Meta.ReadInfo.Reverse);
protoTaskMeta.SetItemsLimit(task.Meta.ReadInfo.ItemsLimit);
-
- if (tableInfo.TableKind == ETableKind::Olap) {
- auto* olapProgram = protoTaskMeta.MutableOlapProgram();
- olapProgram->SetProgram(task.Meta.ReadInfo.OlapProgram.Program);
-
- auto [schema, parameters] = SerializeKqpTasksParametersForOlap(stage, stageInfo, task,
- holderFactory, typeEnv);
-
- olapProgram->SetParametersSchema(schema);
- olapProgram->SetParameters(parameters);
- } else {
- YQL_ENSURE(task.Meta.ReadInfo.OlapProgram.Program.empty());
- }
- }
-
- for (auto& read : *task.Meta.Reads) {
- auto* protoReadMeta = protoTaskMeta.AddReads();
- protoReadMeta->SetShardId(read.ShardId);
- read.Ranges.SerializeTo(protoReadMeta);
-
- YQL_ENSURE((int) read.Columns.size() == protoTaskMeta.GetColumns().size());
- for (ui64 i = 0; i < read.Columns.size(); ++i) {
- YQL_ENSURE(read.Columns[i].Id == protoTaskMeta.GetColumns()[i].GetId());
- YQL_ENSURE(read.Columns[i].Type == protoTaskMeta.GetColumns()[i].GetType());
- }
-
- nShardScans++;
- if (Stats) {
- Stats->AffectedShards.insert(read.ShardId);
- }
- }
-
+
+ if (tableInfo.TableKind == ETableKind::Olap) {
+ auto* olapProgram = protoTaskMeta.MutableOlapProgram();
+ olapProgram->SetProgram(task.Meta.ReadInfo.OlapProgram.Program);
+
+ auto [schema, parameters] = SerializeKqpTasksParametersForOlap(stage, stageInfo, task,
+ holderFactory, typeEnv);
+
+ olapProgram->SetParametersSchema(schema);
+ olapProgram->SetParameters(parameters);
+ } else {
+ YQL_ENSURE(task.Meta.ReadInfo.OlapProgram.Program.empty());
+ }
+ }
+
+ for (auto& read : *task.Meta.Reads) {
+ auto* protoReadMeta = protoTaskMeta.AddReads();
+ protoReadMeta->SetShardId(read.ShardId);
+ read.Ranges.SerializeTo(protoReadMeta);
+
+ YQL_ENSURE((int) read.Columns.size() == protoTaskMeta.GetColumns().size());
+ for (ui64 i = 0; i < read.Columns.size(); ++i) {
+ YQL_ENSURE(read.Columns[i].Id == protoTaskMeta.GetColumns()[i].GetId());
+ YQL_ENSURE(read.Columns[i].Type == protoTaskMeta.GetColumns()[i].GetType());
+ }
+
+ nShardScans++;
+ if (Stats) {
+ Stats->AffectedShards.insert(read.ShardId);
+ }
+ }
+
LOG_D(
"task: " << task.Id <<
", node: " << task.Meta.NodeId <<
", meta: " << protoTaskMeta.ShortDebugString()
);
-
- taskDesc.MutableMeta()->PackFrom(protoTaskMeta);
-
- if (stageInfo.Meta.IsSysView()) {
- computeTasks.emplace_back(std::move(taskDesc));
+
+ taskDesc.MutableMeta()->PackFrom(protoTaskMeta);
+
+ if (stageInfo.Meta.IsSysView()) {
+ computeTasks.emplace_back(std::move(taskDesc));
} else {
- scanTasks[task.Meta.NodeId].emplace_back(std::move(taskDesc));
- nScanTasks++;
+ scanTasks[task.Meta.NodeId].emplace_back(std::move(taskDesc));
+ nScanTasks++;
}
- } else {
- computeTasks.emplace_back(std::move(taskDesc));
+ } else {
+ computeTasks.emplace_back(std::move(taskDesc));
}
}
- if (computeTasks.size() + nScanTasks > Request.MaxComputeActors) {
- LOG_N("Too many compute actors: computeTasks=" << computeTasks.size() << ", scanTasks=" << nScanTasks);
- TBase::ReplyErrorAndDie(Ydb::StatusIds::PRECONDITION_FAILED,
- YqlIssue({}, TIssuesIds::KIKIMR_PRECONDITION_FAILED, TStringBuilder()
- << "Requested too many execution units: " << (computeTasks.size() + nScanTasks)));
- return;
- }
-
- bool fitSize = AllOf(scanTasks, [this](const auto& x){ return ValidateTaskSize(x.second); })
- && ValidateTaskSize(computeTasks);
- if (!fitSize) {
- return;
- }
-
- LOG_D("Total tasks: " << TasksGraph.GetTasks().size() << ", readonly: true"
- << ", " << nScanTasks << " scan tasks on " << scanTasks.size() << " nodes"
- << ", totalShardScans: " << nShardScans << ", execType: Scan"
+ if (computeTasks.size() + nScanTasks > Request.MaxComputeActors) {
+ LOG_N("Too many compute actors: computeTasks=" << computeTasks.size() << ", scanTasks=" << nScanTasks);
+ TBase::ReplyErrorAndDie(Ydb::StatusIds::PRECONDITION_FAILED,
+ YqlIssue({}, TIssuesIds::KIKIMR_PRECONDITION_FAILED, TStringBuilder()
+ << "Requested too many execution units: " << (computeTasks.size() + nScanTasks)));
+ return;
+ }
+
+ bool fitSize = AllOf(scanTasks, [this](const auto& x){ return ValidateTaskSize(x.second); })
+ && ValidateTaskSize(computeTasks);
+ if (!fitSize) {
+ return;
+ }
+
+ LOG_D("Total tasks: " << TasksGraph.GetTasks().size() << ", readonly: true"
+ << ", " << nScanTasks << " scan tasks on " << scanTasks.size() << " nodes"
+ << ", totalShardScans: " << nShardScans << ", execType: Scan"
<< ", snapshot: {" << Request.Snapshot.TxId << ", " << Request.Snapshot.Step << "}");
- ExecuteScanTx(std::move(computeTasks), std::move(scanTasks));
+ ExecuteScanTx(std::move(computeTasks), std::move(scanTasks));
- Become(&TKqpScanExecuter::ExecuteState);
+ Become(&TKqpScanExecuter::ExecuteState);
}
void ExecuteScanTx(TVector<NYql::NDqProto::TDqTask>&& computeTasks, THashMap<ui64, TVector<NYql::NDqProto::TDqTask>>&& scanTasks) {
- LOG_D("Execute scan tx, computeTasks: " << computeTasks.size() << ", scanTasks: " << scanTasks.size());
+ LOG_D("Execute scan tx, computeTasks: " << computeTasks.size() << ", scanTasks: " << scanTasks.size());
for (const auto& [_, tasks]: scanTasks) {
for (const auto& task : tasks) {
PendingComputeTasks.insert(task.GetId());
- }
- }
-
+ }
+ }
+
for (auto& taskDesc : computeTasks) {
PendingComputeTasks.insert(taskDesc.GetId());
}
@@ -925,15 +925,15 @@ private:
RegisterWithSameMailbox(planner);
}
- void Finalize() {
- auto& response = *ResponseEv->Record.MutableResponse();
+ void Finalize() {
+ auto& response = *ResponseEv->Record.MutableResponse();
response.SetStatus(Ydb::StatusIds::SUCCESS);
TKqpProtoBuilder protoBuilder(*AppData()->FunctionRegistry);
for (auto& result : Results) {
- auto* protoResult = response.MutableResult()->AddResults();
+ auto* protoResult = response.MutableResult()->AddResults();
if (result.IsStream) {
// There is no support for multiple streaming results currently
@@ -945,93 +945,93 @@ private:
protoBuilder.BuildValue(result.Data, result.ItemType, protoResult);
}
- if (Stats) {
- ReportEventElapsedTime();
-
- Stats->FinishTs = TInstant::Now();
- Stats->Finish();
+ if (Stats) {
+ ReportEventElapsedTime();
+
+ Stats->FinishTs = TInstant::Now();
+ Stats->Finish();
- if (Request.StatsMode == NYql::NDqProto::DQ_STATS_MODE_PROFILE) {
- const auto& tx = Request.Transactions[0].Body;
- auto planWithStats = AddExecStatsToTxPlan(tx.GetPlan(), response.GetResult().GetStats());
- response.MutableResult()->MutableStats()->AddTxPlansWithStats(planWithStats);
+ if (Request.StatsMode == NYql::NDqProto::DQ_STATS_MODE_PROFILE) {
+ const auto& tx = Request.Transactions[0].Body;
+ auto planWithStats = AddExecStatsToTxPlan(tx.GetPlan(), response.GetResult().GetStats());
+ response.MutableResult()->MutableStats()->AddTxPlansWithStats(planWithStats);
}
}
- LOG_D("Sending response to: " << Target);
- Send(Target, ResponseEv.release());
- PassAway();
+ LOG_D("Sending response to: " << Target);
+ Send(Target, ResponseEv.release());
+ PassAway();
}
-private:
- void ReplyErrorAndDie(Ydb::StatusIds::StatusCode status,
- google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage>* issues) override
- {
- if (!PendingComputeTasks.empty()) {
- LOG_D("terminate pending resources request: " << Ydb::StatusIds::StatusCode_Name(status));
-
- auto ev = MakeHolder<TEvKqpNode::TEvCancelKqpTasksRequest>();
- ev->Record.SetTxId(TxId);
- ev->Record.SetReason(Ydb::StatusIds::StatusCode_Name(status));
-
- Send(MakeKqpNodeServiceID(SelfId().NodeId()), ev.Release());
- }
-
- TBase::ReplyErrorAndDie(status, issues);
+private:
+ void ReplyErrorAndDie(Ydb::StatusIds::StatusCode status,
+ google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage>* issues) override
+ {
+ if (!PendingComputeTasks.empty()) {
+ LOG_D("terminate pending resources request: " << Ydb::StatusIds::StatusCode_Name(status));
+
+ auto ev = MakeHolder<TEvKqpNode::TEvCancelKqpTasksRequest>();
+ ev->Record.SetTxId(TxId);
+ ev->Record.SetReason(Ydb::StatusIds::StatusCode_Name(status));
+
+ Send(MakeKqpNodeServiceID(SelfId().NodeId()), ev.Release());
+ }
+
+ TBase::ReplyErrorAndDie(status, issues);
}
- void PassAway() override {
+ void PassAway() override {
for (auto channelPair: ResultChannelProxies) {
LOG_D("terminate result channel " << channelPair.first << " proxy at " << channelPair.second->SelfId());
-
+
TAutoPtr<IEventHandle> ev = new IEventHandle(
channelPair.second->SelfId(), SelfId(), new TEvents::TEvPoison
);
channelPair.second->Receive(ev, TActivationContext::AsActorContext());
- }
+ }
if (KqpShardsResolverId) {
Send(KqpShardsResolverId, new TEvents::TEvPoison);
}
- for (auto& [shardId, nodeId] : ShardIdToNodeId) {
- Send(TActivationContext::InterconnectProxy(nodeId), new TEvents::TEvUnsubscribe());
- }
-
- auto totalTime = TInstant::Now() - StartTime;
- Counters->Counters->ScanTxTotalTimeHistogram->Collect(totalTime.MilliSeconds());
-
- TBase::PassAway();
+ for (auto& [shardId, nodeId] : ShardIdToNodeId) {
+ Send(TActivationContext::InterconnectProxy(nodeId), new TEvents::TEvUnsubscribe());
+ }
+
+ auto totalTime = TInstant::Now() - StartTime;
+ Counters->Counters->ScanTxTotalTimeHistogram->Collect(totalTime.MilliSeconds());
+
+ TBase::PassAway();
}
private:
- bool CheckExecutionComplete() {
- if (PendingComputeActors.empty() && PendingComputeTasks.empty()) {
- Finalize();
+ bool CheckExecutionComplete() {
+ if (PendingComputeActors.empty() && PendingComputeTasks.empty()) {
+ Finalize();
UpdateResourcesUsage(true);
return true;
}
UpdateResourcesUsage(false);
- if (IsDebugLogEnabled()) {
- TStringBuilder sb;
- sb << "Waiting for: ";
- for (auto ct : PendingComputeTasks) {
- sb << "CT " << ct << ", ";
- }
- for (auto ca : PendingComputeActors) {
+ if (IsDebugLogEnabled()) {
+ TStringBuilder sb;
+ sb << "Waiting for: ";
+ for (auto ct : PendingComputeTasks) {
+ sb << "CT " << ct << ", ";
+ }
+ for (auto ca : PendingComputeActors) {
sb << "CA " << ca.first << ", ";
- }
- LOG_D(sb);
- }
-
+ }
+ LOG_D(sb);
+ }
+
return false;
}
-public:
+public:
void FillEndpointDesc(NYql::NDqProto::TEndpoint& endpoint, const TTask& task) {
- if (task.ComputeActorId) {
+ if (task.ComputeActorId) {
ActorIdToProto(task.ComputeActorId, endpoint.MutableActorId());
}
}
@@ -1045,7 +1045,7 @@ public:
}
proxy = CreateResultStreamChannelProxy(TxId, channel.Id, Results[0].ItemType,
- Results[0].ResultItemType.Get(), Target, Stats.get(), SelfId());
+ Results[0].ResultItemType.Get(), Target, Stats.get(), SelfId());
} else {
YQL_ENSURE(channel.DstInputIndex < Results.size());
@@ -1055,7 +1055,7 @@ public:
return channelIt->second;
}
- proxy = CreateResultDataChannelProxy(TxId, channel.Id, Stats.get(), SelfId(),
+ proxy = CreateResultDataChannelProxy(TxId, channel.Id, Stats.get(), SelfId(),
&Results[channel.DstInputIndex].Data);
}
@@ -1080,25 +1080,25 @@ public:
ActorIdToProto(proxy->SelfId(), channelDesc.MutableDstEndpoint()->MutableActorId());
}
- channelDesc.SetIsPersistent(IsCrossShardChannel(TasksGraph, channel));
- channelDesc.SetInMemory(channel.InMemory);
+ channelDesc.SetIsPersistent(IsCrossShardChannel(TasksGraph, channel));
+ channelDesc.SetInMemory(channel.InMemory);
}
private:
TVector<TKqpExecuterTxResult> Results;
std::unordered_map<ui64, IActor*> ResultChannelProxies;
- THashSet<ui64> PendingComputeTasks; // Not started yet, waiting resources
- TMap<ui64, ui64> ShardIdToNodeId;
- TMap<ui64, TVector<ui64>> ShardsOnNode;
+ THashSet<ui64> PendingComputeTasks; // Not started yet, waiting resources
+ TMap<ui64, ui64> ShardIdToNodeId;
+ TMap<ui64, TVector<ui64>> ShardsOnNode;
};
} // namespace
-IActor* CreateKqpScanExecuter(IKqpGateway::TExecPhysicalRequest&& request, const TString& database,
- const TMaybe<TString>& userToken, TKqpRequestCounters::TPtr counters)
-{
- return new TKqpScanExecuter(std::move(request), database, userToken, counters);
+IActor* CreateKqpScanExecuter(IKqpGateway::TExecPhysicalRequest&& request, const TString& database,
+ const TMaybe<TString>& userToken, TKqpRequestCounters::TPtr counters)
+{
+ return new TKqpScanExecuter(std::move(request), database, userToken, counters);
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/executer/kqp_shards_resolver.cpp b/ydb/core/kqp/executer/kqp_shards_resolver.cpp
index 243e0a6810..506dfcf573 100644
--- a/ydb/core/kqp/executer/kqp_shards_resolver.cpp
+++ b/ydb/core/kqp/executer/kqp_shards_resolver.cpp
@@ -1,126 +1,126 @@
-#include "kqp_shards_resolver.h"
-
+#include "kqp_shards_resolver.h"
+
#include <ydb/core/base/tablet_resolver.h>
#include <ydb/core/kqp/executer/kqp_executer.h>
#include <ydb/core/tx/scheme_cache/scheme_cache.h>
-
-#include <library/cpp/actors/core/actor_bootstrapped.h>
-#include <library/cpp/actors/core/hfunc.h>
-#include <library/cpp/actors/core/log.h>
-
-#include <util/generic/set.h>
-
-
-namespace NKikimr::NKqp {
-
-using namespace NActors;
-using namespace NYql;
-
-namespace {
-
-#define LOG__(prio, stream) LOG_LOG_S(*TlsActivationContext, prio, NKikimrServices::KQP_EXECUTER, \
- "[ShardsResolver] TxId: " << TxId << ". " << stream)
-
-#define LOG_T(stream) LOG__(NActors::NLog::PRI_TRACE, stream)
-#define LOG_D(stream) LOG__(NActors::NLog::PRI_DEBUG, stream)
-#define LOG_W(stream) LOG__(NActors::NLog::PRI_WARN, stream)
-#define LOG_E(stream) LOG__(NActors::NLog::PRI_ERROR, stream)
-#define LOG_C(stream) LOG__(NActors::NLog::PRI_CRIT, stream)
-
-constexpr ui32 MAX_RETRIES_COUNT = 3;
-
-class TKqpShardsResolver : public TActorBootstrapped<TKqpShardsResolver> {
-public:
- static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
+
+#include <library/cpp/actors/core/actor_bootstrapped.h>
+#include <library/cpp/actors/core/hfunc.h>
+#include <library/cpp/actors/core/log.h>
+
+#include <util/generic/set.h>
+
+
+namespace NKikimr::NKqp {
+
+using namespace NActors;
+using namespace NYql;
+
+namespace {
+
+#define LOG__(prio, stream) LOG_LOG_S(*TlsActivationContext, prio, NKikimrServices::KQP_EXECUTER, \
+ "[ShardsResolver] TxId: " << TxId << ". " << stream)
+
+#define LOG_T(stream) LOG__(NActors::NLog::PRI_TRACE, stream)
+#define LOG_D(stream) LOG__(NActors::NLog::PRI_DEBUG, stream)
+#define LOG_W(stream) LOG__(NActors::NLog::PRI_WARN, stream)
+#define LOG_E(stream) LOG__(NActors::NLog::PRI_ERROR, stream)
+#define LOG_C(stream) LOG__(NActors::NLog::PRI_CRIT, stream)
+
+constexpr ui32 MAX_RETRIES_COUNT = 3;
+
+class TKqpShardsResolver : public TActorBootstrapped<TKqpShardsResolver> {
+public:
+ static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
return NKikimrServices::TActivity::KQP_SHARDS_RESOLVER;
- }
-
-public:
+ }
+
+public:
TKqpShardsResolver(const TActorId& owner, ui64 txId, TSet<ui64>&& shardIds, float failRatio)
: Owner(owner)
, TxId(txId)
- , ShardIds(std::move(shardIds))
- , MaxFailedShards(ShardIds.size() * failRatio) {}
-
- void Bootstrap() {
- auto tabletResolver = MakeTabletResolverID();
- auto resolveFlags = GetResolveFlags();
-
+ , ShardIds(std::move(shardIds))
+ , MaxFailedShards(ShardIds.size() * failRatio) {}
+
+ void Bootstrap() {
+ auto tabletResolver = MakeTabletResolverID();
+ auto resolveFlags = GetResolveFlags();
+
Y_ASSERT(ShardIds.size() > 0);
- for (ui64 tabletId : ShardIds) {
- LOG_T("Send request about tabletId: " << tabletId);
- bool sent = Send(tabletResolver, new TEvTabletResolver::TEvForward(tabletId, nullptr, resolveFlags));
- Y_VERIFY_DEBUG(sent);
- }
-
- Become(&TKqpShardsResolver::ResolveState);
- }
-
-private:
- STATEFN(ResolveState) {
- switch (ev->GetTypeRewrite()) {
- hFunc(TEvTabletResolver::TEvForwardResult, HandleResolve);
- cFunc(TEvents::TSystem::Poison, PassAway);
- default: {
- LOG_C("Unexpected event: " << ev->GetTypeRewrite());
+ for (ui64 tabletId : ShardIds) {
+ LOG_T("Send request about tabletId: " << tabletId);
+ bool sent = Send(tabletResolver, new TEvTabletResolver::TEvForward(tabletId, nullptr, resolveFlags));
+ Y_VERIFY_DEBUG(sent);
+ }
+
+ Become(&TKqpShardsResolver::ResolveState);
+ }
+
+private:
+ STATEFN(ResolveState) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvTabletResolver::TEvForwardResult, HandleResolve);
+ cFunc(TEvents::TSystem::Poison, PassAway);
+ default: {
+ LOG_C("Unexpected event: " << ev->GetTypeRewrite());
ReplyErrorAndDie(Ydb::StatusIds::INTERNAL_ERROR, "Unexpected event while resolving shards");
- }
- }
- }
-
- void HandleResolve(TEvTabletResolver::TEvForwardResult::TPtr& ev) {
- auto* msg = ev->Get();
- LOG_T("Got resolve event for tabletId: " << msg->TabletID << ": " << NKikimrProto::EReplyStatus_Name(msg->Status)
- << ", nodeId: " << msg->TabletActor.NodeId());
-
- if (msg->Status == NKikimrProto::EReplyStatus::OK) {
- Result[msg->TabletID] = msg->TabletActor.NodeId();
-
- if (Result.size() + FailedTablets == ShardIds.size()) {
- LOG_D("Done, success: " << Result.size() << ", failed: " << FailedTablets);
+ }
+ }
+ }
+
+ void HandleResolve(TEvTabletResolver::TEvForwardResult::TPtr& ev) {
+ auto* msg = ev->Get();
+ LOG_T("Got resolve event for tabletId: " << msg->TabletID << ": " << NKikimrProto::EReplyStatus_Name(msg->Status)
+ << ", nodeId: " << msg->TabletActor.NodeId());
+
+ if (msg->Status == NKikimrProto::EReplyStatus::OK) {
+ Result[msg->TabletID] = msg->TabletActor.NodeId();
+
+ if (Result.size() + FailedTablets == ShardIds.size()) {
+ LOG_D("Done, success: " << Result.size() << ", failed: " << FailedTablets);
ReplyAndDie();
- return;
- }
-
- return;
- }
-
- auto& state = States[msg->TabletID];
- if (state.Retries > MAX_RETRIES_COUNT) {
- ++FailedTablets;
- if (FailedTablets > MaxFailedShards) {
- LOG_W("Too many failed requests: " << FailedTablets << " (" << ShardIds.size() << ")");
+ return;
+ }
+
+ return;
+ }
+
+ auto& state = States[msg->TabletID];
+ if (state.Retries > MAX_RETRIES_COUNT) {
+ ++FailedTablets;
+ if (FailedTablets > MaxFailedShards) {
+ LOG_W("Too many failed requests: " << FailedTablets << " (" << ShardIds.size() << ")");
ReplyErrorAndDie(Ydb::StatusIds::GENERIC_ERROR, TStringBuilder()
<< "Too many unresolved shards: " << FailedTablets);
- return;
- }
-
- if (FailedTablets + Result.size() == ShardIds.size()) {
- LOG_D("Done, success: " << Result.size() << ", failed: " << FailedTablets);
+ return;
+ }
+
+ if (FailedTablets + Result.size() == ShardIds.size()) {
+ LOG_D("Done, success: " << Result.size() << ", failed: " << FailedTablets);
ReplyAndDie();
- return;
- }
-
- return; // no more retries for this tabletId
- }
-
- state.Retries++;
-
- // todo: backoff
- Send(MakeTabletResolverID(), new TEvTabletResolver::TEvForward(msg->TabletID, nullptr, GetResolveFlags()));
- }
-
- TEvTabletResolver::TEvForward::TResolveFlags GetResolveFlags() {
- TEvTabletResolver::TEvForward::TResolveFlags resolveFlags;
+ return;
+ }
+
+ return; // no more retries for this tabletId
+ }
+
+ state.Retries++;
+
+ // todo: backoff
+ Send(MakeTabletResolverID(), new TEvTabletResolver::TEvForward(msg->TabletID, nullptr, GetResolveFlags()));
+ }
+
+ TEvTabletResolver::TEvForward::TResolveFlags GetResolveFlags() {
+ TEvTabletResolver::TEvForward::TResolveFlags resolveFlags;
resolveFlags.SetAllowFollower(false);
resolveFlags.SetForceFollower(false);
- resolveFlags.SetPreferLocal(true);
- resolveFlags.SetForceLocal(false);
-
- return resolveFlags;
- }
-
+ resolveFlags.SetPreferLocal(true);
+ resolveFlags.SetForceLocal(false);
+
+ return resolveFlags;
+ }
+
void ReplyErrorAndDie(Ydb::StatusIds::StatusCode status, TString&& message) {
auto replyEv = MakeHolder<TEvKqpExecuter::TEvShardsResolveStatus>();
replyEv->Status = status;
@@ -137,25 +137,25 @@ private:
PassAway();
}
-private:
+private:
const TActorId Owner;
- const ui64 TxId;
- const TSet<ui64> ShardIds;
- const ui32 MaxFailedShards;
-
- struct TState {
- ui32 Retries = 0;
- };
- TMap<ui64, TState> States;
- ui32 FailedTablets = 0;
-
- TMap<ui64, ui64> Result;
-};
-
-} // anonymous namespace
-
+ const ui64 TxId;
+ const TSet<ui64> ShardIds;
+ const ui32 MaxFailedShards;
+
+ struct TState {
+ ui32 Retries = 0;
+ };
+ TMap<ui64, TState> States;
+ ui32 FailedTablets = 0;
+
+ TMap<ui64, ui64> Result;
+};
+
+} // anonymous namespace
+
IActor* CreateKqpShardsResolver(const TActorId& owner, ui64 txId, TSet<ui64>&& shardIds, float failRatio) {
return new TKqpShardsResolver(owner, txId, std::move(shardIds), failRatio);
-}
-
-} // namespace NKikimr::NKqp
+}
+
+} // namespace NKikimr::NKqp
diff --git a/ydb/core/kqp/executer/kqp_shards_resolver.h b/ydb/core/kqp/executer/kqp_shards_resolver.h
index e872210fdc..8408d79c7b 100644
--- a/ydb/core/kqp/executer/kqp_shards_resolver.h
+++ b/ydb/core/kqp/executer/kqp_shards_resolver.h
@@ -1,11 +1,11 @@
-#pragma once
-
-#include <library/cpp/actors/core/actor.h>
-#include <util/generic/set.h>
-
-namespace NKikimr::NKqp {
-
+#pragma once
+
+#include <library/cpp/actors/core/actor.h>
+#include <util/generic/set.h>
+
+namespace NKikimr::NKqp {
+
NActors::IActor* CreateKqpShardsResolver(const NActors::TActorId& owner, ui64 txId, TSet<ui64>&& shardIds,
float failRatio);
-
-} // namespace NKikimr::NKqp
+
+} // namespace NKikimr::NKqp
diff --git a/ydb/core/kqp/executer/kqp_table_resolver.cpp b/ydb/core/kqp/executer/kqp_table_resolver.cpp
index 0d8b420755..6d02c3d07c 100644
--- a/ydb/core/kqp/executer/kqp_table_resolver.cpp
+++ b/ydb/core/kqp/executer/kqp_table_resolver.cpp
@@ -1,86 +1,86 @@
-#include "kqp_table_resolver.h"
-
+#include "kqp_table_resolver.h"
+
#include <ydb/core/base/appdata.h>
#include <ydb/core/base/cputime.h>
#include <ydb/core/base/path.h>
#include <ydb/core/kqp/executer/kqp_executer.h>
-
-
-namespace NKikimr::NKqp {
-
-using namespace NActors;
-using namespace NYql;
-using namespace NYql::NDq;
-
-namespace {
-
-#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream)
-#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream)
-#define LOG_C(stream) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream)
-#define LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream)
-
-class TKqpTableResolver : public TActorBootstrapped<TKqpTableResolver> {
-public:
- static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
+
+
+namespace NKikimr::NKqp {
+
+using namespace NActors;
+using namespace NYql;
+using namespace NYql::NDq;
+
+namespace {
+
+#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream)
+#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream)
+#define LOG_C(stream) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream)
+#define LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream)
+
+class TKqpTableResolver : public TActorBootstrapped<TKqpTableResolver> {
+public:
+ static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
return NKikimrServices::TActivity::KQP_TABLE_RESOLVER;
- }
-
+ }
+
TKqpTableResolver(const TActorId& owner, ui64 txId, TMaybe<TString> userToken,
const TVector<IKqpGateway::TPhysicalTxData>& transactions, TKqpTableKeys& tableKeys,
TKqpTasksGraph& tasksGraph)
: Owner(owner)
, TxId(txId)
- , UserToken(userToken)
+ , UserToken(userToken)
, Transactions(transactions)
, TableKeys(tableKeys)
, TasksGraph(tasksGraph) {}
-
- void Bootstrap() {
- ResolveTables();
- Become(&TKqpTableResolver::ResolveTablesState);
- }
-
-private:
- STATEFN(ResolveTablesState) {
- switch (ev->GetTypeRewrite()) {
- hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleResolveTables);
- hFunc(TEvents::TEvPoison, HandleResolveTables);
- default:
- UnexpectedEvent("ResolveTablesState", ev->GetTypeRewrite());
- }
- }
-
- void HandleResolveTables(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) {
- auto timer = std::make_unique<NCpuTime::TCpuTimer>(CpuTime);
-
- const auto& entries = ev->Get()->Request->ResultSet;
- LOG_D("Resolved tables: " << entries.size());
+
+ void Bootstrap() {
+ ResolveTables();
+ Become(&TKqpTableResolver::ResolveTablesState);
+ }
+
+private:
+ STATEFN(ResolveTablesState) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleResolveTables);
+ hFunc(TEvents::TEvPoison, HandleResolveTables);
+ default:
+ UnexpectedEvent("ResolveTablesState", ev->GetTypeRewrite());
+ }
+ }
+
+ void HandleResolveTables(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) {
+ auto timer = std::make_unique<NCpuTime::TCpuTimer>(CpuTime);
+
+ const auto& entries = ev->Get()->Request->ResultSet;
+ LOG_D("Resolved tables: " << entries.size());
YQL_ENSURE(entries.size() == TableKeys.Size());
-
- for (auto& entry : entries) {
- if (entry.Status != NSchemeCache::TSchemeCacheNavigate::EStatus::Ok) {
- timer.reset();
- if (entry.Status == NSchemeCache::TSchemeCacheNavigate::EStatus::PathErrorUnknown) {
- ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE,
- YqlIssue({}, NYql::TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE, TStringBuilder()
- << "Failed to resolve table `" << JoinPath(entry.Path) << "`: " << entry.Status << '.'));
- } else {
- ReplyErrorAndDie(Ydb::StatusIds::SCHEME_ERROR,
- YqlIssue({}, NYql::TIssuesIds::KIKIMR_SCHEME_ERROR, TStringBuilder()
- << "Failed to resolve table `" << JoinPath(entry.Path) << "`: " << entry.Status << '.'));
- }
- return;
- }
-
- auto* table = TableKeys.FindTablePtr(entry.TableId);
- if (!table) {
- timer.reset();
- ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE,
- YqlIssue({}, NYql::TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE, TStringBuilder()
- << "Unresolved table `" << JoinPath(entry.Path) << "` with tableId: " << entry.TableId << "."));
- return;
- }
-
+
+ for (auto& entry : entries) {
+ if (entry.Status != NSchemeCache::TSchemeCacheNavigate::EStatus::Ok) {
+ timer.reset();
+ if (entry.Status == NSchemeCache::TSchemeCacheNavigate::EStatus::PathErrorUnknown) {
+ ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE,
+ YqlIssue({}, NYql::TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE, TStringBuilder()
+ << "Failed to resolve table `" << JoinPath(entry.Path) << "`: " << entry.Status << '.'));
+ } else {
+ ReplyErrorAndDie(Ydb::StatusIds::SCHEME_ERROR,
+ YqlIssue({}, NYql::TIssuesIds::KIKIMR_SCHEME_ERROR, TStringBuilder()
+ << "Failed to resolve table `" << JoinPath(entry.Path) << "`: " << entry.Status << '.'));
+ }
+ return;
+ }
+
+ auto* table = TableKeys.FindTablePtr(entry.TableId);
+ if (!table) {
+ timer.reset();
+ ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE,
+ YqlIssue({}, NYql::TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE, TStringBuilder()
+ << "Unresolved table `" << JoinPath(entry.Path) << "` with tableId: " << entry.TableId << "."));
+ return;
+ }
+
if (entry.Kind == NSchemeCache::TSchemeCacheNavigate::KindOlapTable) {
YQL_ENSURE(entry.OlapTableInfo || entry.OlapStoreInfo);
// NOTE: entry.SysViewInfo might not be empty for OLAP stats virtual tables
@@ -91,223 +91,223 @@ private:
table->TableKind = ETableKind::Datashard;
}
- // TODO: Resolve columns by id
- TMap<TStringBuf, ui32> columnsMap;
- for (auto& [columnId, column] : entry.Columns) {
- auto ret = columnsMap.emplace(column.Name, columnId);
- YQL_ENSURE(ret.second, "" << column.Name);
-
- if (column.KeyOrder >= 0) {
- table->Columns.emplace(column.Name, TKqpTableKeys::TColumn());
-
- if (table->KeyColumns.size() <= (ui32) column.KeyOrder) {
- table->KeyColumns.resize(column.KeyOrder + 1);
- table->KeyColumnTypes.resize(column.KeyOrder + 1);
- }
- table->KeyColumns[column.KeyOrder] = column.Name;
- table->KeyColumnTypes[column.KeyOrder] = column.PType;
- }
- }
-
- for (auto& keyColumn : table->KeyColumns) {
- YQL_ENSURE(!keyColumn.empty());
- }
-
- auto& sysColumns = GetSystemColumns();
- for (auto& [columnName, columnKey] : table->Columns) {
- if (auto* systemColumn = sysColumns.FindPtr(columnName)) {
- columnKey.Id = systemColumn->ColumnId;
- columnKey.Type = systemColumn->TypeId;
- continue;
- }
-
- auto* columnId = columnsMap.FindPtr(columnName);
- if (!columnId) {
- timer.reset();
- ReplyErrorAndDie(Ydb::StatusIds::SCHEME_ERROR,
- YqlIssue({}, NYql::TIssuesIds::KIKIMR_SCHEME_ERROR, TStringBuilder()
- << "Unknown column `" << columnName << "` at table `" << JoinPath(entry.Path) << "`."));
- return;
- }
-
- auto* column = entry.Columns.FindPtr(*columnId);
- YQL_ENSURE(column);
-
- columnKey.Id = column->Id;
- columnKey.Type = column->PType;
- }
- }
-
- ResolveKeys();
- timer.reset();
-
- Become(&TKqpTableResolver::ResolveKeysState);
- }
-
- void HandleResolveTables(TEvents::TEvPoison::TPtr&) {
- PassAway();
- }
-
-private:
- STATEFN(ResolveKeysState) {
- switch (ev->GetTypeRewrite()) {
- hFunc(TEvTxProxySchemeCache::TEvResolveKeySetResult, HandleResolveKeys);
- hFunc(TEvents::TEvPoison, HandleResolveKeys);
- default: {
- LOG_C("ResolveKeysState: unexpected event " << ev->GetTypeRewrite());
- GotUnexpectedEvent = ev->GetTypeRewrite();
- }
- }
- }
-
- void HandleResolveKeys(TEvTxProxySchemeCache::TEvResolveKeySetResult::TPtr &ev) {
- if (ShouldTerminate) {
- PassAway();
- return;
- }
-
- if (GotUnexpectedEvent) {
- ReplyErrorAndDie(Ydb::StatusIds::INTERNAL_ERROR, TIssue(TStringBuilder()
- << "Unexpected event: " << *GotUnexpectedEvent));
- return;
- }
-
- auto timer = std::make_unique<NCpuTime::TCpuTimer>(CpuTime);
-
+ // TODO: Resolve columns by id
+ TMap<TStringBuf, ui32> columnsMap;
+ for (auto& [columnId, column] : entry.Columns) {
+ auto ret = columnsMap.emplace(column.Name, columnId);
+ YQL_ENSURE(ret.second, "" << column.Name);
+
+ if (column.KeyOrder >= 0) {
+ table->Columns.emplace(column.Name, TKqpTableKeys::TColumn());
+
+ if (table->KeyColumns.size() <= (ui32) column.KeyOrder) {
+ table->KeyColumns.resize(column.KeyOrder + 1);
+ table->KeyColumnTypes.resize(column.KeyOrder + 1);
+ }
+ table->KeyColumns[column.KeyOrder] = column.Name;
+ table->KeyColumnTypes[column.KeyOrder] = column.PType;
+ }
+ }
+
+ for (auto& keyColumn : table->KeyColumns) {
+ YQL_ENSURE(!keyColumn.empty());
+ }
+
+ auto& sysColumns = GetSystemColumns();
+ for (auto& [columnName, columnKey] : table->Columns) {
+ if (auto* systemColumn = sysColumns.FindPtr(columnName)) {
+ columnKey.Id = systemColumn->ColumnId;
+ columnKey.Type = systemColumn->TypeId;
+ continue;
+ }
+
+ auto* columnId = columnsMap.FindPtr(columnName);
+ if (!columnId) {
+ timer.reset();
+ ReplyErrorAndDie(Ydb::StatusIds::SCHEME_ERROR,
+ YqlIssue({}, NYql::TIssuesIds::KIKIMR_SCHEME_ERROR, TStringBuilder()
+ << "Unknown column `" << columnName << "` at table `" << JoinPath(entry.Path) << "`."));
+ return;
+ }
+
+ auto* column = entry.Columns.FindPtr(*columnId);
+ YQL_ENSURE(column);
+
+ columnKey.Id = column->Id;
+ columnKey.Type = column->PType;
+ }
+ }
+
+ ResolveKeys();
+ timer.reset();
+
+ Become(&TKqpTableResolver::ResolveKeysState);
+ }
+
+ void HandleResolveTables(TEvents::TEvPoison::TPtr&) {
+ PassAway();
+ }
+
+private:
+ STATEFN(ResolveKeysState) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvTxProxySchemeCache::TEvResolveKeySetResult, HandleResolveKeys);
+ hFunc(TEvents::TEvPoison, HandleResolveKeys);
+ default: {
+ LOG_C("ResolveKeysState: unexpected event " << ev->GetTypeRewrite());
+ GotUnexpectedEvent = ev->GetTypeRewrite();
+ }
+ }
+ }
+
+ void HandleResolveKeys(TEvTxProxySchemeCache::TEvResolveKeySetResult::TPtr &ev) {
+ if (ShouldTerminate) {
+ PassAway();
+ return;
+ }
+
+ if (GotUnexpectedEvent) {
+ ReplyErrorAndDie(Ydb::StatusIds::INTERNAL_ERROR, TIssue(TStringBuilder()
+ << "Unexpected event: " << *GotUnexpectedEvent));
+ return;
+ }
+
+ auto timer = std::make_unique<NCpuTime::TCpuTimer>(CpuTime);
+
auto& results = ev->Get()->Request->ResultSet;
- LOG_D("Resolved key sets: " << results.size());
-
- for (auto& entry : results) {
- if (entry.Status != NSchemeCache::TSchemeCacheRequest::EStatus::OkData) {
- LOG_E("Error resolving keys for entry: " << entry.ToString(*AppData()->TypeRegistry));
-
- auto* table = TableKeys.FindTablePtr(entry.KeyDescription->TableId);
- TStringBuilder path;
- if (table) {
- path << '`' << table->Path << '`';
- } else {
- path << "unresolved `" << entry.KeyDescription->TableId << '`';
- }
-
- timer.reset();
- ReplyErrorAndDie(Ydb::StatusIds::SCHEME_ERROR,
- YqlIssue({}, NYql::TIssuesIds::KIKIMR_SCHEME_ERROR, TStringBuilder()
- << "Failed to resolve table " << path << " keys: " << entry.Status << '.'));
- return;
- }
-
- for (auto& partition : entry.KeyDescription->Partitions) {
- YQL_ENSURE(partition.Range);
- }
-
- LOG_D("Resolved key: " << entry.ToString(*AppData()->TypeRegistry));
+ LOG_D("Resolved key sets: " << results.size());
+
+ for (auto& entry : results) {
+ if (entry.Status != NSchemeCache::TSchemeCacheRequest::EStatus::OkData) {
+ LOG_E("Error resolving keys for entry: " << entry.ToString(*AppData()->TypeRegistry));
+
+ auto* table = TableKeys.FindTablePtr(entry.KeyDescription->TableId);
+ TStringBuilder path;
+ if (table) {
+ path << '`' << table->Path << '`';
+ } else {
+ path << "unresolved `" << entry.KeyDescription->TableId << '`';
+ }
+
+ timer.reset();
+ ReplyErrorAndDie(Ydb::StatusIds::SCHEME_ERROR,
+ YqlIssue({}, NYql::TIssuesIds::KIKIMR_SCHEME_ERROR, TStringBuilder()
+ << "Failed to resolve table " << path << " keys: " << entry.Status << '.'));
+ return;
+ }
+
+ for (auto& partition : entry.KeyDescription->Partitions) {
+ YQL_ENSURE(partition.Range);
+ }
+
+ LOG_D("Resolved key: " << entry.ToString(*AppData()->TypeRegistry));
auto& stageInfo = DecodeStageInfo(entry.UserData);
stageInfo.Meta.ShardKey = std::move(entry.KeyDescription);
stageInfo.Meta.ShardKind = std::move(entry.Kind);
- }
-
- timer.reset();
-
- auto replyEv = std::make_unique<TEvKqpExecuter::TEvTableResolveStatus>();
- replyEv->CpuTime = CpuTime;
-
- Send(Owner, replyEv.release());
- PassAway();
- }
-
- void HandleResolveKeys(TEvents::TEvPoison::TPtr&) {
- ShouldTerminate = true;
- }
-
-private:
- // TODO: Get rid of ResolveTables & TableKeys, get table information from phy tx proto.
- void ResolveTables() {
+ }
+
+ timer.reset();
+
+ auto replyEv = std::make_unique<TEvKqpExecuter::TEvTableResolveStatus>();
+ replyEv->CpuTime = CpuTime;
+
+ Send(Owner, replyEv.release());
+ PassAway();
+ }
+
+ void HandleResolveKeys(TEvents::TEvPoison::TPtr&) {
+ ShouldTerminate = true;
+ }
+
+private:
+ // TODO: Get rid of ResolveTables & TableKeys, get table information from phy tx proto.
+ void ResolveTables() {
for (auto& tx : Transactions) {
- for (auto& stage : tx.Body.GetStages()) {
- for (auto& op : stage.GetTableOps()) {
+ for (auto& stage : tx.Body.GetStages()) {
+ for (auto& op : stage.GetTableOps()) {
auto& table = TableKeys.GetOrAddTable(MakeTableId(op.GetTable()), op.GetTable().GetPath());
- for (auto& column : op.GetColumns()) {
- table.Columns.emplace(column.GetName(), TKqpTableKeys::TColumn());
- }
- }
- }
- }
-
- auto request = MakeHolder<NSchemeCache::TSchemeCacheNavigate>();
+ for (auto& column : op.GetColumns()) {
+ table.Columns.emplace(column.GetName(), TKqpTableKeys::TColumn());
+ }
+ }
+ }
+ }
+
+ auto request = MakeHolder<NSchemeCache::TSchemeCacheNavigate>();
request->ResultSet.reserve(TableKeys.Size());
for (auto& [tableId, table] : TableKeys.Get()) {
- NSchemeCache::TSchemeCacheNavigate::TEntry entry;
- entry.RequestType = NSchemeCache::TSchemeCacheNavigate::TEntry::ERequestType::ByTableId;
- entry.TableId = tableId;
- entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpTable;
+ NSchemeCache::TSchemeCacheNavigate::TEntry entry;
+ entry.RequestType = NSchemeCache::TSchemeCacheNavigate::TEntry::ERequestType::ByTableId;
+ entry.TableId = tableId;
+ entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpTable;
entry.ShowPrivatePath = true;
-
- request->ResultSet.emplace_back(std::move(entry));
- }
-
- auto ev = MakeHolder<TEvTxProxySchemeCache::TEvNavigateKeySet>(request.Release());
- Send(MakeSchemeCacheID(), ev.Release());
- }
-
- void ResolveKeys() {
- FillKqpTasksGraphStages(TasksGraph, Transactions);
-
- auto request = MakeHolder<NSchemeCache::TSchemeCacheRequest>();
+
+ request->ResultSet.emplace_back(std::move(entry));
+ }
+
+ auto ev = MakeHolder<TEvTxProxySchemeCache::TEvNavigateKeySet>(request.Release());
+ Send(MakeSchemeCacheID(), ev.Release());
+ }
+
+ void ResolveKeys() {
+ FillKqpTasksGraphStages(TasksGraph, Transactions);
+
+ auto request = MakeHolder<NSchemeCache::TSchemeCacheRequest>();
request->ResultSet.reserve(TasksGraph.GetStagesInfo().size());
- if (UserToken) {
- request->UserToken = new NACLib::TUserToken(*UserToken);
- }
-
+ if (UserToken) {
+ request->UserToken = new NACLib::TUserToken(*UserToken);
+ }
+
for (auto& pair : TasksGraph.GetStagesInfo()) {
- auto& stageInfo = pair.second;
- if (!stageInfo.Meta.ShardOperations.empty()) {
- YQL_ENSURE(stageInfo.Meta.TableId);
- YQL_ENSURE(stageInfo.Meta.ShardOperations.size() == 1);
- auto operation = *stageInfo.Meta.ShardOperations.begin();
-
+ auto& stageInfo = pair.second;
+ if (!stageInfo.Meta.ShardOperations.empty()) {
+ YQL_ENSURE(stageInfo.Meta.TableId);
+ YQL_ENSURE(stageInfo.Meta.ShardOperations.size() == 1);
+ auto operation = *stageInfo.Meta.ShardOperations.begin();
+
const TKqpTableKeys::TTable* table = TableKeys.FindTablePtr(stageInfo.Meta.TableId);
stageInfo.Meta.TableKind = table->TableKind;
- stageInfo.Meta.ShardKey = ExtractKey(stageInfo.Meta.TableId, operation);
-
+ stageInfo.Meta.ShardKey = ExtractKey(stageInfo.Meta.TableId, operation);
+
auto& entry = request->ResultSet.emplace_back(std::move(stageInfo.Meta.ShardKey));
entry.UserData = EncodeStageInfo(stageInfo);
- switch (operation) {
- case TKeyDesc::ERowOperation::Read:
- entry.Access = NACLib::EAccessRights::SelectRow;
- break;
- case TKeyDesc::ERowOperation::Update:
- entry.Access = NACLib::EAccessRights::UpdateRow;
- break;
- case TKeyDesc::ERowOperation::Erase:
- entry.Access = NACLib::EAccessRights::EraseRow;
- break;
- default:
- YQL_ENSURE(false, "Unsupported row operation mode: " << (ui32)operation);
- }
- }
- }
-
- Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvResolveKeySet(request));
- }
-
-private:
- THolder<TKeyDesc> ExtractKey(const TTableId& table, TKeyDesc::ERowOperation operation) {
- const auto& tableKey = TableKeys.GetTable(table);
- auto range = GetFullRange(tableKey.KeyColumnTypes.size());
-
- return MakeHolder<TKeyDesc>(table, range.ToTableRange(), operation, tableKey.KeyColumnTypes,
+ switch (operation) {
+ case TKeyDesc::ERowOperation::Read:
+ entry.Access = NACLib::EAccessRights::SelectRow;
+ break;
+ case TKeyDesc::ERowOperation::Update:
+ entry.Access = NACLib::EAccessRights::UpdateRow;
+ break;
+ case TKeyDesc::ERowOperation::Erase:
+ entry.Access = NACLib::EAccessRights::EraseRow;
+ break;
+ default:
+ YQL_ENSURE(false, "Unsupported row operation mode: " << (ui32)operation);
+ }
+ }
+ }
+
+ Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvResolveKeySet(request));
+ }
+
+private:
+ THolder<TKeyDesc> ExtractKey(const TTableId& table, TKeyDesc::ERowOperation operation) {
+ const auto& tableKey = TableKeys.GetTable(table);
+ auto range = GetFullRange(tableKey.KeyColumnTypes.size());
+
+ return MakeHolder<TKeyDesc>(table, range.ToTableRange(), operation, tableKey.KeyColumnTypes,
TVector<TKeyDesc::TColumnOp>{});
- }
-
- static TSerializedTableRange GetFullRange(ui32 columnsCount) {
- TVector<TCell> fromValues(columnsCount);
- TVector<TCell> toValues;
-
- return TSerializedTableRange(fromValues, /* inclusiveFrom */ true, toValues, /* inclusiveTo */ false);
- }
-
+ }
+
+ static TSerializedTableRange GetFullRange(ui32 columnsCount) {
+ TVector<TCell> fromValues(columnsCount);
+ TVector<TCell> toValues;
+
+ return TSerializedTableRange(fromValues, /* inclusiveFrom */ true, toValues, /* inclusiveTo */ false);
+ }
+
static uintptr_t EncodeStageInfo(TKqpTasksGraph::TStageInfoType& stageInfo) {
return reinterpret_cast<uintptr_t>(&stageInfo);
}
@@ -316,42 +316,42 @@ private:
return *reinterpret_cast<TKqpTasksGraph::TStageInfoType*>(userData);
}
-private:
- void UnexpectedEvent(const TString& state, ui32 eventType) {
- LOG_C("TKqpTableResolver, unexpected event: " << eventType << ", at state:" << state << ", self: " << SelfId());
+private:
+ void UnexpectedEvent(const TString& state, ui32 eventType) {
+ LOG_C("TKqpTableResolver, unexpected event: " << eventType << ", at state:" << state << ", self: " << SelfId());
auto issue = NYql::YqlIssue({}, NYql::TIssuesIds::UNEXPECTED, "Internal error while executing transaction.");
ReplyErrorAndDie(Ydb::StatusIds::INTERNAL_ERROR, std::move(issue));
- }
-
- void ReplyErrorAndDie(Ydb::StatusIds::StatusCode status, TIssue&& issue) {
- auto replyEv = std::make_unique<TEvKqpExecuter::TEvTableResolveStatus>();
+ }
+
+ void ReplyErrorAndDie(Ydb::StatusIds::StatusCode status, TIssue&& issue) {
+ auto replyEv = std::make_unique<TEvKqpExecuter::TEvTableResolveStatus>();
replyEv->Status = status;
- replyEv->Issues.AddIssue(std::move(issue));
- replyEv->CpuTime = CpuTime;
- Send(Owner, replyEv.release());
- PassAway();
- }
-
-private:
+ replyEv->Issues.AddIssue(std::move(issue));
+ replyEv->CpuTime = CpuTime;
+ Send(Owner, replyEv.release());
+ PassAway();
+ }
+
+private:
const TActorId Owner;
- const ui64 TxId;
- const TMaybe<TString> UserToken;
+ const ui64 TxId;
+ const TMaybe<TString> UserToken;
const TVector<IKqpGateway::TPhysicalTxData>& Transactions;
TKqpTableKeys& TableKeys;
// TODO: TableResolver should not populate TasksGraph as it's not related to its job (bad API).
TKqpTasksGraph& TasksGraph;
- bool ShouldTerminate = false;
- TMaybe<ui32> GotUnexpectedEvent;
- TDuration CpuTime;
-};
-
-} // anonymous namespace
-
+ bool ShouldTerminate = false;
+ TMaybe<ui32> GotUnexpectedEvent;
+ TDuration CpuTime;
+};
+
+} // anonymous namespace
+
NActors::IActor* CreateKqpTableResolver(const TActorId& owner, ui64 txId, TMaybe<TString> userToken,
const TVector<IKqpGateway::TPhysicalTxData>& transactions, TKqpTableKeys& tableKeys, TKqpTasksGraph& tasksGraph) {
return new TKqpTableResolver(owner, txId, userToken, transactions, tableKeys, tasksGraph);
-}
-
-} // namespace NKikimr::NKqp
+}
+
+} // namespace NKikimr::NKqp
diff --git a/ydb/core/kqp/executer/kqp_table_resolver.h b/ydb/core/kqp/executer/kqp_table_resolver.h
index c78d6dacc9..e9618cf41d 100644
--- a/ydb/core/kqp/executer/kqp_table_resolver.h
+++ b/ydb/core/kqp/executer/kqp_table_resolver.h
@@ -1,10 +1,10 @@
-#pragma once
-
+#pragma once
+
#include "kqp_tasks_graph.h"
-
-namespace NKikimr::NKqp {
-
+
+namespace NKikimr::NKqp {
+
NActors::IActor* CreateKqpTableResolver(const TActorId& owner, ui64 txId, TMaybe<TString> userToken,
const TVector<IKqpGateway::TPhysicalTxData>& transactions, TKqpTableKeys& tableKeys, TKqpTasksGraph& tasksGraph);
-
-} // namespace NKikimr::NKqp
+
+} // namespace NKikimr::NKqp
diff --git a/ydb/core/kqp/executer/kqp_tasks_graph.cpp b/ydb/core/kqp/executer/kqp_tasks_graph.cpp
index b005ab6edc..1ccfa26047 100644
--- a/ydb/core/kqp/executer/kqp_tasks_graph.cpp
+++ b/ydb/core/kqp/executer/kqp_tasks_graph.cpp
@@ -16,8 +16,8 @@ using namespace NYql;
using namespace NYql::NDq;
using namespace NYql::NNodes;
-// #define DBG_TRACE
-
+// #define DBG_TRACE
+
void LogStage(const NActors::TActorContext& ctx, const TStageInfo& stageInfo) {
// TODO: Print stage details, including input types and program.
LOG_DEBUG_S(ctx, NKikimrServices::KQP_EXECUTER, "StageInfo: StageId #" << stageInfo.Id
@@ -34,104 +34,104 @@ bool HasWrites(const TStageInfo& stageInfo) {
stageInfo.Meta.ShardOperations.contains(TKeyDesc::ERowOperation::Erase);
}
-void FillKqpTasksGraphStages(TKqpTasksGraph& tasksGraph, const TVector<IKqpGateway::TPhysicalTxData>& txs) {
- for (size_t txIdx = 0; txIdx < txs.size(); ++txIdx) {
- auto& tx = txs[txIdx];
-
- for (ui32 stageIdx = 0; stageIdx < tx.Body.StagesSize(); ++stageIdx) {
- const auto& stage = tx.Body.GetStages(stageIdx);
- NYql::NDq::TStageId stageId(txIdx, stageIdx);
-
- TStageInfoMeta meta(tx);
-
- bool stageAdded = tasksGraph.AddStageInfo(
- TStageInfo(stageId, stage.InputsSize(), stage.GetOutputsCount(), std::move(meta)));
- YQL_ENSURE(stageAdded);
-
- auto& stageInfo = tasksGraph.GetStageInfo(stageId);
- LogStage(TlsActivationContext->AsActorContext(), stageInfo);
-
- THashSet<TTableId> tables;
- for (auto& op : stage.GetTableOps()) {
- if (!stageInfo.Meta.TableId) {
- YQL_ENSURE(!stageInfo.Meta.TablePath);
- stageInfo.Meta.TableId = MakeTableId(op.GetTable());
- stageInfo.Meta.TablePath = op.GetTable().GetPath();
+void FillKqpTasksGraphStages(TKqpTasksGraph& tasksGraph, const TVector<IKqpGateway::TPhysicalTxData>& txs) {
+ for (size_t txIdx = 0; txIdx < txs.size(); ++txIdx) {
+ auto& tx = txs[txIdx];
+
+ for (ui32 stageIdx = 0; stageIdx < tx.Body.StagesSize(); ++stageIdx) {
+ const auto& stage = tx.Body.GetStages(stageIdx);
+ NYql::NDq::TStageId stageId(txIdx, stageIdx);
+
+ TStageInfoMeta meta(tx);
+
+ bool stageAdded = tasksGraph.AddStageInfo(
+ TStageInfo(stageId, stage.InputsSize(), stage.GetOutputsCount(), std::move(meta)));
+ YQL_ENSURE(stageAdded);
+
+ auto& stageInfo = tasksGraph.GetStageInfo(stageId);
+ LogStage(TlsActivationContext->AsActorContext(), stageInfo);
+
+ THashSet<TTableId> tables;
+ for (auto& op : stage.GetTableOps()) {
+ if (!stageInfo.Meta.TableId) {
+ YQL_ENSURE(!stageInfo.Meta.TablePath);
+ stageInfo.Meta.TableId = MakeTableId(op.GetTable());
+ stageInfo.Meta.TablePath = op.GetTable().GetPath();
stageInfo.Meta.TableKind = ETableKind::Unknown;
- tables.insert(MakeTableId(op.GetTable()));
- } else {
- YQL_ENSURE(stageInfo.Meta.TableId == MakeTableId(op.GetTable()));
- YQL_ENSURE(stageInfo.Meta.TablePath == op.GetTable().GetPath());
- }
-
- switch (op.GetTypeCase()) {
- case NKqpProto::TKqpPhyTableOperation::kReadRange:
+ tables.insert(MakeTableId(op.GetTable()));
+ } else {
+ YQL_ENSURE(stageInfo.Meta.TableId == MakeTableId(op.GetTable()));
+ YQL_ENSURE(stageInfo.Meta.TablePath == op.GetTable().GetPath());
+ }
+
+ switch (op.GetTypeCase()) {
+ case NKqpProto::TKqpPhyTableOperation::kReadRange:
case NKqpProto::TKqpPhyTableOperation::kReadRanges:
case NKqpProto::TKqpPhyTableOperation::kReadOlapRange:
- case NKqpProto::TKqpPhyTableOperation::kLookup:
- stageInfo.Meta.ShardOperations.insert(TKeyDesc::ERowOperation::Read);
- break;
- case NKqpProto::TKqpPhyTableOperation::kUpsertRows:
- stageInfo.Meta.ShardOperations.insert(TKeyDesc::ERowOperation::Update);
- break;
- case NKqpProto::TKqpPhyTableOperation::kDeleteRows:
- stageInfo.Meta.ShardOperations.insert(TKeyDesc::ERowOperation::Erase);
- break;
- default:
- YQL_ENSURE(false, "Unexpected table operation: " << (ui32) op.GetTypeCase());
- }
- }
-
- YQL_ENSURE(tables.empty() || tables.size() == 1);
- YQL_ENSURE(!HasReads(stageInfo) || !HasWrites(stageInfo));
- }
- }
-}
-
-void BuildKqpTaskGraphResultChannels(TKqpTasksGraph& tasksGraph, const NKqpProto::TKqpPhyTx& tx, ui64 txIdx) {
- for (ui32 i = 0; i < tx.ResultsSize(); ++i) {
- const auto& result = tx.GetResults(i);
- const auto& connection = result.GetConnection();
- const auto& inputStageInfo = tasksGraph.GetStageInfo(TStageId(txIdx, connection.GetStageIndex()));
- const auto& outputIdx = connection.GetOutputIndex();
-
- YQL_ENSURE(inputStageInfo.Tasks.size() == 1, "actual count: " << inputStageInfo.Tasks.size());
- auto originTaskId = inputStageInfo.Tasks[0];
-
- auto& channel = tasksGraph.AddChannel();
- channel.SrcTask = originTaskId;
- channel.SrcOutputIndex = outputIdx;
- channel.DstTask = 0;
- channel.DstInputIndex = i;
- channel.InMemory = true;
-
- auto& originTask = tasksGraph.GetTask(originTaskId);
-
- auto& taskOutput = originTask.Outputs[outputIdx];
- taskOutput.Type = TTaskOutputType::Map;
- taskOutput.Channels.push_back(channel.Id);
-
- LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "Create result channelId: " << channel.Id
- << " from task: " << originTaskId << " with index: " << outputIdx);
- }
-}
-
+ case NKqpProto::TKqpPhyTableOperation::kLookup:
+ stageInfo.Meta.ShardOperations.insert(TKeyDesc::ERowOperation::Read);
+ break;
+ case NKqpProto::TKqpPhyTableOperation::kUpsertRows:
+ stageInfo.Meta.ShardOperations.insert(TKeyDesc::ERowOperation::Update);
+ break;
+ case NKqpProto::TKqpPhyTableOperation::kDeleteRows:
+ stageInfo.Meta.ShardOperations.insert(TKeyDesc::ERowOperation::Erase);
+ break;
+ default:
+ YQL_ENSURE(false, "Unexpected table operation: " << (ui32) op.GetTypeCase());
+ }
+ }
+
+ YQL_ENSURE(tables.empty() || tables.size() == 1);
+ YQL_ENSURE(!HasReads(stageInfo) || !HasWrites(stageInfo));
+ }
+ }
+}
+
+void BuildKqpTaskGraphResultChannels(TKqpTasksGraph& tasksGraph, const NKqpProto::TKqpPhyTx& tx, ui64 txIdx) {
+ for (ui32 i = 0; i < tx.ResultsSize(); ++i) {
+ const auto& result = tx.GetResults(i);
+ const auto& connection = result.GetConnection();
+ const auto& inputStageInfo = tasksGraph.GetStageInfo(TStageId(txIdx, connection.GetStageIndex()));
+ const auto& outputIdx = connection.GetOutputIndex();
+
+ YQL_ENSURE(inputStageInfo.Tasks.size() == 1, "actual count: " << inputStageInfo.Tasks.size());
+ auto originTaskId = inputStageInfo.Tasks[0];
+
+ auto& channel = tasksGraph.AddChannel();
+ channel.SrcTask = originTaskId;
+ channel.SrcOutputIndex = outputIdx;
+ channel.DstTask = 0;
+ channel.DstInputIndex = i;
+ channel.InMemory = true;
+
+ auto& originTask = tasksGraph.GetTask(originTaskId);
+
+ auto& taskOutput = originTask.Outputs[outputIdx];
+ taskOutput.Type = TTaskOutputType::Map;
+ taskOutput.Channels.push_back(channel.Id);
+
+ LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "Create result channelId: " << channel.Id
+ << " from task: " << originTaskId << " with index: " << outputIdx);
+ }
+}
+
void BuildMapShardChannels(TKqpTasksGraph& graph, const TStageInfo& stageInfo, ui32 inputIndex,
- const TStageInfo& inputStageInfo, ui32 outputIndex, bool enableSpilling, const TChannelLogFunc& logFunc)
-{
+ const TStageInfo& inputStageInfo, ui32 outputIndex, bool enableSpilling, const TChannelLogFunc& logFunc)
+{
YQL_ENSURE(stageInfo.Tasks.size() == inputStageInfo.Tasks.size());
- THashMap<ui64, ui64> shardToTaskMap;
+ THashMap<ui64, ui64> shardToTaskMap;
for (auto& taskId : stageInfo.Tasks) {
auto& task = graph.GetTask(taskId);
- auto result = shardToTaskMap.insert(std::make_pair(task.Meta.ShardId, taskId));
+ auto result = shardToTaskMap.insert(std::make_pair(task.Meta.ShardId, taskId));
YQL_ENSURE(result.second);
}
for (auto& originTaskId : inputStageInfo.Tasks) {
auto& originTask = graph.GetTask(originTaskId);
- auto targetTaskId = shardToTaskMap.FindPtr(originTask.Meta.ShardId);
+ auto targetTaskId = shardToTaskMap.FindPtr(originTask.Meta.ShardId);
YQL_ENSURE(targetTaskId);
auto& targetTask = graph.GetTask(*targetTaskId);
@@ -140,7 +140,7 @@ void BuildMapShardChannels(TKqpTasksGraph& graph, const TStageInfo& stageInfo, u
channel.SrcOutputIndex = outputIndex;
channel.DstTask = targetTask.Id;
channel.DstInputIndex = inputIndex;
- channel.InMemory = !enableSpilling || inputStageInfo.OutputsCount == 1;
+ channel.InMemory = !enableSpilling || inputStageInfo.OutputsCount == 1;
auto& taskInput = targetTask.Inputs[inputIndex];
taskInput.Channels.push_back(channel.Id);
@@ -149,21 +149,21 @@ void BuildMapShardChannels(TKqpTasksGraph& graph, const TStageInfo& stageInfo, u
taskOutput.Type = TTaskOutputType::Map;
taskOutput.Channels.push_back(channel.Id);
- logFunc(channel.Id, originTask.Id, targetTask.Id, "MapShard/Map", !channel.InMemory);
+ logFunc(channel.Id, originTask.Id, targetTask.Id, "MapShard/Map", !channel.InMemory);
}
}
void BuildShuffleShardChannels(TKqpTasksGraph& graph, const TStageInfo& stageInfo, ui32 inputIndex,
- const TStageInfo& inputStageInfo, ui32 outputIndex, const TKqpTableKeys& tableKeys, bool enableSpilling,
- const TChannelLogFunc& logFunc)
-{
+ const TStageInfo& inputStageInfo, ui32 outputIndex, const TKqpTableKeys& tableKeys, bool enableSpilling,
+ const TChannelLogFunc& logFunc)
+{
YQL_ENSURE(stageInfo.Meta.ShardKey);
THashMap<ui64, const TKeyDesc::TPartitionInfo*> partitionsMap;
for (auto& partition : stageInfo.Meta.ShardKey->Partitions) {
partitionsMap[partition.ShardId] = &partition;
}
- auto table = tableKeys.GetTable(stageInfo.Meta.TableId);
+ auto table = tableKeys.GetTable(stageInfo.Meta.TableId);
for (auto& originTaskId : inputStageInfo.Tasks) {
auto& originTask = graph.GetTask(originTaskId);
@@ -182,7 +182,7 @@ void BuildShuffleShardChannels(TKqpTasksGraph& graph, const TStageInfo& stageInf
channel.SrcOutputIndex = outputIndex;
channel.DstTask = targetTask.Id;
channel.DstInputIndex = inputIndex;
- channel.InMemory = !enableSpilling || inputStageInfo.OutputsCount == 1;
+ channel.InMemory = !enableSpilling || inputStageInfo.OutputsCount == 1;
taskOutput.Meta.ShardPartitions.insert(std::make_pair(channel.Id, *targetPartition));
taskOutput.Channels.push_back(channel.Id);
@@ -190,59 +190,59 @@ void BuildShuffleShardChannels(TKqpTasksGraph& graph, const TStageInfo& stageInf
auto& taskInput = targetTask.Inputs[inputIndex];
taskInput.Channels.push_back(channel.Id);
- logFunc(channel.Id, originTask.Id, targetTask.Id, "ShuffleShard/ShardRangePartition", !channel.InMemory);
+ logFunc(channel.Id, originTask.Id, targetTask.Id, "ShuffleShard/ShardRangePartition", !channel.InMemory);
}
}
}
-void BuildKqpStageChannels(TKqpTasksGraph& tasksGraph, const TKqpTableKeys& tableKeys, const TStageInfo& stageInfo,
- ui64 txId, bool enableSpilling)
-{
- auto& stage = GetStage(stageInfo);
-
- if (stage.GetIsEffectsStage()) {
- YQL_ENSURE(stageInfo.OutputsCount == 1);
-
- for (auto& taskId : stageInfo.Tasks) {
- auto& task = tasksGraph.GetTask(taskId);
- auto& taskOutput = task.Outputs[0];
- taskOutput.Type = TTaskOutputType::Effects;
- }
- }
-
- auto log = [&stageInfo, txId](ui64 channel, ui64 from, ui64 to, TStringBuf type, bool spilling) {
- LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << txId << ". "
- << "Stage " << stageInfo.Id << " create channelId: " << channel
- << " from task: " << from << " to task: " << to << " of type " << type
- << (spilling ? " with spilling" : " without spilling"));
- };
-
- for (ui32 inputIdx = 0; inputIdx < stage.InputsSize(); ++inputIdx) {
- const auto& input = stage.GetInputs(inputIdx);
- const auto& inputStageInfo = tasksGraph.GetStageInfo(TStageId(stageInfo.Id.TxId, input.GetStageIndex()));
- const auto& outputIdx = input.GetOutputIndex();
-
- switch (input.GetTypeCase()) {
- case NKqpProto::TKqpPhyConnection::kUnionAll:
- BuildUnionAllChannels(tasksGraph, stageInfo, inputIdx, inputStageInfo, outputIdx, enableSpilling, log);
- break;
- case NKqpProto::TKqpPhyConnection::kHashShuffle:
- BuildHashShuffleChannels(tasksGraph, stageInfo, inputIdx, inputStageInfo, outputIdx,
- input.GetHashShuffle().GetKeyColumns(), enableSpilling, log);
- break;
- case NKqpProto::TKqpPhyConnection::kBroadcast:
- BuildBroadcastChannels(tasksGraph, stageInfo, inputIdx, inputStageInfo, outputIdx, enableSpilling, log);
- break;
- case NKqpProto::TKqpPhyConnection::kMap:
- BuildMapChannels(tasksGraph, stageInfo, inputIdx, inputStageInfo, outputIdx, enableSpilling, log);
- break;
- case NKqpProto::TKqpPhyConnection::kMapShard:
- BuildMapShardChannels(tasksGraph, stageInfo, inputIdx, inputStageInfo, outputIdx, enableSpilling, log);
- break;
- case NKqpProto::TKqpPhyConnection::kShuffleShard:
- BuildShuffleShardChannels(tasksGraph, stageInfo, inputIdx, inputStageInfo, outputIdx, tableKeys,
- enableSpilling, log);
- break;
+void BuildKqpStageChannels(TKqpTasksGraph& tasksGraph, const TKqpTableKeys& tableKeys, const TStageInfo& stageInfo,
+ ui64 txId, bool enableSpilling)
+{
+ auto& stage = GetStage(stageInfo);
+
+ if (stage.GetIsEffectsStage()) {
+ YQL_ENSURE(stageInfo.OutputsCount == 1);
+
+ for (auto& taskId : stageInfo.Tasks) {
+ auto& task = tasksGraph.GetTask(taskId);
+ auto& taskOutput = task.Outputs[0];
+ taskOutput.Type = TTaskOutputType::Effects;
+ }
+ }
+
+ auto log = [&stageInfo, txId](ui64 channel, ui64 from, ui64 to, TStringBuf type, bool spilling) {
+ LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << txId << ". "
+ << "Stage " << stageInfo.Id << " create channelId: " << channel
+ << " from task: " << from << " to task: " << to << " of type " << type
+ << (spilling ? " with spilling" : " without spilling"));
+ };
+
+ for (ui32 inputIdx = 0; inputIdx < stage.InputsSize(); ++inputIdx) {
+ const auto& input = stage.GetInputs(inputIdx);
+ const auto& inputStageInfo = tasksGraph.GetStageInfo(TStageId(stageInfo.Id.TxId, input.GetStageIndex()));
+ const auto& outputIdx = input.GetOutputIndex();
+
+ switch (input.GetTypeCase()) {
+ case NKqpProto::TKqpPhyConnection::kUnionAll:
+ BuildUnionAllChannels(tasksGraph, stageInfo, inputIdx, inputStageInfo, outputIdx, enableSpilling, log);
+ break;
+ case NKqpProto::TKqpPhyConnection::kHashShuffle:
+ BuildHashShuffleChannels(tasksGraph, stageInfo, inputIdx, inputStageInfo, outputIdx,
+ input.GetHashShuffle().GetKeyColumns(), enableSpilling, log);
+ break;
+ case NKqpProto::TKqpPhyConnection::kBroadcast:
+ BuildBroadcastChannels(tasksGraph, stageInfo, inputIdx, inputStageInfo, outputIdx, enableSpilling, log);
+ break;
+ case NKqpProto::TKqpPhyConnection::kMap:
+ BuildMapChannels(tasksGraph, stageInfo, inputIdx, inputStageInfo, outputIdx, enableSpilling, log);
+ break;
+ case NKqpProto::TKqpPhyConnection::kMapShard:
+ BuildMapShardChannels(tasksGraph, stageInfo, inputIdx, inputStageInfo, outputIdx, enableSpilling, log);
+ break;
+ case NKqpProto::TKqpPhyConnection::kShuffleShard:
+ BuildShuffleShardChannels(tasksGraph, stageInfo, inputIdx, inputStageInfo, outputIdx, tableKeys,
+ enableSpilling, log);
+ break;
case NKqpProto::TKqpPhyConnection::kMerge: {
TVector<TSortColumn> sortColumns;
sortColumns.reserve(input.GetMerge().SortColumnsSize());
@@ -252,47 +252,47 @@ void BuildKqpStageChannels(TKqpTasksGraph& tasksGraph, const TKqpTableKeys& tabl
TSortColumn(sortColumn.GetColumn(), sortColumn.GetAscending())
);
}
- // TODO: spilling?
+ // TODO: spilling?
BuildMergeChannels(tasksGraph, stageInfo, inputIdx, inputStageInfo, outputIdx, sortColumns, log);
break;
}
- default:
- YQL_ENSURE(false, "Unexpected stage input type: " << (ui32)input.GetTypeCase());
- }
- }
-}
-
-TVector<TTaskMeta::TColumn> BuildKqpColumns(const NKqpProto::TKqpPhyTableOperation& op, const TKqpTableKeys::TTable& table) {
- TVector<TTaskMeta::TColumn> columns;
- columns.reserve(op.GetColumns().size());
-
- for (const auto& column : op.GetColumns()) {
- TTaskMeta::TColumn c;
- c.Id = column.GetId();
- c.Type = table.Columns.at(column.GetName()).Type;
- c.Name = column.GetName();
-
- columns.emplace_back(std::move(c));
- }
-
- return columns;
-}
-
-bool IsCrossShardChannel(TKqpTasksGraph& tasksGraph, const TChannel& channel) {
- YQL_ENSURE(channel.SrcTask);
-
- if (!channel.DstTask) {
- return false;
- }
-
- ui64 targetShard = tasksGraph.GetTask(channel.DstTask).Meta.ShardId;
- if (!targetShard) {
- return false;
- }
-
- return targetShard != tasksGraph.GetTask(channel.SrcTask).Meta.ShardId;
-}
-
+ default:
+ YQL_ENSURE(false, "Unexpected stage input type: " << (ui32)input.GetTypeCase());
+ }
+ }
+}
+
+TVector<TTaskMeta::TColumn> BuildKqpColumns(const NKqpProto::TKqpPhyTableOperation& op, const TKqpTableKeys::TTable& table) {
+ TVector<TTaskMeta::TColumn> columns;
+ columns.reserve(op.GetColumns().size());
+
+ for (const auto& column : op.GetColumns()) {
+ TTaskMeta::TColumn c;
+ c.Id = column.GetId();
+ c.Type = table.Columns.at(column.GetName()).Type;
+ c.Name = column.GetName();
+
+ columns.emplace_back(std::move(c));
+ }
+
+ return columns;
+}
+
+bool IsCrossShardChannel(TKqpTasksGraph& tasksGraph, const TChannel& channel) {
+ YQL_ENSURE(channel.SrcTask);
+
+ if (!channel.DstTask) {
+ return false;
+ }
+
+ ui64 targetShard = tasksGraph.GetTask(channel.DstTask).Meta.ShardId;
+ if (!targetShard) {
+ return false;
+ }
+
+ return targetShard != tasksGraph.GetTask(channel.SrcTask).Meta.ShardId;
+}
+
const NKqpProto::TKqpPhyStage& GetStage(const TStageInfo& stageInfo) {
auto& txBody = stageInfo.Meta.Tx.Body;
YQL_ENSURE(stageInfo.Id.StageId < txBody.StagesSize());
@@ -300,322 +300,322 @@ const NKqpProto::TKqpPhyStage& GetStage(const TStageInfo& stageInfo) {
return txBody.GetStages(stageInfo.Id.StageId);
}
-void TShardKeyRanges::AddPoint(TSerializedCellVec&& point) {
- if (!IsFullRange()) {
- Ranges.emplace_back(std::move(point));
- }
-}
-
-void TShardKeyRanges::AddRange(TSerializedTableRange&& range) {
- Y_VERIFY_DEBUG(!range.Point);
- if (!IsFullRange()) {
- Ranges.emplace_back(std::move(range));
- }
-}
-
-void TShardKeyRanges::Add(TSerializedPointOrRange&& pointOrRange) {
+void TShardKeyRanges::AddPoint(TSerializedCellVec&& point) {
+ if (!IsFullRange()) {
+ Ranges.emplace_back(std::move(point));
+ }
+}
+
+void TShardKeyRanges::AddRange(TSerializedTableRange&& range) {
+ Y_VERIFY_DEBUG(!range.Point);
+ if (!IsFullRange()) {
+ Ranges.emplace_back(std::move(range));
+ }
+}
+
+void TShardKeyRanges::Add(TSerializedPointOrRange&& pointOrRange) {
+ if (!IsFullRange()) {
+ Ranges.emplace_back(std::move(pointOrRange));
+ if (std::holds_alternative<TSerializedTableRange>(Ranges.back())) {
+ Y_VERIFY_DEBUG(!std::get<TSerializedTableRange>(Ranges.back()).Point);
+ }
+ }
+}
+
+void TShardKeyRanges::CopyFrom(const TVector<TSerializedPointOrRange>& ranges) {
if (!IsFullRange()) {
- Ranges.emplace_back(std::move(pointOrRange));
- if (std::holds_alternative<TSerializedTableRange>(Ranges.back())) {
- Y_VERIFY_DEBUG(!std::get<TSerializedTableRange>(Ranges.back()).Point);
- }
- }
-}
-
-void TShardKeyRanges::CopyFrom(const TVector<TSerializedPointOrRange>& ranges) {
- if (!IsFullRange()) {
Ranges = ranges;
- for (auto& x : Ranges) {
- if (std::holds_alternative<TSerializedTableRange>(x)) {
- Y_VERIFY_DEBUG(!std::get<TSerializedTableRange>(x).Point);
- }
- }
+ for (auto& x : Ranges) {
+ if (std::holds_alternative<TSerializedTableRange>(x)) {
+ Y_VERIFY_DEBUG(!std::get<TSerializedTableRange>(x).Point);
+ }
+ }
}
};
-void TShardKeyRanges::MakeFullRange(TSerializedTableRange&& range) {
- Ranges.clear();
- FullRange.emplace(std::move(range));
-}
-
-void TShardKeyRanges::MakeFullPoint(TSerializedCellVec&& point) {
- Ranges.clear();
- FullRange.emplace(TSerializedTableRange(std::move(point.GetBuffer()), "", true, true));
- FullRange->Point = true;
-}
-
-void TShardKeyRanges::MakeFull(TSerializedPointOrRange&& pointOrRange) {
- if (std::holds_alternative<TSerializedTableRange>(pointOrRange)) {
- MakeFullRange(std::move(std::get<TSerializedTableRange>(pointOrRange)));
- } else {
- MakeFullPoint(std::move(std::get<TSerializedCellVec>(pointOrRange)));
- }
-}
-
-
+void TShardKeyRanges::MakeFullRange(TSerializedTableRange&& range) {
+ Ranges.clear();
+ FullRange.emplace(std::move(range));
+}
+
+void TShardKeyRanges::MakeFullPoint(TSerializedCellVec&& point) {
+ Ranges.clear();
+ FullRange.emplace(TSerializedTableRange(std::move(point.GetBuffer()), "", true, true));
+ FullRange->Point = true;
+}
+
+void TShardKeyRanges::MakeFull(TSerializedPointOrRange&& pointOrRange) {
+ if (std::holds_alternative<TSerializedTableRange>(pointOrRange)) {
+ MakeFullRange(std::move(std::get<TSerializedTableRange>(pointOrRange)));
+ } else {
+ MakeFullPoint(std::move(std::get<TSerializedCellVec>(pointOrRange)));
+ }
+}
+
+
void TShardKeyRanges::MergeWritePoints(TShardKeyRanges&& other, const TVector<NScheme::TTypeId>& keyTypes) {
-#ifdef DBG_TRACE
- Cerr << (TStringBuilder() << "-- merge " << ToString(keyTypes, *AppData()->TypeRegistry)
- << " with " << other.ToString(keyTypes, *AppData()->TypeRegistry) << Endl);
-#endif
-
- if (IsFullRange()) {
- return;
- }
-
- if (other.IsFullRange()) {
- std::swap(Ranges, other.Ranges);
- FullRange.swap(other.FullRange);
- return;
- }
-
- TVector<TSerializedPointOrRange> result;
- result.reserve(Ranges.size() + other.Ranges.size());
-
- ui64 i = 0, j = 0;
- while (true) {
- if (i >= Ranges.size()) {
- while (j < other.Ranges.size()) {
- result.emplace_back(std::move(other.Ranges[j++]));
- }
- break;
- }
- if (j >= other.Ranges.size()) {
- while (i < Ranges.size()) {
- result.emplace_back(std::move(Ranges[i++]));
- }
- break;
- }
-
- auto& x = Ranges[i];
- auto& y = other.Ranges[j];
-
- int cmp = 0;
-
- // ensure `x` and `y` are points
- YQL_ENSURE(std::holds_alternative<TSerializedCellVec>(x));
- YQL_ENSURE(std::holds_alternative<TSerializedCellVec>(y));
-
+#ifdef DBG_TRACE
+ Cerr << (TStringBuilder() << "-- merge " << ToString(keyTypes, *AppData()->TypeRegistry)
+ << " with " << other.ToString(keyTypes, *AppData()->TypeRegistry) << Endl);
+#endif
+
+ if (IsFullRange()) {
+ return;
+ }
+
+ if (other.IsFullRange()) {
+ std::swap(Ranges, other.Ranges);
+ FullRange.swap(other.FullRange);
+ return;
+ }
+
+ TVector<TSerializedPointOrRange> result;
+ result.reserve(Ranges.size() + other.Ranges.size());
+
+ ui64 i = 0, j = 0;
+ while (true) {
+ if (i >= Ranges.size()) {
+ while (j < other.Ranges.size()) {
+ result.emplace_back(std::move(other.Ranges[j++]));
+ }
+ break;
+ }
+ if (j >= other.Ranges.size()) {
+ while (i < Ranges.size()) {
+ result.emplace_back(std::move(Ranges[i++]));
+ }
+ break;
+ }
+
+ auto& x = Ranges[i];
+ auto& y = other.Ranges[j];
+
+ int cmp = 0;
+
+ // ensure `x` and `y` are points
+ YQL_ENSURE(std::holds_alternative<TSerializedCellVec>(x));
+ YQL_ENSURE(std::holds_alternative<TSerializedCellVec>(y));
+
#if 1
// common case for multi-effects transactions
- cmp = CompareTypedCellVectors(
- std::get<TSerializedCellVec>(x).GetCells().data(),
- std::get<TSerializedCellVec>(y).GetCells().data(),
- keyTypes.data(), keyTypes.size());
+ cmp = CompareTypedCellVectors(
+ std::get<TSerializedCellVec>(x).GetCells().data(),
+ std::get<TSerializedCellVec>(y).GetCells().data(),
+ keyTypes.data(), keyTypes.size());
#else
if (x.IsPoint() && y.IsPoint()) {
- // common case for multi-effects transactions
- cmp = CompareTypedCellVectors(x.From.GetCells().data(), y.From.GetCells().data(), keyTypes.data(), keyTypes.size());
+ // common case for multi-effects transactions
+ cmp = CompareTypedCellVectors(x.From.GetCells().data(), y.From.GetCells().data(), keyTypes.data(), keyTypes.size());
} else if (x.IsPoint()) {
- cmp = ComparePointAndRange(x.From.GetCells(), y.ToTableRange(), keyTypes, keyTypes);
+ cmp = ComparePointAndRange(x.From.GetCells(), y.ToTableRange(), keyTypes, keyTypes);
} else if (y.IsPoint()) {
- cmp = -ComparePointAndRange(y.From.GetCells(), x.ToTableRange(), keyTypes, keyTypes);
- } else {
- cmp = CompareRanges(x.ToTableRange(), y.ToTableRange(), keyTypes);
- }
-#endif
-
- if (cmp < 0) {
- result.emplace_back(std::move(x));
- ++i;
- } else if (cmp > 0) {
- result.emplace_back(std::move(y));
- ++j;
- } else {
- result.emplace_back(std::move(x));
- ++i;
- ++j;
-#if 0
- if (CompareTypedCellVectors(x.From.GetCells().data(), y.From.GetCells().data(), keyTypes.data(), keyTypes.size()) == 0 &&
- x.FromInclusive == y.FromInclusive &&
- CompareTypedCellVectors(x.To.GetCells().data(), y.To.GetCells().data(), keyTypes.data(), keyTypes.size()) == 0 &&
- x.ToInclusive == y.ToInclusive)
- {
- result.emplace_back(std::move(x));
- ++i;
- ++j;
- } else if (x.Point) {
- YQL_ENSURE(!y.Point);
- result.emplace_back(TSerializedTableRange(y.From.GetCells(), y.FromInclusive, x.From.GetCells(), true));
- ++i;
- y.From = x.From;
- y.FromInclusive = false;
- } else if (y.Point) {
- YQL_ENSURE(!x.Point);
- result.emplace_back(TSerializedTableRange(x.From.GetCells(), x.FromInclusive, y.From.GetCells(), true));
- ++j;
- x.From = y.From;
- x.FromInclusive = false;
- } else {
- YQL_ENSURE(!x.Point && !y.Point);
-
- int cmpLeft = CompareBorders<true, true>(x.From.GetCells(), y.From.GetCells(), x.FromInclusive, y.FromInclusive, keyTypes);
- int cmpRight = CompareBorders<false, false>(x.From.GetCells(), y.From.GetCells(), x.FromInclusive, y.FromInclusive, keyTypes);
-
- if (cmpLeft <= 0 && cmpRight >= 0) {
- // y \in x
- result.emplace_back(std::move(x));
- ++i;
- ++j;
- } else if (cmpLeft >= 0 && cmpRight <= 0) {
- // x \in y
- result.emplace_back(std::move(y));
- ++i;
- ++j;
- } else if (cmpLeft < 0) {
- result.emplace_back(TSerializedTableRange(x.From.GetCells(), x.FromInclusive, y.From.GetCells(), !y.FromInclusive));
- x.From = y.From;
- x.FromInclusive = y.FromInclusive;
- } else if (cmpLeft > 0) {
- result.emplace_back(TSerializedTableRange(y.From.GetCells(), y.FromInclusive, x.From.GetCells(), !x.FromInclusive));
- y.From = x.From;
- y.FromInclusive = x.FromInclusive;
- } else /* cmpLeft == cmpRight */ {
- if (cmpRight < 0) {
- y.From = x.To;
- y.FromInclusive = !x.ToInclusive;
- result.emplace_back(std::move(x));
- ++i;
- } else if (cmpRight > 0) {
- x.From = y.To;
- x.FromInclusive = !y.ToInclusive;
- result.emplace_back(std::move(y));
- ++j;
- } else {
- // x == y
- YQL_ENSURE(false);
- }
- }
- }
-#endif
- }
- }
-
- Ranges = std::move(result);
-
-#ifdef DBG_TRACE
- Cerr << (TStringBuilder() << "--- merge result: " << ToString(keyTypes, *AppData()->TypeRegistry) << Endl);
-#endif
-}
-
-TString TShardKeyRanges::ToString(const TVector<NScheme::TTypeId>& keyTypes, const NScheme::TTypeRegistry& typeRegistry) const
-{
- TStringBuilder sb;
- sb << "TShardKeyRanges{ ";
- if (IsFullRange()) {
- sb << "full " << DebugPrintRange(keyTypes, FullRange->ToTableRange(), typeRegistry);
- } else {
- if (Ranges.empty()) {
- sb << "<empty> ";
- }
- for (auto& range : Ranges) {
- if (std::holds_alternative<TSerializedCellVec>(range)) {
- sb << DebugPrintPoint(keyTypes, std::get<TSerializedCellVec>(range).GetCells(), typeRegistry) << ", ";
- } else {
- sb << DebugPrintRange(keyTypes, std::get<TSerializedTableRange>(range).ToTableRange(), typeRegistry) << ", ";
- }
- }
- }
- sb << "}";
- return sb;
-}
-
-void TShardKeyRanges::SerializeTo(NKikimrTxDataShard::TKqpTransaction_TDataTaskMeta_TKeyRange* proto) const {
- if (IsFullRange()) {
- auto& protoRange = *proto->MutableFullRange();
- FullRange->Serialize(protoRange);
- } else {
- auto* protoRanges = proto->MutableRanges();
- for (auto& range : Ranges) {
- if (std::holds_alternative<TSerializedCellVec>(range)) {
- const auto& x = std::get<TSerializedCellVec>(range);
- protoRanges->AddKeyPoints(x.GetBuffer());
- } else {
- auto& x = std::get<TSerializedTableRange>(range);
- Y_VERIFY_DEBUG(!x.Point);
- auto& keyRange = *protoRanges->AddKeyRanges();
- x.Serialize(keyRange);
- }
- }
- }
-}
-
-void TShardKeyRanges::SerializeTo(NKikimrTxDataShard::TKqpTransaction_TScanTaskMeta_TReadOpMeta* proto) const {
- if (IsFullRange()) {
- auto& protoRange = *proto->AddKeyRanges();
- FullRange->Serialize(protoRange);
- } else {
- for (auto& range : Ranges) {
+ cmp = -ComparePointAndRange(y.From.GetCells(), x.ToTableRange(), keyTypes, keyTypes);
+ } else {
+ cmp = CompareRanges(x.ToTableRange(), y.ToTableRange(), keyTypes);
+ }
+#endif
+
+ if (cmp < 0) {
+ result.emplace_back(std::move(x));
+ ++i;
+ } else if (cmp > 0) {
+ result.emplace_back(std::move(y));
+ ++j;
+ } else {
+ result.emplace_back(std::move(x));
+ ++i;
+ ++j;
+#if 0
+ if (CompareTypedCellVectors(x.From.GetCells().data(), y.From.GetCells().data(), keyTypes.data(), keyTypes.size()) == 0 &&
+ x.FromInclusive == y.FromInclusive &&
+ CompareTypedCellVectors(x.To.GetCells().data(), y.To.GetCells().data(), keyTypes.data(), keyTypes.size()) == 0 &&
+ x.ToInclusive == y.ToInclusive)
+ {
+ result.emplace_back(std::move(x));
+ ++i;
+ ++j;
+ } else if (x.Point) {
+ YQL_ENSURE(!y.Point);
+ result.emplace_back(TSerializedTableRange(y.From.GetCells(), y.FromInclusive, x.From.GetCells(), true));
+ ++i;
+ y.From = x.From;
+ y.FromInclusive = false;
+ } else if (y.Point) {
+ YQL_ENSURE(!x.Point);
+ result.emplace_back(TSerializedTableRange(x.From.GetCells(), x.FromInclusive, y.From.GetCells(), true));
+ ++j;
+ x.From = y.From;
+ x.FromInclusive = false;
+ } else {
+ YQL_ENSURE(!x.Point && !y.Point);
+
+ int cmpLeft = CompareBorders<true, true>(x.From.GetCells(), y.From.GetCells(), x.FromInclusive, y.FromInclusive, keyTypes);
+ int cmpRight = CompareBorders<false, false>(x.From.GetCells(), y.From.GetCells(), x.FromInclusive, y.FromInclusive, keyTypes);
+
+ if (cmpLeft <= 0 && cmpRight >= 0) {
+ // y \in x
+ result.emplace_back(std::move(x));
+ ++i;
+ ++j;
+ } else if (cmpLeft >= 0 && cmpRight <= 0) {
+ // x \in y
+ result.emplace_back(std::move(y));
+ ++i;
+ ++j;
+ } else if (cmpLeft < 0) {
+ result.emplace_back(TSerializedTableRange(x.From.GetCells(), x.FromInclusive, y.From.GetCells(), !y.FromInclusive));
+ x.From = y.From;
+ x.FromInclusive = y.FromInclusive;
+ } else if (cmpLeft > 0) {
+ result.emplace_back(TSerializedTableRange(y.From.GetCells(), y.FromInclusive, x.From.GetCells(), !x.FromInclusive));
+ y.From = x.From;
+ y.FromInclusive = x.FromInclusive;
+ } else /* cmpLeft == cmpRight */ {
+ if (cmpRight < 0) {
+ y.From = x.To;
+ y.FromInclusive = !x.ToInclusive;
+ result.emplace_back(std::move(x));
+ ++i;
+ } else if (cmpRight > 0) {
+ x.From = y.To;
+ x.FromInclusive = !y.ToInclusive;
+ result.emplace_back(std::move(y));
+ ++j;
+ } else {
+ // x == y
+ YQL_ENSURE(false);
+ }
+ }
+ }
+#endif
+ }
+ }
+
+ Ranges = std::move(result);
+
+#ifdef DBG_TRACE
+ Cerr << (TStringBuilder() << "--- merge result: " << ToString(keyTypes, *AppData()->TypeRegistry) << Endl);
+#endif
+}
+
+TString TShardKeyRanges::ToString(const TVector<NScheme::TTypeId>& keyTypes, const NScheme::TTypeRegistry& typeRegistry) const
+{
+ TStringBuilder sb;
+ sb << "TShardKeyRanges{ ";
+ if (IsFullRange()) {
+ sb << "full " << DebugPrintRange(keyTypes, FullRange->ToTableRange(), typeRegistry);
+ } else {
+ if (Ranges.empty()) {
+ sb << "<empty> ";
+ }
+ for (auto& range : Ranges) {
+ if (std::holds_alternative<TSerializedCellVec>(range)) {
+ sb << DebugPrintPoint(keyTypes, std::get<TSerializedCellVec>(range).GetCells(), typeRegistry) << ", ";
+ } else {
+ sb << DebugPrintRange(keyTypes, std::get<TSerializedTableRange>(range).ToTableRange(), typeRegistry) << ", ";
+ }
+ }
+ }
+ sb << "}";
+ return sb;
+}
+
+void TShardKeyRanges::SerializeTo(NKikimrTxDataShard::TKqpTransaction_TDataTaskMeta_TKeyRange* proto) const {
+ if (IsFullRange()) {
+ auto& protoRange = *proto->MutableFullRange();
+ FullRange->Serialize(protoRange);
+ } else {
+ auto* protoRanges = proto->MutableRanges();
+ for (auto& range : Ranges) {
+ if (std::holds_alternative<TSerializedCellVec>(range)) {
+ const auto& x = std::get<TSerializedCellVec>(range);
+ protoRanges->AddKeyPoints(x.GetBuffer());
+ } else {
+ auto& x = std::get<TSerializedTableRange>(range);
+ Y_VERIFY_DEBUG(!x.Point);
+ auto& keyRange = *protoRanges->AddKeyRanges();
+ x.Serialize(keyRange);
+ }
+ }
+ }
+}
+
+void TShardKeyRanges::SerializeTo(NKikimrTxDataShard::TKqpTransaction_TScanTaskMeta_TReadOpMeta* proto) const {
+ if (IsFullRange()) {
+ auto& protoRange = *proto->AddKeyRanges();
+ FullRange->Serialize(protoRange);
+ } else {
+ for (auto& range : Ranges) {
auto& keyRange = *proto->AddKeyRanges();
- if (std::holds_alternative<TSerializedTableRange>(range)) {
- auto& x = std::get<TSerializedTableRange>(range);
- Y_VERIFY_DEBUG(!x.Point);
- x.Serialize(keyRange);
- } else {
- const auto& x = std::get<TSerializedCellVec>(range);
- keyRange.SetFrom(x.GetBuffer());
- keyRange.SetTo(x.GetBuffer());
- keyRange.SetFromInclusive(true);
- keyRange.SetToInclusive(true);
- }
- }
- }
-}
-
-std::pair<const TSerializedCellVec*, bool> TShardKeyRanges::GetRightBorder() const {
- if (FullRange) {
- return !FullRange->Point ? std::make_pair(&FullRange->To, true) : std::make_pair(&FullRange->From, true);
- }
-
- YQL_ENSURE(!Ranges.empty());
- const auto& last = Ranges.back();
- if (std::holds_alternative<TSerializedCellVec>(last)) {
- return std::make_pair(&std::get<TSerializedCellVec>(last), true);
- }
-
- const auto& lastRange = std::get<TSerializedTableRange>(last);
- return !lastRange.Point ? std::make_pair(&lastRange.To, lastRange.ToInclusive) : std::make_pair(&lastRange.From, true);
-}
-
-TString TTaskMeta::ToString(const TVector<NScheme::TTypeId>& keyTypes, const NScheme::TTypeRegistry& typeRegistry) const
-{
- TStringBuilder sb;
- sb << "TTaskMeta{ ShardId: " << ShardId << ", Params: [";
-
- for (auto& [name, value] : Params) {
- sb << name << ", ";
- }
-
- sb << "], Reads: { ";
-
- if (Reads) {
- for (ui64 i = 0; i < Reads->size(); ++i) {
- auto& read = (*Reads)[i];
- sb << "[" << i << "]: { columns: [";
- for (auto& x : read.Columns) {
- sb << x.Name << ", ";
- }
- sb << "], ranges: " << read.Ranges.ToString(keyTypes, typeRegistry) << " }";
- if (i != Reads->size() - 1) {
- sb << ", ";
- }
- }
- } else {
+ if (std::holds_alternative<TSerializedTableRange>(range)) {
+ auto& x = std::get<TSerializedTableRange>(range);
+ Y_VERIFY_DEBUG(!x.Point);
+ x.Serialize(keyRange);
+ } else {
+ const auto& x = std::get<TSerializedCellVec>(range);
+ keyRange.SetFrom(x.GetBuffer());
+ keyRange.SetTo(x.GetBuffer());
+ keyRange.SetFromInclusive(true);
+ keyRange.SetToInclusive(true);
+ }
+ }
+ }
+}
+
+std::pair<const TSerializedCellVec*, bool> TShardKeyRanges::GetRightBorder() const {
+ if (FullRange) {
+ return !FullRange->Point ? std::make_pair(&FullRange->To, true) : std::make_pair(&FullRange->From, true);
+ }
+
+ YQL_ENSURE(!Ranges.empty());
+ const auto& last = Ranges.back();
+ if (std::holds_alternative<TSerializedCellVec>(last)) {
+ return std::make_pair(&std::get<TSerializedCellVec>(last), true);
+ }
+
+ const auto& lastRange = std::get<TSerializedTableRange>(last);
+ return !lastRange.Point ? std::make_pair(&lastRange.To, lastRange.ToInclusive) : std::make_pair(&lastRange.From, true);
+}
+
+TString TTaskMeta::ToString(const TVector<NScheme::TTypeId>& keyTypes, const NScheme::TTypeRegistry& typeRegistry) const
+{
+ TStringBuilder sb;
+ sb << "TTaskMeta{ ShardId: " << ShardId << ", Params: [";
+
+ for (auto& [name, value] : Params) {
+ sb << name << ", ";
+ }
+
+ sb << "], Reads: { ";
+
+ if (Reads) {
+ for (ui64 i = 0; i < Reads->size(); ++i) {
+ auto& read = (*Reads)[i];
+ sb << "[" << i << "]: { columns: [";
+ for (auto& x : read.Columns) {
+ sb << x.Name << ", ";
+ }
+ sb << "], ranges: " << read.Ranges.ToString(keyTypes, typeRegistry) << " }";
+ if (i != Reads->size() - 1) {
+ sb << ", ";
+ }
+ }
+ } else {
sb << "none";
- }
-
+ }
+
sb << " }, Writes: { ";
- if (Writes) {
+ if (Writes) {
sb << "ranges: " << Writes->Ranges.ToString(keyTypes, typeRegistry);
- } else {
+ } else {
sb << "none";
- }
-
+ }
+
sb << " } }";
- return sb;
-}
-
-} // namespace NKqp
-} // namespace NKikimr
+ return sb;
+}
+
+} // namespace NKqp
+} // namespace NKikimr
diff --git a/ydb/core/kqp/executer/kqp_tasks_graph.h b/ydb/core/kqp/executer/kqp_tasks_graph.h
index 239c6247ee..7c356ab623 100644
--- a/ydb/core/kqp/executer/kqp_tasks_graph.h
+++ b/ydb/core/kqp/executer/kqp_tasks_graph.h
@@ -9,11 +9,11 @@
#include <ydb/library/yql/dq/tasks/dq_connection_builder.h>
#include <ydb/library/yql/dq/tasks/dq_tasks_graph.h>
-namespace NKikimrTxDataShard {
-class TKqpTransaction_TDataTaskMeta_TKeyRange;
-class TKqpTransaction_TScanTaskMeta_TReadOpMeta;
-}
-
+namespace NKikimrTxDataShard {
+class TKqpTransaction_TDataTaskMeta_TKeyRange;
+class TKqpTransaction_TScanTaskMeta_TReadOpMeta;
+}
+
namespace NKikimr {
namespace NKqp {
@@ -28,13 +28,13 @@ struct TTransaction : private TMoveOnly {
struct TStageInfoMeta {
const IKqpGateway::TPhysicalTxData& Tx;
-
- TTableId TableId;
- TString TablePath;
+
+ TTableId TableId;
+ TString TablePath;
ETableKind TableKind;
-
- TVector<bool> SkipNullKeys;
-
+
+ TVector<bool> SkipNullKeys;
+
THashSet<TKeyDesc::ERowOperation> ShardOperations;
THolder<TKeyDesc> ShardKey;
NSchemeCache::TSchemeCacheRequest::EKind ShardKind = NSchemeCache::TSchemeCacheRequest::EKind::KindUnknown;
@@ -49,9 +49,9 @@ struct TStageInfoMeta {
}
bool IsSysView() const {
- if (!ShardKey) {
- return false;
- }
+ if (!ShardKey) {
+ return false;
+ }
YQL_ENSURE((TableKind == ETableKind::SysView) == ShardKey->IsSystemView());
return TableKind == ETableKind::SysView;
}
@@ -68,50 +68,50 @@ struct TTaskOutputMeta {
THashMap<ui64, const TKeyDesc::TPartitionInfo*> ShardPartitions;
};
-struct TShardKeyRanges {
- // ordered ranges and points
- TVector<TSerializedPointOrRange> Ranges;
- std::optional<TSerializedTableRange> FullRange;
-
- void AddPoint(TSerializedCellVec&& point);
- void AddRange(TSerializedTableRange&& range);
- void Add(TSerializedPointOrRange&& pointOrRange);
-
- void CopyFrom(const TVector<TSerializedPointOrRange>& ranges);
-
- void MakeFullRange(TSerializedTableRange&& range);
- void MakeFullPoint(TSerializedCellVec&& range);
- void MakeFull(TSerializedPointOrRange&& pointOrRange);
-
- bool IsFullRange() const { return FullRange.has_value(); }
- TVector<TSerializedPointOrRange>& GetRanges() { return Ranges; }
-
+struct TShardKeyRanges {
+ // ordered ranges and points
+ TVector<TSerializedPointOrRange> Ranges;
+ std::optional<TSerializedTableRange> FullRange;
+
+ void AddPoint(TSerializedCellVec&& point);
+ void AddRange(TSerializedTableRange&& range);
+ void Add(TSerializedPointOrRange&& pointOrRange);
+
+ void CopyFrom(const TVector<TSerializedPointOrRange>& ranges);
+
+ void MakeFullRange(TSerializedTableRange&& range);
+ void MakeFullPoint(TSerializedCellVec&& range);
+ void MakeFull(TSerializedPointOrRange&& pointOrRange);
+
+ bool IsFullRange() const { return FullRange.has_value(); }
+ TVector<TSerializedPointOrRange>& GetRanges() { return Ranges; }
+
void MergeWritePoints(TShardKeyRanges&& other, const TVector<NScheme::TTypeId>& keyTypes);
-
- TString ToString(const TVector<NScheme::TTypeId>& keyTypes, const NScheme::TTypeRegistry& typeRegistry) const;
- void SerializeTo(NKikimrTxDataShard::TKqpTransaction_TDataTaskMeta_TKeyRange* proto) const;
- void SerializeTo(NKikimrTxDataShard::TKqpTransaction_TScanTaskMeta_TReadOpMeta* proto) const;
-
- std::pair<const TSerializedCellVec*, bool> GetRightBorder() const;
-};
-
+
+ TString ToString(const TVector<NScheme::TTypeId>& keyTypes, const NScheme::TTypeRegistry& typeRegistry) const;
+ void SerializeTo(NKikimrTxDataShard::TKqpTransaction_TDataTaskMeta_TKeyRange* proto) const;
+ void SerializeTo(NKikimrTxDataShard::TKqpTransaction_TScanTaskMeta_TReadOpMeta* proto) const;
+
+ std::pair<const TSerializedCellVec*, bool> GetRightBorder() const;
+};
+
// TODO: use two different structs for scans and data queries
struct TTaskMeta {
- ui64 ShardId = 0; // only in case of non-scans (data-query & legacy scans)
- ui64 NodeId = 0; // only in case of scans over persistent snapshots
+ ui64 ShardId = 0; // only in case of non-scans (data-query & legacy scans)
+ ui64 NodeId = 0; // only in case of scans over persistent snapshots
TMap<TString, NYql::NDqProto::TData> Params;
TMap<TString, NKikimr::NMiniKQL::TType*> ParamTypes;
-
- struct TColumn {
- ui32 Id = 0;
- ui32 Type = 0;
- TString Name;
- };
-
+
+ struct TColumn {
+ ui32 Id = 0;
+ ui32 Type = 0;
+ TString Name;
+ };
+
struct TShardReadInfo {
- TShardKeyRanges Ranges;
- TVector<TColumn> Columns;
+ TShardKeyRanges Ranges;
+ TVector<TColumn> Columns;
ui64 ShardId = 0; // in case of persistent scans
};
@@ -121,20 +121,20 @@ struct TTaskMeta {
};
struct TReadInfo {
- ui64 ItemsLimit = 0;
- bool Reverse = false;
+ ui64 ItemsLimit = 0;
+ bool Reverse = false;
TKqpOlapProgram OlapProgram;
- };
-
- struct TWriteInfo {
- TShardKeyRanges Ranges;
- };
-
+ };
+
+ struct TWriteInfo {
+ TShardKeyRanges Ranges;
+ };
+
TReadInfo ReadInfo;
TMaybe<TVector<TShardReadInfo>> Reads; // if not set -> no reads
- TMaybe<TWriteInfo> Writes; // if not set -> no writes
-
- TString ToString(const TVector<NScheme::TTypeId>& keyTypes, const NScheme::TTypeRegistry& typeRegistry) const;
+ TMaybe<TWriteInfo> Writes; // if not set -> no writes
+
+ TString ToString(const TVector<NScheme::TTypeId>& keyTypes, const NScheme::TTypeRegistry& typeRegistry) const;
};
using TStageInfo = NYql::NDq::TStageInfo<TStageInfoMeta>;
@@ -144,12 +144,12 @@ using TTaskInput = NYql::NDq::TTaskInput<TTaskInputMeta>;
using TTask = NYql::NDq::TTask<TStageInfoMeta, TTaskMeta, TTaskInputMeta, TTaskOutputMeta>;
using TKqpTasksGraph = NYql::NDq::TDqTasksGraph<TStageInfoMeta, TTaskMeta, TTaskInputMeta, TTaskOutputMeta>;
-void FillKqpTasksGraphStages(TKqpTasksGraph& tasksGraph, const TVector<IKqpGateway::TPhysicalTxData>& txs);
-void BuildKqpTaskGraphResultChannels(TKqpTasksGraph& tasksGraph, const NKqpProto::TKqpPhyTx& tx, ui64 txIdx);
-void BuildKqpStageChannels(TKqpTasksGraph& tasksGraph, const TKqpTableKeys& tableKeys, const TStageInfo& stageInfo,
- ui64 txId, bool enableSpilling);
-TVector<TTaskMeta::TColumn> BuildKqpColumns(const NKqpProto::TKqpPhyTableOperation& op, const TKqpTableKeys::TTable& table);
-
+void FillKqpTasksGraphStages(TKqpTasksGraph& tasksGraph, const TVector<IKqpGateway::TPhysicalTxData>& txs);
+void BuildKqpTaskGraphResultChannels(TKqpTasksGraph& tasksGraph, const NKqpProto::TKqpPhyTx& tx, ui64 txIdx);
+void BuildKqpStageChannels(TKqpTasksGraph& tasksGraph, const TKqpTableKeys& tableKeys, const TStageInfo& stageInfo,
+ ui64 txId, bool enableSpilling);
+TVector<TTaskMeta::TColumn> BuildKqpColumns(const NKqpProto::TKqpPhyTableOperation& op, const TKqpTableKeys::TTable& table);
+
struct TKqpTaskOutputType {
enum : ui32 {
ShardRangePartition = TTaskOutputType::COMMON_TASK_OUTPUT_TYPE_END
@@ -163,7 +163,7 @@ void LogStage(const NActors::TActorContext& ctx, const TStageInfo& stageInfo);
bool HasReads(const TStageInfo& stageInfo);
bool HasWrites(const TStageInfo& stageInfo);
-bool IsCrossShardChannel(TKqpTasksGraph& tasksGraph, const NYql::NDq::TChannel& channel);
+bool IsCrossShardChannel(TKqpTasksGraph& tasksGraph, const NYql::NDq::TChannel& channel);
} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/executer/kqp_tasks_validate.cpp b/ydb/core/kqp/executer/kqp_tasks_validate.cpp
index b7ebc07275..dea7056ff0 100644
--- a/ydb/core/kqp/executer/kqp_tasks_validate.cpp
+++ b/ydb/core/kqp/executer/kqp_tasks_validate.cpp
@@ -9,16 +9,16 @@ namespace {
class TTasksValidator {
using TGraphType = TKqpTasksGraph;
- using TStageType = TGraphType::TStageInfoType;
+ using TStageType = TGraphType::TStageInfoType;
using TTaskType = TGraphType::TTaskType;
using TInputType = TGraphType::TTaskType::TInputType;
using TOutputType = TGraphType::TTaskType::TOutputType;
public:
- TTasksValidator(const TGraphType& tasksGraph, const EExecType& execType, bool enableSpilling)
+ TTasksValidator(const TGraphType& tasksGraph, const EExecType& execType, bool enableSpilling)
: TasksGraph(tasksGraph)
- , ExecType(execType)
- , EnableSpilling(enableSpilling) {}
+ , ExecType(execType)
+ , EnableSpilling(enableSpilling) {}
void Validate() {
for (auto& task : TasksGraph.GetTasks()) {
@@ -42,11 +42,11 @@ private:
<< ", dstTaskId: " << channel.DstTask);
}
}
-
- if (!EnableSpilling) {
- YQL_ENSURE(channel.InMemory, "With spilling off, all channels should be stored in memory only. "
- << "Not InMemory channelId: " << channelId);
- }
+
+ if (!EnableSpilling) {
+ YQL_ENSURE(channel.InMemory, "With spilling off, all channels should be stored in memory only. "
+ << "Not InMemory channelId: " << channelId);
+ }
}
void ValidateInput(const TInputType& input) {
@@ -89,14 +89,14 @@ private:
private:
const TGraphType& TasksGraph;
EExecType ExecType;
- bool EnableSpilling;
+ bool EnableSpilling;
};
} // namespace
-bool ValidateTasks(const TKqpTasksGraph& tasksGraph, const EExecType& execType, bool enableSpilling, NYql::TIssue& issue) {
+bool ValidateTasks(const TKqpTasksGraph& tasksGraph, const EExecType& execType, bool enableSpilling, NYql::TIssue& issue) {
try {
- TTasksValidator(tasksGraph, execType, enableSpilling).Validate();
+ TTasksValidator(tasksGraph, execType, enableSpilling).Validate();
return true;
} catch (const TYqlPanic& e) {
issue = YqlIssue({}, TIssuesIds::DEFAULT_ERROR, e.what());
diff --git a/ydb/core/kqp/executer/kqp_tasks_validate.h b/ydb/core/kqp/executer/kqp_tasks_validate.h
index fae4f39b73..f7fd4d1672 100644
--- a/ydb/core/kqp/executer/kqp_tasks_validate.h
+++ b/ydb/core/kqp/executer/kqp_tasks_validate.h
@@ -1,12 +1,12 @@
#pragma once
-#include "kqp_executer_impl.h"
+#include "kqp_executer_impl.h"
#include "kqp_tasks_graph.h"
namespace NKikimr {
namespace NKqp {
-bool ValidateTasks(const TKqpTasksGraph& tasksGraph, const EExecType& execType, bool enableSpilling, NYql::TIssue& issue);
+bool ValidateTasks(const TKqpTasksGraph& tasksGraph, const EExecType& execType, bool enableSpilling, NYql::TIssue& issue);
} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/executer/ut/kqp_executer_ut.cpp b/ydb/core/kqp/executer/ut/kqp_executer_ut.cpp
index 47360ca0a5..3148a2ec59 100644
--- a/ydb/core/kqp/executer/ut/kqp_executer_ut.cpp
+++ b/ydb/core/kqp/executer/ut/kqp_executer_ut.cpp
@@ -19,7 +19,7 @@ using namespace NYdb::NTable;
namespace {
-[[maybe_unused]]
+[[maybe_unused]]
NKqpProto::TKqpPhyTx BuildTxPlan(const TString& sql, TIntrusivePtr<IKqpGateway> gateway,
const TKikimrConfiguration::TPtr& config)
{
@@ -39,14 +39,14 @@ NKqpProto::TKqpPhyTx BuildTxPlan(const TString& sql, TIntrusivePtr<IKqpGateway>
return phyQuery.GetTransactions(0);
}
-[[maybe_unused]]
+[[maybe_unused]]
TIntrusivePtr<IKqpGateway> MakeIcGateway(const TKikimrRunner& kikimr) {
auto actorSystem = kikimr.GetTestServer().GetRuntime()->GetAnyNodeActorSystem();
- return CreateKikimrIcGateway(TString(DefaultKikimrClusterName), "/Root", TKqpGatewaySettings(),
- actorSystem, kikimr.GetTestServer().GetRuntime()->GetNodeId(0), TAlignedPagePoolCounters());
+ return CreateKikimrIcGateway(TString(DefaultKikimrClusterName), "/Root", TKqpGatewaySettings(),
+ actorSystem, kikimr.GetTestServer().GetRuntime()->GetNodeId(0), TAlignedPagePoolCounters());
}
-[[maybe_unused]]
+[[maybe_unused]]
TKikimrParamsMap GetParamsMap(const NYdb::TParams& params) {
TKikimrParamsMap paramsMap;
@@ -66,7 +66,7 @@ TKikimrParamsMap GetParamsMap(const NYdb::TParams& params) {
return paramsMap;
}
-[[maybe_unused]]
+[[maybe_unused]]
TKqpParamsRefMap GetParamRefsMap(const TKikimrParamsMap& paramsMap) {
TKqpParamsRefMap refsMap;
diff --git a/ydb/core/kqp/executer/ut/ya.make b/ydb/core/kqp/executer/ut/ya.make
index 1ed5e6767d..7a2024f9bd 100644
--- a/ydb/core/kqp/executer/ut/ya.make
+++ b/ydb/core/kqp/executer/ut/ya.make
@@ -12,7 +12,7 @@ IF (SANITIZER_TYPE OR WITH_VALGRIND)
ENDIF()
SRCS(
- # kqp_executer_ut.cpp
+ # kqp_executer_ut.cpp
)
PEERDIR(
diff --git a/ydb/core/kqp/executer/ya.make b/ydb/core/kqp/executer/ya.make
index 330e40286a..3db082d5d0 100644
--- a/ydb/core/kqp/executer/ya.make
+++ b/ydb/core/kqp/executer/ya.make
@@ -6,18 +6,18 @@ OWNER(
)
SRCS(
- kqp_data_executer.cpp
- kqp_scan_executer.cpp
- kqp_executer_impl.cpp
- kqp_executer_stats.cpp
- kqp_literal_executer.cpp
- kqp_locks_helper.cpp
- kqp_partition_helper.cpp
- kqp_planner.cpp
- kqp_planner_strategy.cpp
+ kqp_data_executer.cpp
+ kqp_scan_executer.cpp
+ kqp_executer_impl.cpp
+ kqp_executer_stats.cpp
+ kqp_literal_executer.cpp
+ kqp_locks_helper.cpp
+ kqp_partition_helper.cpp
+ kqp_planner.cpp
+ kqp_planner_strategy.cpp
kqp_shards_resolver.cpp
- kqp_result_channel.cpp
- kqp_table_resolver.cpp
+ kqp_result_channel.cpp
+ kqp_table_resolver.cpp
kqp_tasks_graph.cpp
kqp_tasks_validate.cpp
)
diff --git a/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json b/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json
index 7d5756f8c0..2faa33d5e1 100644
--- a/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json
+++ b/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json
@@ -42,22 +42,22 @@
]
},
{
- "Name": "TKqlReadTableBase",
+ "Name": "TKqlReadTableBase",
"Base": "TCallable",
- "Match": {"Type": "CallableBase"},
+ "Match": {"Type": "CallableBase"},
"Children": [
{"Index": 0, "Name": "Table", "Type": "TKqpTable"},
{"Index": 1, "Name": "Range", "Type": "TKqlKeyRange"},
- {"Index": 2, "Name": "Columns", "Type": "TCoAtomList"},
- {"Index": 3, "Name": "Settings", "Type": "TCoNameValueTupleList"}
+ {"Index": 2, "Name": "Columns", "Type": "TCoAtomList"},
+ {"Index": 3, "Name": "Settings", "Type": "TCoNameValueTupleList"}
]
},
{
- "Name": "TKqlReadTable",
- "Base": "TKqlReadTableBase",
- "Match": {"Type": "Callable", "Name": "KqlReadTable"}
- },
- {
+ "Name": "TKqlReadTable",
+ "Base": "TKqlReadTableBase",
+ "Match": {"Type": "Callable", "Name": "KqlReadTable"}
+ },
+ {
"Name": "TKqlReadTableIndex",
"Base": "TKqlReadTableBase",
"Match": {"Type": "Callable", "Name": "KqlReadTableIndex"},
@@ -66,11 +66,11 @@
]
},
{
- "Name": "TKqpReadTable",
- "Base": "TKqlReadTableBase",
- "Match": {"Type": "Callable", "Name": "KqpReadTable"}
- },
- {
+ "Name": "TKqpReadTable",
+ "Base": "TKqlReadTableBase",
+ "Match": {"Type": "Callable", "Name": "KqpReadTable"}
+ },
+ {
"Name": "TKqpWideReadTable",
"Base": "TKqlReadTableBase",
"Match": {"Type": "Callable", "Name": "KqpWideReadTable"}
@@ -160,7 +160,7 @@
{
"Name": "TKqlUpsertRowsBase",
"Base": "TKqlTableEffect",
- "Match": {"Type": "CallableBase"},
+ "Match": {"Type": "CallableBase"},
"Children": [
{"Index": 1, "Name": "Input", "Type": "TExprBase"},
{"Index": 2, "Name": "Columns", "Type": "TCoAtomList"}
@@ -170,8 +170,8 @@
"Name": "TKqlUpsertRows",
"Base": "TKqlUpsertRowsBase",
"Match": {"Type": "Callable", "Name": "KqlUpsertRows"}
- },
- {
+ },
+ {
"Name": "TKqlUpsertRowsIndex",
"Base": "TKqlUpsertRowsBase",
"Match": {"Type": "Callable", "Name": "KqlUpsertRowsIndex"}
@@ -183,27 +183,27 @@
"Children": [
{"Index": 3, "Name": "Settings", "Type": "TCoNameValueTupleList"}
]
- },
- {
+ },
+ {
"Name": "TKqlInsertRowsBase",
- "Base": "TKqlTableEffect",
+ "Base": "TKqlTableEffect",
"Match": {"Type": "CallableBase"},
- "Children": [
- {"Index": 1, "Name": "Input", "Type": "TExprBase"},
- {"Index": 2, "Name": "Columns", "Type": "TCoAtomList"},
- {"Index": 3, "Name": "OnConflict", "Type": "TCoAtom", "Description": "'abort or 'revert"}
- ]
- },
- {
+ "Children": [
+ {"Index": 1, "Name": "Input", "Type": "TExprBase"},
+ {"Index": 2, "Name": "Columns", "Type": "TCoAtomList"},
+ {"Index": 3, "Name": "OnConflict", "Type": "TCoAtom", "Description": "'abort or 'revert"}
+ ]
+ },
+ {
"Name": "TKqlUpdateRowsBase",
- "Base": "TKqlTableEffect",
+ "Base": "TKqlTableEffect",
"Match": {"Type": "CallableBase"},
- "Children": [
- {"Index": 1, "Name": "Input", "Type": "TExprBase"},
- {"Index": 2, "Name": "Columns", "Type": "TCoAtomList"}
- ]
- },
- {
+ "Children": [
+ {"Index": 1, "Name": "Input", "Type": "TExprBase"},
+ {"Index": 2, "Name": "Columns", "Type": "TCoAtomList"}
+ ]
+ },
+ {
"Name": "TKqlUpdateRows",
"Base": "TKqlUpdateRowsBase",
"Match": {"Type": "Callable", "Name": "TKqlUpdateRows"}
@@ -260,10 +260,10 @@
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "KqpPhysicalTx"},
"Children": [
- {"Index": 0, "Name": "Stages", "Type": "TDqPhyStageList"},
+ {"Index": 0, "Name": "Stages", "Type": "TDqPhyStageList"},
{"Index": 1, "Name": "Results", "Type": "TExprList"},
{"Index": 2, "Name": "ParamBindings", "Type": "TKqpParamBindingList"},
- {"Index": 3, "Name": "Settings", "Type": "TCoNameValueTupleList"}
+ {"Index": 3, "Name": "Settings", "Type": "TCoNameValueTupleList"}
]
},
{
@@ -276,33 +276,33 @@
"Match": {"Type": "Callable", "Name": "KqpPhysicalQuery"},
"Children": [
{"Index": 0, "Name": "Transactions", "Type": "TKqpPhysicalTxList"},
- {"Index": 1, "Name": "Results", "Type": "TExprList"},
- {"Index": 2, "Name": "Settings", "Type": "TCoNameValueTupleList"}
+ {"Index": 1, "Name": "Results", "Type": "TExprList"},
+ {"Index": 2, "Name": "Settings", "Type": "TCoNameValueTupleList"}
]
},
{
- "Name": "TKqlQueryResult",
- "Base": "TExprBase",
- "Match": {"Type": "Tuple"},
- "Children": [
- {"Index": 0, "Name": "Value", "Type": "TExprBase"},
- {"Index": 1, "Name": "ColumnHints", "Type": "TCoAtomList"}
- ]
- },
- {
- "Name": "TKqlQueryResultList",
- "ListBase": "TKqlQueryResult"
- },
- {
+ "Name": "TKqlQueryResult",
+ "Base": "TExprBase",
+ "Match": {"Type": "Tuple"},
+ "Children": [
+ {"Index": 0, "Name": "Value", "Type": "TExprBase"},
+ {"Index": 1, "Name": "ColumnHints", "Type": "TCoAtomList"}
+ ]
+ },
+ {
+ "Name": "TKqlQueryResultList",
+ "ListBase": "TKqlQueryResult"
+ },
+ {
"Name": "TKqlQuery",
"Base": "TExprBase",
"Match": {"Type": "Tuple"},
"Children": [
- {"Index": 0, "Name": "Results", "Type": "TKqlQueryResultList"},
+ {"Index": 0, "Name": "Results", "Type": "TKqlQueryResultList"},
{"Index": 1, "Name": "Effects", "Type": "TExprList"}
]
- },
- {
+ },
+ {
"Name": "TKqpEffects",
"VarArgBase": "TExprBase",
"Match": {"Type": "Callable", "Name": "KqpEffects"}
@@ -325,15 +325,15 @@
"Name": "TKqpCnShuffleShard",
"Base": "TDqConnection",
"Match": {"Type": "Callable", "Name": "KqpCnShuffleShard"}
- },
- {
- "Name": "TKqpProgram",
- "Base": "TCallable",
- "Match": {"Type": "Callable", "Name": "KqpProgram"},
- "Children": [
- {"Index": 0, "Name": "Lambda", "Type": "TCoLambda"},
- {"Index": 1, "Name": "ArgsType", "Type": "TExprBase"}
- ]
+ },
+ {
+ "Name": "TKqpProgram",
+ "Base": "TCallable",
+ "Match": {"Type": "Callable", "Name": "KqpProgram"},
+ "Children": [
+ {"Index": 0, "Name": "Lambda", "Type": "TCoLambda"},
+ {"Index": 1, "Name": "ArgsType", "Type": "TExprBase"}
+ ]
},
{
"Name": "TKqlDeleteRowsBase",
@@ -415,17 +415,17 @@
"Children": [
{"Index": 1, "Name": "Column", "Type": "TExprBase"}
]
- },
- {
- "Name": "TKqpEnsure",
- "Base": "TCallable",
- "Match": {"Type": "Callable", "Name": "KqpEnsure"},
- "Children": [
- {"Index": 0, "Name": "Value", "Type": "TExprBase"},
- {"Index": 1, "Name": "Predicate", "Type": "TExprBase"},
- {"Index": 2, "Name": "IssueCode", "Type": "TCoAtom"},
- {"Index": 3, "Name": "Message", "Type": "TExprBase"}
- ]
+ },
+ {
+ "Name": "TKqpEnsure",
+ "Base": "TCallable",
+ "Match": {"Type": "Callable", "Name": "KqpEnsure"},
+ "Children": [
+ {"Index": 0, "Name": "Value", "Type": "TExprBase"},
+ {"Index": 1, "Name": "Predicate", "Type": "TExprBase"},
+ {"Index": 2, "Name": "IssueCode", "Type": "TCoAtom"},
+ {"Index": 3, "Name": "Message", "Type": "TExprBase"}
+ ]
}
]
}
diff --git a/ydb/core/kqp/host/kqp_host.cpp b/ydb/core/kqp/host/kqp_host.cpp
index 1046f6097a..44af6d1168 100644
--- a/ydb/core/kqp/host/kqp_host.cpp
+++ b/ydb/core/kqp/host/kqp_host.cpp
@@ -126,7 +126,7 @@ public:
using TResult = IKqpHost::TQueryResult;
TAsyncValidateYqlResult(TExprNode* queryRoot, TIntrusivePtr<TKikimrSessionContext> sessionCtx,
- TExprContext& exprCtx, TAutoPtr<IGraphTransformer> transformer, TMaybe<TSqlVersion> sqlVersion)
+ TExprContext& exprCtx, TAutoPtr<IGraphTransformer> transformer, TMaybe<TSqlVersion> sqlVersion)
: TKqpAsyncResultBase(queryRoot, exprCtx, *transformer.Get())
, SessionCtx(sessionCtx)
, Transformer(transformer)
@@ -134,7 +134,7 @@ public:
void FillResult(TResult& validateResult) const override {
YQL_ENSURE(SessionCtx->Query().PrepareOnly);
- validateResult.PreparedQuery.reset(SessionCtx->Query().PreparingQuery.release());
+ validateResult.PreparedQuery.reset(SessionCtx->Query().PreparingQuery.release());
validateResult.SqlVersion = SqlVersion;
}
@@ -153,7 +153,7 @@ public:
TAsyncExplainYqlResult(TExprNode* queryRoot, TIntrusivePtr<TKikimrSessionContext> sessionCtx,
TExprContext& exprCtx, TAutoPtr<IGraphTransformer> transformer,
- IPlanBuilder& planBuilder, TMaybe<TSqlVersion> sqlVersion, bool useDqExplain = false)
+ IPlanBuilder& planBuilder, TMaybe<TSqlVersion> sqlVersion, bool useDqExplain = false)
: TKqpAsyncResultBase(queryRoot, exprCtx, *transformer.Get())
, SessionCtx(sessionCtx)
, Transformer(transformer)
@@ -195,7 +195,7 @@ public:
TAsyncExecuteYqlResult(TExprNode* queryRoot, TExprContext& exprCtx, IGraphTransformer& transformer,
const TString& cluster, TIntrusivePtr<TKikimrSessionContext> sessionCtx,
const TResultProviderConfig& resultProviderConfig, IPlanBuilder& planBuilder,
- TMaybe<TSqlVersion> sqlVersion)
+ TMaybe<TSqlVersion> sqlVersion)
: TKqpAsyncExecuteResultBase(queryRoot, exprCtx, transformer, sessionCtx->TxPtr())
, Cluster(cluster)
, SessionCtx(sessionCtx)
@@ -246,8 +246,8 @@ public:
TAsyncExecutePreparedResult(TExprNode* queryRoot, TExprContext& exprCtx, IGraphTransformer& transformer,
TIntrusivePtr<TKikimrSessionContext> sessionCtx, const IDataProvider::TFillSettings& fillSettings,
- TIntrusivePtr<TExecuteContext> executeCtx, std::shared_ptr<const NKikimrKqp::TPreparedQuery> query,
- TMaybe<TSqlVersion> sqlVersion)
+ TIntrusivePtr<TExecuteContext> executeCtx, std::shared_ptr<const NKikimrKqp::TPreparedQuery> query,
+ TMaybe<TSqlVersion> sqlVersion)
: TKqpAsyncExecuteResultBase(queryRoot, exprCtx, transformer, sessionCtx->TxPtr())
, SessionCtx(sessionCtx)
, FillSettings(fillSettings)
@@ -256,7 +256,7 @@ public:
, SqlVersion(sqlVersion) {}
void FillResult(TResult& queryResult) const override {
- for (const auto& result : SessionCtx->Query().PreparedQuery->GetResults()) {
+ for (const auto& result : SessionCtx->Query().PreparedQuery->GetResults()) {
YQL_ENSURE(result.GetKqlIndex() < ExecuteCtx->QueryResults.size());
const auto& execResult = ExecuteCtx->QueryResults[result.GetKqlIndex()];
@@ -274,13 +274,13 @@ public:
queryResult.Results.push_back(protoResult);
}
- if (!SessionCtx->Query().PrepareOnly && SessionCtx->Query().PreparedQuery->KqlsSize() == 1) {
- FillAstAndPlan(queryResult, *SessionCtx->Query().PreparedQuery);
+ if (!SessionCtx->Query().PrepareOnly && SessionCtx->Query().PreparedQuery->KqlsSize() == 1) {
+ FillAstAndPlan(queryResult, *SessionCtx->Query().PreparedQuery);
}
- queryResult.QueryAst = SessionCtx->Query().PreparedQuery->GetPhysicalQuery().GetQueryAst();
+ queryResult.QueryAst = SessionCtx->Query().PreparedQuery->GetPhysicalQuery().GetQueryAst();
if (Query) {
- queryResult.PreparedQuery = Query;
+ queryResult.PreparedQuery = Query;
}
/*
@@ -300,32 +300,32 @@ private:
TIntrusivePtr<TKikimrSessionContext> SessionCtx;
IDataProvider::TFillSettings FillSettings;
TIntrusivePtr<TExecuteContext> ExecuteCtx;
- mutable std::shared_ptr<const NKikimrKqp::TPreparedQuery> Query;
+ mutable std::shared_ptr<const NKikimrKqp::TPreparedQuery> Query;
TMaybe<TSqlVersion> SqlVersion;
};
/*
- * Prepare ScanQuery/DataQuery by AST (when called through scripting).
+ * Prepare ScanQuery/DataQuery by AST (when called through scripting).
*/
class TAsyncExecuteKqlResult : public TKqpAsyncExecuteResultBase<IKqpHost::TQueryResult> {
public:
using TResult = IKqpHost::TQueryResult;
TAsyncExecuteKqlResult(TExprNode* queryRoot, TExprContext& exprCtx, IGraphTransformer& transformer,
- TIntrusivePtr<TKikimrSessionContext> sessionCtx, TExecuteContext& executeCtx)
+ TIntrusivePtr<TKikimrSessionContext> sessionCtx, TExecuteContext& executeCtx)
: TKqpAsyncExecuteResultBase(queryRoot, exprCtx, transformer, sessionCtx->TxPtr())
, SessionCtx(sessionCtx)
, ExecuteCtx(executeCtx) {}
void FillResult(TResult& queryResult) const override {
YQL_ENSURE(ExecuteCtx.QueryResults.size() == 1);
- queryResult = std::move(ExecuteCtx.QueryResults[0]);
- queryResult.QueryPlan = SerializeExplainPlan(queryResult.PreparingQuery->GetPhysicalQuery());
+ queryResult = std::move(ExecuteCtx.QueryResults[0]);
+ queryResult.QueryPlan = SerializeExplainPlan(queryResult.PreparingQuery->GetPhysicalQuery());
}
private:
TIntrusivePtr<TKikimrSessionContext> SessionCtx;
- TExecuteContext& ExecuteCtx;
+ TExecuteContext& ExecuteCtx;
};
/*
@@ -336,7 +336,7 @@ public:
using TResult = IKqpHost::TQueryResult;
TAsyncPrepareYqlResult(TExprNode* queryRoot, TExprContext& exprCtx, IGraphTransformer& transformer,
- TIntrusivePtr<TKikimrQueryContext> queryCtx, const TString& queryText, TMaybe<TSqlVersion> sqlVersion)
+ TIntrusivePtr<TKikimrQueryContext> queryCtx, const TString& queryText, TMaybe<TSqlVersion> sqlVersion)
: TKqpAsyncResultBase(queryRoot, exprCtx, transformer)
, QueryCtx(queryCtx)
, QueryText(queryText)
@@ -344,20 +344,20 @@ public:
void FillResult(TResult& prepareResult) const override {
YQL_ENSURE(QueryCtx->PrepareOnly);
- YQL_ENSURE(QueryCtx->PreparingQuery);
-
- prepareResult.PreparingQuery = std::move(QueryCtx->PreparingQuery);
- prepareResult.PreparingQuery->SetText(std::move(QueryText));
+ YQL_ENSURE(QueryCtx->PreparingQuery);
+
+ prepareResult.PreparingQuery = std::move(QueryCtx->PreparingQuery);
+ prepareResult.PreparingQuery->SetText(std::move(QueryText));
prepareResult.SqlVersion = SqlVersion;
- if (prepareResult.PreparingQuery->GetVersion() == NKikimrKqp::TPreparedQuery::VERSION_PHYSICAL_V1) {
- prepareResult.QueryPlan = SerializeExplainPlan(prepareResult.PreparingQuery->GetPhysicalQuery());
- prepareResult.QueryAst = prepareResult.PreparingQuery->GetPhysicalQuery().GetQueryAst();
+ if (prepareResult.PreparingQuery->GetVersion() == NKikimrKqp::TPreparedQuery::VERSION_PHYSICAL_V1) {
+ prepareResult.QueryPlan = SerializeExplainPlan(prepareResult.PreparingQuery->GetPhysicalQuery());
+ prepareResult.QueryAst = prepareResult.PreparingQuery->GetPhysicalQuery().GetQueryAst();
} else {
- FillAstAndPlan(prepareResult, *prepareResult.PreparingQuery);
+ FillAstAndPlan(prepareResult, *prepareResult.PreparingQuery);
}
-
- prepareResult.QueryTraits = QueryCtx->QueryTraits;
+
+ prepareResult.QueryTraits = QueryCtx->QueryTraits;
}
private:
@@ -366,20 +366,20 @@ private:
TMaybe<TSqlVersion> SqlVersion;
};
-class TFailExpressionEvaluation : public TSyncTransformerBase {
-public:
- TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) override {
- output = input;
-
- auto evaluateNode = FindNode(input, [](const TExprNode::TPtr& node) {
+class TFailExpressionEvaluation : public TSyncTransformerBase {
+public:
+ TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) override {
+ output = input;
+
+ auto evaluateNode = FindNode(input, [](const TExprNode::TPtr& node) {
return node->IsCallable({"EvaluateIf!", "EvaluateFor!", "EvaluateAtom"});
- });
-
+ });
+
if (!evaluateNode)
return TStatus::Ok;
-
+
TStringBuilder builder;
-
+
if (evaluateNode->Content() == "EvaluateAtom"sv)
builder << "ATOM evaluation";
else if (evaluateNode->Content() == "EvaluateIf!"sv)
@@ -398,13 +398,13 @@ public:
);
return TStatus::Error;
- }
-};
-
+ }
+};
+
class TPrepareDataQueryAstTransformer : public TGraphTransformerBase {
public:
TPrepareDataQueryAstTransformer(const TString& cluster, const TIntrusivePtr<TExecuteContext>& executeCtx,
- const TIntrusivePtr<TKikimrQueryContext>& queryCtx, const TIntrusivePtr<IKqpRunner>& kqpRunner)
+ const TIntrusivePtr<TKikimrQueryContext>& queryCtx, const TIntrusivePtr<IKqpRunner>& kqpRunner)
: Cluster(cluster)
, ExecuteCtx(executeCtx)
, QueryCtx(queryCtx)
@@ -456,11 +456,11 @@ public:
return TStatus::Error;
}
- IKqpHost::TQueryResult prepareResult;
- prepareResult.SetSuccess();
- prepareResult.PreparingQuery = std::move(QueryCtx->PreparingQuery);
+ IKqpHost::TQueryResult prepareResult;
+ prepareResult.SetSuccess();
+ prepareResult.PreparingQuery = std::move(QueryCtx->PreparingQuery);
- ExecuteCtx->QueryResults.emplace_back(std::move(prepareResult));
+ ExecuteCtx->QueryResults.emplace_back(std::move(prepareResult));
return TStatus::Ok;
}
@@ -521,13 +521,13 @@ NKikimrMiniKQL::TParams* ValidateParameter(const TString& name, const TTypeAnnot
return parameter;
}
-class TCollectParametersTransformer {
+class TCollectParametersTransformer {
public:
- TCollectParametersTransformer(TIntrusivePtr<TKikimrQueryContext> queryCtx)
+ TCollectParametersTransformer(TIntrusivePtr<TKikimrQueryContext> queryCtx)
: QueryCtx(queryCtx) {}
IGraphTransformer::TStatus operator()(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
- if (QueryCtx->PrepareOnly && QueryCtx->PreparingQuery->ParametersSize() > 0) {
+ if (QueryCtx->PrepareOnly && QueryCtx->PreparingQuery->ParametersSize() > 0) {
return IGraphTransformer::TStatus::Ok;
}
@@ -546,7 +546,7 @@ public:
auto expectedType = parameter.Ref().GetTypeAnn();
if (queryCtx->PrepareOnly) {
- auto& paramDesc = *queryCtx->PreparingQuery->AddParameters();
+ auto& paramDesc = *queryCtx->PreparingQuery->AddParameters();
paramDesc.SetName(TString(name));
if (!ExportTypeToKikimrProto(*expectedType, *paramDesc.MutableType(), ctx)) {
ctx.AddError(TIssue(ctx.GetPosition(parameter.Pos()), TStringBuilder()
@@ -563,8 +563,8 @@ public:
return nullptr;
}
- if (queryCtx->Type == EKikimrQueryType::YqlScript ||
- queryCtx->Type == EKikimrQueryType::YqlScriptStreaming)
+ if (queryCtx->Type == EKikimrQueryType::YqlScript ||
+ queryCtx->Type == EKikimrQueryType::YqlScriptStreaming)
{
return ret;
}
@@ -594,7 +594,7 @@ public:
}
static TAutoPtr<IGraphTransformer> Sync(TIntrusivePtr<TKikimrQueryContext> queryCtx) {
- return CreateFunctorTransformer(TCollectParametersTransformer(queryCtx));
+ return CreateFunctorTransformer(TCollectParametersTransformer(queryCtx));
}
private:
@@ -608,11 +608,11 @@ public:
IGraphTransformer::TStatus operator()(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
YQL_ENSURE(!QueryCtx->PrepareOnly);
- YQL_ENSURE(QueryCtx->PreparedQuery);
+ YQL_ENSURE(QueryCtx->PreparedQuery);
YQL_ENSURE(input->Type() == TExprNode::World);
output = input;
- for (const auto& paramDesc : QueryCtx->PreparedQuery->GetParameters()) {
+ for (const auto& paramDesc : QueryCtx->PreparedQuery->GetParameters()) {
TIssueScopeGuard issueScope(ctx.IssueManager, [input, &paramDesc, &ctx]() {
return MakeIntrusive<TIssue>(YqlIssue(ctx.GetPosition(input->Pos()), TIssuesIds::KIKIMR_BAD_REQUEST, TStringBuilder()
<< "Failed to parse parameter type: " << paramDesc.GetName()));
@@ -651,20 +651,20 @@ public:
TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final {
YQL_ENSURE(!QueryCtx->PrepareOnly);
- YQL_ENSURE(QueryCtx->PreparedQuery);
+ YQL_ENSURE(QueryCtx->PreparedQuery);
YQL_ENSURE(input->Type() == TExprNode::World);
output = input;
if (!AsyncResult) {
- const auto& query = *QueryCtx->PreparedQuery;
- YQL_ENSURE(QueryCtx->Type != EKikimrQueryType::Unspecified);
+ const auto& query = *QueryCtx->PreparedQuery;
+ YQL_ENSURE(QueryCtx->Type != EKikimrQueryType::Unspecified);
if (query.GetVersion() == NKikimrKqp::TPreparedQuery::VERSION_PHYSICAL_V1) {
if (CurrentKqlIndex) {
return TStatus::Ok;
}
- const auto& phyQuery = query.GetPhysicalQuery();
+ const auto& phyQuery = query.GetPhysicalQuery();
if (QueryCtx->Type == EKikimrQueryType::Scan) {
AsyncResult = KqpRunner->ExecutePreparedScanQuery(Cluster, input.Get(), phyQuery, ctx,
@@ -679,11 +679,11 @@ public:
return TStatus::Ok;
}
- const auto& kql = query.GetKqls(CurrentKqlIndex);
+ const auto& kql = query.GetKqls(CurrentKqlIndex);
- YQL_ENSURE(QueryCtx->Type == EKikimrQueryType::Dml);
- YQL_ENSURE(!kql.GetSettings().GetNewEngine());
- AsyncResult = KqpRunner->ExecutePreparedDataQuery(Cluster, input.Get(), kql, ctx, ExecuteCtx->Settings);
+ YQL_ENSURE(QueryCtx->Type == EKikimrQueryType::Dml);
+ YQL_ENSURE(!kql.GetSettings().GetNewEngine());
+ AsyncResult = KqpRunner->ExecutePreparedDataQuery(Cluster, input.Get(), kql, ctx, ExecuteCtx->Settings);
}
}
@@ -703,19 +703,19 @@ public:
return Promise.GetFuture();
}
- TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext&) final {
+ TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext&) final {
output = input;
if (!AsyncResult->HasResult()) {
return TStatus::Repeat;
}
- auto queryResult = std::move(AsyncResult->GetResult());
+ auto queryResult = std::move(AsyncResult->GetResult());
if (!queryResult.Success()) {
return TStatus::Error;
}
- ExecuteCtx->QueryResults.emplace_back(std::move(queryResult));
+ ExecuteCtx->QueryResults.emplace_back(std::move(queryResult));
++CurrentKqlIndex;
AsyncResult.Reset();
@@ -796,8 +796,8 @@ public:
, SessionCtx(sessionCtx)
, KqpRunner(kqpRunner) {}
- TIntrusivePtr<TAsyncQueryResult> ExecuteKql(const TString&, const TExprNode::TPtr&, TExprContext&,
- const TExecuteSettings&) override
+ TIntrusivePtr<TAsyncQueryResult> ExecuteKql(const TString&, const TExprNode::TPtr&, TExprContext&,
+ const TExecuteSettings&) override
{
YQL_ENSURE(false, "Unexpected ExecuteKql call.");
return nullptr;
@@ -806,14 +806,14 @@ public:
TIntrusivePtr<TAsyncQueryResult> ExecuteDataQuery(const TString& cluster, const TExprNode::TPtr& query,
TExprContext& ctx, const TExecuteSettings& settings) override
{
- auto queryType = SessionCtx->Query().Type;
+ auto queryType = SessionCtx->Query().Type;
+
+ YQL_ENSURE(!settings.UseScanQuery ||
+ queryType == EKikimrQueryType::YqlScript ||
+ queryType == EKikimrQueryType::YqlScriptStreaming);
- YQL_ENSURE(!settings.UseScanQuery ||
- queryType == EKikimrQueryType::YqlScript ||
- queryType == EKikimrQueryType::YqlScriptStreaming);
-
if (SessionCtx->Query().PrepareOnly) {
- switch (queryType) {
+ switch (queryType) {
case EKikimrQueryType::Dml:
return KqpRunner->PrepareDataQuery(cluster, query, ctx, settings);
case EKikimrQueryType::Scan:
@@ -822,12 +822,12 @@ public:
case EKikimrQueryType::YqlScriptStreaming:
break;
default:
- YQL_ENSURE(false, "Unexpected query type for prepare action: " << queryType);
+ YQL_ENSURE(false, "Unexpected query type for prepare action: " << queryType);
return nullptr;
}
}
- switch (queryType) {
+ switch (queryType) {
case EKikimrQueryType::YqlScript:
case EKikimrQueryType::YqlScriptStreaming:
{
@@ -842,7 +842,7 @@ public:
querySettings.StatsMode = GetStatsMode(settings.StatsMode);
TFuture<TQueryResult> future;
- switch (queryType) {
+ switch (queryType) {
case EKikimrQueryType::YqlScript:
if (useScanQuery) {
ui64 rowsLimit = 0;
@@ -881,7 +881,7 @@ public:
break;
default:
- YQL_ENSURE(false, "Unexpected query type for execute action: " << queryType);
+ YQL_ENSURE(false, "Unexpected query type for execute action: " << queryType);
return nullptr;
}
@@ -889,14 +889,14 @@ public:
}
default:
- YQL_ENSURE(false, "Unexpected query type for execute script action: " << queryType);
+ YQL_ENSURE(false, "Unexpected query type for execute script action: " << queryType);
return nullptr;
}
}
- TIntrusivePtr<TAsyncQueryResult> ExplainDataQuery(const TString&, const TExprNode::TPtr&, TExprContext&) override {
- YQL_ENSURE(false, "Not implemented.");
- return nullptr;
+ TIntrusivePtr<TAsyncQueryResult> ExplainDataQuery(const TString&, const TExprNode::TPtr&, TExprContext&) override {
+ YQL_ENSURE(false, "Not implemented.");
+ return nullptr;
}
private:
@@ -1048,8 +1048,8 @@ public:
TypesCtx->AddDataSource(KikimrProviderName, kikimrDataSource);
TypesCtx->AddDataSink(KikimrProviderName, kikimrDataSink);
TypesCtx->UdfResolver = CreateSimpleUdfResolver(FuncRegistry);
- TypesCtx->TimeProvider = TAppData::TimeProvider;
- TypesCtx->RandomProvider = TAppData::RandomProvider;
+ TypesCtx->TimeProvider = TAppData::TimeProvider;
+ TypesCtx->RandomProvider = TAppData::RandomProvider;
TypesCtx->Modules = ModuleResolver;
TypesCtx->UserDataStorage = MakeIntrusive<TUserDataStorage>(nullptr, TUserDataTable(), nullptr, nullptr);
TypesCtx->JsonQueryReturnsJsonDocument = true;
@@ -1074,14 +1074,14 @@ public:
YqlTransformer = TTransformationPipeline(TypesCtx)
.AddServiceTransformers()
- .Add(TLogExprTransformer::Sync("YqlTransformer", NYql::NLog::EComponent::ProviderKqp,
- NYql::NLog::ELevel::TRACE), "LogYqlTransform")
+ .Add(TLogExprTransformer::Sync("YqlTransformer", NYql::NLog::EComponent::ProviderKqp,
+ NYql::NLog::ELevel::TRACE), "LogYqlTransform")
.AddPreTypeAnnotation()
- // TODO: .AddExpressionEvaluation(*FuncRegistry)
- .Add(new TFailExpressionEvaluation(), "FailExpressionEvaluation")
+ // TODO: .AddExpressionEvaluation(*FuncRegistry)
+ .Add(new TFailExpressionEvaluation(), "FailExpressionEvaluation")
.AddIOAnnotation()
.AddTypeAnnotation()
- .Add(TCollectParametersTransformer::Sync(SessionCtx->QueryPtr()), "CollectParameters")
+ .Add(TCollectParametersTransformer::Sync(SessionCtx->QueryPtr()), "CollectParameters")
.AddPostTypeAnnotation()
.AddOptimization(true, false)
.Add(TLogExprTransformer::Sync("Optimized expr"), "LogExpr")
@@ -1128,7 +1128,7 @@ public:
}
}
- TGenericResult DeleteTransaction(const TString& txId) override {
+ TGenericResult DeleteTransaction(const TString& txId) override {
try {
return DeleteTransactionInternal(txId);
}
@@ -1137,21 +1137,21 @@ public:
}
}
- IAsyncQueryResultPtr RollbackTransaction(const TString& txId,
- const IKikimrQueryExecutor::TExecuteSettings& settings) override
- {
+ IAsyncQueryResultPtr RollbackTransaction(const TString& txId,
+ const IKikimrQueryExecutor::TExecuteSettings& settings) override
+ {
return CheckedProcessQuery(*ExprCtx,
- [this, txId, settings] (TExprContext& ctx) {
- return RollbackTransactionInternal(txId, settings, ctx);
+ [this, txId, settings] (TExprContext& ctx) {
+ return RollbackTransactionInternal(txId, settings, ctx);
});
}
- TQueryResult SyncRollbackTransaction(const TString& txId,
- const IKikimrQueryExecutor::TExecuteSettings& settings) override
- {
+ TQueryResult SyncRollbackTransaction(const TString& txId,
+ const IKikimrQueryExecutor::TExecuteSettings& settings) override
+ {
return CheckedSyncProcessQuery(
- [this, txId, settings] () {
- return RollbackTransaction(txId, settings);
+ [this, txId, settings] () {
+ return RollbackTransaction(txId, settings);
});
}
@@ -1177,18 +1177,18 @@ public:
return ImplicitTransaction->GetInfo();
}
- void ForceTxOldEngine(const TString& txId) override {
- if (auto tx = FindTransaction(txId)) {
- (*tx)->ForceOldEngine();
- }
- }
-
- void ForceTxNewEngine(const TString& txId, ui32 percent, ui32 level) override {
- if (auto tx = FindTransaction(txId)) {
- (*tx)->ForceNewEngine(percent, level);
- }
- }
-
+ void ForceTxOldEngine(const TString& txId) override {
+ if (auto tx = FindTransaction(txId)) {
+ (*tx)->ForceOldEngine();
+ }
+ }
+
+ void ForceTxNewEngine(const TString& txId, ui32 percent, ui32 level) override {
+ if (auto tx = FindTransaction(txId)) {
+ (*tx)->ForceNewEngine(percent, level);
+ }
+ }
+
TMaybe<TKqpTransactionInfo> GetTransactionInfo(const TString& txId) override {
auto tx = FindTransaction(txId);
if (!tx) {
@@ -1229,17 +1229,17 @@ public:
return abortedCount;
}
- IAsyncQueryResultPtr RollbackAborted() override {
+ IAsyncQueryResultPtr RollbackAborted() override {
return CheckedProcessQuery(*ExprCtx,
- [this] (TExprContext& ctx) {
- return RollbackAbortedInternal(ctx);
+ [this] (TExprContext& ctx) {
+ return RollbackAbortedInternal(ctx);
});
}
- TQueryResult SyncRollbackAborted() override {
+ TQueryResult SyncRollbackAborted() override {
return CheckedSyncProcessQuery(
- [this] () {
- return RollbackAborted();
+ [this] () {
+ return RollbackAborted();
});
}
@@ -1266,19 +1266,19 @@ public:
IAsyncQueryResultPtr PrepareDataQuery(const TString& query, const TPrepareSettings& settings) override {
return CheckedProcessQuery(*ExprCtx,
- [this, &query, settings] (TExprContext& ctx) mutable {
+ [this, &query, settings] (TExprContext& ctx) mutable {
return PrepareDataQueryInternal(query, settings, ctx);
});
}
TQueryResult SyncPrepareDataQuery(const TString& query, const TPrepareSettings& settings) override {
return CheckedSyncProcessQuery(
- [this, &query, settings] () mutable {
+ [this, &query, settings] () mutable {
return PrepareDataQuery(query, settings);
});
}
- IAsyncQueryResultPtr ExecuteDataQuery(const TString& txId, std::shared_ptr<const NKikimrKqp::TPreparedQuery>& query,
+ IAsyncQueryResultPtr ExecuteDataQuery(const TString& txId, std::shared_ptr<const NKikimrKqp::TPreparedQuery>& query,
NKikimrMiniKQL::TParams&& parameters, const IKikimrQueryExecutor::TExecuteSettings& settings) override
{
return CheckedProcessQuery(*ExprCtx,
@@ -1287,7 +1287,7 @@ public:
});
}
- TQueryResult SyncExecuteDataQuery(const TString& txId, std::shared_ptr<const NKikimrKqp::TPreparedQuery>& query,
+ TQueryResult SyncExecuteDataQuery(const TString& txId, std::shared_ptr<const NKikimrKqp::TPreparedQuery>& query,
NKikimrMiniKQL::TParams&& parameters, const IKikimrQueryExecutor::TExecuteSettings& settings) override
{
return CheckedSyncProcessQuery(
@@ -1560,7 +1560,7 @@ private:
}
}
- auto txCtx = MakeIntrusive<TKqpTransactionContext>(/* implicit */ false);
+ auto txCtx = MakeIntrusive<TKqpTransactionContext>(/* implicit */ false);
txCtx->EffectiveIsolationLevel = isolationLevel;
txCtx->Readonly = readonly;
@@ -1610,95 +1610,95 @@ private:
*ResultProviderConfig, *PlanBuilder, sqlVersion);
}
- IAsyncQueryResultPtr RollbackTransactionInternal(const TString& txId,
- const IKikimrQueryExecutor::TExecuteSettings& settings, TExprContext& ctx)
- {
- auto maybeTx = FindTransaction(txId);
- if (!maybeTx) {
- return MakeKikimrResultHolder(ResultFromError<TQueryResult>(
- YqlIssue(TPosition(), TIssuesIds::KIKIMR_TRANSACTION_NOT_FOUND, TStringBuilder()
- << "Transaction not found: " << txId)));
- }
-
- YQL_ENSURE(settings.RollbackTx);
-
- auto query = std::make_unique<NKikimrKqp::TPreparedQuery>();
- auto engine = maybeTx->Get()->DeferredEffects.GetEngine();
- if (engine.has_value() && *engine == TKqpTransactionInfo::EEngine::NewEngine) {
- YQL_ENSURE(settings.UseNewEngine.Defined() && *settings.UseNewEngine);
- query->SetVersion(NKikimrKqp::TPreparedQuery::VERSION_PHYSICAL_V1);
- query->MutablePhysicalQuery()->SetType(NKqpProto::TKqpPhyQuery::TYPE_DATA);
- } else {
- YQL_ENSURE(!settings.UseNewEngine.Defined() || !*settings.UseNewEngine);
- query->SetVersion(NKikimrKqp::TPreparedQuery::VERSION_V1);
- query->AddKqls();
- }
-
- YQL_CLOG(INFO, ProviderKqp) << "Rollback tx: " << txId << ", query: " << query->ShortUtf8DebugString();
-
- std::shared_ptr<const NKikimrKqp::TPreparedQuery> q(query.release());
- return ExecuteDataQueryInternal(txId, q, NKikimrMiniKQL::TParams(), settings, {}, {}, ctx);
- }
-
- IAsyncQueryResultPtr RollbackTransactionInternal(TIntrusivePtr<TKqpTransactionContext> tx,
- const IKikimrQueryExecutor::TExecuteSettings& settings, TExprContext& ctx)
- {
- YQL_ENSURE(settings.RollbackTx);
-
- auto query = std::make_unique<NKikimrKqp::TPreparedQuery>();
- auto settings1 = settings;
-
- auto engine = tx->DeferredEffects.GetEngine();
- if (engine.has_value() && *engine == TKqpTransactionInfo::EEngine::NewEngine) {
- YQL_ENSURE(!settings.UseNewEngine.Defined() || *settings.UseNewEngine == true);
- settings1.UseNewEngine = true;
- query->SetVersion(NKikimrKqp::TPreparedQuery::VERSION_PHYSICAL_V1);
- query->MutablePhysicalQuery()->SetType(NKqpProto::TKqpPhyQuery::TYPE_DATA);
- } else {
- YQL_ENSURE(!settings.UseNewEngine.Defined() || *settings.UseNewEngine == false);
- settings1.UseNewEngine = false;
- query->SetVersion(NKikimrKqp::TPreparedQuery::VERSION_V1);
- query->AddKqls();
- }
-
- std::shared_ptr<const NKikimrKqp::TPreparedQuery> q(query.release());
- return ExecuteDataQueryInternal(tx, q, {}, settings1, {}, {}, /* replyPrepared */ false, ctx);
+ IAsyncQueryResultPtr RollbackTransactionInternal(const TString& txId,
+ const IKikimrQueryExecutor::TExecuteSettings& settings, TExprContext& ctx)
+ {
+ auto maybeTx = FindTransaction(txId);
+ if (!maybeTx) {
+ return MakeKikimrResultHolder(ResultFromError<TQueryResult>(
+ YqlIssue(TPosition(), TIssuesIds::KIKIMR_TRANSACTION_NOT_FOUND, TStringBuilder()
+ << "Transaction not found: " << txId)));
+ }
+
+ YQL_ENSURE(settings.RollbackTx);
+
+ auto query = std::make_unique<NKikimrKqp::TPreparedQuery>();
+ auto engine = maybeTx->Get()->DeferredEffects.GetEngine();
+ if (engine.has_value() && *engine == TKqpTransactionInfo::EEngine::NewEngine) {
+ YQL_ENSURE(settings.UseNewEngine.Defined() && *settings.UseNewEngine);
+ query->SetVersion(NKikimrKqp::TPreparedQuery::VERSION_PHYSICAL_V1);
+ query->MutablePhysicalQuery()->SetType(NKqpProto::TKqpPhyQuery::TYPE_DATA);
+ } else {
+ YQL_ENSURE(!settings.UseNewEngine.Defined() || !*settings.UseNewEngine);
+ query->SetVersion(NKikimrKqp::TPreparedQuery::VERSION_V1);
+ query->AddKqls();
+ }
+
+ YQL_CLOG(INFO, ProviderKqp) << "Rollback tx: " << txId << ", query: " << query->ShortUtf8DebugString();
+
+ std::shared_ptr<const NKikimrKqp::TPreparedQuery> q(query.release());
+ return ExecuteDataQueryInternal(txId, q, NKikimrMiniKQL::TParams(), settings, {}, {}, ctx);
+ }
+
+ IAsyncQueryResultPtr RollbackTransactionInternal(TIntrusivePtr<TKqpTransactionContext> tx,
+ const IKikimrQueryExecutor::TExecuteSettings& settings, TExprContext& ctx)
+ {
+ YQL_ENSURE(settings.RollbackTx);
+
+ auto query = std::make_unique<NKikimrKqp::TPreparedQuery>();
+ auto settings1 = settings;
+
+ auto engine = tx->DeferredEffects.GetEngine();
+ if (engine.has_value() && *engine == TKqpTransactionInfo::EEngine::NewEngine) {
+ YQL_ENSURE(!settings.UseNewEngine.Defined() || *settings.UseNewEngine == true);
+ settings1.UseNewEngine = true;
+ query->SetVersion(NKikimrKqp::TPreparedQuery::VERSION_PHYSICAL_V1);
+ query->MutablePhysicalQuery()->SetType(NKqpProto::TKqpPhyQuery::TYPE_DATA);
+ } else {
+ YQL_ENSURE(!settings.UseNewEngine.Defined() || *settings.UseNewEngine == false);
+ settings1.UseNewEngine = false;
+ query->SetVersion(NKikimrKqp::TPreparedQuery::VERSION_V1);
+ query->AddKqls();
+ }
+
+ std::shared_ptr<const NKikimrKqp::TPreparedQuery> q(query.release());
+ return ExecuteDataQueryInternal(tx, q, {}, settings1, {}, {}, /* replyPrepared */ false, ctx);
}
IAsyncQueryResultPtr CommitTransactionInternal(const TString& txId,
const IKikimrQueryExecutor::TExecuteSettings& settings, TExprContext& ctx)
{
- auto maybeTx = FindTransaction(txId);
- if (!maybeTx) {
- return MakeKikimrResultHolder(ResultFromError<TQueryResult>(
- YqlIssue(TPosition(), TIssuesIds::KIKIMR_TRANSACTION_NOT_FOUND, TStringBuilder()
- << "Transaction not found: " << txId)));
- }
+ auto maybeTx = FindTransaction(txId);
+ if (!maybeTx) {
+ return MakeKikimrResultHolder(ResultFromError<TQueryResult>(
+ YqlIssue(TPosition(), TIssuesIds::KIKIMR_TRANSACTION_NOT_FOUND, TStringBuilder()
+ << "Transaction not found: " << txId)));
+ }
YQL_ENSURE(settings.CommitTx);
- auto query = std::make_unique<NKikimrKqp::TPreparedQuery>();
- auto engine = maybeTx->Get()->DeferredEffects.GetEngine();
- if (engine.has_value() && *engine == TKqpTransactionInfo::EEngine::NewEngine) {
- YQL_ENSURE(settings.UseNewEngine.Defined() && *settings.UseNewEngine);
- query->SetVersion(NKikimrKqp::TPreparedQuery::VERSION_PHYSICAL_V1);
- query->MutablePhysicalQuery()->SetType(NKqpProto::TKqpPhyQuery::TYPE_DATA);
- } else {
- YQL_ENSURE(!settings.UseNewEngine.Defined() || !*settings.UseNewEngine);
- query->SetVersion(NKikimrKqp::TPreparedQuery::VERSION_V1);
- query->AddKqls();
- }
-
- std::shared_ptr<const NKikimrKqp::TPreparedQuery> q(query.release());
- return ExecuteDataQueryInternal(txId, q, NKikimrMiniKQL::TParams(), settings, /* issues */ {},
- /* sqlVersion */ {}, ctx);
- }
-
- IAsyncQueryResultPtr RollbackAbortedInternal(TExprContext& ctx) {
+ auto query = std::make_unique<NKikimrKqp::TPreparedQuery>();
+ auto engine = maybeTx->Get()->DeferredEffects.GetEngine();
+ if (engine.has_value() && *engine == TKqpTransactionInfo::EEngine::NewEngine) {
+ YQL_ENSURE(settings.UseNewEngine.Defined() && *settings.UseNewEngine);
+ query->SetVersion(NKikimrKqp::TPreparedQuery::VERSION_PHYSICAL_V1);
+ query->MutablePhysicalQuery()->SetType(NKqpProto::TKqpPhyQuery::TYPE_DATA);
+ } else {
+ YQL_ENSURE(!settings.UseNewEngine.Defined() || !*settings.UseNewEngine);
+ query->SetVersion(NKikimrKqp::TPreparedQuery::VERSION_V1);
+ query->AddKqls();
+ }
+
+ std::shared_ptr<const NKikimrKqp::TPreparedQuery> q(query.release());
+ return ExecuteDataQueryInternal(txId, q, NKikimrMiniKQL::TParams(), settings, /* issues */ {},
+ /* sqlVersion */ {}, ctx);
+ }
+
+ IAsyncQueryResultPtr RollbackAbortedInternal(TExprContext& ctx) {
if (AbortedTransactions.empty()) {
TQueryResult result;
result.SetSuccess();
- return MakeKikimrResultHolder(std::move(result));
+ return MakeKikimrResultHolder(std::move(result));
}
TVector<TIntrusivePtr<TKqpTransactionContext>> txs;
@@ -1709,22 +1709,22 @@ private:
AbortedTransactions.clear();
return MakeIntrusive<TKqpAsyncExecAllResult<TIntrusivePtr<TKqpTransactionContext>, TQueryResult>>(txs,
- [this, &ctx] (const TIntrusivePtr<TKqpTransactionContext>& tx) {
- IKikimrQueryExecutor::TExecuteSettings settings;
- settings.RollbackTx = true;
- settings.Deadlines.TimeoutAt = TInstant::Now() + TDuration::Minutes(1);
-
- auto engine = tx->DeferredEffects.GetEngine();
- settings.UseNewEngine = engine.has_value() && *engine == TKqpTransactionInfo::EEngine::NewEngine;
-
- return RollbackTransactionInternal(tx, settings, ctx);
+ [this, &ctx] (const TIntrusivePtr<TKqpTransactionContext>& tx) {
+ IKikimrQueryExecutor::TExecuteSettings settings;
+ settings.RollbackTx = true;
+ settings.Deadlines.TimeoutAt = TInstant::Now() + TDuration::Minutes(1);
+
+ auto engine = tx->DeferredEffects.GetEngine();
+ settings.UseNewEngine = engine.has_value() && *engine == TKqpTransactionInfo::EEngine::NewEngine;
+
+ return RollbackTransactionInternal(tx, settings, ctx);
});
}
IAsyncQueryResultPtr ExplainDataQueryInternal(const TString& query, bool isSql, TExprContext& ctx) {
- if (isSql) {
+ if (isSql) {
return PrepareDataQueryInternal(query, {}, ctx);
- }
+ }
auto prepareResult = PrepareDataQueryAstInternal(query, {}, ctx);
if (!prepareResult) {
@@ -1732,21 +1732,21 @@ private:
}
return AsyncApplyResult<TQueryResult, TQueryResult>(prepareResult, []
- (TQueryResult&& prepared) -> IAsyncQueryResultPtr {
- if (!prepared.Success()) {
- return MakeKikimrResultHolder(std::move(prepared));
+ (TQueryResult&& prepared) -> IAsyncQueryResultPtr {
+ if (!prepared.Success()) {
+ return MakeKikimrResultHolder(std::move(prepared));
}
TQueryResult explainResult;
explainResult.SetSuccess();
- if (prepared.PreparingQuery->GetVersion() == NKikimrKqp::TPreparedQuery::VERSION_PHYSICAL_V1) {
- explainResult.QueryPlan = std::move(prepared.QueryPlan);
- explainResult.QueryAst = std::move(*prepared.PreparingQuery->MutablePhysicalQuery()->MutableQueryAst());
+ if (prepared.PreparingQuery->GetVersion() == NKikimrKqp::TPreparedQuery::VERSION_PHYSICAL_V1) {
+ explainResult.QueryPlan = std::move(prepared.QueryPlan);
+ explainResult.QueryAst = std::move(*prepared.PreparingQuery->MutablePhysicalQuery()->MutableQueryAst());
} else {
- FillAstAndPlan(explainResult, *prepared.PreparingQuery);
+ FillAstAndPlan(explainResult, *prepared.PreparingQuery);
}
- explainResult.SqlVersion = prepared.SqlVersion;
- return MakeKikimrResultHolder(std::move(explainResult));
+ explainResult.SqlVersion = prepared.SqlVersion;
+ return MakeKikimrResultHolder(std::move(explainResult));
});
}
@@ -1761,21 +1761,21 @@ private:
TExprContext& ctx)
{
// TODO: Use empty tx context
- auto tempTxCtx = MakeIntrusive<TKqpTransactionContext>(/* implicit */ true);
+ auto tempTxCtx = MakeIntrusive<TKqpTransactionContext>(/* implicit */ true);
SetupYqlTransformer(tempTxCtx);
SessionCtx->Query().Type = EKikimrQueryType::Dml;
SessionCtx->Query().PrepareOnly = true;
- SessionCtx->Query().PreparingQuery = std::make_unique<NKikimrKqp::TPreparedQuery>();
- if (settings.UseNewEngine) {
- SessionCtx->Config().UseNewEngine = *settings.UseNewEngine;
- }
+ SessionCtx->Query().PreparingQuery = std::make_unique<NKikimrKqp::TPreparedQuery>();
+ if (settings.UseNewEngine) {
+ SessionCtx->Config().UseNewEngine = *settings.UseNewEngine;
+ }
if (settings.DocumentApiRestricted) {
SessionCtx->Query().DocumentApiRestricted = *settings.DocumentApiRestricted;
}
TMaybe<TSqlVersion> sqlVersion;
- auto queryExpr = CompileYqlQuery(query, /* isSql */ true, /* sqlAutoCommit */ false, ctx, sqlVersion);
+ auto queryExpr = CompileYqlQuery(query, /* isSql */ true, /* sqlAutoCommit */ false, ctx, sqlVersion);
if (!queryExpr) {
return nullptr;
}
@@ -1795,7 +1795,7 @@ private:
SessionCtx->Query().Type = EKikimrQueryType::Dml;
SessionCtx->Query().PrepareOnly = true;
- SessionCtx->Query().PreparingQuery = std::make_unique<NKikimrKqp::TPreparedQuery>();
+ SessionCtx->Query().PreparingQuery = std::make_unique<NKikimrKqp::TPreparedQuery>();
if (settings.DocumentApiRestricted) {
SessionCtx->Query().DocumentApiRestricted = *settings.DocumentApiRestricted;
}
@@ -1818,7 +1818,7 @@ private:
SessionCtx->Query().Type = EKikimrQueryType::Scan;
SessionCtx->Query().PrepareOnly = true;
SessionCtx->Query().StatsMode = statsMode;
- SessionCtx->Query().PreparingQuery = std::make_unique<NKikimrKqp::TPreparedQuery>();
+ SessionCtx->Query().PreparingQuery = std::make_unique<NKikimrKqp::TPreparedQuery>();
TMaybe<TSqlVersion> sqlVersion = 1;
auto queryExpr = CompileYqlQuery(query, true, false, ctx, sqlVersion);
@@ -1836,7 +1836,7 @@ private:
SessionCtx->Query().Type = EKikimrQueryType::Scan;
SessionCtx->Query().PrepareOnly = true;
- SessionCtx->Query().PreparingQuery = std::make_unique<NKikimrKqp::TPreparedQuery>();
+ SessionCtx->Query().PreparingQuery = std::make_unique<NKikimrKqp::TPreparedQuery>();
TMaybe<TSqlVersion> sqlVersion;
auto queryExpr = CompileYqlQuery(queryAst, false, false, ctx, sqlVersion);
@@ -1851,16 +1851,16 @@ private:
}
IAsyncQueryResultPtr ExecuteDataQueryInternal(TIntrusivePtr<TKqpTransactionContext> tx,
- std::shared_ptr<const NKikimrKqp::TPreparedQuery>& preparedQuery, NKikimrMiniKQL::TParams&& parameters,
+ std::shared_ptr<const NKikimrKqp::TPreparedQuery>& preparedQuery, NKikimrMiniKQL::TParams&& parameters,
const IKikimrQueryExecutor::TExecuteSettings& settings, const TIssues& issues,
const TMaybe<TSqlVersion>& sqlVersion, bool replyPrepared, TExprContext& ctx)
{
YQL_ENSURE(!settings.CommitTx || !settings.RollbackTx);
- switch (preparedQuery->GetVersion()) {
+ switch (preparedQuery->GetVersion()) {
case NKikimrKqp::TPreparedQuery::VERSION_V1: {
- YQL_ENSURE(preparedQuery->KqlsSize() == 1);
- auto& kql = preparedQuery->GetKqls(0);
+ YQL_ENSURE(preparedQuery->KqlsSize() == 1);
+ auto& kql = preparedQuery->GetKqls(0);
YQL_ENSURE(!kql.GetSettings().GetCommitTx());
YQL_ENSURE(!kql.GetSettings().GetRollbackTx());
YQL_ENSURE(kql.GetSettings().GetIsolationLevel() == NKikimrKqp::ISOLATION_LEVEL_UNDEFINED);
@@ -1874,7 +1874,7 @@ private:
default:
return MakeKikimrResultHolder(ResultFromError<TQueryResult>(
TIssue(TPosition(), TStringBuilder()
- << "Unexpected query version: " << preparedQuery->GetVersion())));
+ << "Unexpected query version: " << preparedQuery->GetVersion())));
}
SetupExecutePreparedTransformer(settings, tx);
@@ -1891,24 +1891,24 @@ private:
SessionCtx->Query().StatsMode = settings.StatsMode;
SessionCtx->Query().Deadlines = settings.Deadlines;
SessionCtx->Query().Limits = settings.Limits;
- SessionCtx->Query().PreparedQuery = preparedQuery;
+ SessionCtx->Query().PreparedQuery = preparedQuery;
- // moved to kqp_runner.cpp, ExecutePreparedQuery(...)
-// if (preparedQuery->GetVersion() == NKikimrKqp::TPreparedQuery::VERSION_V1) {
-// SessionCtx->Query().PreparedQuery.MutableKqls(0)->MutableSettings()->SetCommitTx(settings.CommitTx);
-// SessionCtx->Query().PreparedQuery.MutableKqls(0)->MutableSettings()->SetRollbackTx(settings.RollbackTx);
-// }
-
- std::shared_ptr<const NKikimrKqp::TPreparedQuery> reply;
- if (replyPrepared) {
- reply = preparedQuery;
+ // moved to kqp_runner.cpp, ExecutePreparedQuery(...)
+// if (preparedQuery->GetVersion() == NKikimrKqp::TPreparedQuery::VERSION_V1) {
+// SessionCtx->Query().PreparedQuery.MutableKqls(0)->MutableSettings()->SetCommitTx(settings.CommitTx);
+// SessionCtx->Query().PreparedQuery.MutableKqls(0)->MutableSettings()->SetRollbackTx(settings.RollbackTx);
+// }
+
+ std::shared_ptr<const NKikimrKqp::TPreparedQuery> reply;
+ if (replyPrepared) {
+ reply = preparedQuery;
}
return MakeIntrusive<TAsyncExecutePreparedResult>(FakeWorld.Get(), ctx, *ExecutePreparedTransformer,
- SessionCtx, FillSettings, ExecuteCtx, reply, sqlVersion);
+ SessionCtx, FillSettings, ExecuteCtx, reply, sqlVersion);
}
- IAsyncQueryResultPtr ExecuteDataQueryInternal(const TString& txId, std::shared_ptr<const NKikimrKqp::TPreparedQuery>& query,
+ IAsyncQueryResultPtr ExecuteDataQueryInternal(const TString& txId, std::shared_ptr<const NKikimrKqp::TPreparedQuery>& query,
NKikimrMiniKQL::TParams&& parameters, const IKikimrQueryExecutor::TExecuteSettings& settings,
const TIssues& issues, const TMaybe<TSqlVersion>& sqlVersion, TExprContext& ctx)
{
@@ -1953,17 +1953,17 @@ private:
auto tx = *maybeTx;
return AsyncApplyResult<TQueryResult, TQueryResult>(prepareResult,
[this, tx, parameters = std::move(parameters), settings, &ctx]
- (TQueryResult&& prepared) mutable -> IAsyncQueryResultPtr {
- if (!prepared.Success()) {
+ (TQueryResult&& prepared) mutable -> IAsyncQueryResultPtr {
+ if (!prepared.Success()) {
tx->Invalidate();
- return MakeKikimrResultHolder(std::move(prepared));
+ return MakeKikimrResultHolder(std::move(prepared));
}
- std::shared_ptr<const NKikimrKqp::TPreparedQuery> query;
- query.reset(prepared.PreparingQuery.release());
-
- return ExecuteDataQueryInternal(tx, query, std::move(parameters), settings, prepared.Issues(),
- prepared.SqlVersion, /* replyPrepared */ true, ctx);
+ std::shared_ptr<const NKikimrKqp::TPreparedQuery> query;
+ query.reset(prepared.PreparingQuery.release());
+
+ return ExecuteDataQueryInternal(tx, query, std::move(parameters), settings, prepared.Issues(),
+ prepared.SqlVersion, /* replyPrepared */ true, ctx);
});
}
@@ -1975,8 +1975,8 @@ private:
SessionCtx->Query().Type = EKikimrQueryType::YqlScript;
SessionCtx->Query().Deadlines = settings.Deadlines;
SessionCtx->Query().StatsMode = settings.StatsMode;
- SessionCtx->Query().PreparingQuery = std::make_unique<NKikimrKqp::TPreparedQuery>();
- SessionCtx->Query().PreparedQuery.reset();
+ SessionCtx->Query().PreparingQuery = std::make_unique<NKikimrKqp::TPreparedQuery>();
+ SessionCtx->Query().PreparedQuery.reset();
TMaybe<TSqlVersion> sqlVersion;
auto scriptExpr = CompileYqlQuery(script, true, true, ctx, sqlVersion);
@@ -2001,8 +2001,8 @@ private:
SessionCtx->Query().Deadlines = settings.Deadlines;
SessionCtx->Query().StatsMode = settings.StatsMode;
SessionCtx->Query().ReplyTarget = target;
- SessionCtx->Query().PreparingQuery = std::make_unique<NKikimrKqp::TPreparedQuery>();
- SessionCtx->Query().PreparedQuery.reset();
+ SessionCtx->Query().PreparingQuery = std::make_unique<NKikimrKqp::TPreparedQuery>();
+ SessionCtx->Query().PreparedQuery.reset();
TMaybe<TSqlVersion> sqlVersion;
auto scriptExpr = CompileYqlQuery(script, true, true, ctx, sqlVersion);
@@ -2023,9 +2023,9 @@ private:
SessionCtx->Query().Type = EKikimrQueryType::YqlScript;
SessionCtx->Query().PrepareOnly = true;
- SessionCtx->Query().SuppressDdlChecks = true;
- SessionCtx->Query().PreparingQuery = std::make_unique<NKikimrKqp::TPreparedQuery>();
- SessionCtx->Query().PreparedQuery.reset();
+ SessionCtx->Query().SuppressDdlChecks = true;
+ SessionCtx->Query().PreparingQuery = std::make_unique<NKikimrKqp::TPreparedQuery>();
+ SessionCtx->Query().PreparedQuery.reset();
TMaybe<TSqlVersion> sqlVersion;
auto scriptExpr = CompileYqlQuery(script, true, true, ctx, sqlVersion);
@@ -2038,7 +2038,7 @@ private:
.AddPreTypeAnnotation()
.AddIOAnnotation()
.AddTypeAnnotation()
- .Add(TCollectParametersTransformer::Sync(SessionCtx->QueryPtr()), "CollectParameters")
+ .Add(TCollectParametersTransformer::Sync(SessionCtx->QueryPtr()), "CollectParameters")
.Build(false);
return MakeIntrusive<TAsyncValidateYqlResult>(scriptExpr.Get(), SessionCtx, ctx, transformer, sqlVersion);
@@ -2049,8 +2049,8 @@ private:
SessionCtx->Query().Type = EKikimrQueryType::YqlScript;
SessionCtx->Query().PrepareOnly = true;
- SessionCtx->Query().SuppressDdlChecks = true;
- SessionCtx->Query().PreparingQuery = std::make_unique<NKikimrKqp::TPreparedQuery>();
+ SessionCtx->Query().SuppressDdlChecks = true;
+ SessionCtx->Query().PreparingQuery = std::make_unique<NKikimrKqp::TPreparedQuery>();
TMaybe<TSqlVersion> sqlVersion;
auto scriptExpr = CompileYqlQuery(script, true, true, ctx, sqlVersion);
@@ -2081,21 +2081,21 @@ private:
}
return AsyncApplyResult<TQueryResult, TQueryResult>(prepareResult,
- [this, parameters = std::move(parameters), target, settings, &ctx]
- (TQueryResult&& prepared) mutable -> IAsyncQueryResultPtr {
- if (!prepared.Success()) {
- return MakeKikimrResultHolder(std::move(prepared));
+ [this, parameters = std::move(parameters), target, settings, &ctx]
+ (TQueryResult&& prepared) mutable -> IAsyncQueryResultPtr {
+ if (!prepared.Success()) {
+ return MakeKikimrResultHolder(std::move(prepared));
}
- std::shared_ptr<const NKikimrKqp::TPreparedQuery> query;
- query.reset(prepared.PreparingQuery.release());
-
- return ExecuteScanQueryInternal(query, std::move(parameters), target, settings, prepared.Issues(),
- prepared.SqlVersion, ctx);
+ std::shared_ptr<const NKikimrKqp::TPreparedQuery> query;
+ query.reset(prepared.PreparingQuery.release());
+
+ return ExecuteScanQueryInternal(query, std::move(parameters), target, settings, prepared.Issues(),
+ prepared.SqlVersion, ctx);
});
}
- IAsyncQueryResultPtr ExecuteScanQueryInternal(std::shared_ptr<const NKikimrKqp::TPreparedQuery>& query,
+ IAsyncQueryResultPtr ExecuteScanQueryInternal(std::shared_ptr<const NKikimrKqp::TPreparedQuery>& query,
NKikimrMiniKQL::TParams&& parameters, const NActors::TActorId& target,
const IKikimrQueryExecutor::TExecuteSettings& settings, const TIssues& issues,
const TMaybe<TSqlVersion>& sqlVersion, TExprContext& ctx)
@@ -2103,8 +2103,8 @@ private:
auto txCtx = MakeIntrusive<TKqpTransactionContext>(true);
txCtx->EffectiveIsolationLevel = NKikimrKqp::ISOLATION_LEVEL_UNDEFINED;
- SetupExecutePreparedTransformer(settings, txCtx);
-
+ SetupExecutePreparedTransformer(settings, txCtx);
+
ExecuteCtx->ReplyTarget = target;
if (issues) {
@@ -2116,14 +2116,14 @@ private:
}
SessionCtx->Query().Type = EKikimrQueryType::Scan;
- SessionCtx->Query().PreparedQuery = query;
- SessionCtx->Query().Deadlines = settings.Deadlines;
+ SessionCtx->Query().PreparedQuery = query;
+ SessionCtx->Query().Deadlines = settings.Deadlines;
SessionCtx->Query().Limits = settings.Limits;
SessionCtx->Query().StatsMode = settings.StatsMode;
SessionCtx->Query().RlPath = settings.RlPath;
return MakeIntrusive<TAsyncExecutePreparedResult>(FakeWorld.Get(), ctx, *ExecutePreparedTransformer,
- SessionCtx, FillSettings, ExecuteCtx, nullptr, sqlVersion);
+ SessionCtx, FillSettings, ExecuteCtx, nullptr, sqlVersion);
}
void SetupSession(TIntrusivePtr<IKikimrTransactionContext> txCtx) {
@@ -2203,25 +2203,25 @@ private:
} // namespace
-NDqProto::EDqStatsMode GetStatsMode(NYql::EKikimrStatsMode statsMode) {
+NDqProto::EDqStatsMode GetStatsMode(NYql::EKikimrStatsMode statsMode) {
switch (statsMode) {
case NYql::EKikimrStatsMode::Basic:
- return NYql::NDqProto::DQ_STATS_MODE_BASIC;
+ return NYql::NDqProto::DQ_STATS_MODE_BASIC;
case NYql::EKikimrStatsMode::Profile:
- return NYql::NDqProto::DQ_STATS_MODE_PROFILE;
+ return NYql::NDqProto::DQ_STATS_MODE_PROFILE;
default:
- return NYql::NDqProto::DQ_STATS_MODE_NONE;
+ return NYql::NDqProto::DQ_STATS_MODE_NONE;
}
}
-TKqpTransactionInfo TKqpTransactionContext::GetInfo() const {
+TKqpTransactionInfo TKqpTransactionContext::GetInfo() const {
TKqpTransactionInfo txInfo;
// Status
if (Invalidated) {
txInfo.Status = TKqpTransactionInfo::EStatus::Aborted;
} else if (Closed) {
- txInfo.Status = TKqpTransactionInfo::EStatus::Committed;
+ txInfo.Status = TKqpTransactionInfo::EStatus::Committed;
} else {
txInfo.Status = TKqpTransactionInfo::EStatus::Active;
}
@@ -2248,9 +2248,9 @@ TKqpTransactionInfo TKqpTransactionContext::GetInfo() const {
txInfo.ServerDuration = QueriesDuration;
txInfo.QueriesCount = QueriesCount;
- txInfo.TxEngine = DeferredEffects.GetEngine();
- txInfo.ForceNewEngineState = ForceNewEngineSettings;
-
+ txInfo.TxEngine = DeferredEffects.GetEngine();
+ txInfo.ForceNewEngineState = ForceNewEngineSettings;
+
return txInfo;
}
@@ -2262,5 +2262,5 @@ TIntrusivePtr<IKqpHost> CreateKqpHost(TIntrusivePtr<IKqpGateway> gateway,
keepConfigChanges);
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/host/kqp_host.h b/ydb/core/kqp/host/kqp_host.h
index 772199c1e5..11282d0ebe 100644
--- a/ydb/core/kqp/host/kqp_host.h
+++ b/ydb/core/kqp/host/kqp_host.h
@@ -27,11 +27,11 @@ public:
struct TPrepareSettings {
TMaybe<bool> UseNewEngine;
TMaybe<bool> DocumentApiRestricted;
-
- TString ToString() const {
- return TStringBuilder() << "TPrepareSettings{ UseNewEngine: " << UseNewEngine
- << ", DocumentApiRestricted: " << DocumentApiRestricted << " }";
- }
+
+ TString ToString() const {
+ return TStringBuilder() << "TPrepareSettings{ UseNewEngine: " << UseNewEngine
+ << ", DocumentApiRestricted: " << DocumentApiRestricted << " }";
+ }
};
struct TExecScriptSettings {
@@ -43,9 +43,9 @@ public:
virtual TMaybe<TKqpTransactionInfo> GetTransactionInfo() = 0;
- virtual void ForceTxOldEngine(const TString& txId) = 0;
- virtual void ForceTxNewEngine(const TString& txId, ui32 percent, ui32 level) = 0;
-
+ virtual void ForceTxOldEngine(const TString& txId) = 0;
+ virtual void ForceTxNewEngine(const TString& txId, ui32 percent, ui32 level) = 0;
+
/* Transaction control */
virtual TBeginTxResult BeginTransaction(NKikimrKqp::EIsolationLevel isolationLevel,
bool readonly) = 0;
@@ -57,18 +57,18 @@ public:
virtual TQueryResult SyncCommitTransaction(const TString& txId,
const NYql::IKikimrQueryExecutor::TExecuteSettings& settings) = 0;
- virtual IAsyncQueryResultPtr RollbackTransaction(const TString& txId,
- const NYql::IKikimrQueryExecutor::TExecuteSettings& settings) = 0;
- virtual TQueryResult SyncRollbackTransaction(const TString& txId,
- const NYql::IKikimrQueryExecutor::TExecuteSettings& settings) = 0;
+ virtual IAsyncQueryResultPtr RollbackTransaction(const TString& txId,
+ const NYql::IKikimrQueryExecutor::TExecuteSettings& settings) = 0;
+ virtual TQueryResult SyncRollbackTransaction(const TString& txId,
+ const NYql::IKikimrQueryExecutor::TExecuteSettings& settings) = 0;
virtual TMaybe<TKqpTransactionInfo> GetTransactionInfo(const TString& txId) = 0;
virtual bool AbortTransaction(const TString& txId) = 0;
virtual ui32 AbortAll() = 0;
- virtual IAsyncQueryResultPtr RollbackAborted() = 0;
- virtual TQueryResult SyncRollbackAborted() = 0;
+ virtual IAsyncQueryResultPtr RollbackAborted() = 0;
+ virtual TQueryResult SyncRollbackAborted() = 0;
/* Data queries */
virtual IAsyncQueryResultPtr ExplainDataQuery(const TString& query, bool isSql) = 0;
@@ -77,10 +77,10 @@ public:
virtual IAsyncQueryResultPtr PrepareDataQuery(const TString& query, const TPrepareSettings& settings) = 0;
virtual TQueryResult SyncPrepareDataQuery(const TString& query, const TPrepareSettings& settings) = 0;
- virtual IAsyncQueryResultPtr ExecuteDataQuery(const TString& txId, std::shared_ptr<const NKikimrKqp::TPreparedQuery>& query,
+ virtual IAsyncQueryResultPtr ExecuteDataQuery(const TString& txId, std::shared_ptr<const NKikimrKqp::TPreparedQuery>& query,
NKikimrMiniKQL::TParams&& parameters, const NYql::IKikimrQueryExecutor::TExecuteSettings& settings) = 0;
- virtual TQueryResult SyncExecuteDataQuery(const TString& txId, std::shared_ptr<const NKikimrKqp::TPreparedQuery>& query,
+ virtual TQueryResult SyncExecuteDataQuery(const TString& txId, std::shared_ptr<const NKikimrKqp::TPreparedQuery>& query,
NKikimrMiniKQL::TParams&& parameters, const NYql::IKikimrQueryExecutor::TExecuteSettings& settings) = 0;
virtual IAsyncQueryResultPtr ExecuteDataQuery(const TString& txId, const TString& query, bool isSql,
@@ -120,5 +120,5 @@ TIntrusivePtr<IKqpHost> CreateKqpHost(TIntrusivePtr<IKqpGateway> gateway,
const TString& cluster, const TString& database, NYql::TKikimrConfiguration::TPtr config, NYql::IModuleResolver::TPtr moduleResolver,
const NKikimr::NMiniKQL::IFunctionRegistry* funcRegistry = nullptr, bool keepConfigChanges = false);
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/host/kqp_host_impl.h b/ydb/core/kqp/host/kqp_host_impl.h
index bc8350e542..fb1fb139cd 100644
--- a/ydb/core/kqp/host/kqp_host_impl.h
+++ b/ydb/core/kqp/host/kqp_host_impl.h
@@ -8,7 +8,7 @@
namespace NKikimr {
namespace NKqp {
-NYql::NDqProto::EDqStatsMode GetStatsMode(NYql::EKikimrStatsMode statsMode);
+NYql::NDqProto::EDqStatsMode GetStatsMode(NYql::EKikimrStatsMode statsMode);
template<typename TResult, bool copyIssues = true>
class TKqpAsyncResultBase : public NYql::IKikimrAsyncResult<TResult> {
@@ -31,7 +31,7 @@ public:
return false;
}
- TResult GetResult() override {
+ TResult GetResult() override {
YQL_ENSURE(HasResult());
if (Status.GetValue() == NYql::IGraphTransformer::TStatus::Error) {
@@ -49,7 +49,7 @@ public:
result.AddIssues(ExprCtx.IssueManager.GetIssues());
}
FillResult(result);
- return std::move(result);
+ return std::move(result);
}
NThreading::TFuture<bool> Continue() override {
@@ -107,7 +107,7 @@ private:
template<typename TResult, typename TApplyResult>
class TKqpAsyncApplyResult : public NYql::IKikimrAsyncResult<TApplyResult> {
public:
- using TCallback = std::function<TIntrusivePtr<NYql::IKikimrAsyncResult<TApplyResult>>(TResult&&)>;
+ using TCallback = std::function<TIntrusivePtr<NYql::IKikimrAsyncResult<TApplyResult>>(TResult&&)>;
TKqpAsyncApplyResult(TIntrusivePtr<NYql::IKikimrAsyncResult<TResult>> result, const TCallback& callback)
: Result(result)
@@ -125,10 +125,10 @@ public:
return ApplyResult->HasResult();
}
- TApplyResult GetResult() override {
+ TApplyResult GetResult() override {
YQL_ENSURE(HasResult());
- return std::move(ApplyResult->GetResult());
+ return std::move(ApplyResult->GetResult());
}
NThreading::TFuture<bool> Continue() override {
@@ -154,7 +154,7 @@ private:
template<typename TItem, typename TExecResult>
class TKqpAsyncExecAllResult : public NYql::IKikimrAsyncResult<TExecResult> {
public:
- using TCallback = std::function<TIntrusivePtr<NYql::IKikimrAsyncResult<TExecResult>>(TItem&&)>;
+ using TCallback = std::function<TIntrusivePtr<NYql::IKikimrAsyncResult<TExecResult>>(TItem&&)>;
TKqpAsyncExecAllResult(const TVector<TItem>& items, const TCallback& callback)
: Items(items)
@@ -167,7 +167,7 @@ public:
return CurrentIndex >= Items.size();
}
- TExecResult GetResult() override {
+ TExecResult GetResult() override {
YQL_ENSURE(HasResult());
TExecResult execResult;
@@ -177,7 +177,7 @@ public:
execResult.AddIssues(Issues);
- return std::move(execResult);
+ return std::move(execResult);
}
NThreading::TFuture<bool> Continue() override {
@@ -186,7 +186,7 @@ public:
}
if (!CurrentResult) {
- CurrentResult.Reset(Callback(std::move(Items[CurrentIndex])));
+ CurrentResult.Reset(Callback(std::move(Items[CurrentIndex])));
}
if (CurrentResult->HasResult()) {
@@ -258,13 +258,13 @@ public:
const NYql::IKikimrQueryExecutor::TExecuteSettings& settings) = 0;
virtual TIntrusivePtr<TAsyncQueryResult> ExecutePreparedDataQuery(const TString& cluster,
- NYql::TExprNode* queryExpr, const NKikimrKqp::TPreparedKql& kql, NYql::TExprContext& ctx,
+ NYql::TExprNode* queryExpr, const NKikimrKqp::TPreparedKql& kql, NYql::TExprContext& ctx,
const NYql::IKikimrQueryExecutor::TExecuteSettings& settings) = 0;
virtual TIntrusivePtr<TAsyncQueryResult> ExecutePreparedQueryNewEngine(const TString& cluster,
const NYql::TExprNode::TPtr& world, const NKqpProto::TKqpPhyQuery& phyQuery, NYql::TExprContext& ctx,
- const NYql::IKikimrQueryExecutor::TExecuteSettings& settings) = 0;
-
+ const NYql::IKikimrQueryExecutor::TExecuteSettings& settings) = 0;
+
virtual TIntrusivePtr<TAsyncQueryResult> ExecutePreparedScanQuery(const TString& cluster,
const NYql::TExprNode::TPtr& world, const NKqpProto::TKqpPhyQuery& phyQuery, NYql::TExprContext& ctx,
const NActors::TActorId& target) = 0;
@@ -293,10 +293,10 @@ TAutoPtr<NYql::IGraphTransformer> CreateKqpExecuteScanTransformer(TIntrusivePtr<
TIntrusivePtr<TKqlTransformContext> transformCtx);
TAutoPtr<NYql::IGraphTransformer> CreateKqpCreateSnapshotTransformer(TIntrusivePtr<IKqpGateway> gateway,
- TIntrusivePtr<TKqlTransformContext> transformCtx, TIntrusivePtr<TKqpTransactionState> txState);
+ TIntrusivePtr<TKqlTransformContext> transformCtx, TIntrusivePtr<TKqpTransactionState> txState);
TAutoPtr<NYql::IGraphTransformer> CreateKqpReleaseSnapshotTransformer(TIntrusivePtr<IKqpGateway> gateway,
TIntrusivePtr<TKqpTransactionState> txState);
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/host/kqp_ne_helper.cpp b/ydb/core/kqp/host/kqp_ne_helper.cpp
index 6dca0a755a..030140717c 100644
--- a/ydb/core/kqp/host/kqp_ne_helper.cpp
+++ b/ydb/core/kqp/host/kqp_ne_helper.cpp
@@ -1,63 +1,63 @@
-#include "kqp_ne_helper.h"
-
+#include "kqp_ne_helper.h"
+
#include <ydb/core/kqp/common/kqp_yql.h>
-
+
#include <ydb/library/yql/core/yql_expr_optimize.h>
#include <ydb/library/yql/utils/log/log.h>
-
-namespace NKikimr {
-namespace NKqp {
-
-using namespace NYql;
-using namespace NYql::NNodes;
-
-bool CanExecuteWithNewEngine(const TKiProgram& program, TExprContext& ctx) {
- bool allow = true;
- TStringBuf forbiddenCallable;
-
- VisitExpr(
- program.Ptr(),
- [&allow](const TExprNode::TPtr&) {
- return allow;
- },
- [&allow, &forbiddenCallable](const TExprNode::TPtr& node) {
-
-#define FORBID_CALLABLE(x) if (x::Match(node.Get())) { allow = false; forbiddenCallable = x::CallableName(); return true; }
-
- FORBID_CALLABLE(TKiUpdateRow);
- FORBID_CALLABLE(TKiEraseRow);
- FORBID_CALLABLE(TKiDeleteTable);
- FORBID_CALLABLE(TKiEffects);
- FORBID_CALLABLE(TKiSelectIndexRange);
- FORBID_CALLABLE(TCoEquiJoin);
- FORBID_CALLABLE(TCoJoin);
- FORBID_CALLABLE(TCoMapJoinCore);
- FORBID_CALLABLE(TCoJoinDict);
- FORBID_CALLABLE(TCoSqlIn);
-
-#undef FORBID_CALLABLE
-
- if (auto selectRange = TMaybeNode<TKiSelectRange>(node)) {
- if (selectRange.Cast().Table().Path().Value().ends_with("/indexImplTable")) {
- forbiddenCallable = "KiSelectIndexRange";
- allow = false;
- return true;
- }
- }
-
- return true;
- });
-
- if (allow) {
- YQL_CLOG(NOTICE, ProviderKqp) << "Query " << KqpExprToPrettyString(program.Ref(), ctx)
- << " can be executed with NewEngine";
- } else {
- YQL_CLOG(NOTICE, ProviderKqp) << "Query " << KqpExprToPrettyString(program.Ref(), ctx)
- << " cannot be executed with NewEngine (blocked by '" << forbiddenCallable << "')";
- }
-
- return allow;
-}
-
-} // namespace NKqp
-} // namespace NKikimr
+
+namespace NKikimr {
+namespace NKqp {
+
+using namespace NYql;
+using namespace NYql::NNodes;
+
+bool CanExecuteWithNewEngine(const TKiProgram& program, TExprContext& ctx) {
+ bool allow = true;
+ TStringBuf forbiddenCallable;
+
+ VisitExpr(
+ program.Ptr(),
+ [&allow](const TExprNode::TPtr&) {
+ return allow;
+ },
+ [&allow, &forbiddenCallable](const TExprNode::TPtr& node) {
+
+#define FORBID_CALLABLE(x) if (x::Match(node.Get())) { allow = false; forbiddenCallable = x::CallableName(); return true; }
+
+ FORBID_CALLABLE(TKiUpdateRow);
+ FORBID_CALLABLE(TKiEraseRow);
+ FORBID_CALLABLE(TKiDeleteTable);
+ FORBID_CALLABLE(TKiEffects);
+ FORBID_CALLABLE(TKiSelectIndexRange);
+ FORBID_CALLABLE(TCoEquiJoin);
+ FORBID_CALLABLE(TCoJoin);
+ FORBID_CALLABLE(TCoMapJoinCore);
+ FORBID_CALLABLE(TCoJoinDict);
+ FORBID_CALLABLE(TCoSqlIn);
+
+#undef FORBID_CALLABLE
+
+ if (auto selectRange = TMaybeNode<TKiSelectRange>(node)) {
+ if (selectRange.Cast().Table().Path().Value().ends_with("/indexImplTable")) {
+ forbiddenCallable = "KiSelectIndexRange";
+ allow = false;
+ return true;
+ }
+ }
+
+ return true;
+ });
+
+ if (allow) {
+ YQL_CLOG(NOTICE, ProviderKqp) << "Query " << KqpExprToPrettyString(program.Ref(), ctx)
+ << " can be executed with NewEngine";
+ } else {
+ YQL_CLOG(NOTICE, ProviderKqp) << "Query " << KqpExprToPrettyString(program.Ref(), ctx)
+ << " cannot be executed with NewEngine (blocked by '" << forbiddenCallable << "')";
+ }
+
+ return allow;
+}
+
+} // namespace NKqp
+} // namespace NKikimr
diff --git a/ydb/core/kqp/host/kqp_ne_helper.h b/ydb/core/kqp/host/kqp_ne_helper.h
index b554de86b5..64b495afcc 100644
--- a/ydb/core/kqp/host/kqp_ne_helper.h
+++ b/ydb/core/kqp/host/kqp_ne_helper.h
@@ -1,13 +1,13 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/kqp/provider/yql_kikimr_expr_nodes.h>
#include <ydb/library/yql/ast/yql_expr.h>
-
-namespace NKikimr {
-namespace NKqp {
-
-bool CanExecuteWithNewEngine(const NYql::NNodes::TKiProgram& program, NYql::TExprContext& ctx);
-
-} // namespace NKqp
-} // namespace NKikimr
+
+namespace NKikimr {
+namespace NKqp {
+
+bool CanExecuteWithNewEngine(const NYql::NNodes::TKiProgram& program, NYql::TExprContext& ctx);
+
+} // namespace NKqp
+} // namespace NKikimr
diff --git a/ydb/core/kqp/host/kqp_run_data.cpp b/ydb/core/kqp/host/kqp_run_data.cpp
index 2d0d72e097..b2bc80d71b 100644
--- a/ydb/core/kqp/host/kqp_run_data.cpp
+++ b/ydb/core/kqp/host/kqp_run_data.cpp
@@ -17,20 +17,20 @@ public:
: TKqpExecutePhysicalTransformerBase(gateway, cluster, txState, transformCtx) {}
protected:
- TStatus DoExecute(const NKqpProto::TKqpPhyTx* tx, bool commit, NYql::TExprContext& ctx) final {
+ TStatus DoExecute(const NKqpProto::TKqpPhyTx* tx, bool commit, NYql::TExprContext& ctx) final {
auto& txState = TxState->Tx();
- auto request = PrepareRequest();
-
- if (!request.Timeout) {
- ctx.AddError(YqlIssue({}, TIssuesIds::KIKIMR_TIMEOUT, "Query request timeout."));
- return TStatus::Error;
- }
-
- if (request.CancelAfter && !*request.CancelAfter) {
- ctx.AddError(YqlIssue({}, TIssuesIds::KIKIMR_OPERATION_CANCELLED, "Query cancelled by timeout."));
- return TStatus::Error;
- }
-
+ auto request = PrepareRequest();
+
+ if (!request.Timeout) {
+ ctx.AddError(YqlIssue({}, TIssuesIds::KIKIMR_TIMEOUT, "Query request timeout."));
+ return TStatus::Error;
+ }
+
+ if (request.CancelAfter && !*request.CancelAfter) {
+ ctx.AddError(YqlIssue({}, TIssuesIds::KIKIMR_OPERATION_CANCELLED, "Query cancelled by timeout."));
+ return TStatus::Error;
+ }
+
if (tx) {
switch (tx->GetType()) {
case NKqpProto::TKqpPhyTx::TYPE_COMPUTE:
@@ -43,7 +43,7 @@ protected:
request.Transactions.emplace_back(*tx, PrepareParameters(*tx));
} else {
YQL_ENSURE(commit);
- if (txState.DeferredEffects.Empty() && !txState.Locks.HasLocks()) {
+ if (txState.DeferredEffects.Empty() && !txState.Locks.HasLocks()) {
return TStatus::Ok;
}
}
@@ -51,22 +51,22 @@ protected:
if (commit) {
Y_VERIFY_DEBUG(txState.DeferredEffects.Empty() || !txState.Locks.Broken());
- for (const auto& effect : txState.DeferredEffects) {
+ for (const auto& effect : txState.DeferredEffects) {
YQL_ENSURE(!effect.Node);
YQL_ENSURE(effect.PhysicalTx.GetType() == NKqpProto::TKqpPhyTx::TYPE_DATA);
request.Transactions.emplace_back(effect.PhysicalTx, GetParamsRefMap(effect.Params));
}
- if (!txState.DeferredEffects.Empty()) {
- request.PerShardKeysSizeLimitBytes = TransformCtx->Config->_CommitPerShardKeysSizeLimitBytes.Get().GetRef();
- }
-
+ if (!txState.DeferredEffects.Empty()) {
+ request.PerShardKeysSizeLimitBytes = TransformCtx->Config->_CommitPerShardKeysSizeLimitBytes.Get().GetRef();
+ }
+
if (txState.Locks.HasLocks()) {
request.ValidateLocks = !(txState.GetSnapshot().IsValid() && txState.DeferredEffects.Empty());
- request.EraseLocks = true;
-
- for (auto& [lockId, lock] : txState.Locks.LocksMap) {
- request.Locks.emplace_back(lock.GetValueRef(txState.Locks.LockType));
+ request.EraseLocks = true;
+
+ for (auto& [lockId, lock] : txState.Locks.LocksMap) {
+ request.Locks.emplace_back(lock.GetValueRef(txState.Locks.LockType));
}
}
} else if (ShouldAcquireLocks()) {
@@ -77,31 +77,31 @@ protected:
return TStatus::Async;
}
- TStatus DoRollback() final {
- auto& txState = TxState->Tx();
-
- YQL_CLOG(DEBUG, ProviderKqp) << "TKqpExecutePhysicalDataTransformer::DoRollback()"
- << ", effects: " << txState.DeferredEffects.Size() << ", locks: " << txState.Locks.Size()
- << ", timeout: " << TransformCtx->QueryCtx->Deadlines.TimeoutAt;
-
- if (!txState.Locks.HasLocks()) {
- ClearTx();
- return TStatus::Ok;
- }
-
- auto request = PrepareRequest();
-
- request.ValidateLocks = false;
- request.EraseLocks = true;
-
- for (auto& [key, lock] : txState.Locks.LocksMap) {
- request.Locks.emplace_back(lock.GetValueRef(txState.Locks.LockType));
- }
-
+ TStatus DoRollback() final {
+ auto& txState = TxState->Tx();
+
+ YQL_CLOG(DEBUG, ProviderKqp) << "TKqpExecutePhysicalDataTransformer::DoRollback()"
+ << ", effects: " << txState.DeferredEffects.Size() << ", locks: " << txState.Locks.Size()
+ << ", timeout: " << TransformCtx->QueryCtx->Deadlines.TimeoutAt;
+
+ if (!txState.Locks.HasLocks()) {
+ ClearTx();
+ return TStatus::Ok;
+ }
+
+ auto request = PrepareRequest();
+
+ request.ValidateLocks = false;
+ request.EraseLocks = true;
+
+ for (auto& [key, lock] : txState.Locks.LocksMap) {
+ request.Locks.emplace_back(lock.GetValueRef(txState.Locks.LockType));
+ }
+
ExecuteFuture = Gateway->ExecutePhysical(std::move(request), {});
- return TStatus::Async;
- }
-
+ return TStatus::Async;
+ }
+
bool OnExecuterResult(NKikimrKqp::TExecuterTxResult&& execResult, TExprContext& ctx, bool commit) override {
if (execResult.HasLocks()) {
YQL_ENSURE(!commit);
@@ -116,31 +116,31 @@ protected:
}
if (commit) {
- ClearTx();
+ ClearTx();
}
return true;
}
private:
- IKqpGateway::TExecPhysicalRequest PrepareRequest() {
- IKqpGateway::TExecPhysicalRequest request;
- auto now = Gateway->GetCurrentTime();
-
- request.Timeout = TransformCtx->QueryCtx->Deadlines.TimeoutAt - now;
- if (auto cancelAt = TransformCtx->QueryCtx->Deadlines.CancelAt) {
- request.CancelAfter = cancelAt - now;
- }
- request.StatsMode = GetStatsMode(TransformCtx->QueryCtx->StatsMode);
- request.MaxAffectedShards = TransformCtx->QueryCtx->Limits.PhaseLimits.AffectedShardsLimit;
- request.TotalReadSizeLimitBytes = TransformCtx->QueryCtx->Limits.PhaseLimits.TotalReadSizeLimitBytes;
- request.MkqlMemoryLimit = TransformCtx->QueryCtx->Limits.PhaseLimits.ComputeNodeMemoryLimitBytes;
+ IKqpGateway::TExecPhysicalRequest PrepareRequest() {
+ IKqpGateway::TExecPhysicalRequest request;
+ auto now = Gateway->GetCurrentTime();
+
+ request.Timeout = TransformCtx->QueryCtx->Deadlines.TimeoutAt - now;
+ if (auto cancelAt = TransformCtx->QueryCtx->Deadlines.CancelAt) {
+ request.CancelAfter = cancelAt - now;
+ }
+ request.StatsMode = GetStatsMode(TransformCtx->QueryCtx->StatsMode);
+ request.MaxAffectedShards = TransformCtx->QueryCtx->Limits.PhaseLimits.AffectedShardsLimit;
+ request.TotalReadSizeLimitBytes = TransformCtx->QueryCtx->Limits.PhaseLimits.TotalReadSizeLimitBytes;
+ request.MkqlMemoryLimit = TransformCtx->QueryCtx->Limits.PhaseLimits.ComputeNodeMemoryLimitBytes;
request.Snapshot = TxState->Tx().GetSnapshot();
request.IsolationLevel = *TxState->Tx().EffectiveIsolationLevel;
-
- return request;
- }
-
+
+ return request;
+ }
+
bool ShouldAcquireLocks() {
if (*TxState->Tx().EffectiveIsolationLevel != NKikimrKqp::ISOLATION_LEVEL_SERIALIZABLE) {
return false;
@@ -150,16 +150,16 @@ private:
return false; // Do not acquire locks after first lock issue
}
- if (!TxState->Tx().DeferredEffects.Empty()) {
+ if (!TxState->Tx().DeferredEffects.Empty()) {
return true; // Acquire locks in read write tx
}
- for (auto& tx : TransformCtx->PhysicalQuery->GetTransactions()) {
- if (tx.GetHasEffects()) {
- return true; // Acquire locks in read write tx
- }
- }
-
+ for (auto& tx : TransformCtx->PhysicalQuery->GetTransactions()) {
+ if (tx.GetHasEffects()) {
+ return true; // Acquire locks in read write tx
+ }
+ }
+
if (!TransformCtx->Settings.GetCommitTx()) {
return true; // Is not a commit tx
}
@@ -189,5 +189,5 @@ TAutoPtr<IGraphTransformer> CreateKqpExecutePhysicalDataTransformer(TIntrusivePt
return new TKqpExecutePhysicalDataTransformer(gateway, cluster, txState, transformCtx);
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/host/kqp_run_physical.cpp b/ydb/core/kqp/host/kqp_run_physical.cpp
index ccb2cdee8b..e6bd601eef 100644
--- a/ydb/core/kqp/host/kqp_run_physical.cpp
+++ b/ydb/core/kqp/host/kqp_run_physical.cpp
@@ -17,53 +17,53 @@ IGraphTransformer::TStatus TKqpExecutePhysicalTransformerBase::DoTransform(TExpr
YQL_ENSURE(TMaybeNode<TCoWorld>(input));
YQL_ENSURE(TransformCtx->PhysicalQuery);
- const auto& query = *TransformCtx->PhysicalQuery;
+ const auto& query = *TransformCtx->PhysicalQuery;
TStatus status = TStatus::Ok;
auto& txState = TxState->Tx();
- const ui64 txsCount = query.TransactionsSize();
-
- if (settings.GetRollbackTx()) {
- if (ExecuteFlags.HasFlags(TKqpExecuteFlag::Rollback)) {
- ClearTx();
- } else {
- status = Rollback();
- }
- } else if (ExecuteFlags.HasFlags(TKqpExecuteFlag::Commit)) {
- YQL_ENSURE(CurrentTxIndex >= txsCount);
+ const ui64 txsCount = query.TransactionsSize();
+
+ if (settings.GetRollbackTx()) {
+ if (ExecuteFlags.HasFlags(TKqpExecuteFlag::Rollback)) {
+ ClearTx();
+ } else {
+ status = Rollback();
+ }
+ } else if (ExecuteFlags.HasFlags(TKqpExecuteFlag::Commit)) {
+ YQL_ENSURE(CurrentTxIndex >= txsCount);
} else {
- if (CurrentTxIndex >= txsCount) {
- if (!txState.DeferredEffects.Empty() && TxState->Tx().Locks.Broken()) {
+ if (CurrentTxIndex >= txsCount) {
+ if (!txState.DeferredEffects.Empty() && TxState->Tx().Locks.Broken()) {
TxState->Tx().Locks.ReportIssues(ctx);
return TStatus::Error;
}
-
- if (settings.GetCommitTx()) {
- status = Execute(nullptr, /* commit */ true, ctx);
- }
- } else {
- const auto& tx = query.GetTransactions(CurrentTxIndex);
-
- if (tx.GetHasEffects()) {
- if (!AddDeferredEffect(tx)) {
- ctx.AddError(YqlIssue({}, TIssuesIds::KIKIMR_BAD_REQUEST,
- "Failed to mix queries with old- and new- engines"));
- return TStatus::Error;
- }
-
- ++CurrentTxIndex;
- return TStatus::Repeat;
+
+ if (settings.GetCommitTx()) {
+ status = Execute(nullptr, /* commit */ true, ctx);
}
+ } else {
+ const auto& tx = query.GetTransactions(CurrentTxIndex);
+
+ if (tx.GetHasEffects()) {
+ if (!AddDeferredEffect(tx)) {
+ ctx.AddError(YqlIssue({}, TIssuesIds::KIKIMR_BAD_REQUEST,
+ "Failed to mix queries with old- and new- engines"));
+ return TStatus::Error;
+ }
+
+ ++CurrentTxIndex;
+ return TStatus::Repeat;
+ }
+
+ // tx without effects
- // tx without effects
-
bool commit = false;
- if (CurrentTxIndex == txsCount - 1 && settings.GetCommitTx() && txState.DeferredEffects.Empty()) {
- // last transaction and no effects (RO-query)
- commit = true;
+ if (CurrentTxIndex == txsCount - 1 && settings.GetCommitTx() && txState.DeferredEffects.Empty()) {
+ // last transaction and no effects (RO-query)
+ commit = true;
}
- status = Execute(&tx, commit, ctx);
+ status = Execute(&tx, commit, ctx);
}
}
@@ -117,10 +117,10 @@ IGraphTransformer::TStatus TKqpExecutePhysicalTransformerBase::DoApplyAsyncChang
return TStatus::Error;
}
- if (ExecuteFlags.HasFlags(TKqpExecuteFlag::Rollback)) {
- return TStatus::Repeat;
- }
-
+ if (ExecuteFlags.HasFlags(TKqpExecuteFlag::Rollback)) {
+ return TStatus::Repeat;
+ }
+
auto& execResult = result.ExecuterResult;
if (ExecuteFlags.HasFlags(TKqpExecuteFlag::Results)) {
TVector<NKikimrMiniKQL::TResult> txResults;
@@ -146,9 +146,9 @@ void TKqpExecutePhysicalTransformerBase::Rewind() {
TransformState->TxResults.clear();
}
-IGraphTransformer::TStatus TKqpExecutePhysicalTransformerBase::Execute(const NKqpProto::TKqpPhyTx* tx, bool commit,
- NYql::TExprContext& ctx)
-{
+IGraphTransformer::TStatus TKqpExecutePhysicalTransformerBase::Execute(const NKqpProto::TKqpPhyTx* tx, bool commit,
+ NYql::TExprContext& ctx)
+{
ExecuteFlags = TKqpExecuteFlags();
if (tx) {
@@ -161,21 +161,21 @@ IGraphTransformer::TStatus TKqpExecutePhysicalTransformerBase::Execute(const NKq
ExecuteFlags |= TKqpExecuteFlag::Commit;
}
- return DoExecute(tx, commit, ctx);
+ return DoExecute(tx, commit, ctx);
+}
+
+IGraphTransformer::TStatus TKqpExecutePhysicalTransformerBase::Rollback() {
+ ExecuteFlags = TKqpExecuteFlags();
+ ExecuteFlags |= TKqpExecuteFlag::Rollback;
+
+ return DoRollback();
}
-IGraphTransformer::TStatus TKqpExecutePhysicalTransformerBase::Rollback() {
- ExecuteFlags = TKqpExecuteFlags();
- ExecuteFlags |= TKqpExecuteFlag::Rollback;
-
- return DoRollback();
-}
-
-bool TKqpExecutePhysicalTransformerBase::AddDeferredEffect(const NKqpProto::TKqpPhyTx& tx) {
- TParamValueMap params;
- PreserveParams(tx, params);
-
- return TxState->Tx().AddDeferredEffect(tx, std::move(params));
+bool TKqpExecutePhysicalTransformerBase::AddDeferredEffect(const NKqpProto::TKqpPhyTx& tx) {
+ TParamValueMap params;
+ PreserveParams(tx, params);
+
+ return TxState->Tx().AddDeferredEffect(tx, std::move(params));
}
NDq::TMkqlValueRef TKqpExecutePhysicalTransformerBase::GetParamValue(
@@ -260,7 +260,7 @@ TKqpParamsMap TKqpExecutePhysicalTransformerBase::PrepareParameters(const NKqpPr
TKqpParamsMap paramsMap(TransformState);
for (const auto& paramBinding : tx.GetParamBindings()) {
auto it = paramsMap.Values.emplace(paramBinding.GetName(), GetParamValue(paramBinding));
- YQL_ENSURE(it.second);
+ YQL_ENSURE(it.second);
}
return paramsMap;
@@ -274,15 +274,15 @@ void TKqpExecutePhysicalTransformerBase::PreserveParams(const NKqpProto::TKqpPhy
param.MutableType()->CopyFrom(paramValueRef.GetType());
param.MutableValue()->CopyFrom(paramValueRef.GetValue());
- YQL_ENSURE(paramsMap.emplace(paramBinding.GetName(), std::move(param)).second);
+ YQL_ENSURE(paramsMap.emplace(paramBinding.GetName(), std::move(param)).second);
}
}
-void TKqpExecutePhysicalTransformerBase::ClearTx() {
- TxState->Tx().ClearDeferredEffects();
- TxState->Tx().Locks.Clear();
- TxState->Tx().Finish();
-}
-
-} // namespace NKqp
+void TKqpExecutePhysicalTransformerBase::ClearTx() {
+ TxState->Tx().ClearDeferredEffects();
+ TxState->Tx().Locks.Clear();
+ TxState->Tx().Finish();
+}
+
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/host/kqp_run_physical.h b/ydb/core/kqp/host/kqp_run_physical.h
index a404dc7f8f..c55fcc59a0 100644
--- a/ydb/core/kqp/host/kqp_run_physical.h
+++ b/ydb/core/kqp/host/kqp_run_physical.h
@@ -12,7 +12,7 @@ namespace NKqp {
enum class TKqpExecuteFlag : ui32 {
Results = 1 << 0,
Commit = 1 << 1,
- Rollback = 1 << 2,
+ Rollback = 1 << 2,
};
Y_DECLARE_FLAGS(TKqpExecuteFlags, TKqpExecuteFlag);
@@ -40,23 +40,23 @@ public:
void Rewind() override;
protected:
- virtual TStatus DoExecute(const NKqpProto::TKqpPhyTx* tx, bool commit, NYql::TExprContext& ctx) = 0;
- virtual TStatus DoRollback() = 0;
+ virtual TStatus DoExecute(const NKqpProto::TKqpPhyTx* tx, bool commit, NYql::TExprContext& ctx) = 0;
+ virtual TStatus DoRollback() = 0;
- virtual bool OnExecuterResult(NKikimrKqp::TExecuterTxResult&& execResult, NYql::TExprContext& ctx, bool commit) = 0;
+ virtual bool OnExecuterResult(NKikimrKqp::TExecuterTxResult&& execResult, NYql::TExprContext& ctx, bool commit) = 0;
protected:
NYql::NDq::TMkqlValueRef GetParamValue(const NKqpProto::TKqpPhyParamBinding& paramBinding);
TKqpParamsMap PrepareParameters(const NKqpProto::TKqpPhyTx& tx);
- void ClearTx();
-
+ void ClearTx();
+
private:
- TStatus Execute(const NKqpProto::TKqpPhyTx* tx, bool commit, NYql::TExprContext& ctx);
- TStatus Rollback();
+ TStatus Execute(const NKqpProto::TKqpPhyTx* tx, bool commit, NYql::TExprContext& ctx);
+ TStatus Rollback();
- bool AddDeferredEffect(const NKqpProto::TKqpPhyTx& tx);
+ bool AddDeferredEffect(const NKqpProto::TKqpPhyTx& tx);
void PreserveParams(const NKqpProto::TKqpPhyTx& tx, TParamValueMap& paramsMap);
@@ -84,5 +84,5 @@ protected:
std::shared_ptr<TTransformState> TransformState;
};
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/host/kqp_run_prepared.cpp b/ydb/core/kqp/host/kqp_run_prepared.cpp
index 44ec61762e..e71a3bacee 100644
--- a/ydb/core/kqp/host/kqp_run_prepared.cpp
+++ b/ydb/core/kqp/host/kqp_run_prepared.cpp
@@ -206,5 +206,5 @@ TAutoPtr<IGraphTransformer> CreateKqpExecutePreparedTransformer(TIntrusivePtr<IK
return new TKqpExecutePreparedTransformer(gateway, cluster, txState, transformCtx);
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/host/kqp_run_scan.cpp b/ydb/core/kqp/host/kqp_run_scan.cpp
index 8633b02412..7c91ae8200 100644
--- a/ydb/core/kqp/host/kqp_run_scan.cpp
+++ b/ydb/core/kqp/host/kqp_run_scan.cpp
@@ -3,7 +3,7 @@
#include <ydb/core/base/kikimr_issue.h>
#include <ydb/core/kqp/common/kqp_yql.h>
-
+
namespace NKikimr {
namespace NKqp {
@@ -18,7 +18,7 @@ public:
: TKqpExecutePhysicalTransformerBase(gateway, cluster, txState, transformCtx) {}
protected:
- TStatus DoExecute(const NKqpProto::TKqpPhyTx* tx, bool commit, NYql::TExprContext&) final {
+ TStatus DoExecute(const NKqpProto::TKqpPhyTx* tx, bool commit, NYql::TExprContext&) final {
YQL_ENSURE(tx);
YQL_ENSURE(!commit);
@@ -26,36 +26,36 @@ protected:
request.Transactions.emplace_back(*tx, PrepareParameters(*tx));
request.RlPath = TransformCtx->QueryCtx->RlPath;
- request.Timeout = TransformCtx->QueryCtx->Deadlines.TimeoutAt - Gateway->GetCurrentTime();
- if (!request.Timeout) {
+ request.Timeout = TransformCtx->QueryCtx->Deadlines.TimeoutAt - Gateway->GetCurrentTime();
+ if (!request.Timeout) {
// TODO: Just cancel request.
- request.Timeout = TDuration::MilliSeconds(1);
- }
- request.MaxComputeActors = TransformCtx->Config->_KqpMaxComputeActors.Get().GetRef();
+ request.Timeout = TDuration::MilliSeconds(1);
+ }
+ request.MaxComputeActors = TransformCtx->Config->_KqpMaxComputeActors.Get().GetRef();
request.StatsMode = GetStatsMode(TransformCtx->QueryCtx->StatsMode);
- request.DisableLlvmForUdfStages = TransformCtx->Config->DisableLlvmForUdfStages();
+ request.DisableLlvmForUdfStages = TransformCtx->Config->DisableLlvmForUdfStages();
request.LlvmEnabled = TransformCtx->Config->GetEnableLlvm() != EOptionalFlag::Disabled;
request.Snapshot = TxState->Tx().GetSnapshot();
switch (tx->GetType()) {
case NKqpProto::TKqpPhyTx::TYPE_COMPUTE:
- ExecuteFuture = Gateway->ExecutePhysical(std::move(request), TransformCtx->ReplyTarget);
+ ExecuteFuture = Gateway->ExecutePhysical(std::move(request), TransformCtx->ReplyTarget);
break;
case NKqpProto::TKqpPhyTx::TYPE_SCAN:
ExecuteFuture = Gateway->ExecuteScanQuery(std::move(request), TransformCtx->ReplyTarget);
break;
default:
- YQL_ENSURE(false, "Unexpected physical tx type in scan query: "
- << NKqpProto::TKqpPhyTx_EType_Name(tx->GetType()));
+ YQL_ENSURE(false, "Unexpected physical tx type in scan query: "
+ << NKqpProto::TKqpPhyTx_EType_Name(tx->GetType()));
}
return TStatus::Async;
}
- TStatus DoRollback() final {
- YQL_ENSURE(false, "Rollback in ScanQuery tx");
- }
-
+ TStatus DoRollback() final {
+ YQL_ENSURE(false, "Rollback in ScanQuery tx");
+ }
+
bool OnExecuterResult(NKikimrKqp::TExecuterTxResult&& execResult, TExprContext& ctx, bool commit) override {
Y_UNUSED(execResult);
Y_UNUSED(ctx);
@@ -68,7 +68,7 @@ protected:
class TKqpCreateSnapshotTransformer : public TGraphTransformerBase {
public:
TKqpCreateSnapshotTransformer(TIntrusivePtr<IKqpGateway> gateway, TIntrusivePtr<TKqlTransformContext> transformCtx,
- TIntrusivePtr<TKqpTransactionState> txState)
+ TIntrusivePtr<TKqpTransactionState> txState)
: Gateway(gateway)
, TransformCtx(transformCtx)
, TxState(txState)
@@ -114,27 +114,27 @@ public:
auto handle = SnapshotFuture.ExtractValue();
if (!handle.Snapshot.IsValid()) {
- YQL_CLOG(NOTICE, ProviderKqp) << "Failed to create persistent snapshot. "
- << "Status: " << NKikimrIssues::TStatusIds_EStatusCode_Name(handle.Status)
+ YQL_CLOG(NOTICE, ProviderKqp) << "Failed to create persistent snapshot. "
+ << "Status: " << NKikimrIssues::TStatusIds_EStatusCode_Name(handle.Status)
<< ", issues: " << handle.Issues().ToString();
-
+
TIssue issue("Failed to create persistent snapshot");
- switch (handle.Status) {
- case NKikimrIssues::TStatusIds::SCHEME_ERROR:
- issue.SetCode(NYql::TIssuesIds::KIKIMR_SCHEME_ERROR, NYql::TSeverityIds::S_ERROR);
- break;
- case NKikimrIssues::TStatusIds::TIMEOUT:
- issue.SetCode(NYql::TIssuesIds::KIKIMR_TIMEOUT, NYql::TSeverityIds::S_ERROR);
- break;
- case NKikimrIssues::TStatusIds::OVERLOADED:
- issue.SetCode(NYql::TIssuesIds::KIKIMR_OVERLOADED, NYql::TSeverityIds::S_ERROR);
- break;
- default:
- // ScanQuery is always RO, so we can return UNAVAILABLE here
- issue.SetCode(NYql::TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE, NYql::TSeverityIds::S_ERROR);
- break;
- }
-
+ switch (handle.Status) {
+ case NKikimrIssues::TStatusIds::SCHEME_ERROR:
+ issue.SetCode(NYql::TIssuesIds::KIKIMR_SCHEME_ERROR, NYql::TSeverityIds::S_ERROR);
+ break;
+ case NKikimrIssues::TStatusIds::TIMEOUT:
+ issue.SetCode(NYql::TIssuesIds::KIKIMR_TIMEOUT, NYql::TSeverityIds::S_ERROR);
+ break;
+ case NKikimrIssues::TStatusIds::OVERLOADED:
+ issue.SetCode(NYql::TIssuesIds::KIKIMR_OVERLOADED, NYql::TSeverityIds::S_ERROR);
+ break;
+ default:
+ // ScanQuery is always RO, so we can return UNAVAILABLE here
+ issue.SetCode(NYql::TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE, NYql::TSeverityIds::S_ERROR);
+ break;
+ }
+
for (const auto& subIssue: handle.Issues()) {
issue.AddSubIssue(MakeIntrusive<TIssue>(subIssue));
}
@@ -186,9 +186,9 @@ TAutoPtr<IGraphTransformer> CreateKqpExecuteScanTransformer(TIntrusivePtr<IKqpGa
}
TAutoPtr<IGraphTransformer> CreateKqpCreateSnapshotTransformer(TIntrusivePtr<IKqpGateway> gateway,
- TIntrusivePtr<TKqlTransformContext> transformCtx, TIntrusivePtr<TKqpTransactionState> txState)
+ TIntrusivePtr<TKqlTransformContext> transformCtx, TIntrusivePtr<TKqpTransactionState> txState)
{
- return new TKqpCreateSnapshotTransformer(gateway, transformCtx, txState);
+ return new TKqpCreateSnapshotTransformer(gateway, transformCtx, txState);
}
TAutoPtr<IGraphTransformer> CreateKqpReleaseSnapshotTransformer(TIntrusivePtr<IKqpGateway> gateway,
@@ -197,5 +197,5 @@ TAutoPtr<IGraphTransformer> CreateKqpReleaseSnapshotTransformer(TIntrusivePtr<IK
return new TKqpReleaseSnapshotTransformer(gateway, txState);
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/host/kqp_runner.cpp b/ydb/core/kqp/host/kqp_runner.cpp
index 3951234923..42b4eca7c6 100644
--- a/ydb/core/kqp/host/kqp_runner.cpp
+++ b/ydb/core/kqp/host/kqp_runner.cpp
@@ -1,5 +1,5 @@
#include "kqp_host_impl.h"
-#include "kqp_ne_helper.h"
+#include "kqp_ne_helper.h"
#include <ydb/core/kqp/common/kqp_yql.h>
#include <ydb/core/kqp/compile/kqp_compile.h>
@@ -43,9 +43,9 @@ public:
using TResult = IKikimrQueryExecutor::TQueryResult;
TAsyncRunResult(const TExprNode::TPtr& queryRoot, TExprContext& exprCtx, IGraphTransformer& transformer,
- const TKqlTransformContext& transformCtx)
+ const TKqlTransformContext& transformCtx)
: TKqpAsyncResultBase(queryRoot, exprCtx, transformer)
- , TransformCtx(transformCtx) {}
+ , TransformCtx(transformCtx) {}
void FillResult(TResult& queryResult) const override {
if (TransformCtx.QueryCtx->PrepareOnly) {
@@ -57,7 +57,7 @@ public:
queryResult.ProtobufArenaPtr.get());
}
- queryResult.QueryTraits = TransformCtx.QueryCtx->QueryTraits;
+ queryResult.QueryTraits = TransformCtx.QueryCtx->QueryTraits;
queryResult.QueryStats.CopyFrom(TransformCtx.QueryStats);
}
@@ -84,7 +84,7 @@ public:
results.push_back(result);
}
- queryResult.QueryTraits = TransformCtx.QueryCtx->QueryTraits;
+ queryResult.QueryTraits = TransformCtx.QueryCtx->QueryTraits;
queryResult.QueryStats.CopyFrom(TransformCtx.QueryStats);
queryResult.Results = std::move(results);
}
@@ -117,8 +117,8 @@ public:
output = input;
if (Iterations > MaxTransformIterations) {
- ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder()
- << "Exceeded maximum allowed KQP iterations: " << MaxTransformIterations));
+ ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder()
+ << "Exceeded maximum allowed KQP iterations: " << MaxTransformIterations));
return TStatus::Error;
}
@@ -156,33 +156,33 @@ public:
KqlTypeAnnTransformer = CreateTypeAnnotationTransformer(CreateExtCallableTypeAnnotationTransformer(*typesCtx),
*typesCtx);
- auto logLevel = NLog::ELevel::TRACE;
- auto logComp = NLog::EComponent::ProviderKqp;
-
+ auto logLevel = NLog::ELevel::TRACE;
+ auto logComp = NLog::EComponent::ProviderKqp;
+
KqlOptimizeTransformer = TTransformationPipeline(typesCtx)
.AddServiceTransformers()
- .Add(TLogExprTransformer::Sync("KqlOptimizeTransformer", logComp, logLevel), "LogKqlOptimize")
+ .Add(TLogExprTransformer::Sync("KqlOptimizeTransformer", logComp, logLevel), "LogKqlOptimize")
.AddTypeAnnotationTransformer()
.AddPostTypeAnnotation()
.AddOptimization(false)
.Build(false);
- KqlPrepareTransformer = TTransformationPipeline(typesCtx)
+ KqlPrepareTransformer = TTransformationPipeline(typesCtx)
.AddServiceTransformers()
- .Add(TLogExprTransformer::Sync("KqlPrepareTransformer", logComp, logLevel), "LogKqlRun")
+ .Add(TLogExprTransformer::Sync("KqlPrepareTransformer", logComp, logLevel), "LogKqlRun")
.Add(new TKqpIterationGuardTransformer(), "IterationGuard")
.AddTypeAnnotationTransformer()
.Add(CreateKqpCheckKiProgramTransformer(), "CheckQuery")
.Add(CreateKqpSimplifyTransformer(), "Simplify")
.Add(CreateKqpAnalyzeTransformer(TransformCtx), "Analyze")
.Add(CreateKqpRewriteTransformer(TransformCtx), "Rewrite")
- .Add(CreateKqpExecTransformer(Gateway, Cluster, TxState, TransformCtx), "Prepare")
+ .Add(CreateKqpExecTransformer(Gateway, Cluster, TxState, TransformCtx), "Prepare")
.Add(CreateKqpSubstituteTransformer(TxState, TransformCtx), "Substitute")
.Add(CreateKqpFinalizeTransformer(Gateway, Cluster, TxState, TransformCtx), "Finalize")
.Build(false);
PreparedRunTransformer = TTransformationPipeline(typesCtx)
- .Add(TLogExprTransformer::Sync("PreparedRun iteration", logComp, logLevel), "KqlPreparedRun")
+ .Add(TLogExprTransformer::Sync("PreparedRun iteration", logComp, logLevel), "KqlPreparedRun")
.Add(CreateKqpAcquireMvccSnapshotTransformer(Gateway, TxState, TransformCtx), "AcquireMvccSnapshot")
.Add(CreateKqpExecutePreparedTransformer(Gateway, Cluster, TxState, TransformCtx), "ExecutePrepared")
.Add(CreateKqpFinalizeTransformer(Gateway, Cluster, TxState, TransformCtx), "Finalize")
@@ -194,7 +194,7 @@ public:
PhysicalOptimizeTransformer = TTransformationPipeline(typesCtx)
.AddServiceTransformers()
- .Add(TLogExprTransformer::Sync("PhysicalOptimizeTransformer", logComp, logLevel), "LogPhysicalOptimize")
+ .Add(TLogExprTransformer::Sync("PhysicalOptimizeTransformer", logComp, logLevel), "LogPhysicalOptimize")
.AddTypeAnnotationTransformer(CreateKqpTypeAnnotationTransformer(Cluster, sessionCtx->TablesPtr(),
*typesCtx, Config))
.Add(CreateKqpCheckQueryTransformer(), "CheckKqlQuery")
@@ -202,27 +202,27 @@ public:
.AddCommonOptimization()
.Add(CreateKqpLogOptTransformer(OptimizeCtx, *typesCtx, Config), "LogicalOptimize")
.Add(CreateKqpPhyOptTransformer(OptimizeCtx, *typesCtx), "PhysicalOptimize")
- .Add(CreateKqpFinalizingOptTransformer(), "FinalizingOptimize")
+ .Add(CreateKqpFinalizingOptTransformer(), "FinalizingOptimize")
.Add(CreateKqpQueryPhasesTransformer(), "QueryPhases")
.Add(CreateKqpQueryEffectsTransformer(OptimizeCtx), "QueryEffects")
- .Add(CreateKqpCheckPhysicalQueryTransformer(), "CheckKqlPhysicalQuery")
+ .Add(CreateKqpCheckPhysicalQueryTransformer(), "CheckKqlPhysicalQuery")
.Build(false);
PhysicalBuildQueryTransformer = TTransformationPipeline(typesCtx)
.AddServiceTransformers()
- .Add(TLogExprTransformer::Sync("PhysicalBuildQueryTransformer", logComp, logLevel), "LogPhysicalBuildQuery")
- .AddTypeAnnotationTransformer(CreateKqpTypeAnnotationTransformer(Cluster, sessionCtx->TablesPtr(), *typesCtx, Config))
+ .Add(TLogExprTransformer::Sync("PhysicalBuildQueryTransformer", logComp, logLevel), "LogPhysicalBuildQuery")
+ .AddTypeAnnotationTransformer(CreateKqpTypeAnnotationTransformer(Cluster, sessionCtx->TablesPtr(), *typesCtx, Config))
.AddPostTypeAnnotation()
- .Add(
- CreateKqpBuildTxsTransformer(
- OptimizeCtx,
- BuildQueryCtx,
- CreateTypeAnnotationTransformer(
- CreateKqpTypeAnnotationTransformer(Cluster, sessionCtx->TablesPtr(), *typesCtx, Config),
- *typesCtx),
- *typesCtx,
- Config),
- "BuildPhysicalTxs")
+ .Add(
+ CreateKqpBuildTxsTransformer(
+ OptimizeCtx,
+ BuildQueryCtx,
+ CreateTypeAnnotationTransformer(
+ CreateKqpTypeAnnotationTransformer(Cluster, sessionCtx->TablesPtr(), *typesCtx, Config),
+ *typesCtx),
+ *typesCtx,
+ Config),
+ "BuildPhysicalTxs")
.Build(false);
PhysicalPeepholeTransformer = TTransformationPipeline(typesCtx)
@@ -243,7 +243,7 @@ public:
.Build(false);
ScanRunQueryTransformer = TTransformationPipeline(typesCtx)
- .Add(CreateKqpCreateSnapshotTransformer(Gateway, TransformCtx, TxState), "CreateSnapshot")
+ .Add(CreateKqpCreateSnapshotTransformer(Gateway, TransformCtx, TxState), "CreateSnapshot")
.Add(CreateKqpExecuteScanTransformer(Gateway, Cluster, TxState, TransformCtx), "ExecuteScan")
.Add(CreateKqpReleaseSnapshotTransformer(Gateway, TxState), "ReleaseSnapshot")
.Build(false);
@@ -254,10 +254,10 @@ public:
{
YQL_ENSURE(TransformCtx->QueryCtx->Type == EKikimrQueryType::Dml);
YQL_ENSURE(TransformCtx->QueryCtx->PrepareOnly);
- YQL_ENSURE(TransformCtx->QueryCtx->PreparingQuery);
- YQL_ENSURE(TMaybeNode<TKiDataQuery>(query));
+ YQL_ENSURE(TransformCtx->QueryCtx->PreparingQuery);
+ YQL_ENSURE(TMaybeNode<TKiDataQuery>(query));
- return PrepareQueryInternal(cluster, TKiDataQuery(query), ctx, settings);
+ return PrepareQueryInternal(cluster, TKiDataQuery(query), ctx, settings);
}
TIntrusivePtr<TAsyncQueryResult> PrepareScanQuery(const TString& cluster, const TExprNode::TPtr& query,
@@ -265,28 +265,28 @@ public:
{
YQL_ENSURE(TransformCtx->QueryCtx->Type == EKikimrQueryType::Scan);
YQL_ENSURE(TransformCtx->QueryCtx->PrepareOnly);
- YQL_ENSURE(TransformCtx->QueryCtx->PreparingQuery);
- YQL_ENSURE(TMaybeNode<TKiDataQuery>(query));
+ YQL_ENSURE(TransformCtx->QueryCtx->PreparingQuery);
+ YQL_ENSURE(TMaybeNode<TKiDataQuery>(query));
TKiDataQuery dataQuery(query);
-
+
if (dataQuery.Results().Size() != 1) {
ctx.AddError(YqlIssue(ctx.GetPosition(dataQuery.Pos()), TIssuesIds::KIKIMR_PRECONDITION_FAILED,
"Scan query should have a single result set."));
- return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues()));
+ return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues()));
}
if (dataQuery.Effects().ArgCount() > 0) {
ctx.AddError(YqlIssue(ctx.GetPosition(dataQuery.Pos()), TIssuesIds::KIKIMR_PRECONDITION_FAILED,
"Scan query cannot have data modifications."));
- return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues()));
+ return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues()));
}
IKikimrQueryExecutor::TExecuteSettings scanSettings(settings);
- return PrepareQueryInternal(cluster, dataQuery, ctx, scanSettings);
+ return PrepareQueryInternal(cluster, dataQuery, ctx, scanSettings);
}
TIntrusivePtr<TAsyncQueryResult> ExecutePreparedDataQuery(const TString& cluster, TExprNode* queryExpr,
- const NKikimrKqp::TPreparedKql& kql, TExprContext& ctx,
+ const NKikimrKqp::TPreparedKql& kql, TExprContext& ctx,
const IKikimrQueryExecutor::TExecuteSettings& settings) override
{
YQL_ENSURE(queryExpr->Type() == TExprNode::World);
@@ -296,8 +296,8 @@ public:
TransformCtx->Reset();
TransformCtx->Settings.CopyFrom(kql.GetSettings());
- TransformCtx->Settings.SetCommitTx(settings.CommitTx);
- TransformCtx->Settings.SetRollbackTx(settings.RollbackTx);
+ TransformCtx->Settings.SetCommitTx(settings.CommitTx);
+ TransformCtx->Settings.SetRollbackTx(settings.RollbackTx);
TransformCtx->PreparedKql = &kql;
YQL_ENSURE(TxState->Tx().EffectiveIsolationLevel);
@@ -305,10 +305,10 @@ public:
bool strictDml = MergeFlagValue(Config->StrictDml.Get(Cluster), settings.StrictDml);
if (!ApplyTableOperations(kql, strictDml, ctx)) {
- return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues()));
+ return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues()));
}
- return MakeIntrusive<TAsyncRunResult>(queryExpr, ctx, *PreparedRunTransformer, *TransformCtx);
+ return MakeIntrusive<TAsyncRunResult>(queryExpr, ctx, *PreparedRunTransformer, *TransformCtx);
}
TIntrusivePtr<TAsyncQueryResult> ExecutePreparedQueryNewEngine(const TString& cluster,
@@ -330,13 +330,13 @@ public:
TIntrusivePtr<TAsyncQueryResult> ExecutePreparedScanQuery(const TString& cluster,
const NYql::TExprNode::TPtr& world, const NKqpProto::TKqpPhyQuery& phyQuery, TExprContext& ctx,
const NActors::TActorId& target) override
- {
- YQL_ENSURE(cluster == Cluster);
+ {
+ YQL_ENSURE(cluster == Cluster);
YQL_ENSURE(phyQuery.GetType() == NKqpProto::TKqpPhyQuery::TYPE_SCAN);
-
+
return ExecutePhysicalScanQuery(world, phyQuery, ctx, target);
- }
-
+ }
+
private:
bool ApplyTableOperations(const TVector<NKqpProto::TKqpTableOp>& operations,
const TVector<NKqpProto::TKqpTableInfo>& tableInfos, bool strictDml, TExprContext& ctx)
@@ -361,17 +361,17 @@ private:
return ApplyTableOperations(operations, tableInfos, strictDml, ctx);
}
- TIntrusivePtr<TAsyncQueryResult> PrepareQueryInternal(const TString& cluster, const TKiDataQuery& dataQuery,
+ TIntrusivePtr<TAsyncQueryResult> PrepareQueryInternal(const TString& cluster, const TKiDataQuery& dataQuery,
TExprContext& ctx, const IKikimrQueryExecutor::TExecuteSettings& settings)
{
YQL_ENSURE(cluster == Cluster);
- auto* queryCtx = TransformCtx->QueryCtx.Get();
-
- if (queryCtx->Type == EKikimrQueryType::Dml) {
+ auto* queryCtx = TransformCtx->QueryCtx.Get();
+
+ if (queryCtx->Type == EKikimrQueryType::Dml) {
ui32 resultsCount = dataQuery.Results().Size();
for (ui32 i = 0; i < resultsCount; ++i) {
- auto& result = *queryCtx->PreparingQuery->AddResults();
+ auto& result = *queryCtx->PreparingQuery->AddResults();
result.SetKqlIndex(0);
result.SetResultIndex(i);
for (const auto& column : dataQuery.Results().Item(i).Columns()) {
@@ -379,43 +379,43 @@ private:
}
result.SetRowsLimit(FromString<ui64>(dataQuery.Results().Item(i).RowsLimit()));
}
- } else {
- // scan query
+ } else {
+ // scan query
}
bool sysColumnsEnabled = TransformCtx->Config->SystemColumnsEnabled();
-
- std::optional<TKqpTransactionInfo::EEngine> engine;
- if (settings.UseNewEngine.Defined()) {
- engine = *settings.UseNewEngine
- ? TKqpTransactionInfo::EEngine::NewEngine
- : TKqpTransactionInfo::EEngine::OldEngine;
- }
- if (!engine.has_value() && Config->UseNewEngine.Get().Defined()) {
- engine = Config->UseNewEngine.Get().Get()
- ? TKqpTransactionInfo::EEngine::NewEngine
- : TKqpTransactionInfo::EEngine::OldEngine;
- }
- if (!engine.has_value() && Config->HasKqpForceNewEngine()) {
- engine = TKqpTransactionInfo::EEngine::NewEngine;
- }
-
- if ((queryCtx->Type == EKikimrQueryType::Scan) ||
- (engine.has_value() && *engine == TKqpTransactionInfo::EEngine::NewEngine))
- {
+
+ std::optional<TKqpTransactionInfo::EEngine> engine;
+ if (settings.UseNewEngine.Defined()) {
+ engine = *settings.UseNewEngine
+ ? TKqpTransactionInfo::EEngine::NewEngine
+ : TKqpTransactionInfo::EEngine::OldEngine;
+ }
+ if (!engine.has_value() && Config->UseNewEngine.Get().Defined()) {
+ engine = Config->UseNewEngine.Get().Get()
+ ? TKqpTransactionInfo::EEngine::NewEngine
+ : TKqpTransactionInfo::EEngine::OldEngine;
+ }
+ if (!engine.has_value() && Config->HasKqpForceNewEngine()) {
+ engine = TKqpTransactionInfo::EEngine::NewEngine;
+ }
+
+ if ((queryCtx->Type == EKikimrQueryType::Scan) ||
+ (engine.has_value() && *engine == TKqpTransactionInfo::EEngine::NewEngine))
+ {
return PrepareQueryNewEngine(cluster, dataQuery, ctx, settings, sysColumnsEnabled);
}
- // OldEngine only
- YQL_ENSURE(!engine.has_value() || *engine == TKqpTransactionInfo::EEngine::OldEngine);
-
+ // OldEngine only
+ YQL_ENSURE(!engine.has_value() || *engine == TKqpTransactionInfo::EEngine::OldEngine);
+
for (const auto& [name, table] : TransformCtx->Tables->GetTables()) {
if (!table.Metadata->SysView.empty()) {
- ctx.AddError(TIssue(ctx.GetPosition(dataQuery.Pos()), TStringBuilder()
+ ctx.AddError(TIssue(ctx.GetPosition(dataQuery.Pos()), TStringBuilder()
<< "Table " << table.Metadata->Name << " is a system view. "
<< "System views are not supported by data queries."
));
- return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues()));
+ return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues()));
}
}
@@ -426,14 +426,14 @@ private:
TExprNode::TPtr optimizedProgram = program.Ptr();
auto status = InstantTransform(*KqlOptimizeTransformer, optimizedProgram, ctx);
if (status != IGraphTransformer::TStatus::Ok || !TMaybeNode<TKiProgram>(optimizedProgram)) {
- ctx.AddError(TIssue(ctx.GetPosition(dataQuery.Pos()), "Failed to optimize KQL query."));
- return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues()));
+ ctx.AddError(TIssue(ctx.GetPosition(dataQuery.Pos()), "Failed to optimize KQL query."));
+ return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues()));
}
YQL_ENSURE(optimizedProgram->GetTypeAnn());
-
- queryCtx->QueryTraits = CollectQueryTraits(TKiProgram(optimizedProgram), ctx);
-
+
+ queryCtx->QueryTraits = CollectQueryTraits(TKiProgram(optimizedProgram), ctx);
+
KqlTypeAnnTransformer->Rewind();
TExprNode::TPtr finalProgram;
@@ -443,8 +443,8 @@ private:
status = PeepHoleOptimizeNode<false>(optimizedProgram, finalProgram, ctx, TypesCtx, KqlTypeAnnTransformer.Get(),
hasNonDeterministicFunctions, peepholeSettings);
if (status != IGraphTransformer::TStatus::Ok) {
- ctx.AddError(TIssue(ctx.GetPosition(dataQuery.Pos()), "Failed to peephole optimize KQL query."));
- return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues()));
+ ctx.AddError(TIssue(ctx.GetPosition(dataQuery.Pos()), "Failed to peephole optimize KQL query."));
+ return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues()));
}
status = ReplaceNonDetFunctionsWithParams(finalProgram, ctx);
@@ -454,7 +454,7 @@ private:
return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues()));
}
- KqlPrepareTransformer->Rewind();
+ KqlPrepareTransformer->Rewind();
NKikimrKqp::TKqlSettings kqlSettings;
kqlSettings.SetCommitTx(settings.CommitTx);
@@ -467,24 +467,24 @@ private:
TxState->Tx().EffectiveIsolationLevel = kqlSettings.GetIsolationLevel();
}
- TransformCtx->QueryCtx->PreparingQuery->SetVersion(NKikimrKqp::TPreparedQuery::VERSION_V1);
- auto kql = TransformCtx->QueryCtx->PreparingQuery->AddKqls();
- kql->MutableSettings()->CopyFrom(TransformCtx->Settings);
+ TransformCtx->QueryCtx->PreparingQuery->SetVersion(NKikimrKqp::TPreparedQuery::VERSION_V1);
+ auto kql = TransformCtx->QueryCtx->PreparingQuery->AddKqls();
+ kql->MutableSettings()->CopyFrom(TransformCtx->Settings);
- FillAstAndPlan(*kql, finalProgram, ctx);
+ FillAstAndPlan(*kql, finalProgram, ctx);
- auto operations = TableOperationsToProto(dataQuery.Operations(), ctx);
- for (auto& op : operations) {
- const auto& tableName = op.GetTable();
+ auto operations = TableOperationsToProto(dataQuery.Operations(), ctx);
+ for (auto& op : operations) {
+ const auto& tableName = op.GetTable();
- kql->AddOperations()->CopyFrom(op);
- const auto& desc = TransformCtx->Tables->GetTable(cluster, tableName);
- TableDescriptionToTableInfo(desc, kql->AddTableInfo());
- }
+ kql->AddOperations()->CopyFrom(op);
+ const auto& desc = TransformCtx->Tables->GetTable(cluster, tableName);
+ TableDescriptionToTableInfo(desc, kql->AddTableInfo());
+ }
- TransformCtx->PreparingKql = kql;
+ TransformCtx->PreparingKql = kql;
- return MakeIntrusive<TAsyncRunResult>(finalProgram, ctx, *KqlPrepareTransformer, *TransformCtx);
+ return MakeIntrusive<TAsyncRunResult>(finalProgram, ctx, *KqlPrepareTransformer, *TransformCtx);
}
TIntrusivePtr<TAsyncQueryResult> PrepareQueryNewEngine(const TString& cluster, const TKiDataQuery& dataQuery,
@@ -495,19 +495,19 @@ private:
YQL_ENSURE(!settings.RollbackTx);
YQL_ENSURE(TransformCtx->QueryCtx->PrepareOnly);
- EKikimrQueryType queryType = TransformCtx->QueryCtx->Type;
- switch (queryType) {
- case EKikimrQueryType::Dml:
+ EKikimrQueryType queryType = TransformCtx->QueryCtx->Type;
+ switch (queryType) {
+ case EKikimrQueryType::Dml:
case EKikimrQueryType::Scan:
- break;
- default:
- YQL_ENSURE(false, "PrepareQueryNewEngine, unexpected query type: " << queryType);
- }
-
- if (!Config->HasAllowKqpNewEngine() && queryType == EKikimrQueryType::Dml) {
+ break;
+ default:
+ YQL_ENSURE(false, "PrepareQueryNewEngine, unexpected query type: " << queryType);
+ }
+
+ if (!Config->HasAllowKqpNewEngine() && queryType == EKikimrQueryType::Dml) {
ctx.AddError(TIssue(ctx.GetPosition(dataQuery.Pos()),
"NewEngine execution is not allowed on this cluster."));
- return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues()));
+ return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues()));
}
auto kqlQuery = BuildKqlQuery(dataQuery, *TransformCtx->Tables, ctx, sysColumnsEnabled, OptimizeCtx);
@@ -516,7 +516,7 @@ private:
}
auto query = kqlQuery->Ptr();
- YQL_CLOG(INFO, ProviderKqp) << "Initial KQL query: " << KqpExprToPrettyString(*query, ctx);
+ YQL_CLOG(INFO, ProviderKqp) << "Initial KQL query: " << KqpExprToPrettyString(*query, ctx);
TransformCtx->Reset();
TransformCtx->Settings = NKikimrKqp::TKqlSettings();
@@ -526,10 +526,10 @@ private:
auto status = InstantTransform(*PhysicalOptimizeTransformer, optimizedQuery, ctx);
if (status != IGraphTransformer::TStatus::Ok) {
ctx.AddError(TIssue(ctx.GetPosition(query->Pos()), "Failed to optimize query."));
- return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues()));
+ return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues()));
}
- YQL_CLOG(INFO, ProviderKqp) << "Optimized KQL query: " << KqpExprToPrettyString(*optimizedQuery, ctx);
+ YQL_CLOG(INFO, ProviderKqp) << "Optimized KQL query: " << KqpExprToPrettyString(*optimizedQuery, ctx);
BuildQueryCtx->Reset();
PhysicalBuildQueryTransformer->Rewind();
@@ -537,7 +537,7 @@ private:
status = InstantTransform(*PhysicalBuildQueryTransformer, builtQuery, ctx);
if (status != IGraphTransformer::TStatus::Ok) {
ctx.AddError(TIssue(ctx.GetPosition(query->Pos()), "Failed to build physical query."));
- return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues()));
+ return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues()));
}
PhysicalPeepholeTransformer->Rewind();
@@ -549,16 +549,16 @@ private:
ctx.IssueManager.GetIssues()));
}
- YQL_CLOG(INFO, ProviderKqp) << "Physical KQL query: " << KqpExprToPrettyString(*builtQuery, ctx);
+ YQL_CLOG(INFO, ProviderKqp) << "Physical KQL query: " << KqpExprToPrettyString(*builtQuery, ctx);
- auto& preparedQuery = *TransformCtx->QueryCtx->PreparingQuery;
+ auto& preparedQuery = *TransformCtx->QueryCtx->PreparingQuery;
TKqpPhysicalQuery physicalQuery(transformedQuery);
auto compiler = CreateKqpQueryCompiler(Cluster, OptimizeCtx->Tables, FuncRegistry);
auto ret = compiler->CompilePhysicalQuery(physicalQuery, dataQuery.Operations(),
*preparedQuery.MutablePhysicalQuery(), ctx);
if (!ret) {
ctx.AddError(TIssue(ctx.GetPosition(query->Pos()), "Failed to compile physical query."));
- return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues()));
+ return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues()));
}
preparedQuery.SetVersion(NKikimrKqp::TPreparedQuery::VERSION_PHYSICAL_V1);
// TODO(sk): only on stats mode or if explain-only
@@ -629,7 +629,7 @@ private:
TAutoPtr<IGraphTransformer> KqlTypeAnnTransformer;
TAutoPtr<IGraphTransformer> KqlOptimizeTransformer;
- TAutoPtr<IGraphTransformer> KqlPrepareTransformer;
+ TAutoPtr<IGraphTransformer> KqlPrepareTransformer;
TAutoPtr<IGraphTransformer> PreparedRunTransformer;
TAutoPtr<IGraphTransformer> PreparedExplainTransformer;
@@ -803,5 +803,5 @@ TIntrusivePtr<IKqpRunner> CreateKqpRunner(TIntrusivePtr<IKqpGateway> gateway, co
return new TKqpRunner(gateway, cluster, typesCtx, sessionCtx, funcRegistry);
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/host/ya.make b/ydb/core/kqp/host/ya.make
index 01f30b7cc4..e0f595515c 100644
--- a/ydb/core/kqp/host/ya.make
+++ b/ydb/core/kqp/host/ya.make
@@ -7,7 +7,7 @@ OWNER(
SRCS(
kqp_host.cpp
- kqp_ne_helper.cpp
+ kqp_ne_helper.cpp
kqp_run_data.cpp
kqp_explain_prepared.cpp
kqp_run_physical.cpp
diff --git a/ydb/core/kqp/kqp.h b/ydb/core/kqp/kqp.h
index 729d670bdd..09f794591d 100644
--- a/ydb/core/kqp/kqp.h
+++ b/ydb/core/kqp/kqp.h
@@ -3,7 +3,7 @@
#include "kqp_query_replay.h"
#include <ydb/core/kqp/common/kqp_common.h>
#include <ydb/core/kqp/counters/kqp_counters.h>
-#include <ydb/core/kqp/provider/yql_kikimr_query_traits.h>
+#include <ydb/core/kqp/provider/yql_kikimr_query_traits.h>
#include <ydb/public/api/protos/ydb_status_codes.pb.h>
#include <ydb/library/yql/dq/actors/dq.h>
@@ -29,25 +29,25 @@ inline NActors::TActorId MakeKqpCompileServiceID(ui32 nodeId) {
}
inline NActors::TActorId MakeKqpResourceManagerServiceID(ui32 nodeId) {
- const char name[12] = "kqp_resman";
+ const char name[12] = "kqp_resman";
return NActors::TActorId(nodeId, TStringBuf(name, 12));
-}
-
-inline NActors::TActorId MakeKqpRmServiceID(ui32 nodeId) {
- const char name[12] = "kqp_rm";
- return NActors::TActorId(nodeId, TStringBuf(name, 12));
-}
-
-inline NActors::TActorId MakeKqpNodeServiceID(ui32 nodeId) {
- const char name[12] = "kqp_node";
- return NActors::TActorId(nodeId, TStringBuf(name, 12));
-}
-
+}
+
+inline NActors::TActorId MakeKqpRmServiceID(ui32 nodeId) {
+ const char name[12] = "kqp_rm";
+ return NActors::TActorId(nodeId, TStringBuf(name, 12));
+}
+
+inline NActors::TActorId MakeKqpNodeServiceID(ui32 nodeId) {
+ const char name[12] = "kqp_node";
+ return NActors::TActorId(nodeId, TStringBuf(name, 12));
+}
+
inline NActors::TActorId MakeKqpLocalFileSpillingServiceID(ui32 nodeId) {
- const char name[12] = "kqp_lfspill";
+ const char name[12] = "kqp_lfspill";
return NActors::TActorId(nodeId, TStringBuf(name, 12));
-}
-
+}
+
class TKqpShutdownController;
class TKqpShutdownState : public TThrRefBase {
@@ -164,8 +164,8 @@ public:
}
};
-using TPreparedQueryConstPtr = std::shared_ptr<const NKikimrKqp::TPreparedQuery>;
-
+using TPreparedQueryConstPtr = std::shared_ptr<const NKikimrKqp::TPreparedQuery>;
+
struct TKqpCompileResult {
using TConstPtr = std::shared_ptr<const TKqpCompileResult>;
@@ -199,9 +199,9 @@ struct TKqpCompileResult {
TMaybe<TKqpQueryId> Query;
TString Uid;
- TPreparedQueryConstPtr PreparedQuery;
- TPreparedQueryConstPtr PreparedQueryNewEngine;
- std::optional<TQueryTraits> QueryTraits;
+ TPreparedQueryConstPtr PreparedQuery;
+ TPreparedQueryConstPtr PreparedQueryNewEngine;
+ std::optional<TQueryTraits> QueryTraits;
};
struct TEvKqp {
@@ -364,7 +364,7 @@ struct TEvKqp {
struct TEvCompileRequest : public TEventLocal<TEvCompileRequest, TKqpEvents::EvCompileRequest> {
TEvCompileRequest(const TString& userToken, const TMaybe<TString>& uid, TMaybe<TKqpQueryId>&& query,
- bool keepInCache, TInstant deadline, TKqpDbCountersPtr dbCounters)
+ bool keepInCache, TInstant deadline, TKqpDbCountersPtr dbCounters)
: UserToken(userToken)
, Uid(uid)
, Query(std::move(query))
@@ -379,7 +379,7 @@ struct TEvKqp {
TMaybe<TString> Uid;
TMaybe<TKqpQueryId> Query;
bool KeepInCache = false;
- // it is allowed for local event to use absolute time (TInstant) instead of time interval (TDuration)
+ // it is allowed for local event to use absolute time (TInstant) instead of time interval (TDuration)
TInstant Deadline;
TKqpDbCountersPtr DbCounters;
TMaybe<bool> DocumentApiRestricted;
@@ -387,7 +387,7 @@ struct TEvKqp {
struct TEvRecompileRequest : public TEventLocal<TEvRecompileRequest, TKqpEvents::EvRecompileRequest> {
TEvRecompileRequest(const TString& userToken, const TString& uid, const TMaybe<TKqpQueryId>& query,
- TInstant deadline, TKqpDbCountersPtr dbCounters)
+ TInstant deadline, TKqpDbCountersPtr dbCounters)
: UserToken(userToken)
, Uid(uid)
, Query(query)
@@ -408,10 +408,10 @@ struct TEvKqp {
TKqpCompileResult::TConstPtr CompileResult;
NKqpProto::TKqpStatsCompile Stats;
- std::optional<TString> ReplayMessage;
-
- ui32 ForceNewEnginePercent = 0;
- ui32 ForceNewEngineLevel = 0;
+ std::optional<TString> ReplayMessage;
+
+ ui32 ForceNewEnginePercent = 0;
+ ui32 ForceNewEngineLevel = 0;
};
struct TEvCompileInvalidateRequest : public TEventLocal<TEvCompileInvalidateRequest,
@@ -424,18 +424,18 @@ struct TEvKqp {
TString Uid;
TKqpDbCountersPtr DbCounters;
};
-
- struct TEvInitiateShutdownRequest : public TEventLocal<TEvInitiateShutdownRequest, TKqpEvents::EvInitiateShutdownRequest> {
- TIntrusivePtr<TKqpShutdownState> ShutdownState;
-
- TEvInitiateShutdownRequest(TIntrusivePtr<TKqpShutdownState> ShutdownState)
- : ShutdownState(ShutdownState)
- {}
- };
-
- using TEvAbortExecution = NYql::NDq::TEvDq::TEvAbortExecution;
-};
-
+
+ struct TEvInitiateShutdownRequest : public TEventLocal<TEvInitiateShutdownRequest, TKqpEvents::EvInitiateShutdownRequest> {
+ TIntrusivePtr<TKqpShutdownState> ShutdownState;
+
+ TEvInitiateShutdownRequest(TIntrusivePtr<TKqpShutdownState> ShutdownState)
+ : ShutdownState(ShutdownState)
+ {}
+ };
+
+ using TEvAbortExecution = NYql::NDq::TEvDq::TEvAbortExecution;
+};
+
class TKqpRequestInfo {
public:
TKqpRequestInfo(const TString& traceId, const TString& sessionId)
@@ -480,7 +480,7 @@ IActor* CreateKqpProxyService(const NKikimrConfig::TLogConfig& logConfig,
TVector<NKikimrKqp::TKqpSetting>&& settings,
std::shared_ptr<IQueryReplayBackendFactory> queryReplayFactory);
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
template<>
diff --git a/ydb/core/kqp/kqp_compile_actor.cpp b/ydb/core/kqp/kqp_compile_actor.cpp
index dc5261a066..27cf296a03 100644
--- a/ydb/core/kqp/kqp_compile_actor.cpp
+++ b/ydb/core/kqp/kqp_compile_actor.cpp
@@ -22,12 +22,12 @@
namespace NKikimr {
namespace NKqp {
-static std::atomic<bool> FailForcedNewEngineCompilationStatus = false;
-
-void FailForcedNewEngineCompilationForTests(bool fail) {
- FailForcedNewEngineCompilationStatus = fail;
-}
-
+static std::atomic<bool> FailForcedNewEngineCompilationStatus = false;
+
+void FailForcedNewEngineCompilationForTests(bool fail) {
+ FailForcedNewEngineCompilationStatus = fail;
+}
+
static const TString YqlName = "CompileActor";
using namespace NKikimrConfig;
@@ -46,7 +46,7 @@ public:
TKqpCompileActor(const TActorId& owner, const TKqpSettings::TConstPtr& kqpSettings,
const TTableServiceConfig& serviceConfig, TIntrusivePtr<TModuleResolverState> moduleResolverState,
TIntrusivePtr<TKqpCounters> counters, const TString& uid, const TKqpQueryId& query, const TString& userToken,
- TKqpDbCountersPtr dbCounters, bool recompileWithNewEngine)
+ TKqpDbCountersPtr dbCounters, bool recompileWithNewEngine)
: Owner(owner)
, ModuleResolverState(moduleResolverState)
, Counters(counters)
@@ -56,9 +56,9 @@ public:
, DbCounters(dbCounters)
, Config(MakeIntrusive<TKikimrConfiguration>())
, CompilationTimeout(TDuration::MilliSeconds(serviceConfig.GetCompileTimeoutMs()))
- , RecompileWithNewEngine(recompileWithNewEngine)
+ , RecompileWithNewEngine(recompileWithNewEngine)
{
- Config->Init(kqpSettings->DefaultSettings.GetDefaultSettings(), Query.Cluster, kqpSettings->Settings, false);
+ Config->Init(kqpSettings->DefaultSettings.GetDefaultSettings(), Query.Cluster, kqpSettings->Settings, false);
if (!Query.Database.empty()) {
Config->_KqpTablePathPrefix = Query.Database;
@@ -85,10 +85,10 @@ public:
TYqlLogScope logScope(ctx, NKikimrServices::KQP_YQL, YqlName, "");
- TKqpRequestCounters::TPtr counters = new TKqpRequestCounters;
- counters->Counters = Counters;
- counters->DbCounters = DbCounters;
- counters->TxProxyMon = new NTxProxy::TTxProxyMon(AppData(ctx)->Counters);
+ TKqpRequestCounters::TPtr counters = new TKqpRequestCounters;
+ counters->Counters = Counters;
+ counters->DbCounters = DbCounters;
+ counters->TxProxyMon = new NTxProxy::TTxProxyMon(AppData(ctx)->Counters);
std::shared_ptr<NYql::IKikimrGateway::IKqpTableMetadataLoader> loader = std::make_shared<TKqpTableMetadataLoader>(TlsActivationContext->ActorSystem(), true);
Gateway = CreateKikimrIcGateway(Query.Cluster, Query.Database, std::move(loader), ctx.ExecutorThread.ActorSystem,
ctx.SelfID.NodeId(), counters, MakeMiniKQLCompileServiceID());
@@ -100,7 +100,7 @@ public:
AppData(ctx)->FunctionRegistry, false);
IKqpHost::TPrepareSettings prepareSettings;
- // prepareSettings.UseNewEngine = use default settings
+ // prepareSettings.UseNewEngine = use default settings
prepareSettings.DocumentApiRestricted = Query.Settings.DocumentApiRestricted;
NCpuTime::TCpuTimer timer(CompileCpuTime);
@@ -134,28 +134,28 @@ private:
}
}
- STFUNC(RecompileNewEngineState) {
- try {
- switch (ev->GetTypeRewrite()) {
- HFunc(TEvKqp::TEvContinueProcess, HandleRecompile);
- CFunc(TEvents::TSystem::Wakeup, HandleRecompileTimeout);
- default: {
- LOG_ERROR_S(ctx, NKikimrServices::KQP_COMPILE_ACTOR, "Internal error in RecompileNewEngineState"
- << ", self: " << ctx.SelfID
- << ", unexpected event: " << ev->GetTypeRewrite());
-
- Reply(KqpCompileResult, ctx);
- }
- }
- } catch (const yexception& e) {
- LOG_ERROR_S(ctx, NKikimrServices::KQP_COMPILE_ACTOR, "Internal error in RecompileNewEngineState"
- << ", self: " << ctx.SelfID
- << ", message: " << e.what());
-
- Reply(KqpCompileResult, ctx);
- }
- }
-
+ STFUNC(RecompileNewEngineState) {
+ try {
+ switch (ev->GetTypeRewrite()) {
+ HFunc(TEvKqp::TEvContinueProcess, HandleRecompile);
+ CFunc(TEvents::TSystem::Wakeup, HandleRecompileTimeout);
+ default: {
+ LOG_ERROR_S(ctx, NKikimrServices::KQP_COMPILE_ACTOR, "Internal error in RecompileNewEngineState"
+ << ", self: " << ctx.SelfID
+ << ", unexpected event: " << ev->GetTypeRewrite());
+
+ Reply(KqpCompileResult, ctx);
+ }
+ }
+ } catch (const yexception& e) {
+ LOG_ERROR_S(ctx, NKikimrServices::KQP_COMPILE_ACTOR, "Internal error in RecompileNewEngineState"
+ << ", self: " << ctx.SelfID
+ << ", message: " << e.what());
+
+ Reply(KqpCompileResult, ctx);
+ }
+ }
+
private:
void Continue(const TActorContext &ctx) {
TActorSystem* actorSystem = ctx.ExecutorThread.ActorSystem;
@@ -248,7 +248,7 @@ private:
return;
}
- auto kqpResult = std::move(AsyncCompileResult->GetResult());
+ auto kqpResult = std::move(AsyncCompileResult->GetResult());
auto status = GetYdbStatus(kqpResult);
auto database = Query.Database;
@@ -263,48 +263,48 @@ private:
KqpCompileResult = TKqpCompileResult::Make(Uid, std::move(Query), status, kqpResult.Issues());
if (status == Ydb::StatusIds::SUCCESS) {
- YQL_ENSURE(kqpResult.PreparingQuery);
- KqpCompileResult->PreparedQuery.reset(kqpResult.PreparingQuery.release());
- KqpCompileResult->QueryTraits = kqpResult.QueryTraits;
+ YQL_ENSURE(kqpResult.PreparingQuery);
+ KqpCompileResult->PreparedQuery.reset(kqpResult.PreparingQuery.release());
+ KqpCompileResult->QueryTraits = kqpResult.QueryTraits;
- auto now = TInstant::Now();
- auto duration = now - StartTime;
+ auto now = TInstant::Now();
+ auto duration = now - StartTime;
Counters->ReportCompileDurations(DbCounters, duration, CompileCpuTime);
LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_ACTOR, "Compilation successful"
<< ", self: " << ctx.SelfID
- << ", duration: " << duration
- << ", traits: " << (KqpCompileResult->QueryTraits ? KqpCompileResult->QueryTraits->ToString() : "<none>"));
-
- if (RecompileWithNewEngine &&
- KqpCompileResult->PreparedQuery->GetVersion() == NKikimrKqp::TPreparedQuery::VERSION_V1)
- {
- LOG_NOTICE_S(ctx, NKikimrServices::KQP_COMPILE_ACTOR, "About to recompile with NewEngine"
- << ", self: " << ctx.SelfID);
-
- RecompileStartTime = now;
-
- IKqpHost::TPrepareSettings prepareSettings;
- prepareSettings.UseNewEngine = true;
- prepareSettings.DocumentApiRestricted = KqpCompileResult->Query->Settings.DocumentApiRestricted;
-
- NCpuTime::TCpuTimer timer(CompileCpuTime);
- AsyncCompileResult = KqpHost->PrepareDataQuery(KqpCompileResult->Query->Text, prepareSettings);
-
- Continue(ctx);
- Become(&TKqpCompileActor::RecompileNewEngineState);
-
- return;
- }
+ << ", duration: " << duration
+ << ", traits: " << (KqpCompileResult->QueryTraits ? KqpCompileResult->QueryTraits->ToString() : "<none>"));
+
+ if (RecompileWithNewEngine &&
+ KqpCompileResult->PreparedQuery->GetVersion() == NKikimrKqp::TPreparedQuery::VERSION_V1)
+ {
+ LOG_NOTICE_S(ctx, NKikimrServices::KQP_COMPILE_ACTOR, "About to recompile with NewEngine"
+ << ", self: " << ctx.SelfID);
+
+ RecompileStartTime = now;
+
+ IKqpHost::TPrepareSettings prepareSettings;
+ prepareSettings.UseNewEngine = true;
+ prepareSettings.DocumentApiRestricted = KqpCompileResult->Query->Settings.DocumentApiRestricted;
+
+ NCpuTime::TCpuTimer timer(CompileCpuTime);
+ AsyncCompileResult = KqpHost->PrepareDataQuery(KqpCompileResult->Query->Text, prepareSettings);
+
+ Continue(ctx);
+ Become(&TKqpCompileActor::RecompileNewEngineState);
+
+ return;
+ }
} else {
- LOG_ERROR_S(ctx, NKikimrServices::KQP_COMPILE_ACTOR, "Compilation failed"
- << ", self: " << ctx.SelfID
- << ", status: " << Ydb::StatusIds_StatusCode_Name(status)
- << ", issues: " << kqpResult.Issues().ToString());
+ LOG_ERROR_S(ctx, NKikimrServices::KQP_COMPILE_ACTOR, "Compilation failed"
+ << ", self: " << ctx.SelfID
+ << ", status: " << Ydb::StatusIds_StatusCode_Name(status)
+ << ", issues: " << kqpResult.Issues().ToString());
Counters->ReportCompileError(DbCounters);
}
- Reply(KqpCompileResult, ctx);
+ Reply(KqpCompileResult, ctx);
}
void HandleTimeout(const TActorContext& ctx) {
@@ -319,55 +319,55 @@ private:
return ReplyError(Ydb::StatusIds::TIMEOUT, {issue}, ctx);
}
- void HandleRecompile(TEvKqp::TEvContinueProcess::TPtr &ev, const TActorContext &ctx) {
- Y_ENSURE(!ev->Get()->QueryId);
-
+ void HandleRecompile(TEvKqp::TEvContinueProcess::TPtr &ev, const TActorContext &ctx) {
+ Y_ENSURE(!ev->Get()->QueryId);
+
TYqlLogScope logScope(ctx, NKikimrServices::KQP_YQL, YqlName, "");
-
- if (!ev->Get()->Finished) {
- NCpuTime::TCpuTimer timer(CompileCpuTime);
- Continue(ctx);
- return;
- }
-
- auto kqpResult = std::move(AsyncCompileResult->GetResult());
- auto status = GetYdbStatus(kqpResult);
-
- if (status == Ydb::StatusIds::SUCCESS && !FailForcedNewEngineCompilationStatus.load(std::memory_order_relaxed)) {
- YQL_ENSURE(kqpResult.PreparingQuery);
- KqpCompileResult->PreparedQueryNewEngine.reset(kqpResult.PreparingQuery.release());
-
- auto duration = TInstant::Now() - RecompileStartTime;
+
+ if (!ev->Get()->Finished) {
+ NCpuTime::TCpuTimer timer(CompileCpuTime);
+ Continue(ctx);
+ return;
+ }
+
+ auto kqpResult = std::move(AsyncCompileResult->GetResult());
+ auto status = GetYdbStatus(kqpResult);
+
+ if (status == Ydb::StatusIds::SUCCESS && !FailForcedNewEngineCompilationStatus.load(std::memory_order_relaxed)) {
+ YQL_ENSURE(kqpResult.PreparingQuery);
+ KqpCompileResult->PreparedQueryNewEngine.reset(kqpResult.PreparingQuery.release());
+
+ auto duration = TInstant::Now() - RecompileStartTime;
Counters->ReportCompileDurations(DbCounters, duration, CompileCpuTime);
-
- LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_ACTOR, "ReCompilation successful"
- << ", self: " << ctx.SelfID
- << ", duration: " << duration);
- } else {
+
+ LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_ACTOR, "ReCompilation successful"
+ << ", self: " << ctx.SelfID
+ << ", duration: " << duration);
+ } else {
Counters->ReportCompileError(DbCounters);
- Counters->ForceNewEngineCompileErrors->Inc();
-
- LOG_ERROR_S(ctx, NKikimrServices::KQP_COMPILE_ACTOR, "ReCompilation failed"
- << ", self: " << ctx.SelfID
- << ", query: " << KqpCompileResult->Query->Text
- << ", status: " << Ydb::StatusIds_StatusCode_Name(status)
- << ", issues: " << kqpResult.Issues().ToString());
- }
-
- Reply(KqpCompileResult, ctx);
- }
-
- void HandleRecompileTimeout(const TActorContext& ctx) {
- LOG_WARN_S(ctx, NKikimrServices::KQP_COMPILE_ACTOR, "ReCompilation timeout"
- << ", self: " << ctx.SelfID
- << ", cluster: " << Query.Cluster
- << ", database: " << Query.Database
- << ", text: \"" << EscapeC(Query.Text) << "\""
- << ", startTime: " << StartTime);
-
- return Reply(KqpCompileResult, ctx);
- }
-
+ Counters->ForceNewEngineCompileErrors->Inc();
+
+ LOG_ERROR_S(ctx, NKikimrServices::KQP_COMPILE_ACTOR, "ReCompilation failed"
+ << ", self: " << ctx.SelfID
+ << ", query: " << KqpCompileResult->Query->Text
+ << ", status: " << Ydb::StatusIds_StatusCode_Name(status)
+ << ", issues: " << kqpResult.Issues().ToString());
+ }
+
+ Reply(KqpCompileResult, ctx);
+ }
+
+ void HandleRecompileTimeout(const TActorContext& ctx) {
+ LOG_WARN_S(ctx, NKikimrServices::KQP_COMPILE_ACTOR, "ReCompilation timeout"
+ << ", self: " << ctx.SelfID
+ << ", cluster: " << Query.Cluster
+ << ", database: " << Query.Database
+ << ", text: \"" << EscapeC(Query.Text) << "\""
+ << ", startTime: " << StartTime);
+
+ return Reply(KqpCompileResult, ctx);
+ }
+
private:
TActorId Owner;
TIntrusivePtr<TModuleResolverState> ModuleResolverState;
@@ -378,10 +378,10 @@ private:
TKqpDbCountersPtr DbCounters;
TKikimrConfiguration::TPtr Config;
TDuration CompilationTimeout;
- bool RecompileWithNewEngine;
+ bool RecompileWithNewEngine;
TInstant StartTime;
TDuration CompileCpuTime;
- TInstant RecompileStartTime;
+ TInstant RecompileStartTime;
TActorId TimeoutTimerActorId;
TIntrusivePtr<IKqpGateway> Gateway;
TIntrusivePtr<IKqpHost> KqpHost;
@@ -403,11 +403,11 @@ void ApplyServiceConfig(TKikimrConfiguration& kqpConfig, const TTableServiceConf
IActor* CreateKqpCompileActor(const TActorId& owner, const TKqpSettings::TConstPtr& kqpSettings,
const TTableServiceConfig& serviceConfig, TIntrusivePtr<TModuleResolverState> moduleResolverState,
TIntrusivePtr<TKqpCounters> counters, const TString& uid, const TKqpQueryId& query, const TString& userToken,
- TKqpDbCountersPtr dbCounters, bool recompileWithNewEngine)
+ TKqpDbCountersPtr dbCounters, bool recompileWithNewEngine)
{
return new TKqpCompileActor(owner, kqpSettings, serviceConfig, moduleResolverState, counters, uid,
- std::move(query), userToken, dbCounters, recompileWithNewEngine);
+ std::move(query), userToken, dbCounters, recompileWithNewEngine);
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/kqp_compile_request.cpp b/ydb/core/kqp/kqp_compile_request.cpp
index cda1d71769..2c18af159b 100644
--- a/ydb/core/kqp/kqp_compile_request.cpp
+++ b/ydb/core/kqp/kqp_compile_request.cpp
@@ -57,7 +57,7 @@ public:
return;
}
- if (!NavigateTables(*compileResult->PreparedQuery, compileResult->Query->Database, ctx)) {
+ if (!NavigateTables(*compileResult->PreparedQuery, compileResult->Query->Database, ctx)) {
ctx.Send(Owner, ev->Release().Release());
Die(ctx);
return;
diff --git a/ydb/core/kqp/kqp_compile_service.cpp b/ydb/core/kqp/kqp_compile_service.cpp
index 1c4b3945ec..da886224a2 100644
--- a/ydb/core/kqp/kqp_compile_service.cpp
+++ b/ydb/core/kqp/kqp_compile_service.cpp
@@ -41,18 +41,18 @@ public:
TItem* item = &const_cast<TItem&>(*it.first);
auto removedItem = List.Insert(item);
- IncBytes(item->Value.CompileResult->PreparedQuery->ByteSize());
+ IncBytes(item->Value.CompileResult->PreparedQuery->ByteSize());
if (item->Value.CompileResult->PreparedQueryNewEngine) {
IncBytes(item->Value.CompileResult->PreparedQueryNewEngine->ByteSize());
- }
+ }
if (removedItem) {
- DecBytes(removedItem->Value.CompileResult->PreparedQuery->ByteSize());
+ DecBytes(removedItem->Value.CompileResult->PreparedQuery->ByteSize());
if (removedItem->Value.CompileResult->PreparedQueryNewEngine) {
DecBytes(removedItem->Value.CompileResult->PreparedQueryNewEngine->ByteSize());
- }
-
+ }
+
QueryIndex.erase(*removedItem->Value.CompileResult->Query);
Index.erase(removedItem->Key);
}
@@ -104,10 +104,10 @@ public:
TItem* item = &const_cast<TItem&>(*it);
List.Erase(item);
- DecBytes(item->Value.CompileResult->PreparedQuery->ByteSize());
+ DecBytes(item->Value.CompileResult->PreparedQuery->ByteSize());
if (item->Value.CompileResult->PreparedQueryNewEngine) {
DecBytes(item->Value.CompileResult->PreparedQueryNewEngine->ByteSize());
- }
+ }
Y_VERIFY(item->Value.CompileResult);
Y_VERIFY(item->Value.CompileResult->Query);
@@ -141,13 +141,13 @@ public:
return prevSize - Size();
}
- void Clear() {
- List = TList(List.GetMaxSize());
- Index.clear();
- QueryIndex.clear();
- ByteSize = 0;
- }
-
+ void Clear() {
+ List = TList(List.GetMaxSize());
+ Index.clear();
+ QueryIndex.clear();
+ ByteSize = 0;
+ }
+
private:
void DecBytes(ui64 bytes) {
if (bytes > ByteSize) {
@@ -314,12 +314,12 @@ public:
Y_UNUSED(ctx);
QueryReplayBackend.Reset(CreateQueryReplayBackend(Config, Counters, QueryReplayFactory));
- // Subscribe for TableService config changes
- ui32 tableServiceConfigKind = (ui32) NKikimrConsole::TConfigItem::TableServiceConfigItem;
- Send(NConsole::MakeConfigsDispatcherID(SelfId().NodeId()),
- new NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest({tableServiceConfigKind}),
- IEventHandle::FlagTrackDelivery);
-
+ // Subscribe for TableService config changes
+ ui32 tableServiceConfigKind = (ui32) NKikimrConsole::TConfigItem::TableServiceConfigItem;
+ Send(NConsole::MakeConfigsDispatcherID(SelfId().NodeId()),
+ new NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest({tableServiceConfigKind}),
+ IEventHandle::FlagTrackDelivery);
+
Become(&TKqpCompileService::MainState);
if (Config.GetCompileQueryCacheTTLSec()) {
StartCheckQueriesTtlTimer(ctx);
@@ -333,67 +333,67 @@ private:
HFunc(TEvKqp::TEvCompileResponse, Handle);
HFunc(TEvKqp::TEvCompileInvalidateRequest, Handle);
HFunc(TEvKqp::TEvRecompileRequest, Handle);
-
- hFunc(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse, HandleConfig);
- hFunc(NConsole::TEvConsole::TEvConfigNotificationRequest, HandleConfig);
- hFunc(TEvents::TEvUndelivered, HandleUndelivery);
-
+
+ hFunc(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse, HandleConfig);
+ hFunc(NConsole::TEvConsole::TEvConfigNotificationRequest, HandleConfig);
+ hFunc(TEvents::TEvUndelivered, HandleUndelivery);
+
CFunc(TEvents::TSystem::Wakeup, HandleTimeout);
- cFunc(TEvents::TEvPoison::EventType, PassAway);
+ cFunc(TEvents::TEvPoison::EventType, PassAway);
default:
Y_FAIL("TKqpCompileService: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite());
}
}
private:
- void HandleConfig(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse::TPtr&) {
- LOG_INFO(*TlsActivationContext, NKikimrServices::KQP_COMPILE_SERVICE, "Subscribed for config changes");
- }
-
- void HandleConfig(NConsole::TEvConsole::TEvConfigNotificationRequest::TPtr& ev) {
- auto &event = ev->Get()->Record;
-
- ui32 prevForceNewEnginePercent = Config.GetForceNewEnginePercent();
- ui32 prevForceNewEngineLevel = Config.GetForceNewEngineLevel();
-
- Config.Swap(event.MutableConfig()->MutableTableServiceConfig());
- LOG_INFO(*TlsActivationContext, NKikimrServices::KQP_COMPILE_SERVICE, "Updated config");
-
- auto responseEv = MakeHolder<NConsole::TEvConsole::TEvConfigNotificationResponse>(event);
- Send(ev->Sender, responseEv.Release(), IEventHandle::FlagTrackDelivery, ev->Cookie);
-
- if (Config.GetForceNewEnginePercent() != prevForceNewEnginePercent ||
- Config.GetForceNewEngineLevel() != prevForceNewEngineLevel)
- {
- LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::KQP_COMPILE_SERVICE,
- "ForceNewEnginePercent/Level was changed from "
- << prevForceNewEnginePercent << '/' << prevForceNewEngineLevel << " to "
- << Config.GetForceNewEnginePercent() << '/' << Config.GetForceNewEngineLevel());
-
- if (prevForceNewEnginePercent == 0 && Config.GetForceNewEnginePercent() != 0) {
- // clear cache only on `enable feature` action
- QueryCache.Clear();
- }
- }
- }
-
- void HandleUndelivery(TEvents::TEvUndelivered::TPtr& ev) {
- switch (ev->Get()->SourceType) {
- case NConsole::TEvConfigsDispatcher::EvSetConfigSubscriptionRequest:
- LOG_CRIT(*TlsActivationContext, NKikimrServices::KQP_COMPILE_SERVICE,
- "Failed to deliver subscription request to config dispatcher");
- break;
- case NConsole::TEvConsole::EvConfigNotificationResponse:
- LOG_ERROR(*TlsActivationContext, NKikimrServices::KQP_COMPILE_SERVICE,
- "Failed to deliver config notification response");
- break;
- default:
- LOG_ERROR(*TlsActivationContext, NKikimrServices::KQP_COMPILE_SERVICE,
- "Undelivered event with unexpected source type: %d", ev->Get()->SourceType);
- break;
- }
- }
-
+ void HandleConfig(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse::TPtr&) {
+ LOG_INFO(*TlsActivationContext, NKikimrServices::KQP_COMPILE_SERVICE, "Subscribed for config changes");
+ }
+
+ void HandleConfig(NConsole::TEvConsole::TEvConfigNotificationRequest::TPtr& ev) {
+ auto &event = ev->Get()->Record;
+
+ ui32 prevForceNewEnginePercent = Config.GetForceNewEnginePercent();
+ ui32 prevForceNewEngineLevel = Config.GetForceNewEngineLevel();
+
+ Config.Swap(event.MutableConfig()->MutableTableServiceConfig());
+ LOG_INFO(*TlsActivationContext, NKikimrServices::KQP_COMPILE_SERVICE, "Updated config");
+
+ auto responseEv = MakeHolder<NConsole::TEvConsole::TEvConfigNotificationResponse>(event);
+ Send(ev->Sender, responseEv.Release(), IEventHandle::FlagTrackDelivery, ev->Cookie);
+
+ if (Config.GetForceNewEnginePercent() != prevForceNewEnginePercent ||
+ Config.GetForceNewEngineLevel() != prevForceNewEngineLevel)
+ {
+ LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::KQP_COMPILE_SERVICE,
+ "ForceNewEnginePercent/Level was changed from "
+ << prevForceNewEnginePercent << '/' << prevForceNewEngineLevel << " to "
+ << Config.GetForceNewEnginePercent() << '/' << Config.GetForceNewEngineLevel());
+
+ if (prevForceNewEnginePercent == 0 && Config.GetForceNewEnginePercent() != 0) {
+ // clear cache only on `enable feature` action
+ QueryCache.Clear();
+ }
+ }
+ }
+
+ void HandleUndelivery(TEvents::TEvUndelivered::TPtr& ev) {
+ switch (ev->Get()->SourceType) {
+ case NConsole::TEvConfigsDispatcher::EvSetConfigSubscriptionRequest:
+ LOG_CRIT(*TlsActivationContext, NKikimrServices::KQP_COMPILE_SERVICE,
+ "Failed to deliver subscription request to config dispatcher");
+ break;
+ case NConsole::TEvConsole::EvConfigNotificationResponse:
+ LOG_ERROR(*TlsActivationContext, NKikimrServices::KQP_COMPILE_SERVICE,
+ "Failed to deliver config notification response");
+ break;
+ default:
+ LOG_ERROR(*TlsActivationContext, NKikimrServices::KQP_COMPILE_SERVICE,
+ "Undelivered event with unexpected source type: %d", ev->Get()->SourceType);
+ break;
+ }
+ }
+
void Handle(TEvKqp::TEvCompileRequest::TPtr& ev, const TActorContext& ctx) {
try {
PerformRequest(ev, ctx);
@@ -652,14 +652,14 @@ private:
break;
}
- if (request->Deadline && request->Deadline < TAppData::TimeProvider->Now()) {
+ if (request->Deadline && request->Deadline < TAppData::TimeProvider->Now()) {
LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Compilation timed out"
<< ", sender: " << request->Sender
<< ", deadline: " << request->Deadline);
Counters->ReportCompileRequestTimeout(request->DbCounters);
- NYql::TIssue issue(NYql::TPosition(), "Compilation timed out.");
+ NYql::TIssue issue(NYql::TPosition(), "Compilation timed out.");
ReplyError(request->Sender, "", Ydb::StatusIds::TIMEOUT, {issue}, ctx);
} else {
StartCompilation(std::move(*request), ctx);
@@ -670,17 +670,17 @@ private:
}
void StartCompilation(TKqpCompileRequest&& request, const TActorContext& ctx) {
- bool recompileWithNewEngine = Config.GetForceNewEnginePercent() > 0;
-
+ bool recompileWithNewEngine = Config.GetForceNewEnginePercent() > 0;
+
auto compileActor = CreateKqpCompileActor(ctx.SelfID, KqpSettings, Config, ModuleResolverState, Counters,
- request.Uid, request.Query, request.UserToken, request.DbCounters, recompileWithNewEngine);
+ request.Uid, request.Query, request.UserToken, request.DbCounters, recompileWithNewEngine);
auto compileActorId = ctx.ExecutorThread.RegisterActor(compileActor, TMailboxType::HTSwap,
AppData(ctx)->UserPoolId);
LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Created compile actor"
<< ", sender: " << request.Sender
- << ", compileActor: " << compileActorId
- << ", recompileWithNewEngine: " << recompileWithNewEngine);
+ << ", compileActor: " << compileActorId
+ << ", recompileWithNewEngine: " << recompileWithNewEngine);
request.CompileActor = compileActorId;
RequestsQueue.AddActiveRequest(std::move(request));
@@ -691,7 +691,7 @@ private:
new IEventHandle(ctx.SelfID, ctx.SelfID, new TEvents::TEvWakeup()));
}
- void Reply(const TActorId& sender, const TKqpCompileResult::TConstPtr& compileResult,
+ void Reply(const TActorId& sender, const TKqpCompileResult::TConstPtr& compileResult,
const NKqpProto::TKqpStatsCompile& compileStats, const TActorContext& ctx)
{
LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Send response"
@@ -701,16 +701,16 @@ private:
auto responseEv = MakeHolder<TEvKqp::TEvCompileResponse>(compileResult);
responseEv->Stats.CopyFrom(compileStats);
-
- if (responseEv->CompileResult && responseEv->CompileResult->PreparedQueryNewEngine) {
- responseEv->ForceNewEnginePercent = Config.GetForceNewEnginePercent();
- responseEv->ForceNewEngineLevel = Config.GetForceNewEngineLevel();
- }
-
+
+ if (responseEv->CompileResult && responseEv->CompileResult->PreparedQueryNewEngine) {
+ responseEv->ForceNewEnginePercent = Config.GetForceNewEnginePercent();
+ responseEv->ForceNewEngineLevel = Config.GetForceNewEngineLevel();
+ }
+
ctx.Send(sender, responseEv.Release());
}
- void ReplyFromCache(const TActorId& sender, const TKqpCompileResult::TConstPtr& compileResult,
+ void ReplyFromCache(const TActorId& sender, const TKqpCompileResult::TConstPtr& compileResult,
const TActorContext& ctx)
{
NKqpProto::TKqpStatsCompile stats;
@@ -719,13 +719,13 @@ private:
Reply(sender, compileResult, stats, ctx);
}
- void ReplyError(const TActorId& sender, const TString& uid, Ydb::StatusIds::StatusCode status,
+ void ReplyError(const TActorId& sender, const TString& uid, Ydb::StatusIds::StatusCode status,
const TIssues& issues, const TActorContext& ctx)
{
Reply(sender, TKqpCompileResult::Make(uid, status, issues), NKqpProto::TKqpStatsCompile(), ctx);
}
- void ReplyInternalError(const TActorId& sender, const TString& uid, const TString& message,
+ void ReplyInternalError(const TActorId& sender, const TString& uid, const TString& message,
const TActorContext& ctx)
{
NYql::TIssue issue(NYql::TPosition(), TStringBuilder() << "Internal error during query compilation.");
@@ -764,5 +764,5 @@ IActor* CreateKqpCompileService(const TTableServiceConfig& serviceConfig, const
std::move(queryReplayFactory));
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/kqp_compute.h b/ydb/core/kqp/kqp_compute.h
index 184f7e067f..943f5adac3 100644
--- a/ydb/core/kqp/kqp_compute.h
+++ b/ydb/core/kqp/kqp_compute.h
@@ -35,7 +35,7 @@ struct TEvKqpCompute {
TVector<TOwnedCellVec> Rows;
std::shared_ptr<arrow::RecordBatch> ArrowBatch;
TOwnedCellVec LastKey;
- TDuration CpuTime;
+ TDuration CpuTime;
TDuration WaitTime;
ui32 PageFaults = 0; // number of page faults occurred when filling in this message
bool Finished = false;
@@ -69,7 +69,7 @@ struct TEvKqpCompute {
auto ev = MakeHolder<TEvScanData>(pbEv->Record.GetScanId());
ev->Generation = pbEv->Record.GetGeneration();
- ev->CpuTime = TDuration::MicroSeconds(pbEv->Record.GetCpuTimeUs());
+ ev->CpuTime = TDuration::MicroSeconds(pbEv->Record.GetCpuTimeUs());
ev->WaitTime = TDuration::MilliSeconds(pbEv->Record.GetWaitTimeMs());
ev->PageFault = pbEv->Record.GetPageFault();
ev->PageFaults = pbEv->Record.GetPageFaults();
@@ -97,7 +97,7 @@ struct TEvKqpCompute {
Remote->Record.SetScanId(ScanId);
Remote->Record.SetGeneration(Generation);
- Remote->Record.SetCpuTimeUs(CpuTime.MicroSeconds());
+ Remote->Record.SetCpuTimeUs(CpuTime.MicroSeconds());
Remote->Record.SetWaitTimeMs(WaitTime.MilliSeconds());
Remote->Record.SetPageFaults(PageFaults);
Remote->Record.SetFinished(Finished);
@@ -186,9 +186,9 @@ struct TEvKqpCompute {
Record.SetGeneration(generation);
}
};
-
- struct TEvKillScanTablet : public NActors::TEventPB<TEvKillScanTablet, NKikimrKqp::TEvKillScanTablet,
- TKqpComputeEvents::EvKillScanTablet> {};
+
+ struct TEvKillScanTablet : public NActors::TEventPB<TEvKillScanTablet, NKikimrKqp::TEvKillScanTablet,
+ TKqpComputeEvents::EvKillScanTablet> {};
};
} // namespace NKikimr::NKqp
diff --git a/ydb/core/kqp/kqp_default_settings.txt b/ydb/core/kqp/kqp_default_settings.txt
index f7b9175f8f..ff501a2db6 100644
--- a/ydb/core/kqp/kqp_default_settings.txt
+++ b/ydb/core/kqp/kqp_default_settings.txt
@@ -60,7 +60,7 @@ DefaultSettings {
DefaultSettings {
Name: "_KqpAllowNewEngine"
- Value: "true"
+ Value: "true"
}
DefaultSettings {
@@ -122,13 +122,13 @@ DefaultSettings {
Name: "OptDisableJoinReverseTableLookup"
Value: "true"
}
-
-DefaultSettings {
- Name: "_KqpMaxComputeActors"
- Value: "10000"
-}
-
-DefaultSettings {
- Name: "_KqpDisableLlvmForUdfStages"
+
+DefaultSettings {
+ Name: "_KqpMaxComputeActors"
+ Value: "10000"
+}
+
+DefaultSettings {
+ Name: "_KqpDisableLlvmForUdfStages"
Value: "false"
-}
+}
diff --git a/ydb/core/kqp/kqp_ic_gateway.cpp b/ydb/core/kqp/kqp_ic_gateway.cpp
index e2fe21facc..47489a9bd5 100644
--- a/ydb/core/kqp/kqp_ic_gateway.cpp
+++ b/ydb/core/kqp/kqp_ic_gateway.cpp
@@ -274,7 +274,7 @@ private:
TActorId ExecuterActorId;
bool HasMeta = false;
Ydb::ResultSet ResultSet;
- TVector<NYql::NDqProto::TDqExecutionStats> Executions;
+ TVector<NYql::NDqProto::TDqExecutionStats> Executions;
};
// Handles data query request for StreamExecuteYqlScript
@@ -476,7 +476,7 @@ public:
private:
TActorId ExecuterActorId;
TActorId TargetActorId;
- TVector<NYql::NDqProto::TDqExecutionStats> Executions;
+ TVector<NYql::NDqProto::TDqExecutionStats> Executions;
};
class TMkqlRequestHandler : public TRequestHandlerBase<
@@ -546,7 +546,7 @@ public:
TResult reply;
reply.SetSuccess();
reply.CompiledProgram = result.CompiledProgram;
- Promise.SetValue(std::move(reply));
+ Promise.SetValue(std::move(reply));
this->Die(ctx);
return;
}
@@ -727,7 +727,7 @@ public:
IKqpGateway::TGenericResult result;
result.SetSuccess();
- Promise.SetValue(std::move(result));
+ Promise.SetValue(std::move(result));
this->Die(ctx);
return;
}
@@ -799,7 +799,7 @@ public:
IKqpGateway::TGenericResult result;
result.SetSuccess();
- Promise.SetValue(std::move(result));
+ Promise.SetValue(std::move(result));
NTabletPipe::CloseClient(ctx, ShemePipeActorId);
this->Die(ctx);
}
@@ -833,8 +833,8 @@ public:
TKqpExecPhysicalRequestHandler(TRequest* request, bool streaming, const TActorId& target, TPromise<TResult> promise)
: Request(request)
- , Streaming(streaming)
- , Executer(request->ExecuterId)
+ , Streaming(streaming)
+ , Executer(request->ExecuterId)
, Target(target)
, Promise(promise) {}
@@ -846,64 +846,64 @@ public:
private:
void Handle(TEvKqpExecuter::TEvStreamData::TPtr &ev, const TActorContext &ctx) {
- if (Streaming) {
- TlsActivationContext->Send(ev->Forward(Target));
- } else {
- HandleUnexpectedEvent(ev->GetTypeRewrite(), ctx);
- }
- }
-
- void Handle(TEvKqpExecuter::TEvStreamDataAck::TPtr &ev, const TActorContext &ctx) {
- if (Streaming) {
- TlsActivationContext->Send(ev->Forward(Executer));
- } else {
- HandleUnexpectedEvent(ev->GetTypeRewrite(), ctx);
- }
- }
-
- void Handle(TEvKqpExecuter::TEvTxResponse::TPtr &ev, const TActorContext &) {
- auto* response = ev->Get()->Record.MutableResponse();
+ if (Streaming) {
+ TlsActivationContext->Send(ev->Forward(Target));
+ } else {
+ HandleUnexpectedEvent(ev->GetTypeRewrite(), ctx);
+ }
+ }
+
+ void Handle(TEvKqpExecuter::TEvStreamDataAck::TPtr &ev, const TActorContext &ctx) {
+ if (Streaming) {
+ TlsActivationContext->Send(ev->Forward(Executer));
+ } else {
+ HandleUnexpectedEvent(ev->GetTypeRewrite(), ctx);
+ }
+ }
+
+ void Handle(TEvKqpExecuter::TEvTxResponse::TPtr &ev, const TActorContext &) {
+ auto* response = ev->Get()->Record.MutableResponse();
TResult result;
- if (response->GetStatus() == Ydb::StatusIds::SUCCESS) {
+ if (response->GetStatus() == Ydb::StatusIds::SUCCESS) {
result.SetSuccess();
}
- for (auto& issue : response->GetIssues()) {
+ for (auto& issue : response->GetIssues()) {
result.AddIssue(NYql::IssueFromMessage(issue));
}
result.ExecuterResult.Swap(response->MutableResult());
if (Target && result.ExecuterResult.HasStats()) {
- auto statsEv = MakeHolder<TEvKqpExecuter::TEvStreamProfile>();
- auto& record = statsEv->Record;
+ auto statsEv = MakeHolder<TEvKqpExecuter::TEvStreamProfile>();
+ auto& record = statsEv->Record;
record.MutableProfile()->Swap(result.ExecuterResult.MutableStats());
- this->Send(Target, statsEv.Release());
- }
-
+ this->Send(Target, statsEv.Release());
+ }
+
Promise.SetValue(std::move(result));
- this->PassAway();
- }
-
- void Handle(TEvKqpExecuter::TEvExecuterProgress::TPtr& ev, const TActorContext&) {
- this->Send(Target, ev->Release().Release());
- }
-
- void Handle(TEvKqp::TEvAbortExecution::TPtr& ev, const TActorContext& ctx) {
- auto& msg = ev->Get()->Record;
-
- LOG_ERROR_S(ctx, NKikimrServices::KQP_GATEWAY,
- "TKqpExecPhysicalRequestHandler, got EvAbortExecution event."
- << " Code: " << Ydb::StatusIds_StatusCode_Name(msg.GetStatusCode())
- << ", reason: " << msg.GetMessage());
-
- auto issueCode = NYql::YqlStatusFromYdbStatus(msg.GetStatusCode());
- Promise.SetValue(ResultFromError<TResult>(YqlIssue({}, issueCode, msg.GetMessage())));
-
- this->PassAway();
- }
-
+ this->PassAway();
+ }
+
+ void Handle(TEvKqpExecuter::TEvExecuterProgress::TPtr& ev, const TActorContext&) {
+ this->Send(Target, ev->Release().Release());
+ }
+
+ void Handle(TEvKqp::TEvAbortExecution::TPtr& ev, const TActorContext& ctx) {
+ auto& msg = ev->Get()->Record;
+
+ LOG_ERROR_S(ctx, NKikimrServices::KQP_GATEWAY,
+ "TKqpExecPhysicalRequestHandler, got EvAbortExecution event."
+ << " Code: " << Ydb::StatusIds_StatusCode_Name(msg.GetStatusCode())
+ << ", reason: " << msg.GetMessage());
+
+ auto issueCode = NYql::YqlStatusFromYdbStatus(msg.GetStatusCode());
+ Promise.SetValue(ResultFromError<TResult>(YqlIssue({}, issueCode, msg.GetMessage())));
+
+ this->PassAway();
+ }
+
void HandleUnexpectedEvent(ui32 eventType, const TActorContext &ctx) {
LOG_CRIT_S(ctx, NKikimrServices::KQP_GATEWAY,
"TKqpExecPhysicalRequestHandler, unexpected event, type: " << eventType);
@@ -911,16 +911,16 @@ private:
Promise.SetValue(ResultFromError<TResult>(YqlIssue({}, TIssuesIds::UNEXPECTED, TStringBuilder()
<< "TKqpExecPhysicalRequestHandler, unexpected event, type: " << eventType)));
- this->PassAway();
+ this->PassAway();
}
STFUNC(ProcessState) {
switch (ev->GetTypeRewrite()) {
HFunc(TEvKqpExecuter::TEvStreamData, Handle);
- HFunc(TEvKqpExecuter::TEvStreamDataAck, Handle);
+ HFunc(TEvKqpExecuter::TEvStreamDataAck, Handle);
HFunc(TEvKqpExecuter::TEvTxResponse, Handle);
- HFunc(TEvKqpExecuter::TEvExecuterProgress, Handle);
- HFunc(TEvKqp::TEvAbortExecution, Handle);
+ HFunc(TEvKqpExecuter::TEvExecuterProgress, Handle);
+ HFunc(TEvKqp::TEvAbortExecution, Handle);
default:
HandleUnexpectedEvent(ev->GetTypeRewrite(), ctx);
}
@@ -928,7 +928,7 @@ private:
private:
THolder<TRequest> Request;
- bool Streaming;
+ bool Streaming;
TActorId Executer;
TActorId Target;
TPromise<TResult> Promise;
@@ -989,7 +989,7 @@ public:
, Database(database)
, ActorSystem(actorSystem)
, NodeId(nodeId)
- , Counters(counters)
+ , Counters(counters)
, MetadataLoader(std::move(metadataLoader))
, MkqlComplileService(mkqlComplileService)
{}
@@ -1666,7 +1666,7 @@ public:
TQueryResult queryResult;
queryResult.ProtobufArenaPtr.reset(new google::protobuf::Arena());
KqpResponseToQueryResult(responseEv.Record.GetRef(), queryResult);
- promise.SetValue(std::move(queryResult));
+ promise.SetValue(std::move(queryResult));
});
}
@@ -1694,8 +1694,8 @@ public:
FillParameters(std::move(params), *ev->Record.MutableRequest()->MutableParameters());
}
- //auto& querySettings = *ev->Record.MutableRequest()->MutableQuerySettings();
- //querySettings.set_use_new_engine(NYql::GetFlagValue(settings.UseNewEngine));
+ //auto& querySettings = *ev->Record.MutableRequest()->MutableQuerySettings();
+ //querySettings.set_use_new_engine(NYql::GetFlagValue(settings.UseNewEngine));
auto& txControl = *ev->Record.MutableRequest()->MutableTxControl();
txControl.mutable_begin_tx()->CopyFrom(txSettings);
@@ -1706,7 +1706,7 @@ public:
TQueryResult queryResult;
queryResult.ProtobufArenaPtr.reset(new google::protobuf::Arena());
KqpResponseToQueryResult(responseEv.Record.GetRef(), queryResult);
- promise.SetValue(std::move(queryResult));
+ promise.SetValue(std::move(queryResult));
});
}
@@ -1738,7 +1738,7 @@ public:
TQueryResult queryResult;
queryResult.ProtobufArenaPtr.reset(new google::protobuf::Arena());
KqpResponseToQueryResult(responseEv.Record.GetRef(), queryResult);
- promise.SetValue(std::move(queryResult));
+ promise.SetValue(std::move(queryResult));
});
}
@@ -1764,7 +1764,7 @@ public:
TQueryResult queryResult;
queryResult.ProtobufArenaPtr.reset(new google::protobuf::Arena());
KqpResponseToQueryResult(responseEv.Record.GetRef(), queryResult);
- promise.SetValue(std::move(queryResult));
+ promise.SetValue(std::move(queryResult));
});
}
@@ -1791,8 +1791,8 @@ public:
FillParameters(std::move(params), *ev->Record.MutableRequest()->MutableParameters());
}
- //auto& querySettings = *ev->Record.MutableRequest()->MutableQuerySettings();
- //querySettings.set_use_new_engine(NYql::GetFlagValue(settings.UseNewEngine));
+ //auto& querySettings = *ev->Record.MutableRequest()->MutableQuerySettings();
+ //querySettings.set_use_new_engine(NYql::GetFlagValue(settings.UseNewEngine));
auto& txControl = *ev->Record.MutableRequest()->MutableTxControl();
txControl.mutable_begin_tx()->CopyFrom(txSettings);
@@ -1803,7 +1803,7 @@ public:
TQueryResult queryResult;
queryResult.ProtobufArenaPtr.reset(new google::protobuf::Arena());
KqpResponseToQueryResult(responseEv.Record.GetRef(), queryResult);
- promise.SetValue(std::move(queryResult));
+ promise.SetValue(std::move(queryResult));
});
}
@@ -1842,7 +1842,7 @@ public:
queryResult.QueryAst = queryResponse.GetQueryAst();
queryResult.QueryPlan = queryResponse.GetQueryPlan();
- promise.SetValue(std::move(queryResult));
+ promise.SetValue(std::move(queryResult));
});
}
@@ -1854,22 +1854,22 @@ public:
auto* snapMgr = CreateKqpSnapshotManager(Database, queryTimeout);
auto snapMgrActorId = RegisterActor(snapMgr);
- auto ev = MakeHolder<TEvKqpSnapshot::TEvCreateSnapshotRequest>(tablePaths);
+ auto ev = MakeHolder<TEvKqpSnapshot::TEvCreateSnapshotRequest>(tablePaths);
return SendActorRequest<
- TEvKqpSnapshot::TEvCreateSnapshotRequest,
- TEvKqpSnapshot::TEvCreateSnapshotResponse,
+ TEvKqpSnapshot::TEvCreateSnapshotRequest,
+ TEvKqpSnapshot::TEvCreateSnapshotResponse,
IKqpGateway::TKqpSnapshotHandle>
(
snapMgrActorId,
ev.Release(),
[snapMgrActorId](TPromise<IKqpGateway::TKqpSnapshotHandle> promise,
- TEvKqpSnapshot::TEvCreateSnapshotResponse&& response) mutable
+ TEvKqpSnapshot::TEvCreateSnapshotResponse&& response) mutable
{
IKqpGateway::TKqpSnapshotHandle handle;
handle.Snapshot = response.Snapshot;
handle.ManagingActor = snapMgrActorId;
- handle.Status = response.Status;
+ handle.Status = response.Status;
handle.AddIssues(response.Issues);
promise.SetValue(handle);
}
@@ -1906,10 +1906,10 @@ public:
ActorSystem->Send(handle.ManagingActor, new TEvKqpSnapshot::TEvDiscardSnapshot(handle.Snapshot));
}
- TInstant GetCurrentTime() const override {
- return TAppData::TimeProvider->Now();
- }
-
+ TInstant GetCurrentTime() const override {
+ return TAppData::TimeProvider->Now();
+ }
+
private:
using TDescribeSchemeResponse = TEvSchemeShard::TEvDescribeSchemeResult;
using TTransactionResponse = TEvTxUserProxy::TEvProposeTransactionStatus;
@@ -1991,7 +1991,7 @@ private:
TKqpParamsMap&& paramsMap, TMkqlRequestHandler::TCallbackFunc callback)
{
auto promise = NewPromise<TMkqlResult>();
- IActor* requestHandler = new TMkqlRequestHandler(Counters->Counters->AllocCounters, request,
+ IActor* requestHandler = new TMkqlRequestHandler(Counters->Counters->AllocCounters, request,
std::move(paramsMap), promise, callback, MkqlComplileService);
RegisterActor(requestHandler);
@@ -2121,23 +2121,23 @@ private:
}
TFuture<TExecPhysicalResult> ExecutePhysicalQueryInternal(TExecPhysicalRequest&& request, const TActorId& target,
- bool streaming)
- {
- auto executerActor = CreateKqpExecuter(std::move(request), Database,
- UserToken ? TMaybe<TString>(UserToken->Serialized) : Nothing(), Counters);
- auto executerId = RegisterActor(executerActor);
-
- LOG_DEBUG_S(*ActorSystem, NKikimrServices::KQP_GATEWAY, "Created new KQP executer: " << executerId);
-
- auto promise = NewPromise<TExecPhysicalResult>();
-
- auto ev = MakeHolder<TEvTxUserProxy::TEvProposeKqpTransaction>(executerId);
+ bool streaming)
+ {
+ auto executerActor = CreateKqpExecuter(std::move(request), Database,
+ UserToken ? TMaybe<TString>(UserToken->Serialized) : Nothing(), Counters);
+ auto executerId = RegisterActor(executerActor);
+
+ LOG_DEBUG_S(*ActorSystem, NKikimrServices::KQP_GATEWAY, "Created new KQP executer: " << executerId);
+
+ auto promise = NewPromise<TExecPhysicalResult>();
+
+ auto ev = MakeHolder<TEvTxUserProxy::TEvProposeKqpTransaction>(executerId);
IActor* requestHandler = new TKqpExecPhysicalRequestHandler(ev.Release(), streaming, target, promise);
- RegisterActor(requestHandler);
-
- return promise.GetFuture();
- }
-
+ RegisterActor(requestHandler);
+
+ return promise.GetFuture();
+ }
+
bool GetDatabaseForLoginOperation(TString& database) {
TAppData* appData = AppData(ActorSystem);
if (appData && appData->AuthConfig.GetDomainLoginOnly()) {
@@ -2641,7 +2641,7 @@ private:
TString Database;
TActorSystem* ActorSystem;
ui32 NodeId;
- TKqpRequestCounters::TPtr Counters;
+ TKqpRequestCounters::TPtr Counters;
TAlignedPagePoolCounters AllocCounters;
TMaybe<TUserTokenData> UserToken;
std::shared_ptr<IKqpTableMetadataLoader> MetadataLoader;
@@ -2657,5 +2657,5 @@ TIntrusivePtr<IKqpGateway> CreateKikimrIcGateway(const TString& cluster, const T
return MakeIntrusive<TKikimrIcGateway>(cluster, database, std::move(metadataLoader), actorSystem, nodeId, counters, mkqlComplileService);
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/kqp_impl.h b/ydb/core/kqp/kqp_impl.h
index 06271113d0..7d6f172899 100644
--- a/ydb/core/kqp/kqp_impl.h
+++ b/ydb/core/kqp/kqp_impl.h
@@ -52,7 +52,7 @@ IActor* CreateKqpCompileService(const NKikimrConfig::TTableServiceConfig& servic
IActor* CreateKqpCompileActor(const TActorId& owner, const TKqpSettings::TConstPtr& kqpSettings,
const NKikimrConfig::TTableServiceConfig& serviceConfig, TIntrusivePtr<TModuleResolverState> moduleResolverState,
TIntrusivePtr<TKqpCounters> counters, const TString& uid, const TKqpQueryId& query, const TString& userToken,
- TKqpDbCountersPtr dbCounters, bool recompileWithNewEngine);
+ TKqpDbCountersPtr dbCounters, bool recompileWithNewEngine);
IActor* CreateKqpCompileRequestActor(const TActorId& owner, const TString& userToken, const TMaybe<TString>& uid,
TMaybe<TKqpQueryId>&& query, bool keepInCache, const TInstant& deadline, TKqpDbCountersPtr dbCounters);
@@ -86,13 +86,13 @@ TIntrusivePtr<IKqpGateway> CreateKikimrIcGateway(const TString& cluster, const T
std::shared_ptr<IKqpGateway::IKqpTableMetadataLoader>&& metadataLoader, NActors::TActorSystem* actorSystem, ui32 nodeId, TKqpRequestCounters::TPtr counters,
const TActorId& MkqlCompileService);
-Ydb::StatusIds::StatusCode GetYdbStatus(const NYql::NCommon::TOperationResult& queryResult);
+Ydb::StatusIds::StatusCode GetYdbStatus(const NYql::NCommon::TOperationResult& queryResult);
void AddQueryIssues(NKikimrKqp::TQueryResponse& response, const NYql::TIssues& issues);
bool HasSchemeOrFatalIssues(const NYql::TIssues& issues);
-// for tests only
-void FailForcedNewEngineCompilationForTests(bool fail = true);
-void FailForcedNewEngineExecutionForTests(bool fail = true);
-
-} // namespace NKqp
+// for tests only
+void FailForcedNewEngineCompilationForTests(bool fail = true);
+void FailForcedNewEngineExecutionForTests(bool fail = true);
+
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/kqp_response.cpp b/ydb/core/kqp/kqp_response.cpp
index 1ad8274603..d3859cd39c 100644
--- a/ydb/core/kqp/kqp_response.cpp
+++ b/ydb/core/kqp/kqp_response.cpp
@@ -59,9 +59,9 @@ TMaybe<Ydb::StatusIds::StatusCode> GetYdbStatus(const TIssue& issue) {
case TIssuesIds::KIKIMR_OPERATION_STATE_UNKNOWN:
return Ydb::StatusIds::UNDETERMINED;
- case TIssuesIds::KIKIMR_PRECONDITION_FAILED:
- return Ydb::StatusIds::PRECONDITION_FAILED;
-
+ case TIssuesIds::KIKIMR_PRECONDITION_FAILED:
+ return Ydb::StatusIds::PRECONDITION_FAILED;
+
case TIssuesIds::KIKIMR_UNSUPPORTED:
return Ydb::StatusIds::UNSUPPORTED;
@@ -114,7 +114,7 @@ bool HasSchemeOrFatalIssues(const TIssue& issue) {
} // namespace
-Ydb::StatusIds::StatusCode GetYdbStatus(const NYql::NCommon::TOperationResult& queryResult) {
+Ydb::StatusIds::StatusCode GetYdbStatus(const NYql::NCommon::TOperationResult& queryResult) {
if (queryResult.Success()) {
return Ydb::StatusIds::SUCCESS;
}
@@ -159,5 +159,5 @@ bool HasSchemeOrFatalIssues(const TIssues& issues) {
return false;
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/kqp_worker_actor.cpp b/ydb/core/kqp/kqp_worker_actor.cpp
index 78e9cd9b29..cccb8a51e8 100644
--- a/ydb/core/kqp/kqp_worker_actor.cpp
+++ b/ydb/core/kqp/kqp_worker_actor.cpp
@@ -4,7 +4,7 @@
#include <ydb/core/actorlib_impl/long_timer.h>
#include <ydb/core/base/appdata.h>
-#include <ydb/core/base/counters.h>
+#include <ydb/core/base/counters.h>
#include <ydb/core/base/cputime.h>
#include <ydb/core/client/minikql_compile/mkql_compile_service.h>
#include <ydb/core/protos/kqp.pb.h>
@@ -34,14 +34,14 @@ using namespace NYql;
using namespace NYql::NDq;
using namespace NRuCalc;
-static std::atomic<bool> FailForcedNewEngineExecution = false;
-void FailForcedNewEngineExecutionForTests(bool fail) {
- FailForcedNewEngineExecution = fail;
-}
-
+static std::atomic<bool> FailForcedNewEngineExecution = false;
+void FailForcedNewEngineExecutionForTests(bool fail) {
+ FailForcedNewEngineExecution = fail;
+}
+
namespace {
-constexpr std::string_view DocumentApiRequestType = "_document_api_request"sv;
+constexpr std::string_view DocumentApiRequestType = "_document_api_request"sv;
using TQueryResult = IKqpHost::TQueryResult;
@@ -63,16 +63,16 @@ struct TKqpQueryState {
TString TxId;
TKqpCompileResult::TConstPtr QueryCompileResult;
NKqpProto::TKqpStatsCompile CompileStats;
- ui32 ReplyFlags = 0;
- bool KeepSession = false;
- bool InteractiveTx = true;
-
- bool OldEngineFallback = false;
- bool NewEngineCompatibleQuery = false;
-
- TKqpForceNewEngineState ForceNewEngineState;
- std::optional<TQueryTraits> QueryTraits;
-
+ ui32 ReplyFlags = 0;
+ bool KeepSession = false;
+ bool InteractiveTx = true;
+
+ bool OldEngineFallback = false;
+ bool NewEngineCompatibleQuery = false;
+
+ TKqpForceNewEngineState ForceNewEngineState;
+ std::optional<TQueryTraits> QueryTraits;
+
TMaybe<NKikimrKqp::TRlPath> RlPath;
};
@@ -122,19 +122,19 @@ enum ETableReadType {
FullScan = 2,
};
-EKikimrStatsMode GetStatsMode(const NKikimrKqp::TQueryRequest& queryRequest, EKikimrStatsMode minMode) {
+EKikimrStatsMode GetStatsMode(const NKikimrKqp::TQueryRequest& queryRequest, EKikimrStatsMode minMode) {
if (queryRequest.GetProfile()) {
// TODO: Deprecate, StatsMode is the new way to enable stats.
return EKikimrStatsMode::Profile;
}
switch (queryRequest.GetStatsMode()) {
- case NYql::NDqProto::DQ_STATS_MODE_BASIC:
+ case NYql::NDqProto::DQ_STATS_MODE_BASIC:
return EKikimrStatsMode::Basic;
- case NYql::NDqProto::DQ_STATS_MODE_PROFILE:
+ case NYql::NDqProto::DQ_STATS_MODE_PROFILE:
return EKikimrStatsMode::Profile;
default:
- return std::max(EKikimrStatsMode::None, minMode);
+ return std::max(EKikimrStatsMode::None, minMode);
}
}
@@ -161,7 +161,7 @@ public:
Y_VERIFY(ModuleResolverState);
Y_VERIFY(ModuleResolverState->ModuleResolver);
- Config->Init(kqpSettings->DefaultSettings.GetDefaultSettings(), Settings.Cluster, kqpSettings->Settings, false);
+ Config->Init(kqpSettings->DefaultSettings.GetDefaultSettings(), Settings.Cluster, kqpSettings->Settings, false);
if (!Settings.Database.empty()) {
Config->_KqpTablePathPrefix = Settings.Database;
@@ -170,11 +170,11 @@ public:
ApplyServiceConfig(*Config, Settings.Service);
Config->FreezeDefaults();
-
- RequestCounters = MakeIntrusive<TKqpRequestCounters>();
- RequestCounters->Counters = Counters;
- RequestCounters->DbCounters = Settings.DbCounters;
- RequestCounters->TxProxyMon = MakeIntrusive<NTxProxy::TTxProxyMon>(AppData()->Counters);
+
+ RequestCounters = MakeIntrusive<TKqpRequestCounters>();
+ RequestCounters->Counters = Counters;
+ RequestCounters->DbCounters = Settings.DbCounters;
+ RequestCounters->TxProxyMon = MakeIntrusive<NTxProxy::TTxProxyMon>(AppData()->Counters);
}
void Bootstrap(const TActorContext& ctx) {
@@ -270,20 +270,20 @@ public:
queryRequest.SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE);
}
- auto now = TAppData::TimeProvider->Now();
-
+ auto now = TAppData::TimeProvider->Now();
+
QueryState->ParametersSize = queryRequest.GetParameters().ByteSize();
QueryState->Sender = ev->Sender;
QueryState->ProxyRequestId = proxyRequestId;
QueryState->KeepSession = Settings.LongSession || queryRequest.GetKeepSession();
QueryState->TraceId = requestInfo.GetTraceId();
QueryState->RequestType = event.GetRequestType();
- QueryState->StartTime = now;
+ QueryState->StartTime = now;
QueryState->ReplyFlags = queryRequest.GetReplyFlags();
QueryState->UserToken = event.GetUserToken();
QueryState->RequestActorId = ActorIdFromProto(event.GetRequestActorId());
- if (GetStatsMode(queryRequest, EKikimrStatsMode::None) > EKikimrStatsMode::Basic) {
+ if (GetStatsMode(queryRequest, EKikimrStatsMode::None) > EKikimrStatsMode::Basic) {
QueryState->ReplyFlags |= NKikimrKqp::QUERY_REPLY_FLAG_AST;
}
@@ -292,14 +292,14 @@ public:
}
NCpuTime::TCpuTimer timer;
-
+
if (queryRequest.GetCancelAfterMs()) {
- QueryState->QueryDeadlines.CancelAt = now + TDuration::MilliSeconds(queryRequest.GetCancelAfterMs());
+ QueryState->QueryDeadlines.CancelAt = now + TDuration::MilliSeconds(queryRequest.GetCancelAfterMs());
}
auto timeoutMs = GetQueryTimeout(queryRequest.GetType(), queryRequest.GetTimeoutMs(), Settings.Service);
QueryState->QueryDeadlines.TimeoutAt = now + timeoutMs;
-
+
auto onError = [this, &ctx, &requestInfo] (Ydb::StatusIds::StatusCode status, const TString& message) {
ReplyProcessError(QueryState->Sender, QueryState->ProxyRequestId, requestInfo, status, message, ctx);
@@ -355,74 +355,74 @@ public:
break;
}
- HandleQueryRequest(timer, false, ctx);
- }
-
- void HandleQueryRequest(NCpuTime::TCpuTimer& timer, bool fallbackToOldEngine, const TActorContext& ctx) {
- auto& queryRequest = QueryState->Request;
-
- if (fallbackToOldEngine) {
- LOG_DEBUG_S(ctx, NKikimrServices::KQP_WORKER, "OldEngine fallback request, proxyRequestId: "
- << QueryState->ProxyRequestId);
-
- QueryState->ForceNewEngineState.ForcedNewEngine = false;
- QueryState->OldEngineFallback = true;
- }
-
+ HandleQueryRequest(timer, false, ctx);
+ }
+
+ void HandleQueryRequest(NCpuTime::TCpuTimer& timer, bool fallbackToOldEngine, const TActorContext& ctx) {
+ auto& queryRequest = QueryState->Request;
+
+ if (fallbackToOldEngine) {
+ LOG_DEBUG_S(ctx, NKikimrServices::KQP_WORKER, "OldEngine fallback request, proxyRequestId: "
+ << QueryState->ProxyRequestId);
+
+ QueryState->ForceNewEngineState.ForcedNewEngine = false;
+ QueryState->OldEngineFallback = true;
+ }
+
if (queryRequest.HasTxControl()) {
- const auto& txControl = queryRequest.GetTxControl();
-
- switch (txControl.tx_selector_case()) {
- case Ydb::Table::TransactionControl::kTxId: {
- QueryState->TxId = txControl.tx_id();
-
- auto txInfo = KqpHost->GetTransactionInfo(QueryState->TxId);
- if (!txInfo) {
- QueryState->AsyncQueryResult = MakeKikimrResultHolder(NCommon::ResultFromError<TQueryResult>(
- YqlIssue(TPosition(), TIssuesIds::KIKIMR_TRANSACTION_NOT_FOUND, TStringBuilder()
- << "Transaction not found: " << QueryState->TxId)));
-
- ContinueQueryProcess(ctx);
- Become(&TKqpWorkerActor::PerformQueryState);
- return;
- }
-
- YQL_ENSURE(!QueryState->OldEngineFallback);
- QueryState->ForceNewEngineState = txInfo->ForceNewEngineState;
- break;
- }
-
- case Ydb::Table::TransactionControl::kBeginTx: {
- if (txControl.commit_tx()) {
- QueryState->InteractiveTx = false;
- }
- if (QueryState->OldEngineFallback) {
- YQL_ENSURE(!QueryState->InteractiveTx);
- }
- break;
- }
-
- case Ydb::Table::TransactionControl::TX_SELECTOR_NOT_SET: {
- Y_VERIFY(false);
- }
+ const auto& txControl = queryRequest.GetTxControl();
+
+ switch (txControl.tx_selector_case()) {
+ case Ydb::Table::TransactionControl::kTxId: {
+ QueryState->TxId = txControl.tx_id();
+
+ auto txInfo = KqpHost->GetTransactionInfo(QueryState->TxId);
+ if (!txInfo) {
+ QueryState->AsyncQueryResult = MakeKikimrResultHolder(NCommon::ResultFromError<TQueryResult>(
+ YqlIssue(TPosition(), TIssuesIds::KIKIMR_TRANSACTION_NOT_FOUND, TStringBuilder()
+ << "Transaction not found: " << QueryState->TxId)));
+
+ ContinueQueryProcess(ctx);
+ Become(&TKqpWorkerActor::PerformQueryState);
+ return;
+ }
+
+ YQL_ENSURE(!QueryState->OldEngineFallback);
+ QueryState->ForceNewEngineState = txInfo->ForceNewEngineState;
+ break;
+ }
+
+ case Ydb::Table::TransactionControl::kBeginTx: {
+ if (txControl.commit_tx()) {
+ QueryState->InteractiveTx = false;
+ }
+ if (QueryState->OldEngineFallback) {
+ YQL_ENSURE(!QueryState->InteractiveTx);
+ }
+ break;
+ }
+
+ case Ydb::Table::TransactionControl::TX_SELECTOR_NOT_SET: {
+ Y_VERIFY(false);
+ }
}
- } else {
- // some kind of internal query? or verify here?
+ } else {
+ // some kind of internal query? or verify here?
}
StopIdleTimer(ctx);
if (CompileQuery(ctx)) {
- if (QueryState) {
+ if (QueryState) {
QueryState->CpuTime += timer.GetTime();
- }
+ }
return;
}
PerformQuery(ctx);
- if (QueryState) {
+ if (QueryState) {
QueryState->CpuTime += timer.GetTime();
- }
+ }
}
void HandleContinueShutdown(TEvKqp::TEvContinueShutdown::TPtr &ev, const TActorContext &ctx) {
@@ -459,8 +459,8 @@ public:
Y_VERIFY(compileResult);
Y_VERIFY(QueryState);
- if (compileResult->Status != Ydb::StatusIds::SUCCESS) {
- if (ReplyQueryCompileError(compileResult, ctx)) {
+ if (compileResult->Status != Ydb::StatusIds::SUCCESS) {
+ if (ReplyQueryCompileError(compileResult, ctx)) {
StartIdleTimer(ctx);
Become(&TKqpWorkerActor::ReadyState);
} else {
@@ -470,20 +470,20 @@ public:
return;
}
- QueryState->QueryTraits = compileResult->QueryTraits;
-
- if (!QueryState->ForceNewEngineState.ForcedNewEngine.has_value()) {
- // first query in tx
- QueryState->ForceNewEngineState.ForceNewEnginePercent = ev->Get()->ForceNewEnginePercent;
- QueryState->ForceNewEngineState.ForceNewEngineLevel = ev->Get()->ForceNewEngineLevel;
- }
-
- auto& queryRequest = QueryState->Request;
-
- QueryState->CompileStats.Swap(&ev->Get()->Stats);
-
- if (queryRequest.GetAction() == NKikimrKqp::QUERY_ACTION_PREPARE) {
- if (ReplyPrepareResult(compileResult, ctx)) {
+ QueryState->QueryTraits = compileResult->QueryTraits;
+
+ if (!QueryState->ForceNewEngineState.ForcedNewEngine.has_value()) {
+ // first query in tx
+ QueryState->ForceNewEngineState.ForceNewEnginePercent = ev->Get()->ForceNewEnginePercent;
+ QueryState->ForceNewEngineState.ForceNewEngineLevel = ev->Get()->ForceNewEngineLevel;
+ }
+
+ auto& queryRequest = QueryState->Request;
+
+ QueryState->CompileStats.Swap(&ev->Get()->Stats);
+
+ if (queryRequest.GetAction() == NKikimrKqp::QUERY_ACTION_PREPARE) {
+ if (ReplyPrepareResult(compileResult, ctx)) {
StartIdleTimer(ctx);
Become(&TKqpWorkerActor::ReadyState);
} else {
@@ -502,17 +502,17 @@ public:
default:
Y_VERIFY_S(false, "Unexpected action on successful compile result: "
- << NKikimrKqp::EQueryAction_Name(queryRequest.GetAction()));
+ << NKikimrKqp::EQueryAction_Name(queryRequest.GetAction()));
break;
}
NCpuTime::TCpuTimer timer;
PerformQuery(ctx);
-
+
// PerformQuery can reset QueryState
- if (QueryState) {
+ if (QueryState) {
QueryState->CpuTime += timer.GetTime();
- }
+ }
}
void HandleCompileQuery(TEvKqp::TEvQueryRequest::TPtr &ev, const TActorContext &ctx) {
@@ -575,39 +575,39 @@ public:
QueryState->QueryResult = QueryState->AsyncQueryResult->GetResult();
QueryState->AsyncQueryResult.Reset();
- if (!QueryState->OldEngineFallback) {
- auto& x = QueryState->ForceNewEngineState;
- if (x.ForcedNewEngine && *x.ForcedNewEngine) {
- auto status = GetYdbStatus(QueryState->QueryResult);
- bool failForTests = FailForcedNewEngineExecution.load(std::memory_order_relaxed);
-
- if (status != Ydb::StatusIds::SUCCESS || failForTests) {
- if (x.ForceNewEngineLevel == 0 || x.ForceNewEngineLevel == 1) {
- bool shouldFallback = status != Ydb::StatusIds::CANCELLED
- && status != Ydb::StatusIds::ABORTED
- && status != Ydb::StatusIds::OVERLOADED
- && status != Ydb::StatusIds::PRECONDITION_FAILED;
- if (shouldFallback) {
- QueryState->ForceNewEngineState = {};
- QueryState->NewEngineCompatibleQuery = false;
-
- GetServiceCounters(AppData()->Counters, "kqp")->GetCounter("Requests/OldEngineFallback", true)->Inc();
-
- NCpuTime::TCpuTimer timer;
- HandleQueryRequest(timer, true, ctx);
- return;
- }
- }
-
- if (failForTests) {
- QueryState->QueryResult.SetStatus(NYql::TIssuesIds::DEFAULT_ERROR);
- QueryState->QueryResult.AddIssue(YqlIssue(TPosition(), TIssuesIds::DEFAULT_ERROR, "Failed for test."));
- GetServiceCounters(AppData()->Counters, "kqp")->GetCounter("Requests/ForceNewEngineExecError", true)->Inc();
- }
- }
- }
- }
-
+ if (!QueryState->OldEngineFallback) {
+ auto& x = QueryState->ForceNewEngineState;
+ if (x.ForcedNewEngine && *x.ForcedNewEngine) {
+ auto status = GetYdbStatus(QueryState->QueryResult);
+ bool failForTests = FailForcedNewEngineExecution.load(std::memory_order_relaxed);
+
+ if (status != Ydb::StatusIds::SUCCESS || failForTests) {
+ if (x.ForceNewEngineLevel == 0 || x.ForceNewEngineLevel == 1) {
+ bool shouldFallback = status != Ydb::StatusIds::CANCELLED
+ && status != Ydb::StatusIds::ABORTED
+ && status != Ydb::StatusIds::OVERLOADED
+ && status != Ydb::StatusIds::PRECONDITION_FAILED;
+ if (shouldFallback) {
+ QueryState->ForceNewEngineState = {};
+ QueryState->NewEngineCompatibleQuery = false;
+
+ GetServiceCounters(AppData()->Counters, "kqp")->GetCounter("Requests/OldEngineFallback", true)->Inc();
+
+ NCpuTime::TCpuTimer timer;
+ HandleQueryRequest(timer, true, ctx);
+ return;
+ }
+ }
+
+ if (failForTests) {
+ QueryState->QueryResult.SetStatus(NYql::TIssuesIds::DEFAULT_ERROR);
+ QueryState->QueryResult.AddIssue(YqlIssue(TPosition(), TIssuesIds::DEFAULT_ERROR, "Failed for test."));
+ GetServiceCounters(AppData()->Counters, "kqp")->GetCounter("Requests/ForceNewEngineExecError", true)->Inc();
+ }
+ }
+ }
+ }
+
QueryCleanup(ctx);
} else {
NCpuTime::TCpuTimer timer(QueryState->CpuTime);
@@ -793,7 +793,7 @@ private:
<< ", type: " << (ui32)queryRequest.GetType()
<< ", query: \"" << queryRequest.GetQuery().substr(0, 1000) << "\"");
- return false;
+ return false;
}
IKqpHost::TBeginTxResult BeginTransaction(const Ydb::Table::TransactionSettings& settings) {
@@ -825,7 +825,7 @@ private:
return KqpHost->BeginTransaction(isolation, readonly);
}
- bool CompileQuery(const TActorContext& ctx) {
+ bool CompileQuery(const TActorContext& ctx) {
if (!Settings.LongSession) {
return false;
}
@@ -883,7 +883,7 @@ private:
return true;
}
- void PerformQuery(const TActorContext& ctx) {
+ void PerformQuery(const TActorContext& ctx) {
Y_VERIFY(QueryState);
auto requestInfo = TKqpRequestInfo(QueryState->TraceId, SessionId);
TYqlLogScope logScope(ctx, NKikimrServices::KQP_YQL, SessionId, QueryState->TraceId);
@@ -916,17 +916,17 @@ private:
onError(Ydb::StatusIds::BAD_REQUEST, message);
};
- bool commit = false;
+ bool commit = false;
if (queryRequest.HasTxControl()) {
- const auto& txControl = queryRequest.GetTxControl();
+ const auto& txControl = queryRequest.GetTxControl();
commit = txControl.commit_tx();
switch (txControl.tx_selector_case()) {
- case Ydb::Table::TransactionControl::kTxId: {
- Y_VERIFY_DEBUG(!QueryState->TxId.empty());
+ case Ydb::Table::TransactionControl::kTxId: {
+ Y_VERIFY_DEBUG(!QueryState->TxId.empty());
break;
- }
+ }
case Ydb::Table::TransactionControl::kBeginTx: {
beginTxResult = BeginTransaction(txControl.begin_tx());
@@ -947,10 +947,10 @@ private:
Counters->ReportTxCreated(Settings.DbCounters);
QueryState->TxId = beginTxResult.TxId;
- if (commit) {
- QueryState->InteractiveTx = false;
- }
-
+ if (commit) {
+ QueryState->InteractiveTx = false;
+ }
+
break;
}
@@ -972,7 +972,7 @@ private:
switch (action) {
case NKikimrKqp::QUERY_ACTION_EXECUTE: {
- if (!ExecuteQuery(queryRequest, queryType, commit, QueryState->RequestActorId)) {
+ if (!ExecuteQuery(queryRequest, queryType, commit, QueryState->RequestActorId)) {
onBadRequest(QueryState->Error);
return;
}
@@ -1012,148 +1012,148 @@ private:
case NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED: {
// NOTE: For compatibility with old clients, remove once not used.
- const TString& query = queryRequest.HasPreparedQuery()
+ const TString& query = queryRequest.HasPreparedQuery()
? queryRequest.GetPreparedQuery()
: queryRequest.GetQuery();
- TPreparedQueryConstPtr preparedQuery;
+ TPreparedQueryConstPtr preparedQuery;
if (QueryState->QueryCompileResult) {
Y_VERIFY(queryType == NKikimrKqp::QUERY_TYPE_PREPARED_DML);
-
- bool newEngineCompatibleTx = !QueryState->OldEngineFallback
- && QueryState->ForceNewEngineState.ForceNewEnginePercent > 0;
-
- if (QueryState->ForceNewEngineState.ForcedNewEngine.has_value() &&
- QueryState->ForceNewEngineState.ForcedNewEngine.value() == false)
- {
- newEngineCompatibleTx = false;
- }
-
- QueryState->NewEngineCompatibleQuery = (bool) QueryState->QueryCompileResult->PreparedQueryNewEngine
- && newEngineCompatibleTx;
-
- if (newEngineCompatibleTx) {
- if (QueryState->ForceNewEngineState.ForceNewEngineLevel == 0) {
- if (QueryState->InteractiveTx || !QueryState->QueryTraits) {
- newEngineCompatibleTx = false;
- QueryState->NewEngineCompatibleQuery = false;
- } else {
- const auto& traits = QueryState->QueryTraits.value();
- if (!traits.ReadOnly || traits.WithJoin || traits.WithSqlIn || traits.WithIndex) {
- newEngineCompatibleTx = false;
- QueryState->NewEngineCompatibleQuery = false;
- }
- }
- } else if (QueryState->ForceNewEngineState.ForceNewEngineLevel == 1) {
- if (QueryState->InteractiveTx || !QueryState->QueryTraits) {
- newEngineCompatibleTx = false;
- QueryState->NewEngineCompatibleQuery = false;
- } else {
- const auto& traits = QueryState->QueryTraits.value();
- if (!traits.ReadOnly) {
- newEngineCompatibleTx = false;
- QueryState->NewEngineCompatibleQuery = false;
- }
- }
- } else if (QueryState->ForceNewEngineState.ForceNewEngineLevel == 2) {
- if (!QueryState->QueryTraits.has_value() || !QueryState->QueryTraits->ReadOnly) {
- QueryState->NewEngineCompatibleQuery = false;
- // but Tx is still NE Compatible, i.e. RO-queries can be executed with NewEngine
- }
- if (commit) {
- // select engine according to deferred effects
- auto engine = KqpHost->GetTransactionInfo(QueryState->TxId)->TxEngine;
- if (engine) {
- if (*engine == TKqpTransactionInfo::EEngine::NewEngine) {
- QueryState->NewEngineCompatibleQuery = true;
- } else {
- QueryState->NewEngineCompatibleQuery = false;
- }
- } else {
- // Y_VERIFY(false);
- }
- }
- } else if (QueryState->ForceNewEngineState.ForceNewEngineLevel == 3) {
- newEngineCompatibleTx = true;
- QueryState->NewEngineCompatibleQuery = true;
- } else {
- YQL_ENSURE(false);
- }
- }
-
- LOG_DEBUG_S(ctx, NKikimrServices::KQP_WORKER, "-- NE Compatible: "
- << " tx: " << newEngineCompatibleTx
- << ", query: " << QueryState->NewEngineCompatibleQuery
- << ", interactive: " << QueryState->InteractiveTx
- << ", preparedNewEngine: " << (bool) QueryState->QueryCompileResult->PreparedQueryNewEngine
- << ", forcedNewEngine: " << (QueryState->ForceNewEngineState.ForcedNewEngine
- ? ToString(*QueryState->ForceNewEngineState.ForcedNewEngine)
- : "<none>")
- << ", traits: " << (QueryState->QueryCompileResult->QueryTraits
- ? QueryState->QueryCompileResult->QueryTraits->ToString()
- : "<none>"));
-
- if (newEngineCompatibleTx) {
- if (QueryState->NewEngineCompatibleQuery) {
- if (QueryState->ForceNewEngineState.ForcedNewEngine && *QueryState->ForceNewEngineState.ForcedNewEngine) {
- preparedQuery = QueryState->QueryCompileResult->PreparedQueryNewEngine;
- LOG_INFO_S(ctx, NKikimrServices::KQP_WORKER, "Force NewEngine query execution (as part of tx)");
- } else if (QueryState->ForceNewEngineState.ForceNewEnginePercent >= RandomNumber((ui32) 100)) {
- preparedQuery = QueryState->QueryCompileResult->PreparedQueryNewEngine;
- QueryState->ForceNewEngineState.ForcedNewEngine = true;
-
- KqpHost->ForceTxNewEngine(
- QueryState->TxId,
- QueryState->ForceNewEngineState.ForceNewEnginePercent,
- QueryState->ForceNewEngineState.ForceNewEngineLevel
- );
- LOG_INFO_S(ctx, NKikimrServices::KQP_WORKER, "Force NewEngine query execution (new tx)");
- } else {
- QueryState->ForceNewEngineState.ForcedNewEngine = false;
- KqpHost->ForceTxOldEngine(QueryState->TxId);
- preparedQuery = QueryState->QueryCompileResult->PreparedQuery;
- LOG_INFO_S(ctx, NKikimrServices::KQP_WORKER, "Force OldEngine query execution (new tx)");
- }
- } else {
- if (!QueryState->ForceNewEngineState.ForcedNewEngine.has_value()) {
- if (QueryState->ForceNewEngineState.ForceNewEnginePercent >= RandomNumber((ui32) 100)) {
- QueryState->ForceNewEngineState.ForcedNewEngine = true;
-
- KqpHost->ForceTxNewEngine(
- QueryState->TxId,
- QueryState->ForceNewEngineState.ForceNewEnginePercent,
- QueryState->ForceNewEngineState.ForceNewEngineLevel
- );
- LOG_INFO_S(ctx, NKikimrServices::KQP_WORKER, "Force NewEngine query execution (new tx)");
- } else {
- QueryState->ForceNewEngineState.ForcedNewEngine = false;
- KqpHost->ForceTxOldEngine(QueryState->TxId);
- }
- }
-
- preparedQuery = QueryState->QueryCompileResult->PreparedQuery;
- }
- } else {
- preparedQuery = QueryState->QueryCompileResult->PreparedQuery;
- }
+
+ bool newEngineCompatibleTx = !QueryState->OldEngineFallback
+ && QueryState->ForceNewEngineState.ForceNewEnginePercent > 0;
+
+ if (QueryState->ForceNewEngineState.ForcedNewEngine.has_value() &&
+ QueryState->ForceNewEngineState.ForcedNewEngine.value() == false)
+ {
+ newEngineCompatibleTx = false;
+ }
+
+ QueryState->NewEngineCompatibleQuery = (bool) QueryState->QueryCompileResult->PreparedQueryNewEngine
+ && newEngineCompatibleTx;
+
+ if (newEngineCompatibleTx) {
+ if (QueryState->ForceNewEngineState.ForceNewEngineLevel == 0) {
+ if (QueryState->InteractiveTx || !QueryState->QueryTraits) {
+ newEngineCompatibleTx = false;
+ QueryState->NewEngineCompatibleQuery = false;
+ } else {
+ const auto& traits = QueryState->QueryTraits.value();
+ if (!traits.ReadOnly || traits.WithJoin || traits.WithSqlIn || traits.WithIndex) {
+ newEngineCompatibleTx = false;
+ QueryState->NewEngineCompatibleQuery = false;
+ }
+ }
+ } else if (QueryState->ForceNewEngineState.ForceNewEngineLevel == 1) {
+ if (QueryState->InteractiveTx || !QueryState->QueryTraits) {
+ newEngineCompatibleTx = false;
+ QueryState->NewEngineCompatibleQuery = false;
+ } else {
+ const auto& traits = QueryState->QueryTraits.value();
+ if (!traits.ReadOnly) {
+ newEngineCompatibleTx = false;
+ QueryState->NewEngineCompatibleQuery = false;
+ }
+ }
+ } else if (QueryState->ForceNewEngineState.ForceNewEngineLevel == 2) {
+ if (!QueryState->QueryTraits.has_value() || !QueryState->QueryTraits->ReadOnly) {
+ QueryState->NewEngineCompatibleQuery = false;
+ // but Tx is still NE Compatible, i.e. RO-queries can be executed with NewEngine
+ }
+ if (commit) {
+ // select engine according to deferred effects
+ auto engine = KqpHost->GetTransactionInfo(QueryState->TxId)->TxEngine;
+ if (engine) {
+ if (*engine == TKqpTransactionInfo::EEngine::NewEngine) {
+ QueryState->NewEngineCompatibleQuery = true;
+ } else {
+ QueryState->NewEngineCompatibleQuery = false;
+ }
+ } else {
+ // Y_VERIFY(false);
+ }
+ }
+ } else if (QueryState->ForceNewEngineState.ForceNewEngineLevel == 3) {
+ newEngineCompatibleTx = true;
+ QueryState->NewEngineCompatibleQuery = true;
+ } else {
+ YQL_ENSURE(false);
+ }
+ }
+
+ LOG_DEBUG_S(ctx, NKikimrServices::KQP_WORKER, "-- NE Compatible: "
+ << " tx: " << newEngineCompatibleTx
+ << ", query: " << QueryState->NewEngineCompatibleQuery
+ << ", interactive: " << QueryState->InteractiveTx
+ << ", preparedNewEngine: " << (bool) QueryState->QueryCompileResult->PreparedQueryNewEngine
+ << ", forcedNewEngine: " << (QueryState->ForceNewEngineState.ForcedNewEngine
+ ? ToString(*QueryState->ForceNewEngineState.ForcedNewEngine)
+ : "<none>")
+ << ", traits: " << (QueryState->QueryCompileResult->QueryTraits
+ ? QueryState->QueryCompileResult->QueryTraits->ToString()
+ : "<none>"));
+
+ if (newEngineCompatibleTx) {
+ if (QueryState->NewEngineCompatibleQuery) {
+ if (QueryState->ForceNewEngineState.ForcedNewEngine && *QueryState->ForceNewEngineState.ForcedNewEngine) {
+ preparedQuery = QueryState->QueryCompileResult->PreparedQueryNewEngine;
+ LOG_INFO_S(ctx, NKikimrServices::KQP_WORKER, "Force NewEngine query execution (as part of tx)");
+ } else if (QueryState->ForceNewEngineState.ForceNewEnginePercent >= RandomNumber((ui32) 100)) {
+ preparedQuery = QueryState->QueryCompileResult->PreparedQueryNewEngine;
+ QueryState->ForceNewEngineState.ForcedNewEngine = true;
+
+ KqpHost->ForceTxNewEngine(
+ QueryState->TxId,
+ QueryState->ForceNewEngineState.ForceNewEnginePercent,
+ QueryState->ForceNewEngineState.ForceNewEngineLevel
+ );
+ LOG_INFO_S(ctx, NKikimrServices::KQP_WORKER, "Force NewEngine query execution (new tx)");
+ } else {
+ QueryState->ForceNewEngineState.ForcedNewEngine = false;
+ KqpHost->ForceTxOldEngine(QueryState->TxId);
+ preparedQuery = QueryState->QueryCompileResult->PreparedQuery;
+ LOG_INFO_S(ctx, NKikimrServices::KQP_WORKER, "Force OldEngine query execution (new tx)");
+ }
+ } else {
+ if (!QueryState->ForceNewEngineState.ForcedNewEngine.has_value()) {
+ if (QueryState->ForceNewEngineState.ForceNewEnginePercent >= RandomNumber((ui32) 100)) {
+ QueryState->ForceNewEngineState.ForcedNewEngine = true;
+
+ KqpHost->ForceTxNewEngine(
+ QueryState->TxId,
+ QueryState->ForceNewEngineState.ForceNewEnginePercent,
+ QueryState->ForceNewEngineState.ForceNewEngineLevel
+ );
+ LOG_INFO_S(ctx, NKikimrServices::KQP_WORKER, "Force NewEngine query execution (new tx)");
+ } else {
+ QueryState->ForceNewEngineState.ForcedNewEngine = false;
+ KqpHost->ForceTxOldEngine(QueryState->TxId);
+ }
+ }
+
+ preparedQuery = QueryState->QueryCompileResult->PreparedQuery;
+ }
+ } else {
+ preparedQuery = QueryState->QueryCompileResult->PreparedQuery;
+ }
} else if (Settings.LongSession) {
onError(Ydb::StatusIds::NOT_FOUND, TStringBuilder() << "Prepared query not found: " << query);
- return;
+ return;
} else {
- NKikimrKqp::TPreparedQuery tmp;
- if (!tmp.ParseFromString(query)) {
+ NKikimrKqp::TPreparedQuery tmp;
+ if (!tmp.ParseFromString(query)) {
onBadRequest("Failed to parse prepared query.");
return;
}
- preparedQuery = std::make_shared<const NKikimrKqp::TPreparedQuery>(std::move(tmp));
+ preparedQuery = std::make_shared<const NKikimrKqp::TPreparedQuery>(std::move(tmp));
}
- YQL_ENSURE(preparedQuery);
+ YQL_ENSURE(preparedQuery);
- QueryState->Request.SetQuery(preparedQuery->GetText());
+ QueryState->Request.SetQuery(preparedQuery->GetText());
- if (!ExecutePreparedQuery(preparedQuery, queryType, std::move(*QueryState->Request.MutableParameters()),
+ if (!ExecutePreparedQuery(preparedQuery, queryType, std::move(*QueryState->Request.MutableParameters()),
commit, GetStatsMode(queryRequest, EKikimrStatsMode::Basic)))
{
onBadRequest(QueryState->Error);
@@ -1165,14 +1165,14 @@ private:
case NKikimrKqp::QUERY_ACTION_BEGIN_TX: {
TQueryResult result;
result.SetSuccess();
- QueryState->AsyncQueryResult = MakeKikimrResultHolder(std::move(result));
+ QueryState->AsyncQueryResult = MakeKikimrResultHolder(std::move(result));
ContinueQueryProcess(ctx);
Become(&TKqpWorkerActor::PerformQueryState);
return;
}
case NKikimrKqp::QUERY_ACTION_COMMIT_TX: {
- if (!CommitTx(commit, GetStatsMode(queryRequest, EKikimrStatsMode::Basic))) {
+ if (!CommitTx(commit, GetStatsMode(queryRequest, EKikimrStatsMode::Basic))) {
onBadRequest(QueryState->Error);
return;
}
@@ -1216,10 +1216,10 @@ private:
auto abortedCount = KqpHost->AbortAll();
Counters->ReportTxAborted(Settings.DbCounters, abortedCount);
}
- CleanupState->AsyncResult = KqpHost->RollbackAborted();
+ CleanupState->AsyncResult = KqpHost->RollbackAborted();
} else {
- if (isFinal && QueryState->TxId) {
- CleanupState->AsyncResult = KqpHost->RollbackTransaction(QueryState->TxId, CreateRollbackSettings());
+ if (isFinal && QueryState->TxId) {
+ CleanupState->AsyncResult = KqpHost->RollbackTransaction(QueryState->TxId, CreateRollbackSettings());
}
}
@@ -1270,13 +1270,13 @@ private:
Cleanup(ctx, true);
}
- bool ExecuteQuery(NKikimrKqp::TQueryRequest& queryRequest, NKikimrKqp::EQueryType type, bool commit,
+ bool ExecuteQuery(NKikimrKqp::TQueryRequest& queryRequest, NKikimrKqp::EQueryType type, bool commit,
const TActorId& requestActorId)
{
- const auto& query = queryRequest.GetQuery();
- auto* parameters = queryRequest.MutableParameters();
- auto statsMode = GetStatsMode(queryRequest, EKikimrStatsMode::Basic);
-
+ const auto& query = queryRequest.GetQuery();
+ auto* parameters = queryRequest.MutableParameters();
+ auto statsMode = GetStatsMode(queryRequest, EKikimrStatsMode::Basic);
+
switch (type) {
case NKikimrKqp::QUERY_TYPE_SQL_DML:
case NKikimrKqp::QUERY_TYPE_AST_DML: {
@@ -1287,27 +1287,27 @@ private:
execSettings.StatsMode = statsMode;
execSettings.Deadlines = QueryState->QueryDeadlines;
execSettings.Limits = GetQueryLimits(Settings);
- execSettings.StrictDml = false;
- execSettings.UseNewEngine = UseNewEngine();
+ execSettings.StrictDml = false;
+ execSettings.UseNewEngine = UseNewEngine();
execSettings.DocumentApiRestricted = IsDocumentApiRestricted(QueryState->RequestType);
QueryState->AsyncQueryResult = KqpHost->ExecuteDataQuery(QueryState->TxId, query, isSql,
- std::move(*parameters), execSettings);
+ std::move(*parameters), execSettings);
break;
}
-
- case NKikimrKqp::QUERY_TYPE_SQL_DDL: {
+
+ case NKikimrKqp::QUERY_TYPE_SQL_DDL: {
QueryState->AsyncQueryResult = KqpHost->ExecuteSchemeQuery(query, true);
break;
- }
+ }
- case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT: {
+ case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT: {
IKqpHost::TExecScriptSettings execSettings;
execSettings.Deadlines = QueryState->QueryDeadlines;
execSettings.StatsMode = statsMode;
QueryState->AsyncQueryResult = KqpHost->ExecuteYqlScript(query, std::move(*parameters), execSettings);
break;
- }
+ }
case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT_STREAMING: {
IKqpHost::TExecScriptSettings execSettings;
@@ -1322,27 +1322,27 @@ private:
case NKikimrKqp::QUERY_TYPE_AST_SCAN: {
bool isSql = (type == NKikimrKqp::QUERY_TYPE_SQL_SCAN);
- NYql::IKikimrQueryExecutor::TExecuteSettings execSettings;
+ NYql::IKikimrQueryExecutor::TExecuteSettings execSettings;
execSettings.StatsMode = statsMode;
- execSettings.Deadlines = QueryState->QueryDeadlines;
+ execSettings.Deadlines = QueryState->QueryDeadlines;
execSettings.Limits = GetQueryLimits(Settings);
execSettings.RlPath = QueryState->RlPath;
QueryState->AsyncQueryResult = KqpHost->ExecuteScanQuery(query, isSql, std::move(*parameters),
- requestActorId, execSettings);
+ requestActorId, execSettings);
break;
- }
+ }
- default: {
+ default: {
QueryState->Error = "Unexpected query type.";
return false;
- }
+ }
}
return true;
}
- bool ExplainQuery(const TActorContext&, const TString& query, NKikimrKqp::EQueryType type) {
+ bool ExplainQuery(const TActorContext&, const TString& query, NKikimrKqp::EQueryType type) {
switch (type) {
case NKikimrKqp::QUERY_TYPE_SQL_DML:
case NKikimrKqp::QUERY_TYPE_AST_DML: {
@@ -1373,7 +1373,7 @@ private:
return true;
}
- bool ValidateQuery(const TActorContext&, const TString& query, NKikimrKqp::EQueryType type) {
+ bool ValidateQuery(const TActorContext&, const TString& query, NKikimrKqp::EQueryType type) {
switch (type) {
case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT:
case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT_STREAMING: {
@@ -1389,7 +1389,7 @@ private:
return true;
}
- bool PrepareQuery(const TActorContext&, const TString& query, NKikimrKqp::EQueryType type, bool sqlAutoCommit) {
+ bool PrepareQuery(const TActorContext&, const TString& query, NKikimrKqp::EQueryType type, bool sqlAutoCommit) {
if (sqlAutoCommit) {
QueryState->Error = "Expected SqlAutoCommit=false for query prepare.";
return false;
@@ -1398,7 +1398,7 @@ private:
switch (type) {
case NKikimrKqp::QUERY_TYPE_SQL_DML: {
IKqpHost::TPrepareSettings prepareSettings;
- // prepareSettings.UseNewEngine = use default settings
+ // prepareSettings.UseNewEngine = use default settings
prepareSettings.DocumentApiRestricted = IsDocumentApiRestricted(QueryState->RequestType);
QueryState->AsyncQueryResult = KqpHost->PrepareDataQuery(query, prepareSettings);
break;
@@ -1412,22 +1412,22 @@ private:
return true;
}
- bool ExecutePreparedQuery(TPreparedQueryConstPtr& query, NKikimrKqp::EQueryType type,
- NKikimrMiniKQL::TParams&& parameters, bool commit, EKikimrStatsMode statsMode)
- {
- if (type == NKikimrKqp::QUERY_TYPE_PREPARED_DML) {
- NYql::IKikimrQueryExecutor::TExecuteSettings execSettings;
- execSettings.CommitTx = commit;
- execSettings.StatsMode = statsMode;
- execSettings.Deadlines = QueryState->QueryDeadlines;
- execSettings.Limits = GetQueryLimits(Settings);
-
- QueryState->AsyncQueryResult = KqpHost->ExecuteDataQuery(QueryState->TxId, query, std::move(parameters),
- execSettings);
- return true;
+ bool ExecutePreparedQuery(TPreparedQueryConstPtr& query, NKikimrKqp::EQueryType type,
+ NKikimrMiniKQL::TParams&& parameters, bool commit, EKikimrStatsMode statsMode)
+ {
+ if (type == NKikimrKqp::QUERY_TYPE_PREPARED_DML) {
+ NYql::IKikimrQueryExecutor::TExecuteSettings execSettings;
+ execSettings.CommitTx = commit;
+ execSettings.StatsMode = statsMode;
+ execSettings.Deadlines = QueryState->QueryDeadlines;
+ execSettings.Limits = GetQueryLimits(Settings);
+
+ QueryState->AsyncQueryResult = KqpHost->ExecuteDataQuery(QueryState->TxId, query, std::move(parameters),
+ execSettings);
+ return true;
} else {
- QueryState->Error = "Unexpected query type.";
- return false;
+ QueryState->Error = "Unexpected query type.";
+ return false;
}
}
@@ -1441,12 +1441,12 @@ private:
return false;
}
- IKikimrQueryExecutor::TExecuteSettings execSettings;
- execSettings.CommitTx = true;
+ IKikimrQueryExecutor::TExecuteSettings execSettings;
+ execSettings.CommitTx = true;
execSettings.StatsMode = statsMode;
execSettings.Deadlines = QueryState->QueryDeadlines;
execSettings.Limits = GetQueryLimits(Settings);
- execSettings.UseNewEngine = UseNewEngine();
+ execSettings.UseNewEngine = UseNewEngine();
QueryState->AsyncQueryResult = KqpHost->CommitTransaction(QueryState->TxId, execSettings);
return true;
@@ -1462,7 +1462,7 @@ private:
return false;
}
- QueryState->AsyncQueryResult = KqpHost->RollbackTransaction(QueryState->TxId, CreateRollbackSettings());
+ QueryState->AsyncQueryResult = KqpHost->RollbackTransaction(QueryState->TxId, CreateRollbackSettings());
return true;
}
@@ -1621,7 +1621,7 @@ private:
if (status == Ydb::StatusIds::SUCCESS) {
Counters->ReportQueryLatency(Settings.DbCounters, queryRequest.GetAction(), queryDuration);
-
+
auto maxReadType = ExtractMostHeavyReadType(queryResult.QueryPlan);
if (maxReadType == ETableReadType::FullScan) {
Counters->ReportQueryWithFullScan(Settings.DbCounters);
@@ -1629,12 +1629,12 @@ private:
Counters->ReportQueryWithRangeScan(Settings.DbCounters);
}
- ui32 affectedShardsCount = 0;
+ ui32 affectedShardsCount = 0;
ui64 readBytesCount = 0;
ui64 readRowsCount = 0;
for (const auto& exec : queryResult.QueryStats.GetExecutions()) {
for (const auto& table : exec.GetTables()) {
- affectedShardsCount = std::max(affectedShardsCount, table.GetAffectedPartitions());
+ affectedShardsCount = std::max(affectedShardsCount, table.GetAffectedPartitions());
readBytesCount += table.GetReadBytes();
readRowsCount += table.GetReadRows();
}
@@ -1647,21 +1647,21 @@ private:
Counters->ReportQueryMaxShardReplySize(Settings.DbCounters, queryResult.QueryStats.GetMaxShardReplySize());
Counters->ReportQueryMaxShardProgramSize(Settings.DbCounters, queryResult.QueryStats.GetMaxShardProgramSize());
- if (QueryState->QueryCompileResult && QueryState->QueryCompileResult->PreparedQueryNewEngine
- && QueryState->NewEngineCompatibleQuery)
- {
- ui64 computeCpuTimeUs = 0;
- for (auto& execution : queryResult.QueryStats.GetExecutions()) {
- computeCpuTimeUs += execution.GetCpuTimeUs();
- }
-
- // query can be executed with NewEngine
- if (QueryState->ForceNewEngineState.ForcedNewEngine && *QueryState->ForceNewEngineState.ForcedNewEngine) {
- Counters->ReportNewEngineForcedQueryStats(queryRequest.GetAction(), queryDuration, computeCpuTimeUs);
- } else {
- Counters->ReportNewEngineCompatibleQueryStats(queryRequest.GetAction(), queryDuration, computeCpuTimeUs);
- }
- }
+ if (QueryState->QueryCompileResult && QueryState->QueryCompileResult->PreparedQueryNewEngine
+ && QueryState->NewEngineCompatibleQuery)
+ {
+ ui64 computeCpuTimeUs = 0;
+ for (auto& execution : queryResult.QueryStats.GetExecutions()) {
+ computeCpuTimeUs += execution.GetCpuTimeUs();
+ }
+
+ // query can be executed with NewEngine
+ if (QueryState->ForceNewEngineState.ForcedNewEngine && *QueryState->ForceNewEngineState.ForcedNewEngine) {
+ Counters->ReportNewEngineForcedQueryStats(queryRequest.GetAction(), queryDuration, computeCpuTimeUs);
+ } else {
+ Counters->ReportNewEngineCompatibleQueryStats(queryRequest.GetAction(), queryDuration, computeCpuTimeUs);
+ }
+ }
}
if (queryResult.SqlVersion) {
@@ -1722,7 +1722,7 @@ private:
});
}
- bool reportStats = (GetStatsMode(queryRequest, EKikimrStatsMode::None) != EKikimrStatsMode::None);
+ bool reportStats = (GetStatsMode(queryRequest, EKikimrStatsMode::None) != EKikimrStatsMode::None);
if (reportStats) {
// TODO: For compatibility with old rpc handlers, deprecate.
@@ -1893,11 +1893,11 @@ private:
response.SetPreparedQuery(compileResult->Uid);
auto& preparedQuery = compileResult->PreparedQuery;
- response.MutableQueryParameters()->CopyFrom(preparedQuery->GetParameters());
+ response.MutableQueryParameters()->CopyFrom(preparedQuery->GetParameters());
- if (preparedQuery->KqlsSize() > 0) {
- response.SetQueryAst(preparedQuery->GetKqls(0).GetAst());
- response.SetQueryPlan(preparedQuery->GetKqls(0).GetPlan());
+ if (preparedQuery->KqlsSize() > 0) {
+ response.SetQueryAst(preparedQuery->GetKqls(0).GetAst());
+ response.SetQueryPlan(preparedQuery->GetKqls(0).GetPlan());
}
}
}
@@ -1907,7 +1907,7 @@ private:
for (auto& execStats : stats.GetExecutions()) {
auto& txStats = *kqlProfile.AddMkqlProfiles()->MutableTxStats();
- txStats.SetDurationUs(execStats.GetDurationUs());
+ txStats.SetDurationUs(execStats.GetDurationUs());
for (auto& tableStats : execStats.GetTables()) {
auto& txTableStats = *txStats.AddTableAccessStats();
@@ -2007,11 +2007,11 @@ private:
if (replyQueryParameters) {
if (QueryState->QueryCompileResult) {
ev.MutableResponse()->MutableQueryParameters()->CopyFrom(
- QueryState->QueryCompileResult->PreparedQuery->GetParameters());
+ QueryState->QueryCompileResult->PreparedQuery->GetParameters());
} else {
- YQL_ENSURE(queryResult.PreparedQuery);
- ev.MutableResponse()->MutableQueryParameters()->CopyFrom(
- queryResult.PreparedQuery->GetParameters());
+ YQL_ENSURE(queryResult.PreparedQuery);
+ ev.MutableResponse()->MutableQueryParameters()->CopyFrom(
+ queryResult.PreparedQuery->GetParameters());
}
}
@@ -2020,8 +2020,8 @@ private:
if (QueryState->QueryCompileResult) {
queryId = QueryState->QueryCompileResult->Uid;
} else {
- YQL_ENSURE(!Settings.LongSession);
- Y_PROTOBUF_SUPPRESS_NODISCARD queryResult.PreparedQuery->SerializeToString(&queryId);
+ YQL_ENSURE(!Settings.LongSession);
+ Y_PROTOBUF_SUPPRESS_NODISCARD queryResult.PreparedQuery->SerializeToString(&queryId);
}
ev.MutableResponse()->SetPreparedQuery(queryId);
@@ -2109,29 +2109,29 @@ private:
IdleTimerActorId = TActorId();
}
- IKikimrQueryExecutor::TExecuteSettings CreateRollbackSettings() {
- YQL_ENSURE(QueryState);
-
- IKikimrQueryExecutor::TExecuteSettings settings;
- settings.RollbackTx = true;
- settings.Deadlines.TimeoutAt = TInstant::Now() + TDuration::Minutes(1);
- settings.UseNewEngine = UseNewEngine();
-
- return settings;
- }
-
- TMaybe<bool> UseNewEngine() const {
- YQL_ENSURE(QueryState);
-
- if (auto txInfo = KqpHost->GetTransactionInfo(QueryState->TxId)) {
- if (auto engine = txInfo->TxEngine; engine.has_value()) {
- return *engine == TKqpTransactionInfo::EEngine::NewEngine;
- }
- }
-
- return Nothing();
- }
-
+ IKikimrQueryExecutor::TExecuteSettings CreateRollbackSettings() {
+ YQL_ENSURE(QueryState);
+
+ IKikimrQueryExecutor::TExecuteSettings settings;
+ settings.RollbackTx = true;
+ settings.Deadlines.TimeoutAt = TInstant::Now() + TDuration::Minutes(1);
+ settings.UseNewEngine = UseNewEngine();
+
+ return settings;
+ }
+
+ TMaybe<bool> UseNewEngine() const {
+ YQL_ENSURE(QueryState);
+
+ if (auto txInfo = KqpHost->GetTransactionInfo(QueryState->TxId)) {
+ if (auto engine = txInfo->TxEngine; engine.has_value()) {
+ return *engine == TKqpTransactionInfo::EEngine::NewEngine;
+ }
+ }
+
+ return Nothing();
+ }
+
static bool IsExecuteAction(const NKikimrKqp::EQueryAction& action) {
switch (action) {
case NKikimrKqp::QUERY_ACTION_EXECUTE:
@@ -2167,7 +2167,7 @@ private:
TKqpWorkerSettings Settings;
TIntrusivePtr<TModuleResolverState> ModuleResolverState;
TIntrusivePtr<TKqpCounters> Counters;
- TIntrusivePtr<TKqpRequestCounters> RequestCounters;
+ TIntrusivePtr<TKqpRequestCounters> RequestCounters;
TKikimrConfiguration::TPtr Config;
TInstant CreationTime;
TIntrusivePtr<IKqpGateway> Gateway;
@@ -2189,5 +2189,5 @@ IActor* CreateKqpWorkerActor(const TActorId& owner, const TString& sessionId,
return new TKqpWorkerActor(owner, sessionId, kqpSettings, workerSettings, moduleResolverState, counters);
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/node/kqp_node.cpp b/ydb/core/kqp/node/kqp_node.cpp
index 09b52b6925..24f8f2ac79 100644
--- a/ydb/core/kqp/node/kqp_node.cpp
+++ b/ydb/core/kqp/node/kqp_node.cpp
@@ -1,301 +1,301 @@
-#include "kqp_node.h"
-#include "kqp_node_state.h"
-
+#include "kqp_node.h"
+#include "kqp_node_state.h"
+
#include <ydb/core/actorlib_impl/long_timer.h>
#include <ydb/core/cms/console/configs_dispatcher.h>
#include <ydb/core/cms/console/console.h>
#include <ydb/core/protos/tx_datashard.pb.h>
#include <ydb/core/mon/mon.h>
-
+
#include <ydb/core/kqp/kqp.h>
#include <ydb/core/kqp/compute_actor/kqp_compute_actor.h>
#include <ydb/core/kqp/rm/kqp_resource_estimation.h>
#include <ydb/core/kqp/rm/kqp_rm.h>
#include <ydb/core/kqp/common/kqp_resolve.h>
-
-#include <library/cpp/actors/core/actor_bootstrapped.h>
-#include <library/cpp/monlib/service/pages/templates.h>
-
+
+#include <library/cpp/actors/core/actor_bootstrapped.h>
+#include <library/cpp/monlib/service/pages/templates.h>
+
#include <util/string/join.h>
-
-namespace NKikimr {
-namespace NKqp {
-
-using namespace NActors;
-
-namespace {
-
-#define LOG_C(stream) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::KQP_NODE, stream)
-#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_NODE, stream)
-#define LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::KQP_NODE, stream)
-#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_NODE, stream)
-#define LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::KQP_NODE, stream)
-#define LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::KQP_NODE, stream)
-
+
+namespace NKikimr {
+namespace NKqp {
+
+using namespace NActors;
+
+namespace {
+
+#define LOG_C(stream) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::KQP_NODE, stream)
+#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_NODE, stream)
+#define LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::KQP_NODE, stream)
+#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_NODE, stream)
+#define LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::KQP_NODE, stream)
+#define LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::KQP_NODE, stream)
+
// Min interval between stats send from scan/compute actor to executor
constexpr TDuration MinStatInterval = TDuration::MilliSeconds(20);
// Max interval in case of no activety
constexpr TDuration MaxStatInterval = TDuration::MilliSeconds(100);
-template <class TTasksCollection>
-TString TasksIdsStr(const TTasksCollection& tasks) {
- TVector<ui64> ids;
- for (auto& task: tasks) {
- ids.push_back(task.GetId());
- }
- return TStringBuilder() << "[" << JoinSeq(", ", ids) << "]";
-}
-
-class TKqpNodeService : public TActorBootstrapped<TKqpNodeService> {
- using TBase = TActorBootstrapped<TKqpNodeService>;
-
- struct TEvPrivate {
- enum EEv {
- EvTimeout = EventSpaceBegin(TEvents::ES_PRIVATE),
- };
-
- struct TEvTimeout : public TEventLocal<TEvTimeout, EEv::EvTimeout> {
- const ui64 TxId;
- const TActorId Requester;
-
- TEvTimeout(ui64 txId, const TActorId& requester)
- : TxId(txId)
- , Requester(requester) {}
- };
- };
-
-public:
- static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
- return NKikimrServices::TActivity::KQP_NODE_SERVICE;
- }
-
- TKqpNodeService(const NKikimrConfig::TTableServiceConfig& config, const TIntrusivePtr<TKqpCounters>& counters,
- IKqpNodeComputeActorFactory* caFactory)
- : Config(config.GetResourceManager())
- , Counters(counters)
- , CaFactory(caFactory) {}
-
- void Bootstrap() {
- LOG_I("Starting KQP Node service");
-
- // Subscribe for TableService config changes
- ui32 tableServiceConfigKind = (ui32) NKikimrConsole::TConfigItem::TableServiceConfigItem;
- Send(NConsole::MakeConfigsDispatcherID(SelfId().NodeId()),
- new NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest({tableServiceConfigKind}),
- IEventHandle::FlagTrackDelivery);
-
- NActors::TMon* mon = AppData()->Mon;
- if (mon) {
- NMonitoring::TIndexMonPage* actorsMonPage = mon->RegisterIndexPage("actors", "Actors");
- mon->RegisterActorPage(actorsMonPage, "kqp_node", "KQP Node", false,
- TlsActivationContext->ExecutorThread.ActorSystem, SelfId());
- }
-
- Become(&TKqpNodeService::WorkState);
- }
-
-private:
- STATEFN(WorkState) {
- switch (ev->GetTypeRewrite()) {
- hFunc(TEvKqpNode::TEvStartKqpTasksRequest, HandleWork);
- hFunc(TEvKqpNode::TEvFinishKqpTask, HandleWork);
- hFunc(TEvKqpNode::TEvCancelKqpTasksRequest, HandleWork);
- hFunc(TEvPrivate::TEvTimeout, HandleWork);
- // misc
- hFunc(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse, HandleWork);
- hFunc(NConsole::TEvConsole::TEvConfigNotificationRequest, HandleWork);
- hFunc(TEvents::TEvUndelivered, HandleWork);
- hFunc(TEvents::TEvPoison, HandleWork);
- hFunc(NMon::TEvHttpInfo, HandleWork);
- default: {
- Y_FAIL("Unexpected event 0x%x for TKqpResourceManagerService", ev->GetTypeRewrite());
- }
- }
- }
-
- void HandleWork(TEvKqpNode::TEvStartKqpTasksRequest::TPtr& ev) {
- auto& msg = ev->Get()->Record;
- auto requester = ev->Sender;
-
- ui64 txId = msg.GetTxId();
- bool isScan = msg.HasSnapshot();
-
- YQL_ENSURE(msg.GetStartAllOrFail()); // todo: support partial start
-
- LOG_D("TxId: " << txId << ", new " << (isScan ? "scan " : "") << "compute tasks request from " << requester
- << " with " << msg.GetTasks().size() << " tasks: " << TasksIdsStr(msg.GetTasks()));
-
- NKqpNode::TTasksRequest request;
- request.Executer = ActorIdFromProto(msg.GetExecuterActorId());
-
- if (State.Exists(txId, requester)) {
- LOG_E("TxId: " << txId << ", requester: " << requester << ", request already exists");
- return ReplyError(txId, request.Executer, msg, NKikimrKqp::TEvStartKqpTasksResponse::INTERNAL_ERROR);
- }
-
- ui32 requestScans = 0;
- ui32 requestChannels = 0;
-
- for (auto& dqTask : *msg.MutableTasks()) {
- ui32 nScans = 0;
-
- if (isScan) {
- NKikimrTxDataShard::TKqpTransaction::TScanTaskMeta protoTaskMeta;
-
- YQL_ENSURE(msg.GetRuntimeSettings().GetExecType() == NYql::NDqProto::TComputeRuntimeSettings::SCAN);
- YQL_ENSURE(msg.GetRuntimeSettings().GetTasksOnNodeCount() == 0); // legacy
-
- dqTask.GetMeta().UnpackTo(&protoTaskMeta);
- nScans = protoTaskMeta.GetReads().size();
- }
-
- auto estimation = EstimateTaskResources(dqTask, nScans, /* dsOnNodeCount */ 0, Config);
-
- NKqpNode::TTaskContext& taskCtx = request.InFlyTasks[dqTask.GetId()];
- YQL_ENSURE(taskCtx.TaskId == 0);
- taskCtx.TaskId = dqTask.GetId();
- taskCtx.Memory = estimation.TotalMemoryLimit;
- taskCtx.Channels = estimation.ChannelBuffersCount;
- taskCtx.ChannelSize = estimation.ChannelBufferMemoryLimit;
-
- LOG_D("TxId: " << txId << ", task: " << taskCtx.TaskId << ", requested memory: " << taskCtx.Memory);
-
- requestScans += estimation.ScanBuffersCount;
- requestChannels += estimation.ChannelBuffersCount;
- request.TotalMemory += taskCtx.Memory;
- }
-
- LOG_D("TxId: " << txId << ", requested scans: " << requestScans << ", channels: " << requestChannels
- << ", computeActors: " << msg.GetTasks().size() << ", memory: " << request.TotalMemory);
-
- ui64 txMemory = State.GetTxMemory(txId, NRm::EKqpMemoryPool::ScanQuery) + request.TotalMemory;
- if (txMemory > Config.GetQueryMemoryLimit()) {
- LOG_N("TxId: " << txId << ", requested too many memory: " << request.TotalMemory
- << "(" << txMemory << " for this Tx), limit: " << Config.GetQueryMemoryLimit());
-
- Counters->RmNotEnoughMemory->Inc();
-
- return ReplyError(txId, request.Executer, msg, NKikimrKqp::TEvStartKqpTasksResponse::QUERY_MEMORY_LIMIT_EXCEEDED,
- TStringBuilder() << "Required: " << txMemory << ", limit: " << Config.GetQueryMemoryLimit());
- }
-
- TVector<ui64> allocatedTasks;
- allocatedTasks.reserve(msg.GetTasks().size());
- for (auto& task : request.InFlyTasks) {
- NRm::TKqpResourcesRequest resourcesRequest;
- resourcesRequest.ExecutionUnits = 1;
- resourcesRequest.MemoryPool = NRm::EKqpMemoryPool::ScanQuery;
- resourcesRequest.Memory = task.second.Memory;
-
- NRm::TKqpNotEnoughResources resourcesResponse;
- if (!ResourceManager()->AllocateResources(txId, task.first, resourcesRequest, &resourcesResponse)) {
- NKikimrKqp::TEvStartKqpTasksResponse::ENotStartedTaskReason failReason;
- TStringBuilder error;
-
- if (resourcesResponse.ExecutionUnits()) {
- error << "TxId: " << txId << ", not enough compute actors, requested " << msg.GetTasks().size();
- LOG_N(error);
-
- failReason = NKikimrKqp::TEvStartKqpTasksResponse::NOT_ENOUGH_EXECUTION_UNITS;
- }
-
- if (resourcesResponse.ScanQueryMemory()) {
- error << "TxId: " << txId << ", not enough memory, requested " << task.second.Memory;
- LOG_N(error);
-
- failReason = NKikimrKqp::TEvStartKqpTasksResponse::NOT_ENOUGH_MEMORY;
- }
-
- if (resourcesResponse.QueryMemoryLimit()) {
- error << "TxId: " << txId << ", memory limit exceeded, requested " << task.second.Memory;
- LOG_N(error);
-
- failReason = NKikimrKqp::TEvStartKqpTasksResponse::QUERY_MEMORY_LIMIT_EXCEEDED;
- }
-
- for (ui64 taskId : allocatedTasks) {
- ResourceManager()->FreeResources(txId, taskId);
- }
-
- ReplyError(txId, request.Executer, msg, failReason, error);
- return;
- }
-
- allocatedTasks.push_back(task.first);
- }
-
- auto reply = MakeHolder<TEvKqpNode::TEvStartKqpTasksResponse>();
- reply->Record.SetTxId(txId);
-
- NYql::NDq::TComputeMemoryLimits memoryLimits;
- memoryLimits.ChannelBufferSize = 0;
- memoryLimits.ScanBufferSize = Config.GetScanBufferSize();
- memoryLimits.MkqlLightProgramMemoryLimit = Config.GetMkqlLightProgramMemoryLimit();
- memoryLimits.MkqlHeavyProgramMemoryLimit = Config.GetMkqlHeavyProgramMemoryLimit();
- if (Config.GetEnableInstantMkqlMemoryAlloc()) {
+template <class TTasksCollection>
+TString TasksIdsStr(const TTasksCollection& tasks) {
+ TVector<ui64> ids;
+ for (auto& task: tasks) {
+ ids.push_back(task.GetId());
+ }
+ return TStringBuilder() << "[" << JoinSeq(", ", ids) << "]";
+}
+
+class TKqpNodeService : public TActorBootstrapped<TKqpNodeService> {
+ using TBase = TActorBootstrapped<TKqpNodeService>;
+
+ struct TEvPrivate {
+ enum EEv {
+ EvTimeout = EventSpaceBegin(TEvents::ES_PRIVATE),
+ };
+
+ struct TEvTimeout : public TEventLocal<TEvTimeout, EEv::EvTimeout> {
+ const ui64 TxId;
+ const TActorId Requester;
+
+ TEvTimeout(ui64 txId, const TActorId& requester)
+ : TxId(txId)
+ , Requester(requester) {}
+ };
+ };
+
+public:
+ static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
+ return NKikimrServices::TActivity::KQP_NODE_SERVICE;
+ }
+
+ TKqpNodeService(const NKikimrConfig::TTableServiceConfig& config, const TIntrusivePtr<TKqpCounters>& counters,
+ IKqpNodeComputeActorFactory* caFactory)
+ : Config(config.GetResourceManager())
+ , Counters(counters)
+ , CaFactory(caFactory) {}
+
+ void Bootstrap() {
+ LOG_I("Starting KQP Node service");
+
+ // Subscribe for TableService config changes
+ ui32 tableServiceConfigKind = (ui32) NKikimrConsole::TConfigItem::TableServiceConfigItem;
+ Send(NConsole::MakeConfigsDispatcherID(SelfId().NodeId()),
+ new NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest({tableServiceConfigKind}),
+ IEventHandle::FlagTrackDelivery);
+
+ NActors::TMon* mon = AppData()->Mon;
+ if (mon) {
+ NMonitoring::TIndexMonPage* actorsMonPage = mon->RegisterIndexPage("actors", "Actors");
+ mon->RegisterActorPage(actorsMonPage, "kqp_node", "KQP Node", false,
+ TlsActivationContext->ExecutorThread.ActorSystem, SelfId());
+ }
+
+ Become(&TKqpNodeService::WorkState);
+ }
+
+private:
+ STATEFN(WorkState) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvKqpNode::TEvStartKqpTasksRequest, HandleWork);
+ hFunc(TEvKqpNode::TEvFinishKqpTask, HandleWork);
+ hFunc(TEvKqpNode::TEvCancelKqpTasksRequest, HandleWork);
+ hFunc(TEvPrivate::TEvTimeout, HandleWork);
+ // misc
+ hFunc(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse, HandleWork);
+ hFunc(NConsole::TEvConsole::TEvConfigNotificationRequest, HandleWork);
+ hFunc(TEvents::TEvUndelivered, HandleWork);
+ hFunc(TEvents::TEvPoison, HandleWork);
+ hFunc(NMon::TEvHttpInfo, HandleWork);
+ default: {
+ Y_FAIL("Unexpected event 0x%x for TKqpResourceManagerService", ev->GetTypeRewrite());
+ }
+ }
+ }
+
+ void HandleWork(TEvKqpNode::TEvStartKqpTasksRequest::TPtr& ev) {
+ auto& msg = ev->Get()->Record;
+ auto requester = ev->Sender;
+
+ ui64 txId = msg.GetTxId();
+ bool isScan = msg.HasSnapshot();
+
+ YQL_ENSURE(msg.GetStartAllOrFail()); // todo: support partial start
+
+ LOG_D("TxId: " << txId << ", new " << (isScan ? "scan " : "") << "compute tasks request from " << requester
+ << " with " << msg.GetTasks().size() << " tasks: " << TasksIdsStr(msg.GetTasks()));
+
+ NKqpNode::TTasksRequest request;
+ request.Executer = ActorIdFromProto(msg.GetExecuterActorId());
+
+ if (State.Exists(txId, requester)) {
+ LOG_E("TxId: " << txId << ", requester: " << requester << ", request already exists");
+ return ReplyError(txId, request.Executer, msg, NKikimrKqp::TEvStartKqpTasksResponse::INTERNAL_ERROR);
+ }
+
+ ui32 requestScans = 0;
+ ui32 requestChannels = 0;
+
+ for (auto& dqTask : *msg.MutableTasks()) {
+ ui32 nScans = 0;
+
+ if (isScan) {
+ NKikimrTxDataShard::TKqpTransaction::TScanTaskMeta protoTaskMeta;
+
+ YQL_ENSURE(msg.GetRuntimeSettings().GetExecType() == NYql::NDqProto::TComputeRuntimeSettings::SCAN);
+ YQL_ENSURE(msg.GetRuntimeSettings().GetTasksOnNodeCount() == 0); // legacy
+
+ dqTask.GetMeta().UnpackTo(&protoTaskMeta);
+ nScans = protoTaskMeta.GetReads().size();
+ }
+
+ auto estimation = EstimateTaskResources(dqTask, nScans, /* dsOnNodeCount */ 0, Config);
+
+ NKqpNode::TTaskContext& taskCtx = request.InFlyTasks[dqTask.GetId()];
+ YQL_ENSURE(taskCtx.TaskId == 0);
+ taskCtx.TaskId = dqTask.GetId();
+ taskCtx.Memory = estimation.TotalMemoryLimit;
+ taskCtx.Channels = estimation.ChannelBuffersCount;
+ taskCtx.ChannelSize = estimation.ChannelBufferMemoryLimit;
+
+ LOG_D("TxId: " << txId << ", task: " << taskCtx.TaskId << ", requested memory: " << taskCtx.Memory);
+
+ requestScans += estimation.ScanBuffersCount;
+ requestChannels += estimation.ChannelBuffersCount;
+ request.TotalMemory += taskCtx.Memory;
+ }
+
+ LOG_D("TxId: " << txId << ", requested scans: " << requestScans << ", channels: " << requestChannels
+ << ", computeActors: " << msg.GetTasks().size() << ", memory: " << request.TotalMemory);
+
+ ui64 txMemory = State.GetTxMemory(txId, NRm::EKqpMemoryPool::ScanQuery) + request.TotalMemory;
+ if (txMemory > Config.GetQueryMemoryLimit()) {
+ LOG_N("TxId: " << txId << ", requested too many memory: " << request.TotalMemory
+ << "(" << txMemory << " for this Tx), limit: " << Config.GetQueryMemoryLimit());
+
+ Counters->RmNotEnoughMemory->Inc();
+
+ return ReplyError(txId, request.Executer, msg, NKikimrKqp::TEvStartKqpTasksResponse::QUERY_MEMORY_LIMIT_EXCEEDED,
+ TStringBuilder() << "Required: " << txMemory << ", limit: " << Config.GetQueryMemoryLimit());
+ }
+
+ TVector<ui64> allocatedTasks;
+ allocatedTasks.reserve(msg.GetTasks().size());
+ for (auto& task : request.InFlyTasks) {
+ NRm::TKqpResourcesRequest resourcesRequest;
+ resourcesRequest.ExecutionUnits = 1;
+ resourcesRequest.MemoryPool = NRm::EKqpMemoryPool::ScanQuery;
+ resourcesRequest.Memory = task.second.Memory;
+
+ NRm::TKqpNotEnoughResources resourcesResponse;
+ if (!ResourceManager()->AllocateResources(txId, task.first, resourcesRequest, &resourcesResponse)) {
+ NKikimrKqp::TEvStartKqpTasksResponse::ENotStartedTaskReason failReason;
+ TStringBuilder error;
+
+ if (resourcesResponse.ExecutionUnits()) {
+ error << "TxId: " << txId << ", not enough compute actors, requested " << msg.GetTasks().size();
+ LOG_N(error);
+
+ failReason = NKikimrKqp::TEvStartKqpTasksResponse::NOT_ENOUGH_EXECUTION_UNITS;
+ }
+
+ if (resourcesResponse.ScanQueryMemory()) {
+ error << "TxId: " << txId << ", not enough memory, requested " << task.second.Memory;
+ LOG_N(error);
+
+ failReason = NKikimrKqp::TEvStartKqpTasksResponse::NOT_ENOUGH_MEMORY;
+ }
+
+ if (resourcesResponse.QueryMemoryLimit()) {
+ error << "TxId: " << txId << ", memory limit exceeded, requested " << task.second.Memory;
+ LOG_N(error);
+
+ failReason = NKikimrKqp::TEvStartKqpTasksResponse::QUERY_MEMORY_LIMIT_EXCEEDED;
+ }
+
+ for (ui64 taskId : allocatedTasks) {
+ ResourceManager()->FreeResources(txId, taskId);
+ }
+
+ ReplyError(txId, request.Executer, msg, failReason, error);
+ return;
+ }
+
+ allocatedTasks.push_back(task.first);
+ }
+
+ auto reply = MakeHolder<TEvKqpNode::TEvStartKqpTasksResponse>();
+ reply->Record.SetTxId(txId);
+
+ NYql::NDq::TComputeMemoryLimits memoryLimits;
+ memoryLimits.ChannelBufferSize = 0;
+ memoryLimits.ScanBufferSize = Config.GetScanBufferSize();
+ memoryLimits.MkqlLightProgramMemoryLimit = Config.GetMkqlLightProgramMemoryLimit();
+ memoryLimits.MkqlHeavyProgramMemoryLimit = Config.GetMkqlHeavyProgramMemoryLimit();
+ if (Config.GetEnableInstantMkqlMemoryAlloc()) {
memoryLimits.AllocateMemoryFn = [rm = ResourceManager()](const auto& txId, ui64 taskId, ui64 memory) {
- NRm::TKqpResourcesRequest resources;
- resources.MemoryPool = NRm::EKqpMemoryPool::ScanQuery;
- resources.Memory = memory;
-
+ NRm::TKqpResourcesRequest resources;
+ resources.MemoryPool = NRm::EKqpMemoryPool::ScanQuery;
+ resources.Memory = memory;
+
if (rm->AllocateResources(std::get<ui64>(txId), taskId, resources)) {
- return true;
- }
-
- LOG_W("Can not allocate memory. TxId: " << txId << ", taskId: " << taskId << ", memory: +" << memory);
- return false;
- };
- }
-
- NYql::NDq::TComputeRuntimeSettings runtimeSettingsBase;
- auto& msgRtSettings = msg.GetRuntimeSettings();
- if (msgRtSettings.GetTimeoutMs() > 0) {
- runtimeSettingsBase.Timeout = TDuration::MilliSeconds(msgRtSettings.GetTimeoutMs());
- request.Deadline = TAppData::TimeProvider->Now() + *runtimeSettingsBase.Timeout;
- }
-
- if (msgRtSettings.GetExecType() == NYql::NDqProto::TComputeRuntimeSettings::SCAN) {
- runtimeSettingsBase.ExtraMemoryAllocationPool = NRm::EKqpMemoryPool::ScanQuery;
- runtimeSettingsBase.FailOnUndelivery = false;
- } else if (msgRtSettings.GetExecType() == NYql::NDqProto::TComputeRuntimeSettings::DATA) {
- runtimeSettingsBase.ExtraMemoryAllocationPool = NRm::EKqpMemoryPool::DataQuery;
- runtimeSettingsBase.FailOnUndelivery = true;
- } else {
- runtimeSettingsBase.ExtraMemoryAllocationPool = NRm::EKqpMemoryPool::Unspecified;
- runtimeSettingsBase.FailOnUndelivery = true;
- }
-
- // TODO: fix me
- YQL_ENSURE(runtimeSettingsBase.ExtraMemoryAllocationPool == NRm::EKqpMemoryPool::ScanQuery);
-
- runtimeSettingsBase.StatsMode = msgRtSettings.GetStatsMode();
- runtimeSettingsBase.UseLLVM = msgRtSettings.GetUseLLVM();
- runtimeSettingsBase.UseSpilling = msgRtSettings.GetUseSpilling();
-
+ return true;
+ }
+
+ LOG_W("Can not allocate memory. TxId: " << txId << ", taskId: " << taskId << ", memory: +" << memory);
+ return false;
+ };
+ }
+
+ NYql::NDq::TComputeRuntimeSettings runtimeSettingsBase;
+ auto& msgRtSettings = msg.GetRuntimeSettings();
+ if (msgRtSettings.GetTimeoutMs() > 0) {
+ runtimeSettingsBase.Timeout = TDuration::MilliSeconds(msgRtSettings.GetTimeoutMs());
+ request.Deadline = TAppData::TimeProvider->Now() + *runtimeSettingsBase.Timeout;
+ }
+
+ if (msgRtSettings.GetExecType() == NYql::NDqProto::TComputeRuntimeSettings::SCAN) {
+ runtimeSettingsBase.ExtraMemoryAllocationPool = NRm::EKqpMemoryPool::ScanQuery;
+ runtimeSettingsBase.FailOnUndelivery = false;
+ } else if (msgRtSettings.GetExecType() == NYql::NDqProto::TComputeRuntimeSettings::DATA) {
+ runtimeSettingsBase.ExtraMemoryAllocationPool = NRm::EKqpMemoryPool::DataQuery;
+ runtimeSettingsBase.FailOnUndelivery = true;
+ } else {
+ runtimeSettingsBase.ExtraMemoryAllocationPool = NRm::EKqpMemoryPool::Unspecified;
+ runtimeSettingsBase.FailOnUndelivery = true;
+ }
+
+ // TODO: fix me
+ YQL_ENSURE(runtimeSettingsBase.ExtraMemoryAllocationPool == NRm::EKqpMemoryPool::ScanQuery);
+
+ runtimeSettingsBase.StatsMode = msgRtSettings.GetStatsMode();
+ runtimeSettingsBase.UseLLVM = msgRtSettings.GetUseLLVM();
+ runtimeSettingsBase.UseSpilling = msgRtSettings.GetUseSpilling();
+
if (msgRtSettings.HasRlPath()) {
runtimeSettingsBase.RlPath = msgRtSettings.GetRlPath();
}
runtimeSettingsBase.ReportStatsSettings = NYql::NDq::TReportStatsSettings{MinStatInterval, MaxStatInterval};
- auto actorSystem = TlsActivationContext->ActorSystem();
-
- // start compute actors
- for (int i = 0; i < msg.GetTasks().size(); ++i) {
- auto& dqTask = *msg.MutableTasks(i);
- auto& taskCtx = request.InFlyTasks[dqTask.GetId()];
- YQL_ENSURE(taskCtx.TaskId != 0);
-
- memoryLimits.ChannelBufferSize = taskCtx.ChannelSize;
- Y_VERIFY_DEBUG(memoryLimits.ChannelBufferSize >= Config.GetMinChannelBufferSize(),
- "actual size: %ld, min: %ld", memoryLimits.ChannelBufferSize, Config.GetMinChannelBufferSize());
-
- auto runtimeSettings = runtimeSettingsBase;
- runtimeSettings.TerminateHandler = [actorSystem, rm = SelfId(), txId, taskId = dqTask.GetId()]
- (bool success, const TString& message) {
- actorSystem->Send(rm, new TEvKqpNode::TEvFinishKqpTask(txId, taskId, success, message));
- };
-
+ auto actorSystem = TlsActivationContext->ActorSystem();
+
+ // start compute actors
+ for (int i = 0; i < msg.GetTasks().size(); ++i) {
+ auto& dqTask = *msg.MutableTasks(i);
+ auto& taskCtx = request.InFlyTasks[dqTask.GetId()];
+ YQL_ENSURE(taskCtx.TaskId != 0);
+
+ memoryLimits.ChannelBufferSize = taskCtx.ChannelSize;
+ Y_VERIFY_DEBUG(memoryLimits.ChannelBufferSize >= Config.GetMinChannelBufferSize(),
+ "actual size: %ld, min: %ld", memoryLimits.ChannelBufferSize, Config.GetMinChannelBufferSize());
+
+ auto runtimeSettings = runtimeSettingsBase;
+ runtimeSettings.TerminateHandler = [actorSystem, rm = SelfId(), txId, taskId = dqTask.GetId()]
+ (bool success, const TString& message) {
+ actorSystem->Send(rm, new TEvKqpNode::TEvFinishKqpTask(txId, taskId, success, message));
+ };
+
ETableKind tableKind = ETableKind::Unknown;
{
NKikimrTxDataShard::TKqpTransaction::TScanTaskMeta meta;
@@ -308,262 +308,262 @@ private:
}
}
- IActor* computeActor;
+ IActor* computeActor;
if (tableKind == ETableKind::Datashard || tableKind == ETableKind::Olap) {
- computeActor = CreateKqpScanComputeActor(msg.GetSnapshot(), request.Executer, txId, std::move(dqTask),
+ computeActor = CreateKqpScanComputeActor(msg.GetSnapshot(), request.Executer, txId, std::move(dqTask),
nullptr, nullptr, runtimeSettings, memoryLimits, Counters);
- taskCtx.ComputeActorId = Register(computeActor);
- } else {
- if (Y_LIKELY(!CaFactory)) {
+ taskCtx.ComputeActorId = Register(computeActor);
+ } else {
+ if (Y_LIKELY(!CaFactory)) {
computeActor = CreateKqpComputeActor(request.Executer, txId, std::move(dqTask), nullptr, nullptr, runtimeSettings,
- memoryLimits);
- taskCtx.ComputeActorId = Register(computeActor);
- } else {
- computeActor = CaFactory->CreateKqpComputeActor(request.Executer, txId, std::move(dqTask),
- runtimeSettings, memoryLimits);
- taskCtx.ComputeActorId = computeActor->SelfId();
- }
- }
-
- LOG_D("TxId: " << txId << ", executing task: " << taskCtx.TaskId << " on compute actor: " << taskCtx.ComputeActorId);
-
- auto* startedTask = reply->Record.AddStartedTasks();
- startedTask->SetTaskId(taskCtx.TaskId);
- ActorIdToProto(taskCtx.ComputeActorId, startedTask->MutableActorId());
- }
-
- if (runtimeSettingsBase.Timeout) {
- request.TimeoutTimer = CreateLongTimer(TlsActivationContext->AsActorContext(),
- *runtimeSettingsBase.Timeout + /* gap */ TDuration::Seconds(5),
- new IEventHandle(SelfId(), SelfId(), new TEvPrivate::TEvTimeout(txId, requester)));
- }
-
- Send(request.Executer, reply.Release(), IEventHandle::FlagTrackDelivery, txId);
-
- State.NewRequest(txId, requester, std::move(request), NRm::EKqpMemoryPool::ScanQuery);
- }
-
- void HandleWork(TEvKqpNode::TEvFinishKqpTask::TPtr& ev) {
- auto& msg = *ev->Get();
-
- LOG_D("TxId: " << msg.TxId << ", finish compute task: " << msg.TaskId << ", success: " << msg.Success
- << ", message: " << msg.Message);
-
- auto task = State.RemoveTask(msg.TxId, msg.TaskId, msg.Success, [this, &msg]
- (const TActorId& requester, const NKqpNode::TTasksRequest& request, const NKqpNode::TTaskContext&, bool finishTx) {
- THolder<IEventBase> ev;
-
- if (request.InFlyTasks.empty()) {
- LOG_D("TxId: " << msg.TxId << ", requester: " << requester << " completed");
-
- if (request.TimeoutTimer) {
- Send(request.TimeoutTimer, new TEvents::TEvPoison);
- }
-
- ResourceManager()->FreeResources(msg.TxId);
- } else {
- LOG_D("TxId: " << msg.TxId << ", finish compute task: " << msg.TaskId
- << (msg.Success ? "" : " (cancelled)")
- << ", remains " << request.InFlyTasks.size() << " compute actors and " << request.TotalMemory
- << " bytes in the current request");
-
- ResourceManager()->FreeResources(msg.TxId, msg.TaskId);
- }
-
- if (finishTx) {
- LOG_D("TxId: " << msg.TxId << ", requester: " << requester << " completed");
- }
- });
-
- if (!task) {
- LOG_E("TxId: " << msg.TxId << ", task: " << msg.TaskId << " unknown task");
- return;
- }
- }
-
- void HandleWork(TEvKqpNode::TEvCancelKqpTasksRequest::TPtr& ev) {
- ui64 txId = ev->Get()->Record.GetTxId();
- auto& reason = ev->Get()->Record.GetReason();
-
- LOG_W("TxId: " << txId << ", terminate transaction, reason: " << reason);
- TerminateTx(txId, reason);
- }
-
- void TerminateTx(ui64 txId, const TString& reason) {
- State.RemoveTx(txId, [this, &txId, &reason](const NKqpNode::TTasksRequest& request) {
- LOG_D("TxId: " << txId << ", cancel granted resources");
-
- if (request.TimeoutTimer) {
- Send(request.TimeoutTimer, new TEvents::TEvPoison);
- }
-
- ResourceManager()->FreeResources(txId);
-
- for (auto& [taskId, task] : request.InFlyTasks) {
- auto abortEv = MakeHolder<TEvKqp::TEvAbortExecution>(Ydb::StatusIds::STATUS_CODE_UNSPECIFIED, reason);
- Send(task.ComputeActorId, abortEv.Release());
- }
- });
- }
-
- void HandleWork(TEvPrivate::TEvTimeout::TPtr& ev) {
- ui64 txId = ev->Get()->TxId;
- TActorId requester = ev->Get()->Requester;
-
- LOG_E("txId: " << txId << ", requester: " << requester << ", execution timeout");
-
- auto request = State.RemoveRequest(txId, requester);
- if (!request) {
- LOG_E("txId: " << txId << ", requester: " << requester << ", unknown request");
- return;
- }
-
- ResourceManager()->FreeResources(txId);
-
- // don't send to executer and compute actors, they have their own timers with smaller timeout
- }
-
-private:
- static void HandleWork(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse::TPtr&) {
- LOG_D("Subscribed for config changes");
- }
-
- void HandleWork(NConsole::TEvConsole::TEvConfigNotificationRequest::TPtr& ev) {
- auto &event = ev->Get()->Record;
-
- if (event.GetConfig().GetTableServiceConfig().GetResourceManager().IsInitialized()) {
- Config.Swap(event.MutableConfig()->MutableTableServiceConfig()->MutableResourceManager());
-
-#define FORCE_VALUE(name) if (!Config.Has ## name ()) Config.Set ## name(Config.Get ## name());
- FORCE_VALUE(ComputeActorsCount)
- FORCE_VALUE(ChannelBufferSize)
- FORCE_VALUE(ScanBufferSize)
- FORCE_VALUE(MkqlLightProgramMemoryLimit)
- FORCE_VALUE(MkqlHeavyProgramMemoryLimit)
- FORCE_VALUE(QueryMemoryLimit)
- FORCE_VALUE(PublishStatisticsIntervalSec);
- FORCE_VALUE(EnableInstantMkqlMemoryAlloc);
- FORCE_VALUE(MaxTotalChannelBuffersSize);
- FORCE_VALUE(MinChannelBufferSize);
- FORCE_VALUE(MaxTotalScanBuffersSize);
- FORCE_VALUE(MinScanBufferSize);
-#undef FORCE_VALUE
-
- LOG_I("Updated table service config: " << Config.DebugString());
- }
-
- auto responseEv = MakeHolder<NConsole::TEvConsole::TEvConfigNotificationResponse>(event);
- Send(ev->Sender, responseEv.Release(), IEventHandle::FlagTrackDelivery, ev->Cookie);
- }
-
- void HandleWork(TEvents::TEvUndelivered::TPtr& ev) {
- switch (ev->Get()->SourceType) {
- case TEvKqpNode::TEvStartKqpTasksResponse::EventType: {
- ui64 txId = ev->Cookie;
- LOG_E("TxId: " << txId << ", executer lost: " << (int) ev->Get()->Reason);
-
- TerminateTx(txId, "executer lost");
- break;
- }
-
- case NConsole::TEvConfigsDispatcher::EvSetConfigSubscriptionRequest:
- LOG_C("Failed to deliver subscription request to config dispatcher");
- break;
-
- case NConsole::TEvConsole::EvConfigNotificationResponse:
- LOG_E("Failed to deliver config notification response");
- break;
-
- default:
- LOG_E("Undelivered event with unexpected source type: " << ev->Get()->SourceType);
- break;
- }
- }
-
- void HandleWork(TEvents::TEvPoison::TPtr&) {
- PassAway();
- }
-
- void HandleWork(NMon::TEvHttpInfo::TPtr& ev) {
- TMap<ui64, TVector<std::pair<const TActorId, const NKqpNode::TTasksRequest*>>> byTx;
- for (auto& [key, request] : State.Requests) {
- byTx[key.first].emplace_back(key.second, &request);
- }
-
- TStringStream str;
- HTML(str) {
- PRE() {
- str << "Current config:" << Endl;
- str << Config.DebugString() << Endl;
- str << Endl;
-
- str << Endl << "Transactions:" << Endl;
- for (auto& [txId, requests] : byTx) {
- auto& meta = State.Meta[txId];
- str << " TxId: " << txId << Endl;
- str << " Memory: " << meta.TotalMemory << Endl;
- str << " MemoryPool: " << (ui32) meta.MemoryPool << Endl;
- str << " Compute actors: " << meta.TotalComputeActors << Endl;
- str << " Start time: " << meta.StartTime << Endl;
- str << " Requests:" << Endl;
- for (auto& [requester, request] : requests) {
- str << " Requester: " << requester << Endl;
- str << " Deadline: " << request->Deadline << Endl;
- str << " Memory: " << request->TotalMemory << Endl;
- str << " In-fly tasks:" << Endl;
- for (auto& [taskId, task] : request->InFlyTasks) {
- str << " Task: " << taskId << Endl;
- str << " Memory: " << task.Memory << Endl;
- str << " Channels: " << task.Channels << Endl;
- str << " Compute actor: " << task.ComputeActorId << Endl;
- }
- }
- }
- }
- }
-
- Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str()));
- }
-
-private:
- void ReplyError(ui64 txId, TActorId executer, const NKikimrKqp::TEvStartKqpTasksRequest& request,
- NKikimrKqp::TEvStartKqpTasksResponse::ENotStartedTaskReason reason, const TString& message = "")
- {
- auto ev = MakeHolder<TEvKqpNode::TEvStartKqpTasksResponse>();
- ev->Record.SetTxId(txId);
- for (auto& task : request.GetTasks()) {
- auto* resp = ev->Record.AddNotStartedTasks();
- resp->SetTaskId(task.GetId());
- resp->SetReason(reason);
- resp->SetMessage(message);
- }
- Send(executer, ev.Release());
- }
-
- NRm::IKqpResourceManager* ResourceManager() {
- if (Y_LIKELY(ResourceManager_)) {
- return ResourceManager_;
- }
- ResourceManager_ = GetKqpResourceManager();
- return ResourceManager_;
- }
-
-private:
- NKikimrConfig::TTableServiceConfig::TResourceManager Config;
- TIntrusivePtr<TKqpCounters> Counters;
- IKqpNodeComputeActorFactory* CaFactory;
- NRm::IKqpResourceManager* ResourceManager_ = nullptr;
- NKqpNode::TState State;
-};
-
-
-} // anonymous namespace
-
-IActor* CreateKqpNodeService(const NKikimrConfig::TTableServiceConfig& tableServiceConfig,
- TIntrusivePtr<TKqpCounters> counters, IKqpNodeComputeActorFactory* caFactory)
-{
- return new TKqpNodeService(tableServiceConfig, counters, caFactory);
-}
-
-} // namespace NKqp
-} // namespace NKikimr
+ memoryLimits);
+ taskCtx.ComputeActorId = Register(computeActor);
+ } else {
+ computeActor = CaFactory->CreateKqpComputeActor(request.Executer, txId, std::move(dqTask),
+ runtimeSettings, memoryLimits);
+ taskCtx.ComputeActorId = computeActor->SelfId();
+ }
+ }
+
+ LOG_D("TxId: " << txId << ", executing task: " << taskCtx.TaskId << " on compute actor: " << taskCtx.ComputeActorId);
+
+ auto* startedTask = reply->Record.AddStartedTasks();
+ startedTask->SetTaskId(taskCtx.TaskId);
+ ActorIdToProto(taskCtx.ComputeActorId, startedTask->MutableActorId());
+ }
+
+ if (runtimeSettingsBase.Timeout) {
+ request.TimeoutTimer = CreateLongTimer(TlsActivationContext->AsActorContext(),
+ *runtimeSettingsBase.Timeout + /* gap */ TDuration::Seconds(5),
+ new IEventHandle(SelfId(), SelfId(), new TEvPrivate::TEvTimeout(txId, requester)));
+ }
+
+ Send(request.Executer, reply.Release(), IEventHandle::FlagTrackDelivery, txId);
+
+ State.NewRequest(txId, requester, std::move(request), NRm::EKqpMemoryPool::ScanQuery);
+ }
+
+ void HandleWork(TEvKqpNode::TEvFinishKqpTask::TPtr& ev) {
+ auto& msg = *ev->Get();
+
+ LOG_D("TxId: " << msg.TxId << ", finish compute task: " << msg.TaskId << ", success: " << msg.Success
+ << ", message: " << msg.Message);
+
+ auto task = State.RemoveTask(msg.TxId, msg.TaskId, msg.Success, [this, &msg]
+ (const TActorId& requester, const NKqpNode::TTasksRequest& request, const NKqpNode::TTaskContext&, bool finishTx) {
+ THolder<IEventBase> ev;
+
+ if (request.InFlyTasks.empty()) {
+ LOG_D("TxId: " << msg.TxId << ", requester: " << requester << " completed");
+
+ if (request.TimeoutTimer) {
+ Send(request.TimeoutTimer, new TEvents::TEvPoison);
+ }
+
+ ResourceManager()->FreeResources(msg.TxId);
+ } else {
+ LOG_D("TxId: " << msg.TxId << ", finish compute task: " << msg.TaskId
+ << (msg.Success ? "" : " (cancelled)")
+ << ", remains " << request.InFlyTasks.size() << " compute actors and " << request.TotalMemory
+ << " bytes in the current request");
+
+ ResourceManager()->FreeResources(msg.TxId, msg.TaskId);
+ }
+
+ if (finishTx) {
+ LOG_D("TxId: " << msg.TxId << ", requester: " << requester << " completed");
+ }
+ });
+
+ if (!task) {
+ LOG_E("TxId: " << msg.TxId << ", task: " << msg.TaskId << " unknown task");
+ return;
+ }
+ }
+
+ void HandleWork(TEvKqpNode::TEvCancelKqpTasksRequest::TPtr& ev) {
+ ui64 txId = ev->Get()->Record.GetTxId();
+ auto& reason = ev->Get()->Record.GetReason();
+
+ LOG_W("TxId: " << txId << ", terminate transaction, reason: " << reason);
+ TerminateTx(txId, reason);
+ }
+
+ void TerminateTx(ui64 txId, const TString& reason) {
+ State.RemoveTx(txId, [this, &txId, &reason](const NKqpNode::TTasksRequest& request) {
+ LOG_D("TxId: " << txId << ", cancel granted resources");
+
+ if (request.TimeoutTimer) {
+ Send(request.TimeoutTimer, new TEvents::TEvPoison);
+ }
+
+ ResourceManager()->FreeResources(txId);
+
+ for (auto& [taskId, task] : request.InFlyTasks) {
+ auto abortEv = MakeHolder<TEvKqp::TEvAbortExecution>(Ydb::StatusIds::STATUS_CODE_UNSPECIFIED, reason);
+ Send(task.ComputeActorId, abortEv.Release());
+ }
+ });
+ }
+
+ void HandleWork(TEvPrivate::TEvTimeout::TPtr& ev) {
+ ui64 txId = ev->Get()->TxId;
+ TActorId requester = ev->Get()->Requester;
+
+ LOG_E("txId: " << txId << ", requester: " << requester << ", execution timeout");
+
+ auto request = State.RemoveRequest(txId, requester);
+ if (!request) {
+ LOG_E("txId: " << txId << ", requester: " << requester << ", unknown request");
+ return;
+ }
+
+ ResourceManager()->FreeResources(txId);
+
+ // don't send to executer and compute actors, they have their own timers with smaller timeout
+ }
+
+private:
+ static void HandleWork(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse::TPtr&) {
+ LOG_D("Subscribed for config changes");
+ }
+
+ void HandleWork(NConsole::TEvConsole::TEvConfigNotificationRequest::TPtr& ev) {
+ auto &event = ev->Get()->Record;
+
+ if (event.GetConfig().GetTableServiceConfig().GetResourceManager().IsInitialized()) {
+ Config.Swap(event.MutableConfig()->MutableTableServiceConfig()->MutableResourceManager());
+
+#define FORCE_VALUE(name) if (!Config.Has ## name ()) Config.Set ## name(Config.Get ## name());
+ FORCE_VALUE(ComputeActorsCount)
+ FORCE_VALUE(ChannelBufferSize)
+ FORCE_VALUE(ScanBufferSize)
+ FORCE_VALUE(MkqlLightProgramMemoryLimit)
+ FORCE_VALUE(MkqlHeavyProgramMemoryLimit)
+ FORCE_VALUE(QueryMemoryLimit)
+ FORCE_VALUE(PublishStatisticsIntervalSec);
+ FORCE_VALUE(EnableInstantMkqlMemoryAlloc);
+ FORCE_VALUE(MaxTotalChannelBuffersSize);
+ FORCE_VALUE(MinChannelBufferSize);
+ FORCE_VALUE(MaxTotalScanBuffersSize);
+ FORCE_VALUE(MinScanBufferSize);
+#undef FORCE_VALUE
+
+ LOG_I("Updated table service config: " << Config.DebugString());
+ }
+
+ auto responseEv = MakeHolder<NConsole::TEvConsole::TEvConfigNotificationResponse>(event);
+ Send(ev->Sender, responseEv.Release(), IEventHandle::FlagTrackDelivery, ev->Cookie);
+ }
+
+ void HandleWork(TEvents::TEvUndelivered::TPtr& ev) {
+ switch (ev->Get()->SourceType) {
+ case TEvKqpNode::TEvStartKqpTasksResponse::EventType: {
+ ui64 txId = ev->Cookie;
+ LOG_E("TxId: " << txId << ", executer lost: " << (int) ev->Get()->Reason);
+
+ TerminateTx(txId, "executer lost");
+ break;
+ }
+
+ case NConsole::TEvConfigsDispatcher::EvSetConfigSubscriptionRequest:
+ LOG_C("Failed to deliver subscription request to config dispatcher");
+ break;
+
+ case NConsole::TEvConsole::EvConfigNotificationResponse:
+ LOG_E("Failed to deliver config notification response");
+ break;
+
+ default:
+ LOG_E("Undelivered event with unexpected source type: " << ev->Get()->SourceType);
+ break;
+ }
+ }
+
+ void HandleWork(TEvents::TEvPoison::TPtr&) {
+ PassAway();
+ }
+
+ void HandleWork(NMon::TEvHttpInfo::TPtr& ev) {
+ TMap<ui64, TVector<std::pair<const TActorId, const NKqpNode::TTasksRequest*>>> byTx;
+ for (auto& [key, request] : State.Requests) {
+ byTx[key.first].emplace_back(key.second, &request);
+ }
+
+ TStringStream str;
+ HTML(str) {
+ PRE() {
+ str << "Current config:" << Endl;
+ str << Config.DebugString() << Endl;
+ str << Endl;
+
+ str << Endl << "Transactions:" << Endl;
+ for (auto& [txId, requests] : byTx) {
+ auto& meta = State.Meta[txId];
+ str << " TxId: " << txId << Endl;
+ str << " Memory: " << meta.TotalMemory << Endl;
+ str << " MemoryPool: " << (ui32) meta.MemoryPool << Endl;
+ str << " Compute actors: " << meta.TotalComputeActors << Endl;
+ str << " Start time: " << meta.StartTime << Endl;
+ str << " Requests:" << Endl;
+ for (auto& [requester, request] : requests) {
+ str << " Requester: " << requester << Endl;
+ str << " Deadline: " << request->Deadline << Endl;
+ str << " Memory: " << request->TotalMemory << Endl;
+ str << " In-fly tasks:" << Endl;
+ for (auto& [taskId, task] : request->InFlyTasks) {
+ str << " Task: " << taskId << Endl;
+ str << " Memory: " << task.Memory << Endl;
+ str << " Channels: " << task.Channels << Endl;
+ str << " Compute actor: " << task.ComputeActorId << Endl;
+ }
+ }
+ }
+ }
+ }
+
+ Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str()));
+ }
+
+private:
+ void ReplyError(ui64 txId, TActorId executer, const NKikimrKqp::TEvStartKqpTasksRequest& request,
+ NKikimrKqp::TEvStartKqpTasksResponse::ENotStartedTaskReason reason, const TString& message = "")
+ {
+ auto ev = MakeHolder<TEvKqpNode::TEvStartKqpTasksResponse>();
+ ev->Record.SetTxId(txId);
+ for (auto& task : request.GetTasks()) {
+ auto* resp = ev->Record.AddNotStartedTasks();
+ resp->SetTaskId(task.GetId());
+ resp->SetReason(reason);
+ resp->SetMessage(message);
+ }
+ Send(executer, ev.Release());
+ }
+
+ NRm::IKqpResourceManager* ResourceManager() {
+ if (Y_LIKELY(ResourceManager_)) {
+ return ResourceManager_;
+ }
+ ResourceManager_ = GetKqpResourceManager();
+ return ResourceManager_;
+ }
+
+private:
+ NKikimrConfig::TTableServiceConfig::TResourceManager Config;
+ TIntrusivePtr<TKqpCounters> Counters;
+ IKqpNodeComputeActorFactory* CaFactory;
+ NRm::IKqpResourceManager* ResourceManager_ = nullptr;
+ NKqpNode::TState State;
+};
+
+
+} // anonymous namespace
+
+IActor* CreateKqpNodeService(const NKikimrConfig::TTableServiceConfig& tableServiceConfig,
+ TIntrusivePtr<TKqpCounters> counters, IKqpNodeComputeActorFactory* caFactory)
+{
+ return new TKqpNodeService(tableServiceConfig, counters, caFactory);
+}
+
+} // namespace NKqp
+} // namespace NKikimr
diff --git a/ydb/core/kqp/node/kqp_node.h b/ydb/core/kqp/node/kqp_node.h
index 662e22147f..8f1096ed31 100644
--- a/ydb/core/kqp/node/kqp_node.h
+++ b/ydb/core/kqp/node/kqp_node.h
@@ -1,70 +1,70 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/kqp/common/kqp_common.h>
#include <ydb/core/kqp/counters/kqp_counters.h>
#include <ydb/core/protos/config.pb.h>
-
+
#include <ydb/library/yql/dq/runtime/dq_tasks_runner.h>
-
-#include <library/cpp/actors/core/actor.h>
-#include <library/cpp/actors/core/event_pb.h>
-
-namespace NYql {
-namespace NDq {
- struct TComputeRuntimeSettings;
- struct TComputeMemoryLimits;
-} // namespace NDq
-} // namespace NYql
-
-namespace NKikimr {
-namespace NKqp {
-
-struct TKqpNodeEvents {
- enum EKqpNodeEvents {
- EvStartKqpTasksRequest = EventSpaceBegin(TKikimrEvents::ES_KQP) + 320,
- EvStartKqpTasksResponse,
- EvFinishKqpTasks,
- EvCancelKqpTasksRequest,
- EvCancelKqpTasksResponse,
- };
-};
-
-struct TEvKqpNode {
- struct TEvStartKqpTasksRequest : public TEventPB<TEvStartKqpTasksRequest,
- NKikimrKqp::TEvStartKqpTasksRequest, TKqpNodeEvents::EvStartKqpTasksRequest> {};
-
- struct TEvStartKqpTasksResponse : public TEventPB<TEvStartKqpTasksResponse,
- NKikimrKqp::TEvStartKqpTasksResponse, TKqpNodeEvents::EvStartKqpTasksResponse> {};
-
- struct TEvFinishKqpTask : public TEventLocal<TEvFinishKqpTask, TKqpNodeEvents::EvFinishKqpTasks> {
- const ui64 TxId;
- const ui64 TaskId;
- const bool Success;
- const TString Message;
-
- TEvFinishKqpTask(ui64 txId, ui64 taskId, bool success, const TString& message = "")
- : TxId(txId)
- , TaskId(taskId)
- , Success(success)
- , Message(message) {}
- };
-
- struct TEvCancelKqpTasksRequest : public TEventPB<TEvCancelKqpTasksRequest,
- NKikimrKqp::TEvCancelKqpTasksRequest, TKqpNodeEvents::EvCancelKqpTasksRequest> {};
-
- struct TEvCancelKqpTasksResponse : public TEventPB<TEvCancelKqpTasksResponse,
- NKikimrKqp::TEvCancelKqpTasksResponse, TKqpNodeEvents::EvCancelKqpTasksResponse> {};
-};
-
-struct IKqpNodeComputeActorFactory {
- virtual ~IKqpNodeComputeActorFactory() = default;
-
- virtual IActor* CreateKqpComputeActor(const TActorId& executerId, ui64 txId, NYql::NDqProto::TDqTask&& task,
- const NYql::NDq::TComputeRuntimeSettings& settings, const NYql::NDq::TComputeMemoryLimits& memoryLimits) = 0;
-};
-
-NActors::IActor* CreateKqpNodeService(const NKikimrConfig::TTableServiceConfig& tableServiceConfig,
- TIntrusivePtr<TKqpCounters> counters, IKqpNodeComputeActorFactory* caFactory = nullptr);
-
-} // namespace NKqp
-} // namespace NKikimr
+
+#include <library/cpp/actors/core/actor.h>
+#include <library/cpp/actors/core/event_pb.h>
+
+namespace NYql {
+namespace NDq {
+ struct TComputeRuntimeSettings;
+ struct TComputeMemoryLimits;
+} // namespace NDq
+} // namespace NYql
+
+namespace NKikimr {
+namespace NKqp {
+
+struct TKqpNodeEvents {
+ enum EKqpNodeEvents {
+ EvStartKqpTasksRequest = EventSpaceBegin(TKikimrEvents::ES_KQP) + 320,
+ EvStartKqpTasksResponse,
+ EvFinishKqpTasks,
+ EvCancelKqpTasksRequest,
+ EvCancelKqpTasksResponse,
+ };
+};
+
+struct TEvKqpNode {
+ struct TEvStartKqpTasksRequest : public TEventPB<TEvStartKqpTasksRequest,
+ NKikimrKqp::TEvStartKqpTasksRequest, TKqpNodeEvents::EvStartKqpTasksRequest> {};
+
+ struct TEvStartKqpTasksResponse : public TEventPB<TEvStartKqpTasksResponse,
+ NKikimrKqp::TEvStartKqpTasksResponse, TKqpNodeEvents::EvStartKqpTasksResponse> {};
+
+ struct TEvFinishKqpTask : public TEventLocal<TEvFinishKqpTask, TKqpNodeEvents::EvFinishKqpTasks> {
+ const ui64 TxId;
+ const ui64 TaskId;
+ const bool Success;
+ const TString Message;
+
+ TEvFinishKqpTask(ui64 txId, ui64 taskId, bool success, const TString& message = "")
+ : TxId(txId)
+ , TaskId(taskId)
+ , Success(success)
+ , Message(message) {}
+ };
+
+ struct TEvCancelKqpTasksRequest : public TEventPB<TEvCancelKqpTasksRequest,
+ NKikimrKqp::TEvCancelKqpTasksRequest, TKqpNodeEvents::EvCancelKqpTasksRequest> {};
+
+ struct TEvCancelKqpTasksResponse : public TEventPB<TEvCancelKqpTasksResponse,
+ NKikimrKqp::TEvCancelKqpTasksResponse, TKqpNodeEvents::EvCancelKqpTasksResponse> {};
+};
+
+struct IKqpNodeComputeActorFactory {
+ virtual ~IKqpNodeComputeActorFactory() = default;
+
+ virtual IActor* CreateKqpComputeActor(const TActorId& executerId, ui64 txId, NYql::NDqProto::TDqTask&& task,
+ const NYql::NDq::TComputeRuntimeSettings& settings, const NYql::NDq::TComputeMemoryLimits& memoryLimits) = 0;
+};
+
+NActors::IActor* CreateKqpNodeService(const NKikimrConfig::TTableServiceConfig& tableServiceConfig,
+ TIntrusivePtr<TKqpCounters> counters, IKqpNodeComputeActorFactory* caFactory = nullptr);
+
+} // namespace NKqp
+} // namespace NKikimr
diff --git a/ydb/core/kqp/node/kqp_node_state.h b/ydb/core/kqp/node/kqp_node_state.h
index a02199ff36..1a07b60b68 100644
--- a/ydb/core/kqp/node/kqp_node_state.h
+++ b/ydb/core/kqp/node/kqp_node_state.h
@@ -1,184 +1,184 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/base/appdata.h>
#include <ydb/core/kqp/runtime/kqp_scan_data.h>
#include <ydb/core/kqp/rm/kqp_rm.h>
#include <ydb/core/util/tuples.h>
-
+
#include <ydb/library/yql/dq/proto/dq_tasks.pb.h>
#include <ydb/library/yql/dq/runtime/dq_tasks_runner.h>
-
-#include <library/cpp/actors/core/actorid.h>
-
-#include <util/str_stl.h>
-
-namespace NKikimr {
-namespace NKqp {
-namespace NKqpNode {
-
-// Task information.
-struct TTaskContext {
- ui64 TaskId = 0;
- ui64 Memory = 0;
- ui32 Channels = 0;
- ui64 ChannelSize = 0;
- TActorId ComputeActorId;
-};
-
-// describes single TEvStartKqpTasksRequest request
-struct TTasksRequest {
- // when task is finished it will be removed from this map
- TMap<ui64, TTaskContext> InFlyTasks;
- TInstant Deadline;
- ui64 TotalMemory = 0;
- TActorId Executer;
- TActorId TimeoutTimer;
- bool ExecutionCancelled = false;
-};
-
-struct TTxMeta {
- ui64 TotalMemory = 0;
- NRm::EKqpMemoryPool MemoryPool = NRm::EKqpMemoryPool::Unspecified;
- ui32 TotalComputeActors = 0;
- TInstant StartTime;
-};
-
-struct TState {
- THashMap<std::pair<ui64, const TActorId>, TTasksRequest> Requests;
- TMap<ui64, TTxMeta> Meta;
-
- bool Exists(ui64 txId, const TActorId& requester) const {
- return Requests.contains(std::make_pair(txId, requester));
- }
-
- TTasksRequest* GetRequest(ui64 txId, const TActorId& requester) {
- return Requests.FindPtr(std::make_pair(txId, requester));
- }
-
- ui64 GetTxMemory(ui64 txId, NRm::EKqpMemoryPool memoryPool) const {
- if (auto* meta = Meta.FindPtr(txId)) {
- return meta->MemoryPool == memoryPool ? meta->TotalMemory : 0;
- }
- return 0;
- }
-
- void NewRequest(ui64 txId, const TActorId& requester, TTasksRequest&& request, NRm::EKqpMemoryPool memoryPool) {
- auto& meta = Meta[txId];
- meta.TotalMemory += request.TotalMemory;
- meta.TotalComputeActors += request.InFlyTasks.size();
- if (!meta.StartTime) {
- meta.StartTime = TAppData::TimeProvider->Now();
- meta.MemoryPool = memoryPool;
- } else {
- YQL_ENSURE(meta.MemoryPool == memoryPool);
- }
- auto ret = Requests.emplace(std::make_pair(txId, requester), std::move(request));
- YQL_ENSURE(ret.second);
- }
-
- std::tuple<TTaskContext*, TActorId, TTasksRequest*, TTxMeta*> GetTask(ui64 txId, ui64 taskId) {
- for (auto& [key, request] : Requests) {
- if (key.first != txId) {
- continue;
- }
- auto taskIt = request.InFlyTasks.find(taskId);
- if (taskIt != request.InFlyTasks.end()) {
- return std::make_tuple(&taskIt->second, key.second, &request, Meta.FindPtr(txId));
- }
- }
-
- return std::make_tuple(nullptr, TActorId(), nullptr, nullptr);
- }
-
- TMaybe<TTaskContext> RemoveTask(ui64 txId, ui64 taskId, bool success,
- std::function<void(const TActorId&, const TTasksRequest&, const TTaskContext&, bool)>&& cb)
- {
- for (auto& [key, request] : Requests) {
- if (key.first == txId && request.InFlyTasks.contains(taskId)) {
- auto task = std::move(request.InFlyTasks[taskId]);
- request.InFlyTasks.erase(taskId);
-
- Y_VERIFY_DEBUG(request.TotalMemory >= task.Memory);
- request.TotalMemory -= task.Memory;
- request.ExecutionCancelled |= !success;
-
- auto& meta = Meta[txId];
- Y_VERIFY_DEBUG(meta.TotalMemory >= task.Memory);
- Y_VERIFY_DEBUG(meta.TotalComputeActors >= 1);
- meta.TotalMemory -= task.Memory;
- meta.TotalComputeActors--;
-
- cb(key.second, request, task, meta.TotalComputeActors == 0);
-
- if (request.InFlyTasks.empty()) {
- Requests.erase(key);
- }
- if (meta.TotalComputeActors == 0) {
- Meta.erase(txId);
- }
-
- return std::move(task);
- }
- }
- return Nothing();
- }
-
- TMaybe<TTasksRequest> RemoveRequest(ui64 txId, const TActorId& requester) {
- auto key = std::make_pair(txId, requester);
- auto* request = Requests.FindPtr(key);
- if (!request) {
- return Nothing();
- }
-
- TMaybe<TTasksRequest> ret = std::move(*request);
- Requests.erase(key);
-
- auto& meta = Meta[txId];
- Y_VERIFY_DEBUG(meta.TotalMemory >= ret->TotalMemory);
- Y_VERIFY_DEBUG(meta.TotalComputeActors >= 1);
- meta.TotalMemory -= ret->TotalMemory;
- meta.TotalComputeActors -= ret->InFlyTasks.size();
-
- if (meta.TotalComputeActors == 0) {
- Meta.erase(txId);
- }
-
- return ret;
- }
-
- bool RemoveTx(ui64 txId, std::function<void(const TTasksRequest&)>&& cb) {
- Meta.erase(txId);
-
- auto removeOne = [&]() {
- for (auto& [key, request] : Requests) {
- if (key.first == txId) {
- cb(request);
- Requests.erase(key);
- return true;
- }
- }
- return false;
- };
-
- ui32 count = 0;
- while (removeOne()) {
- ++count;
- }
-
- return count > 0;
- }
-
- ui64 UsedMemory(NRm::EKqpMemoryPool memoryPool) const {
- ui64 mem = 0;
- for (auto& [_, meta] : Meta) {
- if (meta.MemoryPool == memoryPool) {
- mem += meta.TotalMemory;
- }
- }
- return mem;
- }
-};
-
-} // namespace NKqpNode
-} // namespace NKqp
-} // namespace NKikimr
+
+#include <library/cpp/actors/core/actorid.h>
+
+#include <util/str_stl.h>
+
+namespace NKikimr {
+namespace NKqp {
+namespace NKqpNode {
+
+// Task information.
+struct TTaskContext {
+ ui64 TaskId = 0;
+ ui64 Memory = 0;
+ ui32 Channels = 0;
+ ui64 ChannelSize = 0;
+ TActorId ComputeActorId;
+};
+
+// describes single TEvStartKqpTasksRequest request
+struct TTasksRequest {
+ // when task is finished it will be removed from this map
+ TMap<ui64, TTaskContext> InFlyTasks;
+ TInstant Deadline;
+ ui64 TotalMemory = 0;
+ TActorId Executer;
+ TActorId TimeoutTimer;
+ bool ExecutionCancelled = false;
+};
+
+struct TTxMeta {
+ ui64 TotalMemory = 0;
+ NRm::EKqpMemoryPool MemoryPool = NRm::EKqpMemoryPool::Unspecified;
+ ui32 TotalComputeActors = 0;
+ TInstant StartTime;
+};
+
+struct TState {
+ THashMap<std::pair<ui64, const TActorId>, TTasksRequest> Requests;
+ TMap<ui64, TTxMeta> Meta;
+
+ bool Exists(ui64 txId, const TActorId& requester) const {
+ return Requests.contains(std::make_pair(txId, requester));
+ }
+
+ TTasksRequest* GetRequest(ui64 txId, const TActorId& requester) {
+ return Requests.FindPtr(std::make_pair(txId, requester));
+ }
+
+ ui64 GetTxMemory(ui64 txId, NRm::EKqpMemoryPool memoryPool) const {
+ if (auto* meta = Meta.FindPtr(txId)) {
+ return meta->MemoryPool == memoryPool ? meta->TotalMemory : 0;
+ }
+ return 0;
+ }
+
+ void NewRequest(ui64 txId, const TActorId& requester, TTasksRequest&& request, NRm::EKqpMemoryPool memoryPool) {
+ auto& meta = Meta[txId];
+ meta.TotalMemory += request.TotalMemory;
+ meta.TotalComputeActors += request.InFlyTasks.size();
+ if (!meta.StartTime) {
+ meta.StartTime = TAppData::TimeProvider->Now();
+ meta.MemoryPool = memoryPool;
+ } else {
+ YQL_ENSURE(meta.MemoryPool == memoryPool);
+ }
+ auto ret = Requests.emplace(std::make_pair(txId, requester), std::move(request));
+ YQL_ENSURE(ret.second);
+ }
+
+ std::tuple<TTaskContext*, TActorId, TTasksRequest*, TTxMeta*> GetTask(ui64 txId, ui64 taskId) {
+ for (auto& [key, request] : Requests) {
+ if (key.first != txId) {
+ continue;
+ }
+ auto taskIt = request.InFlyTasks.find(taskId);
+ if (taskIt != request.InFlyTasks.end()) {
+ return std::make_tuple(&taskIt->second, key.second, &request, Meta.FindPtr(txId));
+ }
+ }
+
+ return std::make_tuple(nullptr, TActorId(), nullptr, nullptr);
+ }
+
+ TMaybe<TTaskContext> RemoveTask(ui64 txId, ui64 taskId, bool success,
+ std::function<void(const TActorId&, const TTasksRequest&, const TTaskContext&, bool)>&& cb)
+ {
+ for (auto& [key, request] : Requests) {
+ if (key.first == txId && request.InFlyTasks.contains(taskId)) {
+ auto task = std::move(request.InFlyTasks[taskId]);
+ request.InFlyTasks.erase(taskId);
+
+ Y_VERIFY_DEBUG(request.TotalMemory >= task.Memory);
+ request.TotalMemory -= task.Memory;
+ request.ExecutionCancelled |= !success;
+
+ auto& meta = Meta[txId];
+ Y_VERIFY_DEBUG(meta.TotalMemory >= task.Memory);
+ Y_VERIFY_DEBUG(meta.TotalComputeActors >= 1);
+ meta.TotalMemory -= task.Memory;
+ meta.TotalComputeActors--;
+
+ cb(key.second, request, task, meta.TotalComputeActors == 0);
+
+ if (request.InFlyTasks.empty()) {
+ Requests.erase(key);
+ }
+ if (meta.TotalComputeActors == 0) {
+ Meta.erase(txId);
+ }
+
+ return std::move(task);
+ }
+ }
+ return Nothing();
+ }
+
+ TMaybe<TTasksRequest> RemoveRequest(ui64 txId, const TActorId& requester) {
+ auto key = std::make_pair(txId, requester);
+ auto* request = Requests.FindPtr(key);
+ if (!request) {
+ return Nothing();
+ }
+
+ TMaybe<TTasksRequest> ret = std::move(*request);
+ Requests.erase(key);
+
+ auto& meta = Meta[txId];
+ Y_VERIFY_DEBUG(meta.TotalMemory >= ret->TotalMemory);
+ Y_VERIFY_DEBUG(meta.TotalComputeActors >= 1);
+ meta.TotalMemory -= ret->TotalMemory;
+ meta.TotalComputeActors -= ret->InFlyTasks.size();
+
+ if (meta.TotalComputeActors == 0) {
+ Meta.erase(txId);
+ }
+
+ return ret;
+ }
+
+ bool RemoveTx(ui64 txId, std::function<void(const TTasksRequest&)>&& cb) {
+ Meta.erase(txId);
+
+ auto removeOne = [&]() {
+ for (auto& [key, request] : Requests) {
+ if (key.first == txId) {
+ cb(request);
+ Requests.erase(key);
+ return true;
+ }
+ }
+ return false;
+ };
+
+ ui32 count = 0;
+ while (removeOne()) {
+ ++count;
+ }
+
+ return count > 0;
+ }
+
+ ui64 UsedMemory(NRm::EKqpMemoryPool memoryPool) const {
+ ui64 mem = 0;
+ for (auto& [_, meta] : Meta) {
+ if (meta.MemoryPool == memoryPool) {
+ mem += meta.TotalMemory;
+ }
+ }
+ return mem;
+ }
+};
+
+} // namespace NKqpNode
+} // namespace NKqp
+} // namespace NKikimr
diff --git a/ydb/core/kqp/node/kqp_node_ut.cpp b/ydb/core/kqp/node/kqp_node_ut.cpp
index b0a2e1029a..4decee6bd3 100644
--- a/ydb/core/kqp/node/kqp_node_ut.cpp
+++ b/ydb/core/kqp/node/kqp_node_ut.cpp
@@ -4,681 +4,681 @@
#include <ydb/core/kqp/node/kqp_node.h>
#include <ydb/core/kqp/rm/kqp_rm.h>
#include <ydb/core/tablet/resource_broker_impl.h>
-
+
#include <ydb/core/testlib/actor_helpers.h>
#include <ydb/core/testlib/tablet_helpers.h>
#include <ydb/core/testlib/tenant_runtime.h>
-
-#include <library/cpp/testing/unittest/registar.h>
+
+#include <library/cpp/testing/unittest/registar.h>
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h>
-
-#ifndef NDEBUG
-const bool DETAILED_LOG = true;
-#else
-const bool DETAILED_LOG = false;
-#endif
-
-namespace NKikimr {
-namespace NKqp {
-
-using namespace NKikimrResourceBroker;
-using namespace NResourceBroker;
-
-namespace {
-
-TTenantTestConfig MakeTenantTestConfig() {
- TTenantTestConfig cfg = {
- // Domains {name, schemeshard {{ subdomain_names }}}
- {{{DOMAIN1_NAME, SCHEME_SHARD1_ID, {{TENANT1_1_NAME, TENANT1_2_NAME}}}}},
- // HiveId
- HIVE_ID,
- // FakeTenantSlotBroker
- true,
- // FakeSchemeShard
- true,
- // CreateConsole
- false,
- // Nodes
- {{
- // Node0
- {
- // TenantPoolConfig
- {
- // Static slots {tenant, {cpu, memory, network}}
- {{{DOMAIN1_NAME, {1, 1, 1}}}},
- // Dynamic slots {id, type, domain, tenant, {cpu, memory, network}}
- {{{DOMAIN1_SLOT1, SLOT1_TYPE, DOMAIN1_NAME, "", {1, 1, 1}},
- {DOMAIN1_SLOT2, SLOT2_TYPE, DOMAIN1_NAME, "", {2, 2, 2}},
- {DOMAIN1_SLOT3, SLOT3_TYPE, DOMAIN1_NAME, "", {3, 3, 3}}}},
- "node-type"
- }
- }
- }},
- // DataCenterCount
- 1
- };
- return cfg;
-}
-
-TResourceBrokerConfig MakeResourceBrokerTestConfig() {
- TResourceBrokerConfig config;
-
- auto queue = config.AddQueues();
- queue->SetName("queue_default");
- queue->SetWeight(5);
- queue->MutableLimit()->AddResource(4);
-
- queue = config.AddQueues();
- queue->SetName("queue_kqp_resource_manager");
- queue->SetWeight(20);
- queue->MutableLimit()->AddResource(4);
- queue->MutableLimit()->AddResource(50'000);
-
- auto task = config.AddTasks();
- task->SetName("unknown");
- task->SetQueueName("queue_default");
- task->SetDefaultDuration(TDuration::Seconds(5).GetValue());
-
- task = config.AddTasks();
- task->SetName(NLocalDb::KqpResourceManagerTaskName);
- task->SetQueueName("queue_kqp_resource_manager");
- task->SetDefaultDuration(TDuration::Seconds(5).GetValue());
-
- config.MutableResourceLimit()->AddResource(10);
- config.MutableResourceLimit()->AddResource(100'000);
-
- return config;
-}
-
-NKikimrConfig::TTableServiceConfig MakeKqpResourceManagerConfig() {
- NKikimrConfig::TTableServiceConfig config;
-
- config.MutableResourceManager()->SetComputeActorsCount(100);
- config.MutableResourceManager()->SetChannelBufferSize(10);
- config.MutableResourceManager()->SetMinChannelBufferSize(10);
- config.MutableResourceManager()->SetScanBufferSize(50);
- config.MutableResourceManager()->SetMkqlLightProgramMemoryLimit(1'000);
- config.MutableResourceManager()->SetMkqlHeavyProgramMemoryLimit(10'000);
- config.MutableResourceManager()->SetQueryMemoryLimit(30'000);
- config.MutableResourceManager()->SetPublishStatisticsIntervalSec(0);
- config.MutableResourceManager()->SetEnableInstantMkqlMemoryAlloc(true);
-
- return config;
-}
-
-}
-
-struct TMockComputeActor {
- TActorId ActorId;
- TActorId ExecuterId;
- ui64 TxId;
- NYql::NDqProto::TDqTask Task;
+
+#ifndef NDEBUG
+const bool DETAILED_LOG = true;
+#else
+const bool DETAILED_LOG = false;
+#endif
+
+namespace NKikimr {
+namespace NKqp {
+
+using namespace NKikimrResourceBroker;
+using namespace NResourceBroker;
+
+namespace {
+
+TTenantTestConfig MakeTenantTestConfig() {
+ TTenantTestConfig cfg = {
+ // Domains {name, schemeshard {{ subdomain_names }}}
+ {{{DOMAIN1_NAME, SCHEME_SHARD1_ID, {{TENANT1_1_NAME, TENANT1_2_NAME}}}}},
+ // HiveId
+ HIVE_ID,
+ // FakeTenantSlotBroker
+ true,
+ // FakeSchemeShard
+ true,
+ // CreateConsole
+ false,
+ // Nodes
+ {{
+ // Node0
+ {
+ // TenantPoolConfig
+ {
+ // Static slots {tenant, {cpu, memory, network}}
+ {{{DOMAIN1_NAME, {1, 1, 1}}}},
+ // Dynamic slots {id, type, domain, tenant, {cpu, memory, network}}
+ {{{DOMAIN1_SLOT1, SLOT1_TYPE, DOMAIN1_NAME, "", {1, 1, 1}},
+ {DOMAIN1_SLOT2, SLOT2_TYPE, DOMAIN1_NAME, "", {2, 2, 2}},
+ {DOMAIN1_SLOT3, SLOT3_TYPE, DOMAIN1_NAME, "", {3, 3, 3}}}},
+ "node-type"
+ }
+ }
+ }},
+ // DataCenterCount
+ 1
+ };
+ return cfg;
+}
+
+TResourceBrokerConfig MakeResourceBrokerTestConfig() {
+ TResourceBrokerConfig config;
+
+ auto queue = config.AddQueues();
+ queue->SetName("queue_default");
+ queue->SetWeight(5);
+ queue->MutableLimit()->AddResource(4);
+
+ queue = config.AddQueues();
+ queue->SetName("queue_kqp_resource_manager");
+ queue->SetWeight(20);
+ queue->MutableLimit()->AddResource(4);
+ queue->MutableLimit()->AddResource(50'000);
+
+ auto task = config.AddTasks();
+ task->SetName("unknown");
+ task->SetQueueName("queue_default");
+ task->SetDefaultDuration(TDuration::Seconds(5).GetValue());
+
+ task = config.AddTasks();
+ task->SetName(NLocalDb::KqpResourceManagerTaskName);
+ task->SetQueueName("queue_kqp_resource_manager");
+ task->SetDefaultDuration(TDuration::Seconds(5).GetValue());
+
+ config.MutableResourceLimit()->AddResource(10);
+ config.MutableResourceLimit()->AddResource(100'000);
+
+ return config;
+}
+
+NKikimrConfig::TTableServiceConfig MakeKqpResourceManagerConfig() {
+ NKikimrConfig::TTableServiceConfig config;
+
+ config.MutableResourceManager()->SetComputeActorsCount(100);
+ config.MutableResourceManager()->SetChannelBufferSize(10);
+ config.MutableResourceManager()->SetMinChannelBufferSize(10);
+ config.MutableResourceManager()->SetScanBufferSize(50);
+ config.MutableResourceManager()->SetMkqlLightProgramMemoryLimit(1'000);
+ config.MutableResourceManager()->SetMkqlHeavyProgramMemoryLimit(10'000);
+ config.MutableResourceManager()->SetQueryMemoryLimit(30'000);
+ config.MutableResourceManager()->SetPublishStatisticsIntervalSec(0);
+ config.MutableResourceManager()->SetEnableInstantMkqlMemoryAlloc(true);
+
+ return config;
+}
+
+}
+
+struct TMockComputeActor {
+ TActorId ActorId;
+ TActorId ExecuterId;
+ ui64 TxId;
+ NYql::NDqProto::TDqTask Task;
NYql::NDq::TComputeRuntimeSettings Settings;
NYql::NDq::TComputeMemoryLimits MemoryLimits;
-};
-
-struct TMockKqpComputeActorFactory : public IKqpNodeComputeActorFactory {
- TTestBasicRuntime& Runtime;
- TMap<ui64, TMockComputeActor> Task2Actor;
-
- TMockKqpComputeActorFactory(TTestBasicRuntime& runtime)
- : Runtime(runtime) {}
-
- IActor* CreateKqpComputeActor(const TActorId& executerId, ui64 txId, NYql::NDqProto::TDqTask&& task,
+};
+
+struct TMockKqpComputeActorFactory : public IKqpNodeComputeActorFactory {
+ TTestBasicRuntime& Runtime;
+ TMap<ui64, TMockComputeActor> Task2Actor;
+
+ TMockKqpComputeActorFactory(TTestBasicRuntime& runtime)
+ : Runtime(runtime) {}
+
+ IActor* CreateKqpComputeActor(const TActorId& executerId, ui64 txId, NYql::NDqProto::TDqTask&& task,
const NYql::NDq::TComputeRuntimeSettings& settings, const NYql::NDq::TComputeMemoryLimits& memoryLimits) override
- {
- auto actorId = Runtime.AllocateEdgeActor();
- auto& mock = Task2Actor[task.GetId()];
- mock.ActorId = actorId;
- mock.ExecuterId = executerId;
- mock.TxId = txId;
- mock.Task.Swap(&task);
- mock.Settings = settings;
- mock.MemoryLimits = memoryLimits;
- return Runtime.FindActor(actorId);
- }
-};
-
-class KqpNode : public TTestBase {
-public:
- void SetUp() override {
+ {
+ auto actorId = Runtime.AllocateEdgeActor();
+ auto& mock = Task2Actor[task.GetId()];
+ mock.ActorId = actorId;
+ mock.ExecuterId = executerId;
+ mock.TxId = txId;
+ mock.Task.Swap(&task);
+ mock.Settings = settings;
+ mock.MemoryLimits = memoryLimits;
+ return Runtime.FindActor(actorId);
+ }
+};
+
+class KqpNode : public TTestBase {
+public:
+ void SetUp() override {
Runtime.Reset(new TTenantTestRuntime(MakeTenantTestConfig()));
-
- NActors::NLog::EPriority priority = DETAILED_LOG ? NLog::PRI_DEBUG : NLog::PRI_ERROR;
- Runtime->SetLogPriority(NKikimrServices::RESOURCE_BROKER, priority);
- Runtime->SetLogPriority(NKikimrServices::KQP_RESOURCE_MANAGER, priority);
- Runtime->SetLogPriority(NKikimrServices::KQP_NODE, priority);
-
- auto now = Now();
- Runtime->UpdateCurrentTime(now);
-
- Counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
- KqpCounters = MakeIntrusive<TKqpCounters>(Counters);
-
- auto resourceBrokerConfig = MakeResourceBrokerTestConfig();
- auto broker = CreateResourceBrokerActor(resourceBrokerConfig, Counters);
- ResourceBrokerActorId = Runtime->Register(broker);
- WaitForBootstrap();
-
+
+ NActors::NLog::EPriority priority = DETAILED_LOG ? NLog::PRI_DEBUG : NLog::PRI_ERROR;
+ Runtime->SetLogPriority(NKikimrServices::RESOURCE_BROKER, priority);
+ Runtime->SetLogPriority(NKikimrServices::KQP_RESOURCE_MANAGER, priority);
+ Runtime->SetLogPriority(NKikimrServices::KQP_NODE, priority);
+
+ auto now = Now();
+ Runtime->UpdateCurrentTime(now);
+
+ Counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
+ KqpCounters = MakeIntrusive<TKqpCounters>(Counters);
+
+ auto resourceBrokerConfig = MakeResourceBrokerTestConfig();
+ auto broker = CreateResourceBrokerActor(resourceBrokerConfig, Counters);
+ ResourceBrokerActorId = Runtime->Register(broker);
+ WaitForBootstrap();
+
CompFactory.Reset(new TMockKqpComputeActorFactory(*Runtime));
- }
-
- void WaitForBootstrap() {
- TDispatchOptions options;
- options.FinalEvents.emplace_back(TEvents::TSystem::Bootstrap, 1);
- UNIT_ASSERT(Runtime->DispatchEvents(options));
- }
-
- IActor* CreateKqpNode(const NKikimrConfig::TTableServiceConfig& config = {}) {
- auto rm = CreateKqpResourceManagerActor(config.GetResourceManager(), KqpCounters, ResourceBrokerActorId);
+ }
+
+ void WaitForBootstrap() {
+ TDispatchOptions options;
+ options.FinalEvents.emplace_back(TEvents::TSystem::Bootstrap, 1);
+ UNIT_ASSERT(Runtime->DispatchEvents(options));
+ }
+
+ IActor* CreateKqpNode(const NKikimrConfig::TTableServiceConfig& config = {}) {
+ auto rm = CreateKqpResourceManagerActor(config.GetResourceManager(), KqpCounters, ResourceBrokerActorId);
ResourceManagerActorId = Runtime->Register(rm);
Runtime->EnableScheduleForActor(ResourceManagerActorId, true);
- WaitForBootstrap();
-
- auto kqpNode = CreateKqpNodeService(config, KqpCounters, CompFactory.Get());
- KqpNodeActorId = Runtime->Register(kqpNode);
- Runtime->EnableScheduleForActor(KqpNodeActorId, true);
- WaitForBootstrap();
- return kqpNode;
- }
-
- void SendStartTasksRequest(const TActorId& requester, ui64 txId, const TVector<ui64>& taskIds,
- const TActorId& executer = {})
- {
- auto ev = MakeHolder<TEvKqpNode::TEvStartKqpTasksRequest>();
- ev->Record.SetTxId(txId);
- ActorIdToProto(executer ? executer : requester, ev->Record.MutableExecuterActorId());
- ev->Record.SetStartAllOrFail(true);
+ WaitForBootstrap();
+
+ auto kqpNode = CreateKqpNodeService(config, KqpCounters, CompFactory.Get());
+ KqpNodeActorId = Runtime->Register(kqpNode);
+ Runtime->EnableScheduleForActor(KqpNodeActorId, true);
+ WaitForBootstrap();
+ return kqpNode;
+ }
+
+ void SendStartTasksRequest(const TActorId& requester, ui64 txId, const TVector<ui64>& taskIds,
+ const TActorId& executer = {})
+ {
+ auto ev = MakeHolder<TEvKqpNode::TEvStartKqpTasksRequest>();
+ ev->Record.SetTxId(txId);
+ ActorIdToProto(executer ? executer : requester, ev->Record.MutableExecuterActorId());
+ ev->Record.SetStartAllOrFail(true);
ev->Record.MutableRuntimeSettings()->SetExecType(NYql::NDqProto::TComputeRuntimeSettings::SCAN);
-
- for (ui64 taskId : taskIds) {
- auto* task = ev->Record.AddTasks();
- task->SetId(taskId);
- task->AddInputs()->AddChannels();
- task->AddOutputs()->AddChannels();
- }
-
- Runtime->Send(new IEventHandle(KqpNodeActorId, requester, ev.Release()));
- }
-
- void SendFinishTask(const TActorId& computeActorId, ui64 txId, ui64 taskId, bool success = true,
- const TString& message = "")
- {
- auto ev = new TEvKqpNode::TEvFinishKqpTask(txId, taskId, success, message);
- Runtime->Send(new IEventHandle(KqpNodeActorId, computeActorId, ev));
- }
-
- void DispatchKqpNodePostponedEvents(const TActorId& edge) {
- auto ev = new NConsole::TEvConsole::TEvConfigNotificationRequest;
- Runtime->Send(new IEventHandle(KqpNodeActorId, edge, ev), 0, true);
- Runtime->GrabEdgeEvent<NConsole::TEvConsole::TEvConfigNotificationResponse>(edge);
- }
-
- void AssertResourceBrokerSensors(i64 cpu, i64 mem, i64 enqueued, i64 finished, i64 infly) {
- auto q = Counters->GetSubgroup("queue", "queue_kqp_resource_manager");
-// Cerr << "-- queue_kqp_resource_manager\n";
-// q->OutputPlainText(Cerr, " ");
- UNIT_ASSERT_VALUES_EQUAL(q->GetCounter("CPUConsumption")->Val(), cpu);
- UNIT_ASSERT_VALUES_EQUAL(q->GetCounter("MemoryConsumption")->Val(), mem);
- UNIT_ASSERT_VALUES_EQUAL(q->GetCounter("EnqueuedTasks")->Val(), enqueued);
- UNIT_ASSERT_VALUES_EQUAL(q->GetCounter("FinishedTasks")->Val(), finished);
- UNIT_ASSERT_VALUES_EQUAL(q->GetCounter("InFlyTasks")->Val(), infly);
-
- auto t = Counters->GetSubgroup("task", "kqp_query");
-// Cerr << "-- kqp_query\n";
-// t->OutputPlainText(Cerr, " ");
- UNIT_ASSERT_VALUES_EQUAL(t->GetCounter("CPUConsumption")->Val(), cpu);
- UNIT_ASSERT_VALUES_EQUAL(t->GetCounter("MemoryConsumption")->Val(), mem);
- UNIT_ASSERT_VALUES_EQUAL(t->GetCounter("EnqueuedTasks")->Val(), enqueued);
- UNIT_ASSERT_VALUES_EQUAL(t->GetCounter("FinishedTasks")->Val(), finished);
- UNIT_ASSERT_VALUES_EQUAL(t->GetCounter("InFlyTasks")->Val(), infly);
- }
-
-
- UNIT_TEST_SUITE(KqpNode);
- UNIT_TEST(CommonCase);
- UNIT_TEST(ExtraAllocation);
- UNIT_TEST(NotEnoughMemory);
- UNIT_TEST(NotEnoughMemory_Extra);
- UNIT_TEST(NotEnoughComputeActors);
- UNIT_TEST(ResourceBrokerNotEnoughResources);
- UNIT_TEST(ResourceBrokerNotEnoughResources_Extra);
- UNIT_TEST(ExecuterLost);
- UNIT_TEST(TerminateTx);
- UNIT_TEST_SUITE_END();
-
- void CommonCase();
- void ExtraAllocation();
- void NotEnoughMemory();
- void NotEnoughMemory_Extra();
- void NotEnoughComputeActors();
- void ResourceBrokerNotEnoughResources();
- void ResourceBrokerNotEnoughResources_Extra();
- void ExecuterLost();
- void TerminateTx();
-
-private:
- THolder<TTestBasicRuntime> Runtime;
- TIntrusivePtr<NMonitoring::TDynamicCounters> Counters;
- TIntrusivePtr<TKqpCounters> KqpCounters;
- THolder<TMockKqpComputeActorFactory> CompFactory;
- TActorId ResourceBrokerActorId;
+
+ for (ui64 taskId : taskIds) {
+ auto* task = ev->Record.AddTasks();
+ task->SetId(taskId);
+ task->AddInputs()->AddChannels();
+ task->AddOutputs()->AddChannels();
+ }
+
+ Runtime->Send(new IEventHandle(KqpNodeActorId, requester, ev.Release()));
+ }
+
+ void SendFinishTask(const TActorId& computeActorId, ui64 txId, ui64 taskId, bool success = true,
+ const TString& message = "")
+ {
+ auto ev = new TEvKqpNode::TEvFinishKqpTask(txId, taskId, success, message);
+ Runtime->Send(new IEventHandle(KqpNodeActorId, computeActorId, ev));
+ }
+
+ void DispatchKqpNodePostponedEvents(const TActorId& edge) {
+ auto ev = new NConsole::TEvConsole::TEvConfigNotificationRequest;
+ Runtime->Send(new IEventHandle(KqpNodeActorId, edge, ev), 0, true);
+ Runtime->GrabEdgeEvent<NConsole::TEvConsole::TEvConfigNotificationResponse>(edge);
+ }
+
+ void AssertResourceBrokerSensors(i64 cpu, i64 mem, i64 enqueued, i64 finished, i64 infly) {
+ auto q = Counters->GetSubgroup("queue", "queue_kqp_resource_manager");
+// Cerr << "-- queue_kqp_resource_manager\n";
+// q->OutputPlainText(Cerr, " ");
+ UNIT_ASSERT_VALUES_EQUAL(q->GetCounter("CPUConsumption")->Val(), cpu);
+ UNIT_ASSERT_VALUES_EQUAL(q->GetCounter("MemoryConsumption")->Val(), mem);
+ UNIT_ASSERT_VALUES_EQUAL(q->GetCounter("EnqueuedTasks")->Val(), enqueued);
+ UNIT_ASSERT_VALUES_EQUAL(q->GetCounter("FinishedTasks")->Val(), finished);
+ UNIT_ASSERT_VALUES_EQUAL(q->GetCounter("InFlyTasks")->Val(), infly);
+
+ auto t = Counters->GetSubgroup("task", "kqp_query");
+// Cerr << "-- kqp_query\n";
+// t->OutputPlainText(Cerr, " ");
+ UNIT_ASSERT_VALUES_EQUAL(t->GetCounter("CPUConsumption")->Val(), cpu);
+ UNIT_ASSERT_VALUES_EQUAL(t->GetCounter("MemoryConsumption")->Val(), mem);
+ UNIT_ASSERT_VALUES_EQUAL(t->GetCounter("EnqueuedTasks")->Val(), enqueued);
+ UNIT_ASSERT_VALUES_EQUAL(t->GetCounter("FinishedTasks")->Val(), finished);
+ UNIT_ASSERT_VALUES_EQUAL(t->GetCounter("InFlyTasks")->Val(), infly);
+ }
+
+
+ UNIT_TEST_SUITE(KqpNode);
+ UNIT_TEST(CommonCase);
+ UNIT_TEST(ExtraAllocation);
+ UNIT_TEST(NotEnoughMemory);
+ UNIT_TEST(NotEnoughMemory_Extra);
+ UNIT_TEST(NotEnoughComputeActors);
+ UNIT_TEST(ResourceBrokerNotEnoughResources);
+ UNIT_TEST(ResourceBrokerNotEnoughResources_Extra);
+ UNIT_TEST(ExecuterLost);
+ UNIT_TEST(TerminateTx);
+ UNIT_TEST_SUITE_END();
+
+ void CommonCase();
+ void ExtraAllocation();
+ void NotEnoughMemory();
+ void NotEnoughMemory_Extra();
+ void NotEnoughComputeActors();
+ void ResourceBrokerNotEnoughResources();
+ void ResourceBrokerNotEnoughResources_Extra();
+ void ExecuterLost();
+ void TerminateTx();
+
+private:
+ THolder<TTestBasicRuntime> Runtime;
+ TIntrusivePtr<NMonitoring::TDynamicCounters> Counters;
+ TIntrusivePtr<TKqpCounters> KqpCounters;
+ THolder<TMockKqpComputeActorFactory> CompFactory;
+ TActorId ResourceBrokerActorId;
TActorId ResourceManagerActorId;
- TActorId KqpNodeActorId;
-};
-UNIT_TEST_SUITE_REGISTRATION(KqpNode);
-
-void KqpNode::CommonCase() {
- auto cfg = MakeKqpResourceManagerConfig();
- CreateKqpNode(cfg);
-
- TActorId sender1 = Runtime->AllocateEdgeActor();
- TActorId sender2 = Runtime->AllocateEdgeActor();
-
- const ui64 taskSize = 1'000 + 2 * 10;
-
- // first request
- SendStartTasksRequest(sender1, /* txId */ 1, /* taskIds */ {1, 2});
- {
- auto answer = Runtime->GrabEdgeEvent<TEvKqpNode::TEvStartKqpTasksResponse>(sender1);
- auto& record = answer->Get()->Record;
-
- UNIT_ASSERT_VALUES_EQUAL(1, record.GetTxId());
- UNIT_ASSERT_VALUES_EQUAL(2, record.GetStartedTasks().size());
- UNIT_ASSERT_VALUES_EQUAL(0, record.GetNotStartedTasks().size());
- UNIT_ASSERT_VALUES_EQUAL(2, CompFactory->Task2Actor.size());
-
- UNIT_ASSERT(CompFactory->Task2Actor.contains(1));
- UNIT_ASSERT(CompFactory->Task2Actor.contains(2));
-
- auto& memoryLimits = CompFactory->Task2Actor.begin()->second.MemoryLimits;
- UNIT_ASSERT_VALUES_EQUAL(10, memoryLimits.ChannelBufferSize);
- UNIT_ASSERT_VALUES_EQUAL(50, memoryLimits.ScanBufferSize);
- UNIT_ASSERT_VALUES_EQUAL(1'000, memoryLimits.MkqlLightProgramMemoryLimit);
- UNIT_ASSERT_VALUES_EQUAL(10'000, memoryLimits.MkqlHeavyProgramMemoryLimit);
-
- UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmComputeActors->Val(), 2);
- UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmMemory->Val(), 2 * taskSize);
-
- AssertResourceBrokerSensors(0, 2 * taskSize, 0, 0, 2);
- }
-
- Runtime->DispatchEvents(TDispatchOptions(), TDuration::Seconds(1));
-
- {
- TVector<NKikimrKqp::TKqpNodeResources> snapshot;
- std::atomic<int> ready = 0;
+ TActorId KqpNodeActorId;
+};
+UNIT_TEST_SUITE_REGISTRATION(KqpNode);
+
+void KqpNode::CommonCase() {
+ auto cfg = MakeKqpResourceManagerConfig();
+ CreateKqpNode(cfg);
+
+ TActorId sender1 = Runtime->AllocateEdgeActor();
+ TActorId sender2 = Runtime->AllocateEdgeActor();
+
+ const ui64 taskSize = 1'000 + 2 * 10;
+
+ // first request
+ SendStartTasksRequest(sender1, /* txId */ 1, /* taskIds */ {1, 2});
+ {
+ auto answer = Runtime->GrabEdgeEvent<TEvKqpNode::TEvStartKqpTasksResponse>(sender1);
+ auto& record = answer->Get()->Record;
+
+ UNIT_ASSERT_VALUES_EQUAL(1, record.GetTxId());
+ UNIT_ASSERT_VALUES_EQUAL(2, record.GetStartedTasks().size());
+ UNIT_ASSERT_VALUES_EQUAL(0, record.GetNotStartedTasks().size());
+ UNIT_ASSERT_VALUES_EQUAL(2, CompFactory->Task2Actor.size());
+
+ UNIT_ASSERT(CompFactory->Task2Actor.contains(1));
+ UNIT_ASSERT(CompFactory->Task2Actor.contains(2));
+
+ auto& memoryLimits = CompFactory->Task2Actor.begin()->second.MemoryLimits;
+ UNIT_ASSERT_VALUES_EQUAL(10, memoryLimits.ChannelBufferSize);
+ UNIT_ASSERT_VALUES_EQUAL(50, memoryLimits.ScanBufferSize);
+ UNIT_ASSERT_VALUES_EQUAL(1'000, memoryLimits.MkqlLightProgramMemoryLimit);
+ UNIT_ASSERT_VALUES_EQUAL(10'000, memoryLimits.MkqlHeavyProgramMemoryLimit);
+
+ UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmComputeActors->Val(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmMemory->Val(), 2 * taskSize);
+
+ AssertResourceBrokerSensors(0, 2 * taskSize, 0, 0, 2);
+ }
+
+ Runtime->DispatchEvents(TDispatchOptions(), TDuration::Seconds(1));
+
+ {
+ TVector<NKikimrKqp::TKqpNodeResources> snapshot;
+ std::atomic<int> ready = 0;
GetKqpResourceManager(ResourceManagerActorId.NodeId())->RequestClusterResourcesInfo(
[&](TVector<NKikimrKqp::TKqpNodeResources>&& resources) {
snapshot = std::move(resources);
ready = 1;
});
-
- while (ready.load() != 1) {
- Runtime->DispatchEvents(TDispatchOptions(), TDuration::MilliSeconds(100));
- }
-
- UNIT_ASSERT_VALUES_EQUAL(1, snapshot.size());
-
- NKikimrKqp::TKqpNodeResources& payload = snapshot[0];
- UNIT_ASSERT_VALUES_EQUAL(1, payload.GetNodeId());
- UNIT_ASSERT_VALUES_EQUAL(98, payload.GetExecutionUnits());
- UNIT_ASSERT_VALUES_EQUAL(1, payload.GetMemory().size());
- UNIT_ASSERT_VALUES_EQUAL((ui32) NRm::EKqpMemoryPool::ScanQuery, payload.GetMemory()[0].GetPool());
- UNIT_ASSERT_VALUES_EQUAL(cfg.GetResourceManager().GetQueryMemoryLimit() - 2 * taskSize,
- payload.GetMemory()[0].GetAvailable());
- }
-
- // attempt to request resources for the same txId/requester
- SendStartTasksRequest(sender1, /* txId */ 1, /* taskIds */ {3, 4});
- {
- auto answer = Runtime->GrabEdgeEvent<TEvKqpNode::TEvStartKqpTasksResponse>(sender1);
- auto& record = answer->Get()->Record;
-
- UNIT_ASSERT_VALUES_EQUAL(1, record.GetTxId());
- UNIT_ASSERT_EQUAL(2, record.GetNotStartedTasks().size());
- for (auto& notStartedTask : record.GetNotStartedTasks()) {
- UNIT_ASSERT_EQUAL(NKikimrKqp::TEvStartKqpTasksResponse::INTERNAL_ERROR, notStartedTask.GetReason());
- }
-
- AssertResourceBrokerSensors(0, 2 * taskSize, 0, 0, 2);
- }
-
- // second request
- SendStartTasksRequest(sender2, /* txId */ 2, /* taskIds */ {3, 4});
- {
- auto answer = Runtime->GrabEdgeEvent<TEvKqpNode::TEvStartKqpTasksResponse>(sender2);
- auto& record = answer->Get()->Record;
-
- UNIT_ASSERT_VALUES_EQUAL(2, record.GetTxId());
- UNIT_ASSERT_VALUES_EQUAL(2, record.GetStartedTasks().size());
- UNIT_ASSERT_VALUES_EQUAL(0, record.GetNotStartedTasks().size());
- UNIT_ASSERT_VALUES_EQUAL(4, CompFactory->Task2Actor.size());
-
- UNIT_ASSERT(CompFactory->Task2Actor.contains(1));
- UNIT_ASSERT(CompFactory->Task2Actor.contains(2));
- UNIT_ASSERT(CompFactory->Task2Actor.contains(3));
- UNIT_ASSERT(CompFactory->Task2Actor.contains(4));
-
- UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmComputeActors->Val(), 4);
- UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmMemory->Val(), 4 * taskSize);
-
- AssertResourceBrokerSensors(0, 4 * taskSize, 0, 0, 4);
- }
-
- // request extra resources for taskId 4
- {
- NKikimr::TActorSystemStub stub;
-
- auto& task4ExtraAlloc = CompFactory->Task2Actor[4].MemoryLimits.AllocateMemoryFn;
+
+ while (ready.load() != 1) {
+ Runtime->DispatchEvents(TDispatchOptions(), TDuration::MilliSeconds(100));
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(1, snapshot.size());
+
+ NKikimrKqp::TKqpNodeResources& payload = snapshot[0];
+ UNIT_ASSERT_VALUES_EQUAL(1, payload.GetNodeId());
+ UNIT_ASSERT_VALUES_EQUAL(98, payload.GetExecutionUnits());
+ UNIT_ASSERT_VALUES_EQUAL(1, payload.GetMemory().size());
+ UNIT_ASSERT_VALUES_EQUAL((ui32) NRm::EKqpMemoryPool::ScanQuery, payload.GetMemory()[0].GetPool());
+ UNIT_ASSERT_VALUES_EQUAL(cfg.GetResourceManager().GetQueryMemoryLimit() - 2 * taskSize,
+ payload.GetMemory()[0].GetAvailable());
+ }
+
+ // attempt to request resources for the same txId/requester
+ SendStartTasksRequest(sender1, /* txId */ 1, /* taskIds */ {3, 4});
+ {
+ auto answer = Runtime->GrabEdgeEvent<TEvKqpNode::TEvStartKqpTasksResponse>(sender1);
+ auto& record = answer->Get()->Record;
+
+ UNIT_ASSERT_VALUES_EQUAL(1, record.GetTxId());
+ UNIT_ASSERT_EQUAL(2, record.GetNotStartedTasks().size());
+ for (auto& notStartedTask : record.GetNotStartedTasks()) {
+ UNIT_ASSERT_EQUAL(NKikimrKqp::TEvStartKqpTasksResponse::INTERNAL_ERROR, notStartedTask.GetReason());
+ }
+
+ AssertResourceBrokerSensors(0, 2 * taskSize, 0, 0, 2);
+ }
+
+ // second request
+ SendStartTasksRequest(sender2, /* txId */ 2, /* taskIds */ {3, 4});
+ {
+ auto answer = Runtime->GrabEdgeEvent<TEvKqpNode::TEvStartKqpTasksResponse>(sender2);
+ auto& record = answer->Get()->Record;
+
+ UNIT_ASSERT_VALUES_EQUAL(2, record.GetTxId());
+ UNIT_ASSERT_VALUES_EQUAL(2, record.GetStartedTasks().size());
+ UNIT_ASSERT_VALUES_EQUAL(0, record.GetNotStartedTasks().size());
+ UNIT_ASSERT_VALUES_EQUAL(4, CompFactory->Task2Actor.size());
+
+ UNIT_ASSERT(CompFactory->Task2Actor.contains(1));
+ UNIT_ASSERT(CompFactory->Task2Actor.contains(2));
+ UNIT_ASSERT(CompFactory->Task2Actor.contains(3));
+ UNIT_ASSERT(CompFactory->Task2Actor.contains(4));
+
+ UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmComputeActors->Val(), 4);
+ UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmMemory->Val(), 4 * taskSize);
+
+ AssertResourceBrokerSensors(0, 4 * taskSize, 0, 0, 4);
+ }
+
+ // request extra resources for taskId 4
+ {
+ NKikimr::TActorSystemStub stub;
+
+ auto& task4ExtraAlloc = CompFactory->Task2Actor[4].MemoryLimits.AllocateMemoryFn;
bool allocated = task4ExtraAlloc(/* txId */ (ui64)2, /* taskId */ 4, /* memory */ 100);
- UNIT_ASSERT(allocated);
- DispatchKqpNodePostponedEvents(sender1);
- UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmComputeActors->Val(), 4);
- UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmMemory->Val(), 4 * taskSize + 100);
- AssertResourceBrokerSensors(0, 4 * taskSize + 100, 0, 1, 4);
- }
-
- // complete tasks
- for (ui64 taskId : {1, 2, 3, 4}) {
- ui64 extraMem = taskId < 4 ? 100 : 0;
-
- auto mockCA = CompFactory->Task2Actor[taskId];
- CompFactory->Task2Actor.erase(taskId);
-
- SendFinishTask(mockCA.ActorId, taskId < 3 ? 1 : 2, taskId);
- {
- UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmComputeActors->Val(), (i64) CompFactory->Task2Actor.size());
- UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmMemory->Val(), (i64) (4 - taskId) * taskSize + extraMem);
-
- AssertResourceBrokerSensors(0, (4 - taskId) * taskSize + extraMem, 0, 1 + taskId, 4 - taskId);
- }
- }
-
- AssertResourceBrokerSensors(0, 0, 0, 5, 0);
-}
-
-void KqpNode::ExtraAllocation() {
- auto cfg = MakeKqpResourceManagerConfig();
- cfg.MutableResourceManager()->SetQueryMemoryLimit(100'000);
- CreateKqpNode(cfg);
-
- TActorId sender1 = Runtime->AllocateEdgeActor();
-
- const ui64 taskSize = 1'000 + 2 * 10;
-
- SendStartTasksRequest(sender1, /* txId */ 1, /* taskIds */ {1, 2});
- Runtime->GrabEdgeEvent<TEvKqpNode::TEvStartKqpTasksResponse>(sender1);
-
- auto& task1ExtraAlloc = CompFactory->Task2Actor[1].MemoryLimits.AllocateMemoryFn;
-
- // memory granted
- {
- NKikimr::TActorSystemStub stub;
-
+ UNIT_ASSERT(allocated);
+ DispatchKqpNodePostponedEvents(sender1);
+ UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmComputeActors->Val(), 4);
+ UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmMemory->Val(), 4 * taskSize + 100);
+ AssertResourceBrokerSensors(0, 4 * taskSize + 100, 0, 1, 4);
+ }
+
+ // complete tasks
+ for (ui64 taskId : {1, 2, 3, 4}) {
+ ui64 extraMem = taskId < 4 ? 100 : 0;
+
+ auto mockCA = CompFactory->Task2Actor[taskId];
+ CompFactory->Task2Actor.erase(taskId);
+
+ SendFinishTask(mockCA.ActorId, taskId < 3 ? 1 : 2, taskId);
+ {
+ UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmComputeActors->Val(), (i64) CompFactory->Task2Actor.size());
+ UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmMemory->Val(), (i64) (4 - taskId) * taskSize + extraMem);
+
+ AssertResourceBrokerSensors(0, (4 - taskId) * taskSize + extraMem, 0, 1 + taskId, 4 - taskId);
+ }
+ }
+
+ AssertResourceBrokerSensors(0, 0, 0, 5, 0);
+}
+
+void KqpNode::ExtraAllocation() {
+ auto cfg = MakeKqpResourceManagerConfig();
+ cfg.MutableResourceManager()->SetQueryMemoryLimit(100'000);
+ CreateKqpNode(cfg);
+
+ TActorId sender1 = Runtime->AllocateEdgeActor();
+
+ const ui64 taskSize = 1'000 + 2 * 10;
+
+ SendStartTasksRequest(sender1, /* txId */ 1, /* taskIds */ {1, 2});
+ Runtime->GrabEdgeEvent<TEvKqpNode::TEvStartKqpTasksResponse>(sender1);
+
+ auto& task1ExtraAlloc = CompFactory->Task2Actor[1].MemoryLimits.AllocateMemoryFn;
+
+ // memory granted
+ {
+ NKikimr::TActorSystemStub stub;
+
bool allocated = task1ExtraAlloc(/* txId */ (ui64)1, /* taskId */ 1, /* memory */ 100);
- UNIT_ASSERT(allocated);
- DispatchKqpNodePostponedEvents(sender1);
-
- UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmComputeActors->Val(), 2);
- UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmMemory->Val(), 2 * taskSize + 100);
- AssertResourceBrokerSensors(0, 2 * taskSize + 100, 0, 1, 2);
- }
-
- // too big request
- {
- NKikimr::TActorSystemStub stub;
-
+ UNIT_ASSERT(allocated);
+ DispatchKqpNodePostponedEvents(sender1);
+
+ UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmComputeActors->Val(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmMemory->Val(), 2 * taskSize + 100);
+ AssertResourceBrokerSensors(0, 2 * taskSize + 100, 0, 1, 2);
+ }
+
+ // too big request
+ {
+ NKikimr::TActorSystemStub stub;
+
bool allocated = task1ExtraAlloc(/* txId */ (ui64)1, /* taskId */ 1, /* memory */ 50'000);
- UNIT_ASSERT(!allocated);
- DispatchKqpNodePostponedEvents(sender1);
-
- UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmComputeActors->Val(), 2);
- UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmMemory->Val(), 2 * taskSize + 100);
- UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmNotEnoughMemory->Val(), 1);
- UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmNotEnoughComputeActors->Val(), 0);
- AssertResourceBrokerSensors(0, 2 * taskSize + 100, 0, 1, 2);
- }
-}
-
-void KqpNode::NotEnoughMemory() {
- auto cfg = MakeKqpResourceManagerConfig();
- cfg.MutableResourceManager()->SetChannelBufferSize(100'000);
- CreateKqpNode(cfg);
-
- TActorId sender1 = Runtime->AllocateEdgeActor();
-
- SendStartTasksRequest(sender1, 1, {1});
- {
- auto answer = Runtime->GrabEdgeEvent<TEvKqpNode::TEvStartKqpTasksResponse>(sender1);
- auto& record = answer->Get()->Record;
-
- UNIT_ASSERT_VALUES_EQUAL(1, record.GetTxId());
- UNIT_ASSERT_VALUES_EQUAL(1, record.GetNotStartedTasks().size());
- auto& task = record.GetNotStartedTasks()[0];
- UNIT_ASSERT_EQUAL(NKikimrKqp::TEvStartKqpTasksResponse::QUERY_MEMORY_LIMIT_EXCEEDED, task.GetReason());
- UNIT_ASSERT_STRINGS_EQUAL("Required: 201000, limit: 30000", task.GetMessage());
- }
-
- AssertResourceBrokerSensors(0, 0, 0, 0, 0);
-}
-
-void KqpNode::NotEnoughMemory_Extra() {
- CreateKqpNode(MakeKqpResourceManagerConfig());
-
- TActorId sender1 = Runtime->AllocateEdgeActor();
-
- const ui64 taskSize = 1'000 + 2 * 10;
-
- // first request
+ UNIT_ASSERT(!allocated);
+ DispatchKqpNodePostponedEvents(sender1);
+
+ UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmComputeActors->Val(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmMemory->Val(), 2 * taskSize + 100);
+ UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmNotEnoughMemory->Val(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmNotEnoughComputeActors->Val(), 0);
+ AssertResourceBrokerSensors(0, 2 * taskSize + 100, 0, 1, 2);
+ }
+}
+
+void KqpNode::NotEnoughMemory() {
+ auto cfg = MakeKqpResourceManagerConfig();
+ cfg.MutableResourceManager()->SetChannelBufferSize(100'000);
+ CreateKqpNode(cfg);
+
+ TActorId sender1 = Runtime->AllocateEdgeActor();
+
+ SendStartTasksRequest(sender1, 1, {1});
+ {
+ auto answer = Runtime->GrabEdgeEvent<TEvKqpNode::TEvStartKqpTasksResponse>(sender1);
+ auto& record = answer->Get()->Record;
+
+ UNIT_ASSERT_VALUES_EQUAL(1, record.GetTxId());
+ UNIT_ASSERT_VALUES_EQUAL(1, record.GetNotStartedTasks().size());
+ auto& task = record.GetNotStartedTasks()[0];
+ UNIT_ASSERT_EQUAL(NKikimrKqp::TEvStartKqpTasksResponse::QUERY_MEMORY_LIMIT_EXCEEDED, task.GetReason());
+ UNIT_ASSERT_STRINGS_EQUAL("Required: 201000, limit: 30000", task.GetMessage());
+ }
+
+ AssertResourceBrokerSensors(0, 0, 0, 0, 0);
+}
+
+void KqpNode::NotEnoughMemory_Extra() {
+ CreateKqpNode(MakeKqpResourceManagerConfig());
+
+ TActorId sender1 = Runtime->AllocateEdgeActor();
+
+ const ui64 taskSize = 1'000 + 2 * 10;
+
+ // first request
SendStartTasksRequest(sender1, /* txId */ (ui64)1, /* taskIds */ {1, 2});
- {
- auto answer = Runtime->GrabEdgeEvent<TEvKqpNode::TEvStartKqpTasksResponse>(sender1);
- auto& record = answer->Get()->Record;
-
- UNIT_ASSERT_VALUES_EQUAL(1, record.GetTxId());
- UNIT_ASSERT_VALUES_EQUAL(0, record.GetNotStartedTasks().size());
- UNIT_ASSERT_VALUES_EQUAL(2, record.GetStartedTasks().size());
- UNIT_ASSERT_VALUES_EQUAL(2, CompFactory->Task2Actor.size());
-
- UNIT_ASSERT(CompFactory->Task2Actor.contains(1));
- UNIT_ASSERT(CompFactory->Task2Actor.contains(2));
-
- auto& memoryLimits = CompFactory->Task2Actor.begin()->second.MemoryLimits;
- UNIT_ASSERT_VALUES_EQUAL(10, memoryLimits.ChannelBufferSize);
- UNIT_ASSERT_VALUES_EQUAL(50, memoryLimits.ScanBufferSize);
- UNIT_ASSERT_VALUES_EQUAL(1'000, memoryLimits.MkqlLightProgramMemoryLimit);
- UNIT_ASSERT_VALUES_EQUAL(10'000, memoryLimits.MkqlHeavyProgramMemoryLimit);
-
- UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmComputeActors->Val(), 2);
- UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmMemory->Val(), 2 * taskSize);
-
- AssertResourceBrokerSensors(0, 2 * taskSize, 0, 0, 2);
- }
-
- {
- NKikimr::TActorSystemStub stub;
-
- auto& task1ExtraAlloc = CompFactory->Task2Actor[1].MemoryLimits.AllocateMemoryFn;
+ {
+ auto answer = Runtime->GrabEdgeEvent<TEvKqpNode::TEvStartKqpTasksResponse>(sender1);
+ auto& record = answer->Get()->Record;
+
+ UNIT_ASSERT_VALUES_EQUAL(1, record.GetTxId());
+ UNIT_ASSERT_VALUES_EQUAL(0, record.GetNotStartedTasks().size());
+ UNIT_ASSERT_VALUES_EQUAL(2, record.GetStartedTasks().size());
+ UNIT_ASSERT_VALUES_EQUAL(2, CompFactory->Task2Actor.size());
+
+ UNIT_ASSERT(CompFactory->Task2Actor.contains(1));
+ UNIT_ASSERT(CompFactory->Task2Actor.contains(2));
+
+ auto& memoryLimits = CompFactory->Task2Actor.begin()->second.MemoryLimits;
+ UNIT_ASSERT_VALUES_EQUAL(10, memoryLimits.ChannelBufferSize);
+ UNIT_ASSERT_VALUES_EQUAL(50, memoryLimits.ScanBufferSize);
+ UNIT_ASSERT_VALUES_EQUAL(1'000, memoryLimits.MkqlLightProgramMemoryLimit);
+ UNIT_ASSERT_VALUES_EQUAL(10'000, memoryLimits.MkqlHeavyProgramMemoryLimit);
+
+ UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmComputeActors->Val(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmMemory->Val(), 2 * taskSize);
+
+ AssertResourceBrokerSensors(0, 2 * taskSize, 0, 0, 2);
+ }
+
+ {
+ NKikimr::TActorSystemStub stub;
+
+ auto& task1ExtraAlloc = CompFactory->Task2Actor[1].MemoryLimits.AllocateMemoryFn;
bool allocated = task1ExtraAlloc((ui64)1, 1, 1'000'000);
- UNIT_ASSERT(!allocated);
- }
-
- DispatchKqpNodePostponedEvents(sender1);
-
- UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmComputeActors->Val(), 2);
- UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmMemory->Val(), 2 * taskSize);
- UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmNotEnoughMemory->Val(), 1);
- UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmNotEnoughComputeActors->Val(), 0);
-
- AssertResourceBrokerSensors(0, 2 * taskSize, 0, 0, 2);
-}
-
-void KqpNode::NotEnoughComputeActors() {
- auto cfg = MakeKqpResourceManagerConfig();
- cfg.MutableResourceManager()->SetComputeActorsCount(4);
- CreateKqpNode(cfg);
-
- TActorId sender1 = Runtime->AllocateEdgeActor();
-
+ UNIT_ASSERT(!allocated);
+ }
+
+ DispatchKqpNodePostponedEvents(sender1);
+
+ UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmComputeActors->Val(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmMemory->Val(), 2 * taskSize);
+ UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmNotEnoughMemory->Val(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmNotEnoughComputeActors->Val(), 0);
+
+ AssertResourceBrokerSensors(0, 2 * taskSize, 0, 0, 2);
+}
+
+void KqpNode::NotEnoughComputeActors() {
+ auto cfg = MakeKqpResourceManagerConfig();
+ cfg.MutableResourceManager()->SetComputeActorsCount(4);
+ CreateKqpNode(cfg);
+
+ TActorId sender1 = Runtime->AllocateEdgeActor();
+
SendStartTasksRequest(sender1, /* txId */ (ui64)1, /* taskIds */ {1, 2, 3, 4, 5});
- {
- auto answer = Runtime->GrabEdgeEvent<TEvKqpNode::TEvStartKqpTasksResponse>(sender1);
- auto& record = answer->Get()->Record;
-
- UNIT_ASSERT_VALUES_EQUAL(1, record.GetTxId());
- UNIT_ASSERT_VALUES_EQUAL(5, record.GetNotStartedTasks().size());
- for (auto& task : record.GetNotStartedTasks()) {
- UNIT_ASSERT_EQUAL(NKikimrKqp::TEvStartKqpTasksResponse::NOT_ENOUGH_EXECUTION_UNITS, task.GetReason());
- }
- }
-
- AssertResourceBrokerSensors(0, 0, 0, 4, 0);
-}
-
-void KqpNode::ResourceBrokerNotEnoughResources() {
- auto cfg = MakeKqpResourceManagerConfig();
- cfg.MutableResourceManager()->SetChannelBufferSize(6'000);
- cfg.MutableResourceManager()->SetQueryMemoryLimit(100'000);
- CreateKqpNode(cfg);
-
- TActorId sender1 = Runtime->AllocateEdgeActor();
- TActorId sender2 = Runtime->AllocateEdgeActor();
-
- SendStartTasksRequest(sender1, 1, {1, 2});
- {
- auto answer = Runtime->GrabEdgeEvent<TEvKqpNode::TEvStartKqpTasksResponse>(sender1);
- Y_UNUSED(answer);
- AssertResourceBrokerSensors(0, 26'000, 0, 0, 2);
- }
-
- SendStartTasksRequest(sender2, 2, {3, 4});
- {
- auto answer = Runtime->GrabEdgeEvent<TEvKqpNode::TEvStartKqpTasksResponse>(sender2);
- auto& record = answer->Get()->Record;
-
- UNIT_ASSERT_VALUES_EQUAL(2, record.GetTxId());
- UNIT_ASSERT_VALUES_EQUAL(2, record.GetNotStartedTasks().size());
- for (auto& task : record.GetNotStartedTasks()) {
- UNIT_ASSERT_EQUAL(NKikimrKqp::TEvStartKqpTasksResponse::NOT_ENOUGH_MEMORY, task.GetReason());
- }
- }
-
- AssertResourceBrokerSensors(0, 26'000, 0, 1, 2);
-}
-
-void KqpNode::ResourceBrokerNotEnoughResources_Extra() {
- auto cfg = MakeKqpResourceManagerConfig();
- cfg.MutableResourceManager()->SetChannelBufferSize(6'000);
- cfg.MutableResourceManager()->SetQueryMemoryLimit(100'000);
- CreateKqpNode(cfg);
-
- TActorId sender1 = Runtime->AllocateEdgeActor();
-
- SendStartTasksRequest(sender1, 1, {1, 2});
- {
- auto answer = Runtime->GrabEdgeEvent<TEvKqpNode::TEvStartKqpTasksResponse>(sender1);
- Y_UNUSED(answer);
- AssertResourceBrokerSensors(0, 26'000, 0, 0, 2);
- }
-
- {
- NKikimr::TActorSystemStub stub;
-
- auto& task1ExtraAlloc = CompFactory->Task2Actor[1].MemoryLimits.AllocateMemoryFn;
+ {
+ auto answer = Runtime->GrabEdgeEvent<TEvKqpNode::TEvStartKqpTasksResponse>(sender1);
+ auto& record = answer->Get()->Record;
+
+ UNIT_ASSERT_VALUES_EQUAL(1, record.GetTxId());
+ UNIT_ASSERT_VALUES_EQUAL(5, record.GetNotStartedTasks().size());
+ for (auto& task : record.GetNotStartedTasks()) {
+ UNIT_ASSERT_EQUAL(NKikimrKqp::TEvStartKqpTasksResponse::NOT_ENOUGH_EXECUTION_UNITS, task.GetReason());
+ }
+ }
+
+ AssertResourceBrokerSensors(0, 0, 0, 4, 0);
+}
+
+void KqpNode::ResourceBrokerNotEnoughResources() {
+ auto cfg = MakeKqpResourceManagerConfig();
+ cfg.MutableResourceManager()->SetChannelBufferSize(6'000);
+ cfg.MutableResourceManager()->SetQueryMemoryLimit(100'000);
+ CreateKqpNode(cfg);
+
+ TActorId sender1 = Runtime->AllocateEdgeActor();
+ TActorId sender2 = Runtime->AllocateEdgeActor();
+
+ SendStartTasksRequest(sender1, 1, {1, 2});
+ {
+ auto answer = Runtime->GrabEdgeEvent<TEvKqpNode::TEvStartKqpTasksResponse>(sender1);
+ Y_UNUSED(answer);
+ AssertResourceBrokerSensors(0, 26'000, 0, 0, 2);
+ }
+
+ SendStartTasksRequest(sender2, 2, {3, 4});
+ {
+ auto answer = Runtime->GrabEdgeEvent<TEvKqpNode::TEvStartKqpTasksResponse>(sender2);
+ auto& record = answer->Get()->Record;
+
+ UNIT_ASSERT_VALUES_EQUAL(2, record.GetTxId());
+ UNIT_ASSERT_VALUES_EQUAL(2, record.GetNotStartedTasks().size());
+ for (auto& task : record.GetNotStartedTasks()) {
+ UNIT_ASSERT_EQUAL(NKikimrKqp::TEvStartKqpTasksResponse::NOT_ENOUGH_MEMORY, task.GetReason());
+ }
+ }
+
+ AssertResourceBrokerSensors(0, 26'000, 0, 1, 2);
+}
+
+void KqpNode::ResourceBrokerNotEnoughResources_Extra() {
+ auto cfg = MakeKqpResourceManagerConfig();
+ cfg.MutableResourceManager()->SetChannelBufferSize(6'000);
+ cfg.MutableResourceManager()->SetQueryMemoryLimit(100'000);
+ CreateKqpNode(cfg);
+
+ TActorId sender1 = Runtime->AllocateEdgeActor();
+
+ SendStartTasksRequest(sender1, 1, {1, 2});
+ {
+ auto answer = Runtime->GrabEdgeEvent<TEvKqpNode::TEvStartKqpTasksResponse>(sender1);
+ Y_UNUSED(answer);
+ AssertResourceBrokerSensors(0, 26'000, 0, 0, 2);
+ }
+
+ {
+ NKikimr::TActorSystemStub stub;
+
+ auto& task1ExtraAlloc = CompFactory->Task2Actor[1].MemoryLimits.AllocateMemoryFn;
bool allocated = task1ExtraAlloc((ui64)1, 1, 26'000);
- UNIT_ASSERT(!allocated);
- }
-
- AssertResourceBrokerSensors(0, 26'000, 0, 0, 2);
-}
-
-void KqpNode::ExecuterLost() {
- CreateKqpNode(MakeKqpResourceManagerConfig());
-
- TActorId sender1 = Runtime->AllocateEdgeActor();
- SendStartTasksRequest(sender1, 1, {1, 2});
-
- Runtime->GrabEdgeEvent<TEvKqpNode::TEvStartKqpTasksResponse>(sender1);
-
- {
- NKikimr::TActorSystemStub stub;
-
- auto& task1ExtraAlloc = CompFactory->Task2Actor[1].MemoryLimits.AllocateMemoryFn;
+ UNIT_ASSERT(!allocated);
+ }
+
+ AssertResourceBrokerSensors(0, 26'000, 0, 0, 2);
+}
+
+void KqpNode::ExecuterLost() {
+ CreateKqpNode(MakeKqpResourceManagerConfig());
+
+ TActorId sender1 = Runtime->AllocateEdgeActor();
+ SendStartTasksRequest(sender1, 1, {1, 2});
+
+ Runtime->GrabEdgeEvent<TEvKqpNode::TEvStartKqpTasksResponse>(sender1);
+
+ {
+ NKikimr::TActorSystemStub stub;
+
+ auto& task1ExtraAlloc = CompFactory->Task2Actor[1].MemoryLimits.AllocateMemoryFn;
bool allocated = task1ExtraAlloc((ui64)1, 1, 100);
- UNIT_ASSERT(allocated);
- DispatchKqpNodePostponedEvents(sender1);
- }
-
- Runtime->Send(new IEventHandle(KqpNodeActorId, {},
- new TEvents::TEvUndelivered(
- TEvKqpNode::TEvStartKqpTasksResponse::EventType,
- TEvents::TEvUndelivered::EReason::ReasonActorUnknown),
- 0, 1));
-
- for (auto& [taskId, computeActor] : CompFactory->Task2Actor) {
- auto abortEvent = Runtime->GrabEdgeEvent<TEvKqp::TEvAbortExecution>(computeActor.ActorId);
- UNIT_ASSERT_VALUES_EQUAL("executer lost", abortEvent->Get()->Record.GetMessage());
- }
-
- UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmComputeActors->Val(), 0);
- UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmMemory->Val(), 0);
- UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmNotEnoughMemory->Val(), 0);
- UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmNotEnoughComputeActors->Val(), 0);
-
- AssertResourceBrokerSensors(0, 0, 0, 3, 0);
-}
-
-void KqpNode::TerminateTx() {
- CreateKqpNode(MakeKqpResourceManagerConfig());
-
- TActorId executer = Runtime->AllocateEdgeActor();
- SendStartTasksRequest(executer, 1, {1});
-
- TActorId sender1 = Runtime->AllocateEdgeActor();
- SendStartTasksRequest(sender1, 1, {2}, executer);
-
- TActorId sender2 = Runtime->AllocateEdgeActor();
- SendStartTasksRequest(sender2, 1, {3}, executer);
-
- {
- NKikimr::TActorSystemStub stub;
-
- auto& task1ExtraAlloc = CompFactory->Task2Actor[1].MemoryLimits.AllocateMemoryFn;
+ UNIT_ASSERT(allocated);
+ DispatchKqpNodePostponedEvents(sender1);
+ }
+
+ Runtime->Send(new IEventHandle(KqpNodeActorId, {},
+ new TEvents::TEvUndelivered(
+ TEvKqpNode::TEvStartKqpTasksResponse::EventType,
+ TEvents::TEvUndelivered::EReason::ReasonActorUnknown),
+ 0, 1));
+
+ for (auto& [taskId, computeActor] : CompFactory->Task2Actor) {
+ auto abortEvent = Runtime->GrabEdgeEvent<TEvKqp::TEvAbortExecution>(computeActor.ActorId);
+ UNIT_ASSERT_VALUES_EQUAL("executer lost", abortEvent->Get()->Record.GetMessage());
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmComputeActors->Val(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmMemory->Val(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmNotEnoughMemory->Val(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmNotEnoughComputeActors->Val(), 0);
+
+ AssertResourceBrokerSensors(0, 0, 0, 3, 0);
+}
+
+void KqpNode::TerminateTx() {
+ CreateKqpNode(MakeKqpResourceManagerConfig());
+
+ TActorId executer = Runtime->AllocateEdgeActor();
+ SendStartTasksRequest(executer, 1, {1});
+
+ TActorId sender1 = Runtime->AllocateEdgeActor();
+ SendStartTasksRequest(sender1, 1, {2}, executer);
+
+ TActorId sender2 = Runtime->AllocateEdgeActor();
+ SendStartTasksRequest(sender2, 1, {3}, executer);
+
+ {
+ NKikimr::TActorSystemStub stub;
+
+ auto& task1ExtraAlloc = CompFactory->Task2Actor[1].MemoryLimits.AllocateMemoryFn;
bool allocated = task1ExtraAlloc((ui64)1, 1, 100);
- UNIT_ASSERT(allocated);
- DispatchKqpNodePostponedEvents(sender1);
- }
-
- const ui64 taskSize = 1'000 + 2 * 10;
-
- UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmComputeActors->Val(), 3);
- UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmMemory->Val(), 3 * taskSize + 100);
- UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmNotEnoughMemory->Val(), 0);
- UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmNotEnoughComputeActors->Val(), 0);
-
- AssertResourceBrokerSensors(0, 3 * taskSize + 100, 0, 1, 3);
-
- {
- // terminate tx
- auto cancelEvent = MakeHolder<TEvKqpNode::TEvCancelKqpTasksRequest>();
- cancelEvent->Record.SetTxId(1);
- cancelEvent->Record.SetReason("terminate");
- Runtime->Send(new IEventHandle(KqpNodeActorId, executer, cancelEvent.Release()));
-
- for (auto&[taskId, computeActor] : CompFactory->Task2Actor) {
- auto abortEvent = Runtime->GrabEdgeEvent<TEvKqp::TEvAbortExecution>(computeActor.ActorId);
- UNIT_ASSERT_VALUES_EQUAL("terminate", abortEvent->Get()->Record.GetMessage());
- }
- }
-
- UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmComputeActors->Val(), 0);
- UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmMemory->Val(), 0);
- UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmNotEnoughMemory->Val(), 0);
- UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmNotEnoughComputeActors->Val(), 0);
-
- AssertResourceBrokerSensors(0, 0, 0, 4, 0);
-}
-
-} // namespace NKqp
-} // namespace NKikimr
+ UNIT_ASSERT(allocated);
+ DispatchKqpNodePostponedEvents(sender1);
+ }
+
+ const ui64 taskSize = 1'000 + 2 * 10;
+
+ UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmComputeActors->Val(), 3);
+ UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmMemory->Val(), 3 * taskSize + 100);
+ UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmNotEnoughMemory->Val(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmNotEnoughComputeActors->Val(), 0);
+
+ AssertResourceBrokerSensors(0, 3 * taskSize + 100, 0, 1, 3);
+
+ {
+ // terminate tx
+ auto cancelEvent = MakeHolder<TEvKqpNode::TEvCancelKqpTasksRequest>();
+ cancelEvent->Record.SetTxId(1);
+ cancelEvent->Record.SetReason("terminate");
+ Runtime->Send(new IEventHandle(KqpNodeActorId, executer, cancelEvent.Release()));
+
+ for (auto&[taskId, computeActor] : CompFactory->Task2Actor) {
+ auto abortEvent = Runtime->GrabEdgeEvent<TEvKqp::TEvAbortExecution>(computeActor.ActorId);
+ UNIT_ASSERT_VALUES_EQUAL("terminate", abortEvent->Get()->Record.GetMessage());
+ }
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmComputeActors->Val(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmMemory->Val(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmNotEnoughMemory->Val(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(KqpCounters->RmNotEnoughComputeActors->Val(), 0);
+
+ AssertResourceBrokerSensors(0, 0, 0, 4, 0);
+}
+
+} // namespace NKqp
+} // namespace NKikimr
diff --git a/ydb/core/kqp/node/ut/ya.make b/ydb/core/kqp/node/ut/ya.make
index cb8aad521a..282abd4126 100644
--- a/ydb/core/kqp/node/ut/ya.make
+++ b/ydb/core/kqp/node/ut/ya.make
@@ -1,21 +1,21 @@
UNITTEST_FOR(ydb/core/kqp/node)
-
-OWNER(g:kikimr)
-
-FORK_SUBTESTS()
-
-IF (SANITIZER_TYPE OR WITH_VALGRIND)
- SIZE(MEDIUM)
-ENDIF()
-
-SRCS(
- kqp_node_ut.cpp
-)
-
-PEERDIR(
+
+OWNER(g:kikimr)
+
+FORK_SUBTESTS()
+
+IF (SANITIZER_TYPE OR WITH_VALGRIND)
+ SIZE(MEDIUM)
+ENDIF()
+
+SRCS(
+ kqp_node_ut.cpp
+)
+
+PEERDIR(
ydb/core/kqp/ut/common
-)
-
-YQL_LAST_ABI_VERSION()
-
-END()
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/ydb/core/kqp/node/ya.make b/ydb/core/kqp/node/ya.make
index 1d629edd99..6d2c60982b 100644
--- a/ydb/core/kqp/node/ya.make
+++ b/ydb/core/kqp/node/ya.make
@@ -1,12 +1,12 @@
-LIBRARY()
-
+LIBRARY()
+
OWNER(g:kikimr)
-
-SRCS(
- kqp_node.cpp
-)
-
-PEERDIR(
+
+SRCS(
+ kqp_node.cpp
+)
+
+PEERDIR(
library/cpp/actors/core
ydb/core/actorlib_impl
ydb/core/base
@@ -17,12 +17,12 @@ PEERDIR(
ydb/core/protos
ydb/core/tablet
ydb/library/yql/dq/actors/compute
-)
-
-YQL_LAST_ABI_VERSION()
-
-END()
-
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
+
RECURSE_FOR_TESTS(
ut
)
diff --git a/ydb/core/kqp/opt/kqp_opt.cpp b/ydb/core/kqp/opt/kqp_opt.cpp
index 98501bb3ce..1bd771ece1 100644
--- a/ydb/core/kqp/opt/kqp_opt.cpp
+++ b/ydb/core/kqp/opt/kqp_opt.cpp
@@ -50,10 +50,10 @@ bool IsKqpPureLambda(const TCoLambda& lambda) {
return true;
}
- if (TMaybeNode<TKqlLookupTableBase>(node)) {
- return true;
- }
-
+ if (TMaybeNode<TKqlLookupTableBase>(node)) {
+ return true;
+ }
+
if (TMaybeNode<TKqlTableEffect>(node)) {
return true;
}
diff --git a/ydb/core/kqp/opt/kqp_opt.h b/ydb/core/kqp/opt/kqp_opt.h
index db6825cd37..a86561d0c4 100644
--- a/ydb/core/kqp/opt/kqp_opt.h
+++ b/ydb/core/kqp/opt/kqp_opt.h
@@ -7,7 +7,7 @@
namespace NKikimr::NKqp::NOpt {
-struct TKqpOptimizeContext : public TSimpleRefCount<TKqpOptimizeContext> {
+struct TKqpOptimizeContext : public TSimpleRefCount<TKqpOptimizeContext> {
TKqpOptimizeContext(const TString& cluster, const NYql::TKikimrConfiguration::TPtr& config,
const TIntrusivePtr<NYql::TKikimrQueryContext> queryCtx, const TIntrusivePtr<NYql::TKikimrTablesData>& tables)
: Cluster(cluster)
@@ -48,13 +48,13 @@ bool IsKqpEffectsStage(const NYql::NNodes::TDqStageBase& stage);
TMaybe<NYql::NNodes::TKqlQuery> BuildKqlQuery(NYql::NNodes::TKiDataQuery query, const NYql::TKikimrTablesData& tablesData,
NYql::TExprContext& ctx, bool withSystemColumns, const TIntrusivePtr<TKqpOptimizeContext>& kqpCtx);
-TAutoPtr<NYql::IGraphTransformer> CreateKqpFinalizingOptTransformer();
+TAutoPtr<NYql::IGraphTransformer> CreateKqpFinalizingOptTransformer();
TAutoPtr<NYql::IGraphTransformer> CreateKqpQueryPhasesTransformer();
TAutoPtr<NYql::IGraphTransformer> CreateKqpQueryEffectsTransformer(const TIntrusivePtr<TKqpOptimizeContext>& kqpCtx);
-TAutoPtr<NYql::IGraphTransformer> CreateKqpCheckPhysicalQueryTransformer();
+TAutoPtr<NYql::IGraphTransformer> CreateKqpCheckPhysicalQueryTransformer();
-TAutoPtr<NYql::IGraphTransformer> CreateKqpBuildTxsTransformer(const TIntrusivePtr<TKqpOptimizeContext>& kqpCtx,
- const TIntrusivePtr<TKqpBuildQueryContext>& buildCtx, TAutoPtr<NYql::IGraphTransformer>&& typeAnnTransformer,
- NYql::TTypeAnnotationContext& typesCtx, NYql::TKikimrConfiguration::TPtr& config);
+TAutoPtr<NYql::IGraphTransformer> CreateKqpBuildTxsTransformer(const TIntrusivePtr<TKqpOptimizeContext>& kqpCtx,
+ const TIntrusivePtr<TKqpBuildQueryContext>& buildCtx, TAutoPtr<NYql::IGraphTransformer>&& typeAnnTransformer,
+ NYql::TTypeAnnotationContext& typesCtx, NYql::TKikimrConfiguration::TPtr& config);
} // namespace NKikimr::NKqp::NOpt
diff --git a/ydb/core/kqp/opt/kqp_opt_build_txs.cpp b/ydb/core/kqp/opt/kqp_opt_build_txs.cpp
index 48307b369f..f35ba23f66 100644
--- a/ydb/core/kqp/opt/kqp_opt_build_txs.cpp
+++ b/ydb/core/kqp/opt/kqp_opt_build_txs.cpp
@@ -1,60 +1,60 @@
-#include "kqp_opt_impl.h"
-
+#include "kqp_opt_impl.h"
+
#include <ydb/core/kqp/common/kqp_yql.h>
-
+
#include <ydb/library/yql/core/yql_expr_optimize.h>
#include <ydb/library/yql/dq/opt/dq_opt.h>
#include <ydb/library/yql/dq/opt/dq_opt_build.h>
#include <ydb/library/yql/core/services/yql_out_transformers.h>
#include <ydb/library/yql/core/services/yql_transform_pipeline.h>
#include <ydb/library/yql/providers/common/provider/yql_provider.h>
-
+
namespace NKikimr::NKqp::NOpt {
-
-using namespace NYql;
-using namespace NYql::NNodes;
-
-using TStatus = IGraphTransformer::TStatus;
-
-namespace {
-
+
+using namespace NYql;
+using namespace NYql::NNodes;
+
+using TStatus = IGraphTransformer::TStatus;
+
+namespace {
+
TAutoPtr<NYql::IGraphTransformer> CreateKqpBuildPhyStagesTransformer(bool allowDependantConsumers) {
return NDq::CreateDqBuildPhyStagesTransformer(allowDependantConsumers);
-}
-
-class TKqpBuildTxTransformer : public TSyncTransformerBase {
-public:
- TKqpBuildTxTransformer()
- : QueryType(EKikimrQueryType::Dml)
- , IsPrecompute(false) {}
-
- void Init(EKikimrQueryType queryType, bool isPrecompute) {
- QueryType = queryType;
- IsPrecompute = isPrecompute;
- }
-
- TStatus DoTransform(TExprNode::TPtr inputExpr, TExprNode::TPtr& outputExpr, TExprContext& ctx) final {
- if (TKqpPhysicalTx::Match(inputExpr.Get())) {
- outputExpr = inputExpr;
- return TStatus::Ok;
- }
-
- YQL_ENSURE(inputExpr->IsList() && inputExpr->ChildrenSize() > 0);
-
- if (TKqlQueryResult::Match(inputExpr->Child(0))) {
- YQL_CLOG(DEBUG, ProviderKqp) << ">>> TKqpBuildTxTransformer[" << QueryType << "/"
- << (IsPrecompute ? "precompute" : "result") << "]: " << KqpExprToPrettyString(*inputExpr, ctx);
-
- return DoBuildTxResults(inputExpr, outputExpr, ctx);
- } else {
- YQL_CLOG(DEBUG, ProviderKqp) << ">>> TKqpBuildTxTransformer[effects]: "
- << KqpExprToPrettyString(*inputExpr, ctx);
-
- return DoBuildTxEffects(inputExpr, outputExpr, ctx);
- }
- }
-
-private:
+}
+
+class TKqpBuildTxTransformer : public TSyncTransformerBase {
+public:
+ TKqpBuildTxTransformer()
+ : QueryType(EKikimrQueryType::Dml)
+ , IsPrecompute(false) {}
+
+ void Init(EKikimrQueryType queryType, bool isPrecompute) {
+ QueryType = queryType;
+ IsPrecompute = isPrecompute;
+ }
+
+ TStatus DoTransform(TExprNode::TPtr inputExpr, TExprNode::TPtr& outputExpr, TExprContext& ctx) final {
+ if (TKqpPhysicalTx::Match(inputExpr.Get())) {
+ outputExpr = inputExpr;
+ return TStatus::Ok;
+ }
+
+ YQL_ENSURE(inputExpr->IsList() && inputExpr->ChildrenSize() > 0);
+
+ if (TKqlQueryResult::Match(inputExpr->Child(0))) {
+ YQL_CLOG(DEBUG, ProviderKqp) << ">>> TKqpBuildTxTransformer[" << QueryType << "/"
+ << (IsPrecompute ? "precompute" : "result") << "]: " << KqpExprToPrettyString(*inputExpr, ctx);
+
+ return DoBuildTxResults(inputExpr, outputExpr, ctx);
+ } else {
+ YQL_CLOG(DEBUG, ProviderKqp) << ">>> TKqpBuildTxTransformer[effects]: "
+ << KqpExprToPrettyString(*inputExpr, ctx);
+
+ return DoBuildTxEffects(inputExpr, outputExpr, ctx);
+ }
+ }
+
+private:
EPhysicalTxType GetPhyTxType(bool allStagesArePure) {
if (QueryType == EKikimrQueryType::Scan) {
if (IsPrecompute && allStagesArePure) {
@@ -71,192 +71,192 @@ private:
return EPhysicalTxType::Data;
}
- TStatus DoBuildTxResults(TExprNode::TPtr inputExpr, TExprNode::TPtr& outputExpr, TExprContext& ctx) {
- auto stages = CollectStages(inputExpr, ctx);
- Y_VERIFY_DEBUG(!stages.empty());
-
- auto results = TKqlQueryResultList(inputExpr);
- auto txResults = BuildTxResults(results, stages, ctx);
- if (!txResults) {
- return TStatus::Error;
- }
-
- TKqpPhyTxSettings txSettings;
+ TStatus DoBuildTxResults(TExprNode::TPtr inputExpr, TExprNode::TPtr& outputExpr, TExprContext& ctx) {
+ auto stages = CollectStages(inputExpr, ctx);
+ Y_VERIFY_DEBUG(!stages.empty());
+
+ auto results = TKqlQueryResultList(inputExpr);
+ auto txResults = BuildTxResults(results, stages, ctx);
+ if (!txResults) {
+ return TStatus::Error;
+ }
+
+ TKqpPhyTxSettings txSettings;
txSettings.Type = GetPhyTxType(AreAllStagesKqpPure(stages));
- txSettings.WithEffects = false;
-
- auto tx = Build<TKqpPhysicalTx>(ctx, inputExpr->Pos())
- .Stages()
- .Add(stages)
- .Build()
- .Results(txResults.Cast())
- .ParamBindings()
- .Build()
- .Settings(txSettings.BuildNode(ctx, inputExpr->Pos()))
- .Done();
-
- auto newTx = ExtractParamsFromTx(tx, ctx);
- if (!newTx) {
- return TStatus::Error;
- }
-
- outputExpr = newTx.Cast().Ptr();
- return TStatus(TStatus::Repeat, true);
- }
-
- TStatus DoBuildTxEffects(TExprNode::TPtr inputExpr, TExprNode::TPtr& outputExpr, TExprContext& ctx) {
- auto stages = CollectStages(inputExpr, ctx);
- Y_VERIFY_DEBUG(!stages.empty());
-
- TKqpPhyTxSettings txSettings;
- txSettings.Type = EPhysicalTxType::Data;
- txSettings.WithEffects = true;
-
- auto tx = Build<TKqpPhysicalTx>(ctx, inputExpr->Pos())
- .Stages()
- .Add(stages)
- .Build()
- .Results()
- .Build()
- .ParamBindings()
- .Build()
- .Settings(txSettings.BuildNode(ctx, inputExpr->Pos()))
- .Done();
-
- auto newTx = ExtractParamsFromTx(tx, ctx);
- if (!newTx) {
- return TStatus::Error;
- }
-
- outputExpr = newTx.Cast().Ptr();
- return TStatus(TStatus::Repeat, true);
- }
-
-private:
- static TVector<TDqPhyStage> CollectStages(const TExprNode::TPtr& node, TExprContext& /* ctx */) {
- TVector<TDqPhyStage> stages;
+ txSettings.WithEffects = false;
+
+ auto tx = Build<TKqpPhysicalTx>(ctx, inputExpr->Pos())
+ .Stages()
+ .Add(stages)
+ .Build()
+ .Results(txResults.Cast())
+ .ParamBindings()
+ .Build()
+ .Settings(txSettings.BuildNode(ctx, inputExpr->Pos()))
+ .Done();
+
+ auto newTx = ExtractParamsFromTx(tx, ctx);
+ if (!newTx) {
+ return TStatus::Error;
+ }
+
+ outputExpr = newTx.Cast().Ptr();
+ return TStatus(TStatus::Repeat, true);
+ }
+
+ TStatus DoBuildTxEffects(TExprNode::TPtr inputExpr, TExprNode::TPtr& outputExpr, TExprContext& ctx) {
+ auto stages = CollectStages(inputExpr, ctx);
+ Y_VERIFY_DEBUG(!stages.empty());
+
+ TKqpPhyTxSettings txSettings;
+ txSettings.Type = EPhysicalTxType::Data;
+ txSettings.WithEffects = true;
+
+ auto tx = Build<TKqpPhysicalTx>(ctx, inputExpr->Pos())
+ .Stages()
+ .Add(stages)
+ .Build()
+ .Results()
+ .Build()
+ .ParamBindings()
+ .Build()
+ .Settings(txSettings.BuildNode(ctx, inputExpr->Pos()))
+ .Done();
+
+ auto newTx = ExtractParamsFromTx(tx, ctx);
+ if (!newTx) {
+ return TStatus::Error;
+ }
+
+ outputExpr = newTx.Cast().Ptr();
+ return TStatus(TStatus::Repeat, true);
+ }
+
+private:
+ static TVector<TDqPhyStage> CollectStages(const TExprNode::TPtr& node, TExprContext& /* ctx */) {
+ TVector<TDqPhyStage> stages;
auto filter = [](const TExprNode::TPtr& exprNode) {
- return !exprNode->IsLambda();
+ return !exprNode->IsLambda();
};
auto collector = [&stages](const TExprNode::TPtr& exprNode) {
if (TDqPhyStage::Match(exprNode.Get())) {
stages.emplace_back(TDqPhyStage(exprNode));
- } else {
- YQL_ENSURE(!TDqStage::Match(exprNode.Get()));
+ } else {
+ YQL_ENSURE(!TDqStage::Match(exprNode.Get()));
}
return true;
};
VisitExpr(node, filter, collector);
- return stages;
- }
-
- static bool AreAllStagesKqpPure(const TVector<TDqPhyStage>& stages) {
- return std::all_of(stages.begin(), stages.end(), [](const auto& x) { return IsKqpPureLambda(x.Program()); });
- }
-
- static TMaybeNode<TExprList> BuildTxResults(const TKqlQueryResultList& results, TVector<TDqPhyStage>& stages,
- TExprContext& ctx)
- {
- if (NLog::YqlLogger().NeedToLog(NLog::EComponent::ProviderKqp, NLog::ELevel::TRACE)) {
- TStringBuilder sb;
- sb << "-- BuildTxResults" << Endl;
- sb << " results:" << Endl;
- for (const auto& r : results) {
- sb << " * [" << r.Raw()->UniqueId() << "] " << KqpExprToPrettyString(r.Value(), ctx) << Endl;
- }
- YQL_CLOG(TRACE, ProviderKqp) << sb;
- }
-
- TVector<TExprBase> builtResults;
- builtResults.reserve(results.Size());
-
- for (const auto& result : results) {
- if (auto maybeUnionAll = result.Value().Maybe<TDqCnUnionAll>()) {
- auto resultConnection = maybeUnionAll.Cast();
- auto resultStage = resultConnection.Output().Stage().Cast<TDqPhyStage>();
- ui32 resultIndex = FromString<ui32>(resultConnection.Output().Index());
-
-// if (resultIndex != 0) {
-// ctx.AddError(TIssue(ctx.GetPosition(result.Pos()), TStringBuilder()
-// << "Unexpected result index: " << resultIndex));
-// return {};
-// }
-
- TMaybeNode<TDqPhyStage> collectStage;
- // TODO: This is a temporary workaround.
- // If result stage has single UnionAll input, we don't have to build a separate stage
- // for results collection as it's already in single partition.
- // Proper check should use partitioning information for results stage via opt constraints.
- if (resultStage.Inputs().Size() == 1 && resultStage.Inputs().Item(0).Maybe<TDqCnUnionAll>()) {
- collectStage = resultStage;
+ return stages;
+ }
+
+ static bool AreAllStagesKqpPure(const TVector<TDqPhyStage>& stages) {
+ return std::all_of(stages.begin(), stages.end(), [](const auto& x) { return IsKqpPureLambda(x.Program()); });
+ }
+
+ static TMaybeNode<TExprList> BuildTxResults(const TKqlQueryResultList& results, TVector<TDqPhyStage>& stages,
+ TExprContext& ctx)
+ {
+ if (NLog::YqlLogger().NeedToLog(NLog::EComponent::ProviderKqp, NLog::ELevel::TRACE)) {
+ TStringBuilder sb;
+ sb << "-- BuildTxResults" << Endl;
+ sb << " results:" << Endl;
+ for (const auto& r : results) {
+ sb << " * [" << r.Raw()->UniqueId() << "] " << KqpExprToPrettyString(r.Value(), ctx) << Endl;
+ }
+ YQL_CLOG(TRACE, ProviderKqp) << sb;
+ }
+
+ TVector<TExprBase> builtResults;
+ builtResults.reserve(results.Size());
+
+ for (const auto& result : results) {
+ if (auto maybeUnionAll = result.Value().Maybe<TDqCnUnionAll>()) {
+ auto resultConnection = maybeUnionAll.Cast();
+ auto resultStage = resultConnection.Output().Stage().Cast<TDqPhyStage>();
+ ui32 resultIndex = FromString<ui32>(resultConnection.Output().Index());
+
+// if (resultIndex != 0) {
+// ctx.AddError(TIssue(ctx.GetPosition(result.Pos()), TStringBuilder()
+// << "Unexpected result index: " << resultIndex));
+// return {};
+// }
+
+ TMaybeNode<TDqPhyStage> collectStage;
+ // TODO: This is a temporary workaround.
+ // If result stage has single UnionAll input, we don't have to build a separate stage
+ // for results collection as it's already in single partition.
+ // Proper check should use partitioning information for results stage via opt constraints.
+ if (resultStage.Inputs().Size() == 1 && resultStage.Inputs().Item(0).Maybe<TDqCnUnionAll>()) {
+ collectStage = resultStage;
} else if (resultStage.Inputs().Size() == 1 && resultStage.Inputs().Item(0).Maybe<TDqCnMerge>()) {
collectStage = resultStage;
} else if (resultStage.Inputs().Size() == 1 && resultStage.Inputs().Item(0).Maybe<TDqPhyPrecompute>()) {
- collectStage = resultStage;
- } else if (resultStage.Inputs().Empty() && IsKqpPureLambda(resultStage.Program())) {
- collectStage = resultStage;
- } else {
- collectStage = Build<TDqPhyStage>(ctx, results.Pos())
- .Inputs()
- .Add(resultConnection)
- .Build()
- .Program()
- .Args({"row"})
- .Body("row")
- .Build()
- .Settings(NDq::TDqStageSettings::New().BuildNode(ctx, results.Pos()))
- .Done();
- resultIndex = 0;
- stages.emplace_back(collectStage.Cast());
- }
-
- YQL_ENSURE(collectStage);
- auto newResult = Build<TDqCnResult>(ctx, results.Pos())
- .Output()
- .Stage(collectStage.Cast())
- .Index().Build(ToString(resultIndex))
- .Build()
- .ColumnHints(result.ColumnHints())
- .Done();
-
- builtResults.emplace_back(newResult);
- continue;
- } // DqCnUnionAll
-
- if (result.Value().Maybe<TDqCnValue>()) {
- builtResults.emplace_back(result.Value());
- continue;
- } // DqCnValue
-
- if (auto maybeConnection = result.Value().Maybe<TDqConnection>()) {
- ctx.AddError(TIssue(ctx.GetPosition(result.Pos()), TStringBuilder()
- << "Unexpected connection in results transaction: " << maybeConnection.Cast().CallableName()));
- return {};
- } // any other DqConnection
-
- ctx.AddError(TIssue(ctx.GetPosition(result.Pos()), TStringBuilder()
- << "Unexpected node in results: " << KqpExprToPrettyString(result.Value(), ctx)));
- return {};
- }
-
- return Build<TExprList>(ctx, results.Pos())
- .Add(builtResults)
- .Done();
- }
-
- static TMaybeNode<TDqPhyStage> ExtractParamsFromStage(const TDqPhyStage& stage, const TNodeOnNodeOwnedMap& stagesMap,
- TMap<TString, TKqpParamBinding>& bindingsMap, TExprContext& ctx)
- {
+ collectStage = resultStage;
+ } else if (resultStage.Inputs().Empty() && IsKqpPureLambda(resultStage.Program())) {
+ collectStage = resultStage;
+ } else {
+ collectStage = Build<TDqPhyStage>(ctx, results.Pos())
+ .Inputs()
+ .Add(resultConnection)
+ .Build()
+ .Program()
+ .Args({"row"})
+ .Body("row")
+ .Build()
+ .Settings(NDq::TDqStageSettings::New().BuildNode(ctx, results.Pos()))
+ .Done();
+ resultIndex = 0;
+ stages.emplace_back(collectStage.Cast());
+ }
+
+ YQL_ENSURE(collectStage);
+ auto newResult = Build<TDqCnResult>(ctx, results.Pos())
+ .Output()
+ .Stage(collectStage.Cast())
+ .Index().Build(ToString(resultIndex))
+ .Build()
+ .ColumnHints(result.ColumnHints())
+ .Done();
+
+ builtResults.emplace_back(newResult);
+ continue;
+ } // DqCnUnionAll
+
+ if (result.Value().Maybe<TDqCnValue>()) {
+ builtResults.emplace_back(result.Value());
+ continue;
+ } // DqCnValue
+
+ if (auto maybeConnection = result.Value().Maybe<TDqConnection>()) {
+ ctx.AddError(TIssue(ctx.GetPosition(result.Pos()), TStringBuilder()
+ << "Unexpected connection in results transaction: " << maybeConnection.Cast().CallableName()));
+ return {};
+ } // any other DqConnection
+
+ ctx.AddError(TIssue(ctx.GetPosition(result.Pos()), TStringBuilder()
+ << "Unexpected node in results: " << KqpExprToPrettyString(result.Value(), ctx)));
+ return {};
+ }
+
+ return Build<TExprList>(ctx, results.Pos())
+ .Add(builtResults)
+ .Done();
+ }
+
+ static TMaybeNode<TDqPhyStage> ExtractParamsFromStage(const TDqPhyStage& stage, const TNodeOnNodeOwnedMap& stagesMap,
+ TMap<TString, TKqpParamBinding>& bindingsMap, TExprContext& ctx)
+ {
auto bindingsBuilder = [&bindingsMap, &ctx] (const TExprNode::TPtr& node) {
auto maybeParam = TMaybeNode<TCoParameter>(node);
if (!maybeParam.IsValid()) {
return true;
- }
+ }
auto param = maybeParam.Cast();
@@ -272,21 +272,21 @@ private:
bindingsMap.emplace(std::move(paramName), std::move(paramBinding));
- return true;
+ return true;
};
-
+
VisitExpr(stage.Program().Body().Ptr(), bindingsBuilder);
- TVector<TExprBase> newInputs;
- TVector<TCoArgument> newArgs;
- TNodeOnNodeOwnedMap argsMap;
+ TVector<TExprBase> newInputs;
+ TVector<TCoArgument> newArgs;
+ TNodeOnNodeOwnedMap argsMap;
+
+ for (ui32 i = 0; i < stage.Inputs().Size(); ++i) {
+ const auto& input = stage.Inputs().Item(i);
+ const auto& inputArg = stage.Program().Args().Arg(i);
- for (ui32 i = 0; i < stage.Inputs().Size(); ++i) {
- const auto& input = stage.Inputs().Item(i);
- const auto& inputArg = stage.Program().Args().Arg(i);
-
auto maybeBinding = input.Maybe<TKqpTxResultBinding>();
-
+
if (!maybeBinding.IsValid()) {
auto newArg = ctx.NewArgument(inputArg.Pos(), inputArg.Name());
newInputs.push_back(input);
@@ -294,30 +294,30 @@ private:
argsMap.emplace(inputArg.Raw(), std::move(newArg));
continue;
}
-
+
auto binding = maybeBinding.Cast();
-
+
TString paramName = TStringBuilder() << ParamNamePrefix
<< "tx_result_binding_" << binding.TxIndex().Value() << "_" << binding.ResultIndex().Value();
-
+
auto type = binding.Type().Ref().GetTypeAnn();
YQL_ENSURE(type);
YQL_ENSURE(type->GetKind() == ETypeAnnotationKind::Type);
type = type->Cast<TTypeExprType>()->GetType();
YQL_ENSURE(type);
-
+
TExprBase parameter = Build<TCoParameter>(ctx, input.Pos())
.Name().Build(paramName)
.Type(ExpandType(input.Pos(), *type, ctx))
.Done();
- // TODO: (Iterator|ToStream (Parameter ...)) -> (ToFlow (Parameter ...))
-// if (type->GetKind() == ETypeAnnotationKind::List) {
-// parameter = Build<TCoToFlow>(ctx, input.Pos()) // TODO: TDqInputReader?
-// .Input(parameter)
-// .Done();
-// }
-
+ // TODO: (Iterator|ToStream (Parameter ...)) -> (ToFlow (Parameter ...))
+// if (type->GetKind() == ETypeAnnotationKind::List) {
+// parameter = Build<TCoToFlow>(ctx, input.Pos()) // TODO: TDqInputReader?
+// .Input(parameter)
+// .Done();
+// }
+
auto paramBinding = Build<TKqpParamBinding>(ctx, input.Pos())
.Name().Build(paramName)
.Binding(binding)
@@ -329,182 +329,182 @@ private:
"duplicated parameter " << paramName
<< ", first: " << KqpExprToPrettyString(inserted.first->second.Binding().Ref(), ctx)
<< ", second: " << KqpExprToPrettyString(binding, ctx));
- }
+ }
argsMap.emplace(inputArg.Raw(), parameter.Ptr());
- }
-
- auto inputs = Build<TExprList>(ctx, stage.Pos())
- .Add(newInputs)
- .Done();
-
- return Build<TDqPhyStage>(ctx, stage.Pos())
- .Inputs(ctx.ReplaceNodes(inputs.Ptr(), stagesMap))
- .Program()
- .Args(newArgs)
- .Body(ctx.ReplaceNodes(stage.Program().Body().Ptr(), argsMap))
- .Build()
- .Settings(stage.Settings())
- .Done();
- }
-
- static TMaybeNode<TKqpPhysicalTx> ExtractParamsFromTx(TKqpPhysicalTx& tx, TExprContext& ctx) {
- TVector<TDqPhyStage> newStages;
- newStages.reserve(tx.Stages().Size());
- TNodeOnNodeOwnedMap stagesMap;
- TMap<TString, TKqpParamBinding> bindingsMap;
-
- for (const auto& stage : tx.Stages()) {
- auto newStage = ExtractParamsFromStage(stage, stagesMap, bindingsMap, ctx);
- if (!newStage) {
- return {};
- }
-
- newStages.emplace_back(newStage.Cast());
- stagesMap.emplace(stage.Raw(), newStage.Cast().Ptr());
- }
-
- TVector<TKqpParamBinding> bindings;
- bindings.reserve(bindingsMap.size());
- for (auto& pair : bindingsMap) {
- bindings.push_back(pair.second);
- }
-
- return Build<TKqpPhysicalTx>(ctx, tx.Pos())
- .Stages()
- .Add(newStages)
- .Build()
- .Results(ctx.ReplaceNodes(tx.Results().Ptr(), stagesMap))
- .ParamBindings()
- .Add(bindings)
- .Build()
- .Settings(tx.Settings())
- .Done();
- }
-
-private:
- EKikimrQueryType QueryType;
- bool IsPrecompute;
-};
-
-class TKqpBuildTxsTransformer : public TSyncTransformerBase {
-public:
- TKqpBuildTxsTransformer(const TIntrusivePtr<TKqpOptimizeContext>& kqpCtx,
- const TIntrusivePtr<TKqpBuildQueryContext>& buildCtx, TAutoPtr<IGraphTransformer>&& typeAnnTransformer,
- TTypeAnnotationContext& typesCtx, TKikimrConfiguration::TPtr& config)
- : KqpCtx(kqpCtx)
- , BuildCtx(buildCtx)
- , TypeAnnTransformer(std::move(typeAnnTransformer))
- {
- BuildTxTransformer = new TKqpBuildTxTransformer();
-
- DataTxTransformer = TTransformationPipeline(&typesCtx)
- .AddServiceTransformers()
- .Add(TExprLogTransformer::Sync("TxOpt", NLog::EComponent::ProviderKqp, NLog::ELevel::TRACE), "TxOpt")
- .Add(*TypeAnnTransformer, "TypeAnnotation")
- .AddPostTypeAnnotation(/* forSubgraph */ true)
+ }
+
+ auto inputs = Build<TExprList>(ctx, stage.Pos())
+ .Add(newInputs)
+ .Done();
+
+ return Build<TDqPhyStage>(ctx, stage.Pos())
+ .Inputs(ctx.ReplaceNodes(inputs.Ptr(), stagesMap))
+ .Program()
+ .Args(newArgs)
+ .Body(ctx.ReplaceNodes(stage.Program().Body().Ptr(), argsMap))
+ .Build()
+ .Settings(stage.Settings())
+ .Done();
+ }
+
+ static TMaybeNode<TKqpPhysicalTx> ExtractParamsFromTx(TKqpPhysicalTx& tx, TExprContext& ctx) {
+ TVector<TDqPhyStage> newStages;
+ newStages.reserve(tx.Stages().Size());
+ TNodeOnNodeOwnedMap stagesMap;
+ TMap<TString, TKqpParamBinding> bindingsMap;
+
+ for (const auto& stage : tx.Stages()) {
+ auto newStage = ExtractParamsFromStage(stage, stagesMap, bindingsMap, ctx);
+ if (!newStage) {
+ return {};
+ }
+
+ newStages.emplace_back(newStage.Cast());
+ stagesMap.emplace(stage.Raw(), newStage.Cast().Ptr());
+ }
+
+ TVector<TKqpParamBinding> bindings;
+ bindings.reserve(bindingsMap.size());
+ for (auto& pair : bindingsMap) {
+ bindings.push_back(pair.second);
+ }
+
+ return Build<TKqpPhysicalTx>(ctx, tx.Pos())
+ .Stages()
+ .Add(newStages)
+ .Build()
+ .Results(ctx.ReplaceNodes(tx.Results().Ptr(), stagesMap))
+ .ParamBindings()
+ .Add(bindings)
+ .Build()
+ .Settings(tx.Settings())
+ .Done();
+ }
+
+private:
+ EKikimrQueryType QueryType;
+ bool IsPrecompute;
+};
+
+class TKqpBuildTxsTransformer : public TSyncTransformerBase {
+public:
+ TKqpBuildTxsTransformer(const TIntrusivePtr<TKqpOptimizeContext>& kqpCtx,
+ const TIntrusivePtr<TKqpBuildQueryContext>& buildCtx, TAutoPtr<IGraphTransformer>&& typeAnnTransformer,
+ TTypeAnnotationContext& typesCtx, TKikimrConfiguration::TPtr& config)
+ : KqpCtx(kqpCtx)
+ , BuildCtx(buildCtx)
+ , TypeAnnTransformer(std::move(typeAnnTransformer))
+ {
+ BuildTxTransformer = new TKqpBuildTxTransformer();
+
+ DataTxTransformer = TTransformationPipeline(&typesCtx)
+ .AddServiceTransformers()
+ .Add(TExprLogTransformer::Sync("TxOpt", NLog::EComponent::ProviderKqp, NLog::ELevel::TRACE), "TxOpt")
+ .Add(*TypeAnnTransformer, "TypeAnnotation")
+ .AddPostTypeAnnotation(/* forSubgraph */ true)
.Add(CreateKqpBuildPhyStagesTransformer(/* allowDependantConsumers */ false), "BuildPhysicalStages")
- .Add(*BuildTxTransformer, "BuildPhysicalTx")
- .Add(CreateKqpTxPeepholeTransformer(TypeAnnTransformer.Get(), typesCtx, config), "Peephole")
- .Build(false);
-
- ScanTxTransformer = TTransformationPipeline(&typesCtx)
- .AddServiceTransformers()
- .Add(TExprLogTransformer::Sync("TxOpt", NLog::EComponent::ProviderKqp, NLog::ELevel::TRACE), "TxOpt")
- .Add(*TypeAnnTransformer, "TypeAnnotation")
- .AddPostTypeAnnotation(/* forSubgraph */ true)
- .Add(CreateKqpBuildPhyStagesTransformer(config->SpillingEnabled()), "BuildPhysicalStages")
- .Add(*BuildTxTransformer, "BuildPhysicalTx")
+ .Add(*BuildTxTransformer, "BuildPhysicalTx")
+ .Add(CreateKqpTxPeepholeTransformer(TypeAnnTransformer.Get(), typesCtx, config), "Peephole")
+ .Build(false);
+
+ ScanTxTransformer = TTransformationPipeline(&typesCtx)
+ .AddServiceTransformers()
+ .Add(TExprLogTransformer::Sync("TxOpt", NLog::EComponent::ProviderKqp, NLog::ELevel::TRACE), "TxOpt")
+ .Add(*TypeAnnTransformer, "TypeAnnotation")
+ .AddPostTypeAnnotation(/* forSubgraph */ true)
+ .Add(CreateKqpBuildPhyStagesTransformer(config->SpillingEnabled()), "BuildPhysicalStages")
+ .Add(*BuildTxTransformer, "BuildPhysicalTx")
.Add(CreateKqpTxPeepholeTransformer(TypeAnnTransformer.Get(), typesCtx, config, /* withFinalStageRules */ false), "Peephole")
- .Build(false);
- }
-
- TStatus DoTransform(TExprNode::TPtr inputExpr, TExprNode::TPtr& outputExpr, TExprContext& ctx) final {
- if (TKqpPhysicalQuery::Match(inputExpr.Get())) {
- outputExpr = inputExpr;
- return TStatus::Ok;
- }
-
- YQL_CLOG(DEBUG, ProviderKqp) << ">>> TKqpBuildTxsTransformer: " << KqpExprToPrettyString(*inputExpr, ctx);
-
- TKqlQuery query(inputExpr);
-
- if (auto status = TryBuildPrecomputeTx(query, outputExpr, ctx)) {
- return *status;
- }
-
- TVector<TExprBase> queryResults;
- if (!query.Results().Empty()) {
- auto tx = BuildTx(query.Results().Ptr(), ctx, false);
- if (!tx) {
- return TStatus::Error;
- }
-
- BuildCtx->PhysicalTxs.emplace_back(tx.Cast());
-
- for (ui32 i = 0; i < query.Results().Size(); ++i) {
- const auto& result = query.Results().Item(i);
- auto binding = Build<TKqpTxResultBinding>(ctx, query.Pos())
- .Type(ExpandType(query.Pos(), *result.Value().Ref().GetTypeAnn(), ctx))
- .TxIndex()
- .Build(ToString(BuildCtx->PhysicalTxs.size() - 1))
- .ResultIndex()
- .Build(ToString(i))
- .Done();
-
- queryResults.emplace_back(std::move(binding));
- }
- }
-
- if (!query.Effects().Empty()) {
- auto tx = BuildTx(query.Effects().Ptr(), ctx, /* isPrecompute */ false);
- if (!tx) {
- return TStatus::Error;
- }
-
+ .Build(false);
+ }
+
+ TStatus DoTransform(TExprNode::TPtr inputExpr, TExprNode::TPtr& outputExpr, TExprContext& ctx) final {
+ if (TKqpPhysicalQuery::Match(inputExpr.Get())) {
+ outputExpr = inputExpr;
+ return TStatus::Ok;
+ }
+
+ YQL_CLOG(DEBUG, ProviderKqp) << ">>> TKqpBuildTxsTransformer: " << KqpExprToPrettyString(*inputExpr, ctx);
+
+ TKqlQuery query(inputExpr);
+
+ if (auto status = TryBuildPrecomputeTx(query, outputExpr, ctx)) {
+ return *status;
+ }
+
+ TVector<TExprBase> queryResults;
+ if (!query.Results().Empty()) {
+ auto tx = BuildTx(query.Results().Ptr(), ctx, false);
+ if (!tx) {
+ return TStatus::Error;
+ }
+
+ BuildCtx->PhysicalTxs.emplace_back(tx.Cast());
+
+ for (ui32 i = 0; i < query.Results().Size(); ++i) {
+ const auto& result = query.Results().Item(i);
+ auto binding = Build<TKqpTxResultBinding>(ctx, query.Pos())
+ .Type(ExpandType(query.Pos(), *result.Value().Ref().GetTypeAnn(), ctx))
+ .TxIndex()
+ .Build(ToString(BuildCtx->PhysicalTxs.size() - 1))
+ .ResultIndex()
+ .Build(ToString(i))
+ .Done();
+
+ queryResults.emplace_back(std::move(binding));
+ }
+ }
+
+ if (!query.Effects().Empty()) {
+ auto tx = BuildTx(query.Effects().Ptr(), ctx, /* isPrecompute */ false);
+ if (!tx) {
+ return TStatus::Error;
+ }
+
if (!CheckEffectsTx(tx.Cast(), ctx)) {
return TStatus::Error;
}
- BuildCtx->PhysicalTxs.emplace_back(tx.Cast());
- }
-
- TKqpPhyQuerySettings querySettings;
-
- switch (KqpCtx->QueryCtx->Type) {
- case EKikimrQueryType::Dml: {
- querySettings.Type = EPhysicalQueryType::Data;
- break;
- }
- case EKikimrQueryType::Scan: {
- querySettings.Type = EPhysicalQueryType::Scan;
- break;
- }
- default: {
- YQL_ENSURE(false, "Unexpected query type: " << KqpCtx->QueryCtx->Type);
- }
- }
-
- auto phyQuery = Build<TKqpPhysicalQuery>(ctx, query.Pos())
- .Transactions()
- .Add(BuildCtx->PhysicalTxs)
- .Build()
- .Results()
- .Add(queryResults)
- .Build()
- .Settings(querySettings.BuildNode(ctx, query.Pos()))
- .Done();
-
- outputExpr = phyQuery.Ptr();
- return TStatus(TStatus::Repeat, true);
- }
-
- void Rewind() final {
- TSyncTransformerBase::Rewind();
- DataTxTransformer->Rewind();
- ScanTxTransformer->Rewind();
- }
-
-private:
+ BuildCtx->PhysicalTxs.emplace_back(tx.Cast());
+ }
+
+ TKqpPhyQuerySettings querySettings;
+
+ switch (KqpCtx->QueryCtx->Type) {
+ case EKikimrQueryType::Dml: {
+ querySettings.Type = EPhysicalQueryType::Data;
+ break;
+ }
+ case EKikimrQueryType::Scan: {
+ querySettings.Type = EPhysicalQueryType::Scan;
+ break;
+ }
+ default: {
+ YQL_ENSURE(false, "Unexpected query type: " << KqpCtx->QueryCtx->Type);
+ }
+ }
+
+ auto phyQuery = Build<TKqpPhysicalQuery>(ctx, query.Pos())
+ .Transactions()
+ .Add(BuildCtx->PhysicalTxs)
+ .Build()
+ .Results()
+ .Add(queryResults)
+ .Build()
+ .Settings(querySettings.BuildNode(ctx, query.Pos()))
+ .Done();
+
+ outputExpr = phyQuery.Ptr();
+ return TStatus(TStatus::Repeat, true);
+ }
+
+ void Rewind() final {
+ TSyncTransformerBase::Rewind();
+ DataTxTransformer->Rewind();
+ ScanTxTransformer->Rewind();
+ }
+
+private:
bool CheckEffectsTx(TKqpPhysicalTx tx, TExprContext& ctx) const {
TMaybeNode<TExprBase> blackistedNode;
VisitExpr(tx.Ptr(), [&blackistedNode](const TExprNode::TPtr& exprNode) {
@@ -515,9 +515,9 @@ private:
if (auto maybeCallable = TMaybeNode<TCallable>(exprNode)) {
auto callable = maybeCallable.Cast();
- if (callable.Maybe<TCoUdf>() || callable.Maybe<TCoScriptUdf>() ||
+ if (callable.Maybe<TCoUdf>() || callable.Maybe<TCoScriptUdf>() ||
callable.Maybe<TCoUnwrap>() ||
- callable.Maybe<TCoEnsure>() || callable.Maybe<TKqpEnsure>())
+ callable.Maybe<TCoEnsure>() || callable.Maybe<TKqpEnsure>())
{
blackistedNode = callable;
return false;
@@ -536,23 +536,23 @@ private:
return true;
}
- std::pair<TNodeOnNodeOwnedMap, TNodeOnNodeOwnedMap> GatherPrecomputeDependencies(const TKqlQuery& query) {
+ std::pair<TNodeOnNodeOwnedMap, TNodeOnNodeOwnedMap> GatherPrecomputeDependencies(const TKqlQuery& query) {
TNodeOnNodeOwnedMap precomputes;
TNodeOnNodeOwnedMap dependencies;
-
+
auto filter = [](const TExprNode::TPtr& exprNode) {
return !exprNode->IsLambda();
};
-
- auto gather = [&precomputes, &dependencies](const TExprNode::TPtr& exprNode) {
+
+ auto gather = [&precomputes, &dependencies](const TExprNode::TPtr& exprNode) {
TExprBase node(exprNode);
-
+
auto maybeStage = node.Maybe<TDqStage>();
-
+
if (!maybeStage.IsValid()) {
- return true;
- }
-
+ return true;
+ }
+
auto stage = maybeStage.Cast();
for (const auto& input : stage.Inputs()) {
@@ -582,55 +582,55 @@ private:
VisitExpr(query.Ptr(), filter, gather);
- return std::make_pair(std::move(precomputes), std::move(dependencies));
+ return std::make_pair(std::move(precomputes), std::move(dependencies));
}
TMaybe<TStatus> TryBuildPrecomputeTx(const TKqlQuery& query, TExprNode::TPtr& output, TExprContext& ctx) {
- auto [precomputeStagesMap, dependantStagesMap] = GatherPrecomputeDependencies(query);
-
- if (precomputeStagesMap.empty()) {
- return {};
- }
-
- TNodeOnNodeOwnedMap phaseStagesMap;
- TVector<TKqlQueryResult> phaseResults;
+ auto [precomputeStagesMap, dependantStagesMap] = GatherPrecomputeDependencies(query);
+
+ if (precomputeStagesMap.empty()) {
+ return {};
+ }
+
+ TNodeOnNodeOwnedMap phaseStagesMap;
+ TVector<TKqlQueryResult> phaseResults;
TVector<TDqPhyPrecompute> computedInputs;
- TNodeSet computedInputsSet;
+ TNodeSet computedInputsSet;
- // Gather all Precompute stages, that are independent of any other stage and form phase of execution
- for (auto [raw, ptr] : precomputeStagesMap) {
- if (dependantStagesMap.contains(raw)) {
+ // Gather all Precompute stages, that are independent of any other stage and form phase of execution
+ for (auto [raw, ptr] : precomputeStagesMap) {
+ if (dependantStagesMap.contains(raw)) {
continue;
}
- // precompute stage _NOT_IN_ dependant stages
- YQL_ENSURE(!IsKqpEffectsStage(TDqStage(ptr)));
- phaseStagesMap.emplace(raw, ptr);
- }
-
- if (phaseStagesMap.empty()) {
- output = query.Ptr();
- ctx.AddError(TIssue(ctx.GetPosition(query.Pos()), "Phase stages is empty"));
- return TStatus::Error;
- }
-
- for (auto& [_, stagePtr] : dependantStagesMap) {
- TDqStage stage(stagePtr);
-
- for (const auto& input : stage.Inputs()) {
+ // precompute stage _NOT_IN_ dependant stages
+ YQL_ENSURE(!IsKqpEffectsStage(TDqStage(ptr)));
+ phaseStagesMap.emplace(raw, ptr);
+ }
+
+ if (phaseStagesMap.empty()) {
+ output = query.Ptr();
+ ctx.AddError(TIssue(ctx.GetPosition(query.Pos()), "Phase stages is empty"));
+ return TStatus::Error;
+ }
+
+ for (auto& [_, stagePtr] : dependantStagesMap) {
+ TDqStage stage(stagePtr);
+
+ for (const auto& input : stage.Inputs()) {
auto maybePrecompute = input.Maybe<TDqPhyPrecompute>();
-
+
if (!maybePrecompute.IsValid()) {
continue;
}
-
+
auto precompute = maybePrecompute.Cast();
auto precomputeConnection = precompute.Connection();
auto precomputeStage = precomputeConnection.Output().Stage();
-
+
if (!phaseStagesMap.contains(precomputeStage.Raw())) {
continue;
- }
+ }
if (computedInputsSet.contains(precompute.Raw())) {
continue;
@@ -638,84 +638,84 @@ private:
auto result = Build<TKqlQueryResult>(ctx, precompute.Pos())
.Value(precomputeConnection)
- .ColumnHints() // no column hints on intermediate phases
+ .ColumnHints() // no column hints on intermediate phases
.Build()
.Done();
phaseResults.emplace_back(result);
- computedInputs.emplace_back(precompute);
+ computedInputs.emplace_back(precompute);
computedInputsSet.insert(precompute.Raw());
- }
- }
- Y_VERIFY_DEBUG(phaseResults.size() == computedInputs.size());
-
- auto phaseResultsNode = Build<TKqlQueryResultList>(ctx, query.Pos())
- .Add(phaseResults)
- .Done();
-
- auto tx = BuildTx(phaseResultsNode.Ptr(), ctx, /* isPrecompute */ true);
+ }
+ }
+ Y_VERIFY_DEBUG(phaseResults.size() == computedInputs.size());
+
+ auto phaseResultsNode = Build<TKqlQueryResultList>(ctx, query.Pos())
+ .Add(phaseResults)
+ .Done();
+
+ auto tx = BuildTx(phaseResultsNode.Ptr(), ctx, /* isPrecompute */ true);
if (!tx.IsValid()) {
- return TStatus::Error;
- }
-
- BuildCtx->PhysicalTxs.emplace_back(tx.Cast());
-
- TNodeOnNodeOwnedMap replaceMap;
- for (ui64 i = 0; i < computedInputs.size(); ++i) {
- // N.B.: each precompute stage is stored in the `phaseResults` and `computedInputs` at the same index
- auto& input = computedInputs[i];
- auto newInput = Build<TKqpTxResultBinding>(ctx, input.Pos())
- .Type(ExpandType(input.Pos(), *input.Ref().GetTypeAnn(), ctx))
- .TxIndex().Build(ToString(BuildCtx->PhysicalTxs.size() - 1))
- .ResultIndex().Build(ToString(i))
- .Done();
-
- replaceMap.emplace(input.Raw(), newInput.Ptr());
- }
-
- output = ctx.ReplaceNodes(query.Ptr(), replaceMap);
-
- return TStatus(TStatus::Repeat, true);
- }
-
- TMaybeNode<TKqpPhysicalTx> BuildTx(const TExprNode::TPtr& result, TExprContext& ctx, bool isPrecompute) {
- YQL_CLOG(TRACE, ProviderKqp) << "[BuildTx] " << KqpExprToPrettyString(*result, ctx)
- << ", isPrecompute: " << isPrecompute;
-
- auto& transformer = KqpCtx->IsDataQuery() ? *DataTxTransformer : *ScanTxTransformer;
- transformer.Rewind();
- BuildTxTransformer->Init(KqpCtx->QueryCtx->Type, isPrecompute);
- auto expr = result;
-
- while (true) {
- auto status = InstantTransform(transformer, expr, ctx);
- if (status == TStatus::Error) {
- return {};
- }
- if (status == TStatus::Ok) {
- break;
- }
- }
- return TKqpPhysicalTx(expr);
- }
-
-private:
- TIntrusivePtr<TKqpOptimizeContext> KqpCtx;
- TIntrusivePtr<TKqpBuildQueryContext> BuildCtx;
- TAutoPtr<IGraphTransformer> TypeAnnTransformer;
- TAutoPtr<TKqpBuildTxTransformer> BuildTxTransformer;
- TAutoPtr<IGraphTransformer> DataTxTransformer;
- TAutoPtr<IGraphTransformer> ScanTxTransformer;
-};
-
-} // namespace
-
-TAutoPtr<IGraphTransformer> CreateKqpBuildTxsTransformer(const TIntrusivePtr<TKqpOptimizeContext>& kqpCtx,
- const TIntrusivePtr<TKqpBuildQueryContext>& buildCtx, TAutoPtr<IGraphTransformer>&& typeAnnTransformer,
- TTypeAnnotationContext& typesCtx, TKikimrConfiguration::TPtr& config)
-{
- return new TKqpBuildTxsTransformer(kqpCtx, buildCtx, std::move(typeAnnTransformer), typesCtx, config);
-}
-
+ return TStatus::Error;
+ }
+
+ BuildCtx->PhysicalTxs.emplace_back(tx.Cast());
+
+ TNodeOnNodeOwnedMap replaceMap;
+ for (ui64 i = 0; i < computedInputs.size(); ++i) {
+ // N.B.: each precompute stage is stored in the `phaseResults` and `computedInputs` at the same index
+ auto& input = computedInputs[i];
+ auto newInput = Build<TKqpTxResultBinding>(ctx, input.Pos())
+ .Type(ExpandType(input.Pos(), *input.Ref().GetTypeAnn(), ctx))
+ .TxIndex().Build(ToString(BuildCtx->PhysicalTxs.size() - 1))
+ .ResultIndex().Build(ToString(i))
+ .Done();
+
+ replaceMap.emplace(input.Raw(), newInput.Ptr());
+ }
+
+ output = ctx.ReplaceNodes(query.Ptr(), replaceMap);
+
+ return TStatus(TStatus::Repeat, true);
+ }
+
+ TMaybeNode<TKqpPhysicalTx> BuildTx(const TExprNode::TPtr& result, TExprContext& ctx, bool isPrecompute) {
+ YQL_CLOG(TRACE, ProviderKqp) << "[BuildTx] " << KqpExprToPrettyString(*result, ctx)
+ << ", isPrecompute: " << isPrecompute;
+
+ auto& transformer = KqpCtx->IsDataQuery() ? *DataTxTransformer : *ScanTxTransformer;
+ transformer.Rewind();
+ BuildTxTransformer->Init(KqpCtx->QueryCtx->Type, isPrecompute);
+ auto expr = result;
+
+ while (true) {
+ auto status = InstantTransform(transformer, expr, ctx);
+ if (status == TStatus::Error) {
+ return {};
+ }
+ if (status == TStatus::Ok) {
+ break;
+ }
+ }
+ return TKqpPhysicalTx(expr);
+ }
+
+private:
+ TIntrusivePtr<TKqpOptimizeContext> KqpCtx;
+ TIntrusivePtr<TKqpBuildQueryContext> BuildCtx;
+ TAutoPtr<IGraphTransformer> TypeAnnTransformer;
+ TAutoPtr<TKqpBuildTxTransformer> BuildTxTransformer;
+ TAutoPtr<IGraphTransformer> DataTxTransformer;
+ TAutoPtr<IGraphTransformer> ScanTxTransformer;
+};
+
+} // namespace
+
+TAutoPtr<IGraphTransformer> CreateKqpBuildTxsTransformer(const TIntrusivePtr<TKqpOptimizeContext>& kqpCtx,
+ const TIntrusivePtr<TKqpBuildQueryContext>& buildCtx, TAutoPtr<IGraphTransformer>&& typeAnnTransformer,
+ TTypeAnnotationContext& typesCtx, TKikimrConfiguration::TPtr& config)
+{
+ return new TKqpBuildTxsTransformer(kqpCtx, buildCtx, std::move(typeAnnTransformer), typesCtx, config);
+}
+
} // namespace NKikimr::NKqp::NOpt
diff --git a/ydb/core/kqp/opt/kqp_opt_effects.cpp b/ydb/core/kqp/opt/kqp_opt_effects.cpp
index 55edc36aaa..f5f2401ce9 100644
--- a/ydb/core/kqp/opt/kqp_opt_effects.cpp
+++ b/ydb/core/kqp/opt/kqp_opt_effects.cpp
@@ -28,12 +28,12 @@ bool IsMapWrite(const TKikimrTableDescription& table, TExprBase input) {
// TODO: Check for non-deterministic & unsafe functions (like UDF).
// TODO: Once we have partitioning constraints implemented in query optimizer,
// use them to detect map writes.
- if (!input.Maybe<TCoFlatMap>().Input().Maybe<TKqlReadTableBase>()) {
+ if (!input.Maybe<TCoFlatMap>().Input().Maybe<TKqlReadTableBase>()) {
return false;
}
auto flatmap = input.Cast<TCoFlatMap>();
- auto read = flatmap.Input().Cast<TKqlReadTableBase>();
+ auto read = flatmap.Input().Cast<TKqlReadTableBase>();
if (table.Metadata->PathId.ToString() != read.Table().PathId().Value()) {
return false;
@@ -56,48 +56,48 @@ bool IsMapWrite(const TKikimrTableDescription& table, TExprBase input) {
}
TDqPhyPrecompute BuildPrecomputeStage(TExprBase expr, TExprContext& ctx) {
- Y_VERIFY_DEBUG(IsDqPureExpr(expr));
-
- auto pureStage = Build<TDqStage>(ctx, expr.Pos())
- .Inputs()
- .Build()
- .Program()
- .Args({})
- .Body<TCoToStream>()
- .Input<TCoJust>()
- .Input(expr)
- .Build()
- .Build()
- .Build()
+ Y_VERIFY_DEBUG(IsDqPureExpr(expr));
+
+ auto pureStage = Build<TDqStage>(ctx, expr.Pos())
+ .Inputs()
+ .Build()
+ .Program()
+ .Args({})
+ .Body<TCoToStream>()
+ .Input<TCoJust>()
+ .Input(expr)
+ .Build()
+ .Build()
+ .Build()
.Settings().Build()
- .Done();
-
- auto dqValue = Build<TDqCnValue>(ctx, expr.Pos())
- .Output()
- .Stage(pureStage)
- .Index().Build("0")
- .Build()
- .Done();
-
+ .Done();
+
+ auto dqValue = Build<TDqCnValue>(ctx, expr.Pos())
+ .Output()
+ .Stage(pureStage)
+ .Index().Build("0")
+ .Build()
+ .Done();
+
return Build<TDqPhyPrecompute>(ctx, expr.Pos())
- .Connection(dqValue)
- .Done();
-}
-
+ .Connection(dqValue)
+ .Done();
+}
+
bool BuildUpsertRowsEffect(const TKqlUpsertRows& node, TExprContext& ctx, const TKqpOptimizeContext& kqpCtx,
const TCoArgument& inputArg, TMaybeNode<TExprBase>& stageInput, TMaybeNode<TExprBase>& effect)
{
if (IsDqPureExpr(node.Input())) {
- stageInput = BuildPrecomputeStage(node.Input(), ctx);
-
+ stageInput = BuildPrecomputeStage(node.Input(), ctx);
+
effect = Build<TKqpUpsertRows>(ctx, node.Pos())
- .Table(node.Table())
- .Input<TCoIterator>()
- .List(inputArg)
- .Build()
- .Columns(node.Columns())
+ .Table(node.Table())
+ .Input<TCoIterator>()
+ .List(inputArg)
+ .Build()
+ .Columns(node.Columns())
.Settings().Build()
- .Done();
+ .Done();
return true;
}
@@ -111,37 +111,37 @@ bool BuildUpsertRowsEffect(const TKqlUpsertRows& node, TExprContext& ctx, const
auto input = dqUnion.Output().Stage().Program().Body();
if (InplaceUpdateEnabled(*kqpCtx.Config) && IsMapWrite(table, input)) {
- stageInput = Build<TKqpCnMapShard>(ctx, node.Pos())
+ stageInput = Build<TKqpCnMapShard>(ctx, node.Pos())
.Output()
.Stage(dqUnion.Output().Stage())
.Index(dqUnion.Output().Index())
.Build()
.Done();
-
+
TKqpUpsertRowsSettings settings;
settings.SetInplace();
- effect = Build<TKqpUpsertRows>(ctx, node.Pos())
- .Table(node.Table())
- .Input<TCoFromFlow>()
- .Input(inputArg)
- .Build()
- .Columns(node.Columns())
+ effect = Build<TKqpUpsertRows>(ctx, node.Pos())
+ .Table(node.Table())
+ .Input<TCoFromFlow>()
+ .Input(inputArg)
+ .Build()
+ .Columns(node.Columns())
.Settings(settings.BuildNode(ctx, node.Pos()))
- .Done();
+ .Done();
} else {
stageInput = Build<TDqPhyPrecompute>(ctx, node.Pos())
.Connection(dqUnion)
.Done();
-
- effect = Build<TKqpUpsertRows>(ctx, node.Pos())
- .Table(node.Table())
- .Input<TCoIterator>()
- .List(inputArg)
- .Build()
- .Columns(node.Columns())
+
+ effect = Build<TKqpUpsertRows>(ctx, node.Pos())
+ .Table(node.Table())
+ .Input<TCoIterator>()
+ .List(inputArg)
+ .Build()
+ .Columns(node.Columns())
.Settings().Build()
- .Done();
+ .Done();
}
return true;
@@ -151,13 +151,13 @@ bool BuildDeleteRowsEffect(const TKqlDeleteRows& node, TExprContext& ctx, const
const TCoArgument& inputArg, TMaybeNode<TExprBase>& stageInput, TMaybeNode<TExprBase>& effect)
{
if (IsDqPureExpr(node.Input())) {
- stageInput = BuildPrecomputeStage(node.Input(), ctx);
-
+ stageInput = BuildPrecomputeStage(node.Input(), ctx);
+
effect = Build<TKqpDeleteRows>(ctx, node.Pos())
.Table(node.Table())
- .Input<TCoIterator>()
- .List(inputArg)
- .Build()
+ .Input<TCoIterator>()
+ .List(inputArg)
+ .Build()
.Done();
return true;
}
@@ -178,24 +178,24 @@ bool BuildDeleteRowsEffect(const TKqlDeleteRows& node, TExprContext& ctx, const
.Index(dqUnion.Output().Index())
.Build()
.Done();
-
- effect = Build<TKqpDeleteRows>(ctx, node.Pos())
- .Table(node.Table())
- .Input<TCoFromFlow>()
- .Input(inputArg)
- .Build()
- .Done();
+
+ effect = Build<TKqpDeleteRows>(ctx, node.Pos())
+ .Table(node.Table())
+ .Input<TCoFromFlow>()
+ .Input(inputArg)
+ .Build()
+ .Done();
} else {
stageInput = Build<TDqPhyPrecompute>(ctx, node.Pos())
.Connection(dqUnion)
.Done();
-
- effect = Build<TKqpDeleteRows>(ctx, node.Pos())
- .Table(node.Table())
- .Input<TCoIterator>()
- .List(inputArg)
- .Build()
- .Done();
+
+ effect = Build<TKqpDeleteRows>(ctx, node.Pos())
+ .Table(node.Table())
+ .Input<TCoIterator>()
+ .List(inputArg)
+ .Build()
+ .Done();
}
return true;
@@ -254,7 +254,7 @@ bool BuildEffects(TPositionHandle pos, const TVector<TKqlTableEffect>& effects,
for (ui32 i = 0; i < newEffects.size(); ++i) {
auto effect = Build<TDqOutput>(ctx, pos)
.Stage(stage)
- .Index().Build(ToString(0))
+ .Index().Build(ToString(0))
.Done();
builtEffects.push_back(effect);
@@ -263,14 +263,14 @@ bool BuildEffects(TPositionHandle pos, const TVector<TKqlTableEffect>& effects,
return true;
}
-template <bool GroupEffectsByTable>
+template <bool GroupEffectsByTable>
TMaybeNode<TKqlQuery> BuildEffects(const TKqlQuery& query, TExprContext& ctx,
const TKqpOptimizeContext& kqpCtx)
{
- TVector<TExprBase> builtEffects;
+ TVector<TExprBase> builtEffects;
- if constexpr (GroupEffectsByTable) {
- TMap<TStringBuf, TVector<TKqlTableEffect>> tableEffectsMap;
+ if constexpr (GroupEffectsByTable) {
+ TMap<TStringBuf, TVector<TKqlTableEffect>> tableEffectsMap;
for (const auto& maybeEffect: query.Effects()) {
if (const auto maybeList = maybeEffect.Maybe<TExprList>()) {
for (const auto effect : maybeList.Cast()) {
@@ -286,15 +286,15 @@ TMaybeNode<TKqlQuery> BuildEffects(const TKqlQuery& query, TExprContext& ctx,
tableEffectsMap[tableEffect.Table().Path()].push_back(tableEffect);
}
}
-
- for (const auto& pair: tableEffectsMap) {
- if (!BuildEffects(query.Pos(), pair.second, ctx, kqpCtx, builtEffects)) {
- return {};
- }
- }
- } else {
+
+ for (const auto& pair: tableEffectsMap) {
+ if (!BuildEffects(query.Pos(), pair.second, ctx, kqpCtx, builtEffects)) {
+ return {};
+ }
+ }
+ } else {
builtEffects.reserve(query.Effects().Size() * 2);
-
+
for (const auto& maybeEffect : query.Effects()) {
if (const auto maybeList = maybeEffect.Maybe<TExprList>()) {
for (const auto effect : maybeList.Cast()) {
@@ -312,8 +312,8 @@ TMaybeNode<TKqlQuery> BuildEffects(const TKqlQuery& query, TExprContext& ctx,
if (!BuildEffects(query.Pos(), {tableEffect}, ctx, kqpCtx, builtEffects)) {
return {};
}
- }
- }
+ }
+ }
}
return Build<TKqlQuery>(ctx, query.Pos())
@@ -358,7 +358,7 @@ TAutoPtr<IGraphTransformer> CreateKqpQueryEffectsTransformer(const TIntrusivePtr
TParentsMap parentsMap;
GatherParents(*input, parentsMap);
- auto result = BuildEffects<false>(query, ctx, *kqpCtx);
+ auto result = BuildEffects<false>(query, ctx, *kqpCtx);
if (!result) {
return TStatus::Error;
}
diff --git a/ydb/core/kqp/opt/kqp_opt_impl.h b/ydb/core/kqp/opt/kqp_opt_impl.h
index 0ab55409b7..927aa910e8 100644
--- a/ydb/core/kqp/opt/kqp_opt_impl.h
+++ b/ydb/core/kqp/opt/kqp_opt_impl.h
@@ -14,19 +14,19 @@ static inline void DumpAppliedRule(const TString& name, const NYql::TExprNode::T
{
//#define KQP_ENABLE_DUMP_APPLIED_RULE
#ifdef KQP_ENABLE_DUMP_APPLIED_RULE
- if (input != output) {
- auto builder = TStringBuilder() << "Rule applied: " << name << Endl;
- builder << "Expression before rule application: " << Endl;
- builder << KqpExprToPrettyString(*input, ctx) << Endl;
- builder << "Expression after rule application: " << Endl;
- builder << KqpExprToPrettyString(*output, ctx);
- YQL_CLOG(INFO, ProviderKqp) << builder;
- }
+ if (input != output) {
+ auto builder = TStringBuilder() << "Rule applied: " << name << Endl;
+ builder << "Expression before rule application: " << Endl;
+ builder << KqpExprToPrettyString(*input, ctx) << Endl;
+ builder << "Expression after rule application: " << Endl;
+ builder << KqpExprToPrettyString(*output, ctx);
+ YQL_CLOG(INFO, ProviderKqp) << builder;
+ }
#else
Y_UNUSED(ctx);
- if (input != output) {
- YQL_CLOG(INFO, ProviderKqp) << name;
- }
+ if (input != output) {
+ YQL_CLOG(INFO, ProviderKqp) << name;
+ }
#endif
}
@@ -48,7 +48,7 @@ NYql::NNodes::TKqpTable BuildTableMeta(const NYql::TKikimrTableMetadata& tableMe
NYql::NNodes::TExprBase KqpBuildJoin(const NYql::NNodes::TExprBase& node, NYql::TExprContext& ctx,
const TKqpOptimizeContext& kqpCtx, NYql::IOptimizationContext& optCtx, const NYql::TParentsMap& parentsMap,
bool allowStageMultiUsage);
-
+
TIntrusivePtr<NYql::TKikimrTableMetadata> GetIndexMetadata(const NYql::NNodes::TKqlReadTableIndex& index,
const NYql::TKikimrTablesData& tables, TStringBuf cluster);
diff --git a/ydb/core/kqp/opt/kqp_opt_join.cpp b/ydb/core/kqp/opt/kqp_opt_join.cpp
index ff58235a3c..650dc64104 100644
--- a/ydb/core/kqp/opt/kqp_opt_join.cpp
+++ b/ydb/core/kqp/opt/kqp_opt_join.cpp
@@ -1,118 +1,118 @@
-#include "kqp_opt_impl.h"
-
+#include "kqp_opt_impl.h"
+
#include <ydb/core/kqp/common/kqp_yql.h>
#include <ydb/library/yql/dq/opt/dq_opt_phy.h>
-
+
namespace NKikimr::NKqp::NOpt {
-
-using namespace NYql;
+
+using namespace NYql;
using namespace NYql::NDq;
-using namespace NYql::NNodes;
-
-namespace {
-
-// left input should be DqCnUnionAll (with single usage)
-// right input should be either DqCnUnionAll (with single usage) or DqPure expression
-bool ValidateJoinInputs(const TExprBase& left, const TExprBase& right, const TParentsMap& parentsMap,
- bool allowStageMultiUsage)
-{
- if (!left.Maybe<TDqCnUnionAll>()) {
- return false;
- }
- if (!IsSingleConsumerConnection(left.Cast<TDqCnUnionAll>(), parentsMap, allowStageMultiUsage)) {
- return false;
- }
-
- if (right.Maybe<TDqCnUnionAll>()) {
- if (!IsSingleConsumerConnection(right.Cast<TDqCnUnionAll>(), parentsMap, allowStageMultiUsage)) {
- return false;
- }
- } else if (IsDqPureExpr(right, /* isPrecomputePure */ true)) {
- // pass
- } else {
- return false;
- }
-
- return true;
-}
-
-TMaybeNode<TDqJoin> FlipJoin(const TDqJoin& join, TExprContext& ctx) {
- auto joinType = join.JoinType().Value();
-
+using namespace NYql::NNodes;
+
+namespace {
+
+// left input should be DqCnUnionAll (with single usage)
+// right input should be either DqCnUnionAll (with single usage) or DqPure expression
+bool ValidateJoinInputs(const TExprBase& left, const TExprBase& right, const TParentsMap& parentsMap,
+ bool allowStageMultiUsage)
+{
+ if (!left.Maybe<TDqCnUnionAll>()) {
+ return false;
+ }
+ if (!IsSingleConsumerConnection(left.Cast<TDqCnUnionAll>(), parentsMap, allowStageMultiUsage)) {
+ return false;
+ }
+
+ if (right.Maybe<TDqCnUnionAll>()) {
+ if (!IsSingleConsumerConnection(right.Cast<TDqCnUnionAll>(), parentsMap, allowStageMultiUsage)) {
+ return false;
+ }
+ } else if (IsDqPureExpr(right, /* isPrecomputePure */ true)) {
+ // pass
+ } else {
+ return false;
+ }
+
+ return true;
+}
+
+TMaybeNode<TDqJoin> FlipJoin(const TDqJoin& join, TExprContext& ctx) {
+ auto joinType = join.JoinType().Value();
+
if (joinType == "Inner"sv || joinType == "Full"sv || joinType == "Exclusion"sv || joinType == "Cross"sv) {
// pass
- } else if (joinType == "Right"sv) {
- joinType = "Left"sv;
- } else if (joinType == "Left"sv) {
- joinType = "Right"sv;
- } else if (joinType == "RightSemi"sv) {
- joinType = "LeftSemi"sv;
- } else if (joinType == "LeftSemi"sv) {
- joinType = "RightSemi"sv;
- } else if (joinType == "RightOnly"sv) {
- joinType = "LeftOnly"sv;
- } else if (joinType == "LeftOnly"sv) {
- joinType = "RightOnly"sv;
- } else {
- return {};
- }
-
- auto joinKeysBuilder = Build<TDqJoinKeyTupleList>(ctx, join.Pos());
- for (const auto& keys : join.JoinKeys()) {
- joinKeysBuilder.Add<TDqJoinKeyTuple>()
- .LeftLabel(keys.RightLabel())
- .LeftColumn(keys.RightColumn())
- .RightLabel(keys.LeftLabel())
- .RightColumn(keys.LeftColumn())
- .Build();
- }
-
- return Build<TDqJoin>(ctx, join.Pos())
- .LeftInput(join.RightInput())
- .LeftLabel(join.RightLabel())
- .RightInput(join.LeftInput())
- .RightLabel(join.LeftLabel())
- .JoinType().Build(joinType)
- .JoinKeys(joinKeysBuilder.Done())
- .Done();
-}
-
-} // anonymous namespace
-
+ } else if (joinType == "Right"sv) {
+ joinType = "Left"sv;
+ } else if (joinType == "Left"sv) {
+ joinType = "Right"sv;
+ } else if (joinType == "RightSemi"sv) {
+ joinType = "LeftSemi"sv;
+ } else if (joinType == "LeftSemi"sv) {
+ joinType = "RightSemi"sv;
+ } else if (joinType == "RightOnly"sv) {
+ joinType = "LeftOnly"sv;
+ } else if (joinType == "LeftOnly"sv) {
+ joinType = "RightOnly"sv;
+ } else {
+ return {};
+ }
+
+ auto joinKeysBuilder = Build<TDqJoinKeyTupleList>(ctx, join.Pos());
+ for (const auto& keys : join.JoinKeys()) {
+ joinKeysBuilder.Add<TDqJoinKeyTuple>()
+ .LeftLabel(keys.RightLabel())
+ .LeftColumn(keys.RightColumn())
+ .RightLabel(keys.LeftLabel())
+ .RightColumn(keys.LeftColumn())
+ .Build();
+ }
+
+ return Build<TDqJoin>(ctx, join.Pos())
+ .LeftInput(join.RightInput())
+ .LeftLabel(join.RightLabel())
+ .RightInput(join.LeftInput())
+ .RightLabel(join.LeftLabel())
+ .JoinType().Build(joinType)
+ .JoinKeys(joinKeysBuilder.Done())
+ .Done();
+}
+
+} // anonymous namespace
+
TExprBase KqpBuildJoin(const TExprBase& node, TExprContext& ctx, const TKqpOptimizeContext& kqpCtx,
IOptimizationContext& optCtx, const TParentsMap& parentsMap, bool allowStageMultiUsage)
-{
+{
if (!node.Maybe<TDqJoin>()) {
- return node;
- }
-
+ return node;
+ }
+
auto join = node.Cast<TDqJoin>();
-
- if (ValidateJoinInputs(join.LeftInput(), join.RightInput(), parentsMap, allowStageMultiUsage)) {
- // pass
- } else if (ValidateJoinInputs(join.RightInput(), join.LeftInput(), parentsMap, allowStageMultiUsage)) {
- auto maybeFlipJoin = FlipJoin(join, ctx);
- if (!maybeFlipJoin) {
- return node;
- }
- join = maybeFlipJoin.Cast();
- } else {
- return node;
- }
-
+
+ if (ValidateJoinInputs(join.LeftInput(), join.RightInput(), parentsMap, allowStageMultiUsage)) {
+ // pass
+ } else if (ValidateJoinInputs(join.RightInput(), join.LeftInput(), parentsMap, allowStageMultiUsage)) {
+ auto maybeFlipJoin = FlipJoin(join, ctx);
+ if (!maybeFlipJoin) {
+ return node;
+ }
+ join = maybeFlipJoin.Cast();
+ } else {
+ return node;
+ }
+
auto joinType = join.JoinType().Value();
- if (joinType == "Full"sv || joinType == "Exclusion"sv) {
+ if (joinType == "Full"sv || joinType == "Exclusion"sv) {
return DqBuildJoinDict(join, ctx);
}
// NOTE: We don't want to broadcast table data via readsets for data queries, so we need to create a
// separate stage to receive data from both sides of join.
// TODO: We can push MapJoin to existing stage for data query, if it doesn't have table reads. This
- // requires some additional knowledge, probably with use of constraints.
+ // requires some additional knowledge, probably with use of constraints.
bool pushLeftStage = !kqpCtx.IsDataQuery();
- return DqBuildPhyJoin(join, pushLeftStage, ctx, optCtx);
-}
-
+ return DqBuildPhyJoin(join, pushLeftStage, ctx, optCtx);
+}
+
} // namespace NKikimr::NKqp::NOpt
diff --git a/ydb/core/kqp/opt/kqp_opt_kql.cpp b/ydb/core/kqp/opt/kqp_opt_kql.cpp
index 3bcf5f1b20..1eeb95daee 100644
--- a/ydb/core/kqp/opt/kqp_opt_kql.cpp
+++ b/ydb/core/kqp/opt/kqp_opt_kql.cpp
@@ -171,21 +171,21 @@ TExprBase BuildReplaceTableWithIndex(const TKiWriteTable& write, const TCoAtomLi
return effect;
}
-TExprBase BuildInsertTable(const TKiWriteTable& write, bool abort, const TCoAtomList& inputColumns,
- const TKikimrTableDescription& tableData, TExprContext& ctx)
-{
- auto effect = Build<TKqlInsertRows>(ctx, write.Pos())
- .Table(BuildTableMeta(tableData, write.Pos(), ctx))
- .Input(write.Input())
- .Columns(inputColumns)
- .OnConflict()
- .Value(abort ? "abort"sv : "revert"sv)
- .Build()
- .Done();
-
- return effect;
-}
-
+TExprBase BuildInsertTable(const TKiWriteTable& write, bool abort, const TCoAtomList& inputColumns,
+ const TKikimrTableDescription& tableData, TExprContext& ctx)
+{
+ auto effect = Build<TKqlInsertRows>(ctx, write.Pos())
+ .Table(BuildTableMeta(tableData, write.Pos(), ctx))
+ .Input(write.Input())
+ .Columns(inputColumns)
+ .OnConflict()
+ .Value(abort ? "abort"sv : "revert"sv)
+ .Build()
+ .Done();
+
+ return effect;
+}
+
TExprBase BuildInsertTableWithIndex(const TKiWriteTable& write, bool abort, const TCoAtomList& inputColumns,
const TKikimrTableDescription& tableData, TExprContext& ctx)
{
@@ -201,16 +201,16 @@ TExprBase BuildInsertTableWithIndex(const TKiWriteTable& write, bool abort, cons
return effect;
}
-TExprBase BuildUpdateOnTable(const TKiWriteTable& write, const TCoAtomList& inputColumns,
- const TKikimrTableDescription& tableData, TExprContext& ctx)
-{
- return Build<TKqlUpdateRows>(ctx, write.Pos())
- .Table(BuildTableMeta(tableData, write.Pos(), ctx))
- .Input(write.Input())
- .Columns(inputColumns)
- .Done();
-}
-
+TExprBase BuildUpdateOnTable(const TKiWriteTable& write, const TCoAtomList& inputColumns,
+ const TKikimrTableDescription& tableData, TExprContext& ctx)
+{
+ return Build<TKqlUpdateRows>(ctx, write.Pos())
+ .Table(BuildTableMeta(tableData, write.Pos(), ctx))
+ .Input(write.Input())
+ .Columns(inputColumns)
+ .Done();
+}
+
TExprBase BuildUpdateOnTableWithIndex(const TKiWriteTable& write, const TCoAtomList& inputColumns,
const TKikimrTableDescription& tableData, TExprContext& ctx)
{
@@ -241,7 +241,7 @@ TExprBase BuildDeleteTableWithIndex(const TKiWriteTable& write, const TKikimrTab
TExprBase BuildRowsToDelete(const TKikimrTableDescription& tableData, bool withSystemColumns, const TCoLambda& filter,
const TPositionHandle pos, TExprContext& ctx)
-{
+{
const auto tableMeta = BuildTableMeta(tableData, pos, ctx);
const auto tableColumns = BuildColumnsList(tableData, pos, ctx, withSystemColumns);
@@ -254,8 +254,8 @@ TExprBase BuildRowsToDelete(const TKikimrTableDescription& tableData, bool withS
.Build()
.Build()
.Columns(tableColumns)
- .Settings()
- .Build()
+ .Settings()
+ .Build()
.Done();
return Build<TCoFilter>(ctx, pos)
@@ -277,7 +277,7 @@ TExprBase BuildDeleteTable(const TKiDeleteTable& del, const TKikimrTableDescript
}
TVector<TExprBase> BuildDeleteTableWithIndex(const TKiDeleteTable& del, const TKikimrTableDescription& tableData,
- bool withSystemColumns, TExprContext& ctx)
+ bool withSystemColumns, TExprContext& ctx)
{
auto rowsToDelete = BuildRowsToDelete(tableData, withSystemColumns, del.Filter(), del.Pos(), ctx);
@@ -335,8 +335,8 @@ TExprBase BuildRowsToUpdate(const TKikimrTableDescription& tableData, bool withS
.Build()
.Build()
.Columns(BuildColumnsList(tableData, pos, ctx, withSystemColumns))
- .Settings()
- .Build()
+ .Settings()
+ .Build()
.Done();
return Build<TCoFilter>(ctx, pos)
@@ -574,13 +574,13 @@ TExprBase WriteTableSimple(const TKiWriteTable& write, const TCoAtomList& inputC
case TYdbOperation::InsertRevert:
return BuildInsertTable(write, op == TYdbOperation::InsertAbort, inputColumns, tableData, ctx);
case TYdbOperation::UpdateOn:
- return BuildUpdateOnTable(write, inputColumns, tableData, ctx);
+ return BuildUpdateOnTable(write, inputColumns, tableData, ctx);
case TYdbOperation::Delete:
return BuildDeleteTable(write, tableData, ctx);
case TYdbOperation::DeleteOn:
return BuildDeleteTable(write, tableData, ctx);
default:
- YQL_ENSURE(false, "Unsupported table operation: " << op << ", table: " << tableData.Metadata->Name);
+ YQL_ENSURE(false, "Unsupported table operation: " << op << ", table: " << tableData.Metadata->Name);
}
}
@@ -623,8 +623,8 @@ TExprBase HandleWriteTable(const TKiWriteTable& write, TExprContext& ctx, const
TVector<TExprBase> HandleUpdateTable(const TKiUpdateTable& update, TExprContext& ctx,
const TKikimrTablesData& tablesData, bool withSystemColumns)
-{
- const auto& tableData = GetTableData(tablesData, update.DataSink().Cluster(), update.Table().Value());
+{
+ const auto& tableData = GetTableData(tablesData, update.DataSink().Cluster(), update.Table().Value());
if (HasIndexesToWrite(tableData)) {
return BuildUpdateTableWithIndex(update, tableData, withSystemColumns, ctx);
@@ -634,8 +634,8 @@ TVector<TExprBase> HandleUpdateTable(const TKiUpdateTable& update, TExprContext&
}
TVector<TExprBase> HandleDeleteTable(const TKiDeleteTable& del, TExprContext& ctx, const TKikimrTablesData& tablesData,
- bool withSystemColumns)
-{
+ bool withSystemColumns)
+{
auto& tableData = GetTableData(tablesData, del.DataSink().Cluster(), del.Table().Value());
if (HasIndexesToWrite(tableData)) {
return BuildDeleteTableWithIndex(del, tableData, withSystemColumns, ctx);
@@ -672,7 +672,7 @@ TMaybe<TKqlQuery> BuildKqlQuery(TKiDataQuery query, const TKikimrTablesData& tab
auto result = HandleWriteTable(maybeWrite.Cast(), ctx, tablesData);
kqlEffects.push_back(result);
}
-
+
if (auto maybeUpdate = effect.Maybe<TKiUpdateTable>()) {
auto results = HandleUpdateTable(maybeUpdate.Cast(), ctx, tablesData, withSystemColumns);
kqlEffects.insert(kqlEffects.end(), results.begin(), results.end());
@@ -688,10 +688,10 @@ TMaybe<TKqlQuery> BuildKqlQuery(TKiDataQuery query, const TKikimrTablesData& tab
kqlResults.reserve(query.Results().Size());
for (const auto& kiResult : query.Results()) {
kqlResults.emplace_back(
- Build<TKqlQueryResult>(ctx, kiResult.Pos())
- .Value(kiResult.Value())
- .ColumnHints(kiResult.Columns())
- .Done());
+ Build<TKqlQueryResult>(ctx, kiResult.Pos())
+ .Value(kiResult.Value())
+ .ColumnHints(kiResult.Columns())
+ .Done());
}
TKqlQuery kqlQuery = Build<TKqlQuery>(ctx, query.Pos())
diff --git a/ydb/core/kqp/opt/kqp_opt_phy_check.cpp b/ydb/core/kqp/opt/kqp_opt_phy_check.cpp
index 355246113b..c34a094ff3 100644
--- a/ydb/core/kqp/opt/kqp_opt_phy_check.cpp
+++ b/ydb/core/kqp/opt/kqp_opt_phy_check.cpp
@@ -1,126 +1,126 @@
-#include "kqp_opt_impl.h"
-
+#include "kqp_opt_impl.h"
+
#include <ydb/core/kqp/common/kqp_yql.h>
#include <ydb/library/yql/core/yql_expr_optimize.h>
-
+
namespace NKikimr::NKqp::NOpt {
-
-using namespace NYql;
-using namespace NYql::NDq;
-using namespace NYql::NNodes;
-
-using TStatus = IGraphTransformer::TStatus;
-
-TAutoPtr<IGraphTransformer> CreateKqpCheckPhysicalQueryTransformer() {
- return CreateFunctorTransformer(
- [](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) -> TStatus {
- output = input;
-
- YQL_ENSURE(TMaybeNode<TKqlQuery>(input));
- auto query = TKqlQuery(input);
- YQL_ENSURE(query.Ref().GetTypeAnn());
-
- for (const auto& result : query.Effects()) {
- if (!result.Maybe<TDqOutput>()) {
- ctx.AddError(TIssue(ctx.GetPosition(result.Pos()), "Failed to build query effects."));
- return TStatus::Error;
- }
- }
-
- TParentsMap parentsMap;
- GatherParents(*input, parentsMap);
-
- bool hasMultipleConsumers = false;
- bool hasBrokenStage = false;
-
- VisitExpr(input, [&](const TExprNode::TPtr& expr) {
- TExprBase node{expr};
-
- if (auto maybeConnection = node.Maybe<TDqConnection>()) {
- auto connection = maybeConnection.Cast();
-
- if (!IsSingleConsumerConnection(connection, parentsMap)) {
- hasMultipleConsumers = true;
- YQL_CLOG(ERROR, ProviderKqp) << "Connection #" << connection.Ref().UniqueId()
- << " (" << connection.CallableName() << ") has multiple consumers.";
- return false;
- }
- }
-
- if (auto maybeOutput = node.Maybe<TDqOutput>()) {
- auto output = maybeOutput.Cast();
-
- // Suppose that particular stage output is used only through single connection
- // i.e. it's not allowed to consume particular stage output via several connections
- if (!IsSingleConsumer(output, parentsMap)) {
- hasMultipleConsumers = true;
- TStringBuilder sb;
- sb << "Stage #" << output.Stage().Ref().UniqueId()
- << " output " << output.Index().Value() << " has multiple consumers: " << Endl
- << " output: " << KqpExprToPrettyString(output, ctx) << Endl;
- for (const auto& consumer : GetConsumers(output, parentsMap)) {
- sb << "consumer: " << KqpExprToPrettyString(*consumer, ctx) << Endl;
- }
- YQL_CLOG(ERROR, ProviderKqp) << sb;
- return false;
- }
- }
-
- if (auto maybeStage = node.Maybe<TDqStage>()) {
- auto stage = maybeStage.Cast();
- auto stageType = stage.Ref().GetTypeAnn();
- YQL_ENSURE(stageType);
- auto stageResultType = stageType->Cast<TTupleExprType>();
- const auto& stageConsumers = GetConsumers(stage, parentsMap);
-
- TDynBitMap usedOutputs;
- for (auto consumer : stageConsumers) {
- if (auto maybeOutput = TExprBase(consumer).Maybe<TDqOutput>()) {
- auto output = maybeOutput.Cast();
- auto outputIndex = FromString<ui32>(output.Index().Value());
- if (usedOutputs.Test(outputIndex)) {
- hasMultipleConsumers = true;
- YQL_CLOG(ERROR, ProviderKqp) << "Stage #" << node.Ref().UniqueId()
- << ", output " << outputIndex << " has multiple consumers";
- return false;
- }
- usedOutputs.Set(outputIndex);
- } else {
- YQL_ENSURE(false, "Stage #" << PrintKqpStageOnly(stage, ctx) << " has unexpected consumer: "
- << consumer->Content());
- }
- }
-
- for (size_t i = 0; i < stageResultType->GetSize(); ++i) {
- if (!usedOutputs.Test(i)) {
- hasBrokenStage = true;
- YQL_CLOG(ERROR, ProviderKqp) << "Stage #" << PrintKqpStageOnly(stage, ctx)
- << ", output " << i << " (" << FormatType(stageResultType->GetItems()[i]) << ")"
- << " not used";
- return false;
- }
- }
- }
-
- YQL_ENSURE(!node.Maybe<TDqPhyStage>());
-
- return true;
- });
-
- if (hasMultipleConsumers) {
- ctx.AddError(TIssue(ctx.GetPosition(input->Pos()),
- "Failed to build physical query: some connection(s) have several consumers"));
- return TStatus::Error;
- }
-
- if (hasBrokenStage) {
- ctx.AddError(TIssue(ctx.GetPosition(input->Pos()),
- "Failed to build physical query: some stages are broken"));
- return TStatus::Error;
- }
-
- return TStatus::Ok;
- });
-}
-
+
+using namespace NYql;
+using namespace NYql::NDq;
+using namespace NYql::NNodes;
+
+using TStatus = IGraphTransformer::TStatus;
+
+TAutoPtr<IGraphTransformer> CreateKqpCheckPhysicalQueryTransformer() {
+ return CreateFunctorTransformer(
+ [](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) -> TStatus {
+ output = input;
+
+ YQL_ENSURE(TMaybeNode<TKqlQuery>(input));
+ auto query = TKqlQuery(input);
+ YQL_ENSURE(query.Ref().GetTypeAnn());
+
+ for (const auto& result : query.Effects()) {
+ if (!result.Maybe<TDqOutput>()) {
+ ctx.AddError(TIssue(ctx.GetPosition(result.Pos()), "Failed to build query effects."));
+ return TStatus::Error;
+ }
+ }
+
+ TParentsMap parentsMap;
+ GatherParents(*input, parentsMap);
+
+ bool hasMultipleConsumers = false;
+ bool hasBrokenStage = false;
+
+ VisitExpr(input, [&](const TExprNode::TPtr& expr) {
+ TExprBase node{expr};
+
+ if (auto maybeConnection = node.Maybe<TDqConnection>()) {
+ auto connection = maybeConnection.Cast();
+
+ if (!IsSingleConsumerConnection(connection, parentsMap)) {
+ hasMultipleConsumers = true;
+ YQL_CLOG(ERROR, ProviderKqp) << "Connection #" << connection.Ref().UniqueId()
+ << " (" << connection.CallableName() << ") has multiple consumers.";
+ return false;
+ }
+ }
+
+ if (auto maybeOutput = node.Maybe<TDqOutput>()) {
+ auto output = maybeOutput.Cast();
+
+ // Suppose that particular stage output is used only through single connection
+ // i.e. it's not allowed to consume particular stage output via several connections
+ if (!IsSingleConsumer(output, parentsMap)) {
+ hasMultipleConsumers = true;
+ TStringBuilder sb;
+ sb << "Stage #" << output.Stage().Ref().UniqueId()
+ << " output " << output.Index().Value() << " has multiple consumers: " << Endl
+ << " output: " << KqpExprToPrettyString(output, ctx) << Endl;
+ for (const auto& consumer : GetConsumers(output, parentsMap)) {
+ sb << "consumer: " << KqpExprToPrettyString(*consumer, ctx) << Endl;
+ }
+ YQL_CLOG(ERROR, ProviderKqp) << sb;
+ return false;
+ }
+ }
+
+ if (auto maybeStage = node.Maybe<TDqStage>()) {
+ auto stage = maybeStage.Cast();
+ auto stageType = stage.Ref().GetTypeAnn();
+ YQL_ENSURE(stageType);
+ auto stageResultType = stageType->Cast<TTupleExprType>();
+ const auto& stageConsumers = GetConsumers(stage, parentsMap);
+
+ TDynBitMap usedOutputs;
+ for (auto consumer : stageConsumers) {
+ if (auto maybeOutput = TExprBase(consumer).Maybe<TDqOutput>()) {
+ auto output = maybeOutput.Cast();
+ auto outputIndex = FromString<ui32>(output.Index().Value());
+ if (usedOutputs.Test(outputIndex)) {
+ hasMultipleConsumers = true;
+ YQL_CLOG(ERROR, ProviderKqp) << "Stage #" << node.Ref().UniqueId()
+ << ", output " << outputIndex << " has multiple consumers";
+ return false;
+ }
+ usedOutputs.Set(outputIndex);
+ } else {
+ YQL_ENSURE(false, "Stage #" << PrintKqpStageOnly(stage, ctx) << " has unexpected consumer: "
+ << consumer->Content());
+ }
+ }
+
+ for (size_t i = 0; i < stageResultType->GetSize(); ++i) {
+ if (!usedOutputs.Test(i)) {
+ hasBrokenStage = true;
+ YQL_CLOG(ERROR, ProviderKqp) << "Stage #" << PrintKqpStageOnly(stage, ctx)
+ << ", output " << i << " (" << FormatType(stageResultType->GetItems()[i]) << ")"
+ << " not used";
+ return false;
+ }
+ }
+ }
+
+ YQL_ENSURE(!node.Maybe<TDqPhyStage>());
+
+ return true;
+ });
+
+ if (hasMultipleConsumers) {
+ ctx.AddError(TIssue(ctx.GetPosition(input->Pos()),
+ "Failed to build physical query: some connection(s) have several consumers"));
+ return TStatus::Error;
+ }
+
+ if (hasBrokenStage) {
+ ctx.AddError(TIssue(ctx.GetPosition(input->Pos()),
+ "Failed to build physical query: some stages are broken"));
+ return TStatus::Error;
+ }
+
+ return TStatus::Ok;
+ });
+}
+
} // namespace NKikimr::NKqp::NOpt
diff --git a/ydb/core/kqp/opt/kqp_opt_phy_finalize.cpp b/ydb/core/kqp/opt/kqp_opt_phy_finalize.cpp
index e0b586161c..ee827ddb39 100644
--- a/ydb/core/kqp/opt/kqp_opt_phy_finalize.cpp
+++ b/ydb/core/kqp/opt/kqp_opt_phy_finalize.cpp
@@ -1,155 +1,155 @@
-#include "kqp_opt_impl.h"
-
+#include "kqp_opt_impl.h"
+
#include <ydb/library/yql/core/yql_expr_optimize.h>
#include <ydb/library/yql/dq/opt/dq_opt_phy.h>
#include <ydb/library/yql/dq/opt/dq_opt_phy_finalizing.h>
-
+
namespace NKikimr::NKqp::NOpt {
-
-using namespace NYql;
-using namespace NYql::NDq;
-using namespace NYql::NNodes;
-
-using TStatus = IGraphTransformer::TStatus;
-
-namespace {
-
-TStatus KqpBuildPureExprStagesResult(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
- TExprBase inputExpr(input);
- auto query = inputExpr.Cast<TKqlQuery>();
- TNodeOnNodeOwnedMap replaces;
-
- for (const auto& queryResult: query.Results()) {
- TExprBase node(queryResult.Value());
-
- auto result = DqBuildPureExprStage(node, ctx);
- if (result.Raw() != node.Raw()) {
- YQL_CLOG(DEBUG, ProviderKqp) << "Building stage out of pure query #" << node.Raw()->UniqueId();
- replaces[node.Raw()] = result.Ptr();
- }
- }
- output = ctx.ReplaceNodes(TExprNode::TPtr(input), replaces);
-
- return TStatus::Ok;
-}
-
-TStatus KqpBuildUnionResult(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
- TExprBase inputExpr(input);
- auto query = inputExpr.Cast<TKqlQuery>();
- TNodeOnNodeOwnedMap replaces;
-
- for (const auto& queryResult: query.Results()) {
- TExprBase node(queryResult.Value());
-
+
+using namespace NYql;
+using namespace NYql::NDq;
+using namespace NYql::NNodes;
+
+using TStatus = IGraphTransformer::TStatus;
+
+namespace {
+
+TStatus KqpBuildPureExprStagesResult(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
+ TExprBase inputExpr(input);
+ auto query = inputExpr.Cast<TKqlQuery>();
+ TNodeOnNodeOwnedMap replaces;
+
+ for (const auto& queryResult: query.Results()) {
+ TExprBase node(queryResult.Value());
+
+ auto result = DqBuildPureExprStage(node, ctx);
+ if (result.Raw() != node.Raw()) {
+ YQL_CLOG(DEBUG, ProviderKqp) << "Building stage out of pure query #" << node.Raw()->UniqueId();
+ replaces[node.Raw()] = result.Ptr();
+ }
+ }
+ output = ctx.ReplaceNodes(TExprNode::TPtr(input), replaces);
+
+ return TStatus::Ok;
+}
+
+TStatus KqpBuildUnionResult(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
+ TExprBase inputExpr(input);
+ auto query = inputExpr.Cast<TKqlQuery>();
+ TNodeOnNodeOwnedMap replaces;
+
+ for (const auto& queryResult: query.Results()) {
+ TExprBase node(queryResult.Value());
+
auto result = DqBuildExtendStage(node, ctx);
- if (result.Raw() != node.Raw()) {
- YQL_CLOG(DEBUG, ProviderKqp) << "Building stage out of union #" << node.Raw()->UniqueId();
- replaces[node.Raw()] = result.Ptr();
- }
- }
- output = ctx.ReplaceNodes(TExprNode::TPtr(input), replaces);
-
- return TStatus::Ok;
-}
-
-TStatus KqpDuplicateResults(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
- output = input;
-
- TExprBase inputExpr(input);
- auto query = inputExpr.Cast<TKqlQuery>();
-
- const size_t resultsCount = query.Results().Size();
-
- if (resultsCount <= 1) {
- return TStatus::Ok;
- }
-
- struct TKqlQueryResultInfo {
- TMaybeNode<TKqlQueryResult> Node;
- TVector<size_t> Indexes;
- TMaybeNode<TDqStage> ReplicateStage;
- size_t NextIndex = 0;
- };
-
- TNodeMap<TKqlQueryResultInfo> kqlQueryResults;
- bool hasDups = false;
-
- for (size_t i = 0; i < resultsCount; ++i) {
- TKqlQueryResult result = query.Results().Item(i);
-
- if (!result.Value().Maybe<TDqConnection>()) {
- return TStatus::Ok;
- }
-
- auto& info = kqlQueryResults[result.Raw()];
- if (info.Indexes.empty()) {
- info.Node = result;
- } else {
- hasDups = true;
- }
- info.Indexes.push_back(i);
- }
-
- if (!hasDups) {
- return TStatus::Ok;
- }
-
- auto identityLambda = NDq::BuildIdentityLambda(query.Pos(), ctx);
-
- for (auto& [ptr, info] : kqlQueryResults) {
- if (info.Indexes.size() > 1) {
- // YQL_CLOG(TRACE, ProviderKqp) << " * " << KqpExprToPrettyString(TExprBase(ptr), ctx)
- // << " --> [" << JoinSeq(',', info.Indexes) << "]";
-
- info.ReplicateStage = Build<TDqStage>(ctx, query.Pos())
- .Inputs()
- .Add(info.Node.Cast().Value())
- .Build()
- .Program()
- .Args({"stream"})
- .Body<TDqReplicate>()
- .Input("stream")
- .FreeArgs()
- .Add(TVector<TExprBase>(info.Indexes.size(), identityLambda))
- .Build()
- .Build()
- .Build()
- .Settings().Build()
- .Done();
- }
- }
-
- TVector<TExprNode::TPtr> results(resultsCount);
-
- for (size_t i = 0; i < resultsCount; ++i) {
- auto& info = kqlQueryResults.at(query.Results().Item(i).Raw());
-
- if (info.Indexes.size() == 1) {
- results[i] = info.Node.Cast().Ptr();
- } else {
- results[i] = Build<TKqlQueryResult>(ctx, query.Pos())
- .Value<TDqCnUnionAll>()
- .Output()
- .Stage(info.ReplicateStage.Cast())
- .Index().Build(ToString(info.NextIndex))
- .Build()
- .Build()
- .ColumnHints(info.Node.Cast().ColumnHints())
- .Done().Ptr();
- info.NextIndex++;
- }
- }
-
- output = Build<TKqlQuery>(ctx, query.Pos())
- .Results()
- .Add(results)
- .Build()
- .Effects(query.Effects())
- .Done().Ptr();
-
- return TStatus::Ok;
-}
-
+ if (result.Raw() != node.Raw()) {
+ YQL_CLOG(DEBUG, ProviderKqp) << "Building stage out of union #" << node.Raw()->UniqueId();
+ replaces[node.Raw()] = result.Ptr();
+ }
+ }
+ output = ctx.ReplaceNodes(TExprNode::TPtr(input), replaces);
+
+ return TStatus::Ok;
+}
+
+TStatus KqpDuplicateResults(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
+ output = input;
+
+ TExprBase inputExpr(input);
+ auto query = inputExpr.Cast<TKqlQuery>();
+
+ const size_t resultsCount = query.Results().Size();
+
+ if (resultsCount <= 1) {
+ return TStatus::Ok;
+ }
+
+ struct TKqlQueryResultInfo {
+ TMaybeNode<TKqlQueryResult> Node;
+ TVector<size_t> Indexes;
+ TMaybeNode<TDqStage> ReplicateStage;
+ size_t NextIndex = 0;
+ };
+
+ TNodeMap<TKqlQueryResultInfo> kqlQueryResults;
+ bool hasDups = false;
+
+ for (size_t i = 0; i < resultsCount; ++i) {
+ TKqlQueryResult result = query.Results().Item(i);
+
+ if (!result.Value().Maybe<TDqConnection>()) {
+ return TStatus::Ok;
+ }
+
+ auto& info = kqlQueryResults[result.Raw()];
+ if (info.Indexes.empty()) {
+ info.Node = result;
+ } else {
+ hasDups = true;
+ }
+ info.Indexes.push_back(i);
+ }
+
+ if (!hasDups) {
+ return TStatus::Ok;
+ }
+
+ auto identityLambda = NDq::BuildIdentityLambda(query.Pos(), ctx);
+
+ for (auto& [ptr, info] : kqlQueryResults) {
+ if (info.Indexes.size() > 1) {
+ // YQL_CLOG(TRACE, ProviderKqp) << " * " << KqpExprToPrettyString(TExprBase(ptr), ctx)
+ // << " --> [" << JoinSeq(',', info.Indexes) << "]";
+
+ info.ReplicateStage = Build<TDqStage>(ctx, query.Pos())
+ .Inputs()
+ .Add(info.Node.Cast().Value())
+ .Build()
+ .Program()
+ .Args({"stream"})
+ .Body<TDqReplicate>()
+ .Input("stream")
+ .FreeArgs()
+ .Add(TVector<TExprBase>(info.Indexes.size(), identityLambda))
+ .Build()
+ .Build()
+ .Build()
+ .Settings().Build()
+ .Done();
+ }
+ }
+
+ TVector<TExprNode::TPtr> results(resultsCount);
+
+ for (size_t i = 0; i < resultsCount; ++i) {
+ auto& info = kqlQueryResults.at(query.Results().Item(i).Raw());
+
+ if (info.Indexes.size() == 1) {
+ results[i] = info.Node.Cast().Ptr();
+ } else {
+ results[i] = Build<TKqlQueryResult>(ctx, query.Pos())
+ .Value<TDqCnUnionAll>()
+ .Output()
+ .Stage(info.ReplicateStage.Cast())
+ .Index().Build(ToString(info.NextIndex))
+ .Build()
+ .Build()
+ .ColumnHints(info.Node.Cast().ColumnHints())
+ .Done().Ptr();
+ info.NextIndex++;
+ }
+ }
+
+ output = Build<TKqlQuery>(ctx, query.Pos())
+ .Results()
+ .Add(results)
+ .Build()
+ .Effects(query.Effects())
+ .Done().Ptr();
+
+ return TStatus::Ok;
+}
+
template <typename TFunctor>
NYql::IGraphTransformer::TStatus PerformGlobalRule(const TString& ruleName, const NYql::TExprNode::TPtr& input,
NYql::TExprNode::TPtr& output, NYql::TExprContext& ctx, TFunctor func)
@@ -166,49 +166,49 @@ NYql::IGraphTransformer::TStatus PerformGlobalRule(const TString& ruleName, cons
return NYql::IGraphTransformer::TStatus(NYql::IGraphTransformer::TStatus::Repeat, true);
}
-} // anonymous namespace
-
-#define PERFORM_GLOBAL_RULE(id, input, output, ctx, ...) \
- do { \
- auto status = PerformGlobalRule(id, input, output, ctx, __VA_ARGS__); \
- if (status != IGraphTransformer::TStatus::Ok) { return status; } \
- } while (0)
-
-TAutoPtr<IGraphTransformer> CreateKqpFinalizingOptTransformer() {
- return CreateFunctorTransformer(
- [](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) -> TStatus {
+} // anonymous namespace
+
+#define PERFORM_GLOBAL_RULE(id, input, output, ctx, ...) \
+ do { \
+ auto status = PerformGlobalRule(id, input, output, ctx, __VA_ARGS__); \
+ if (status != IGraphTransformer::TStatus::Ok) { return status; } \
+ } while (0)
+
+TAutoPtr<IGraphTransformer> CreateKqpFinalizingOptTransformer() {
+ return CreateFunctorTransformer(
+ [](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) -> TStatus {
PERFORM_GLOBAL_RULE("ReplicateMultiUsedConnection", input, output, ctx,
- [](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
- YQL_ENSURE(TKqlQuery::Match(input.Get()));
- return NDq::DqReplicateStageMultiOutput(input, output, ctx);
- });
-
+ [](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
+ YQL_ENSURE(TKqlQuery::Match(input.Get()));
+ return NDq::DqReplicateStageMultiOutput(input, output, ctx);
+ });
+
PERFORM_GLOBAL_RULE("BuildPureExprStages", input, output, ctx,
- [](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
- return KqpBuildPureExprStagesResult(input, output, ctx);
- });
-
+ [](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
+ return KqpBuildPureExprStagesResult(input, output, ctx);
+ });
+
PERFORM_GLOBAL_RULE("BuildUnion", input, output, ctx,
- [] (const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
- return KqpBuildUnionResult(input, output, ctx);
- });
-
+ [] (const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
+ return KqpBuildUnionResult(input, output, ctx);
+ });
+
PERFORM_GLOBAL_RULE("ExtractPrecomputeToInputs", input, output, ctx,
- [] (const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
- return DqExtractPrecomputeToStageInput(input, output, ctx);
- });
-
- PERFORM_GLOBAL_RULE("DuplicateResults", input, output, ctx,
- [] (const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
- return KqpDuplicateResults(input, output, ctx);
- });
-
- YQL_CLOG(INFO, ProviderKqp) << "FinalizingOptimized KQL query: " << KqpExprToPrettyString(*input, ctx);
-
- return TStatus::Ok;
- });
-}
-
-#undef PERFORM_GLOBAL_RULE
-
+ [] (const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
+ return DqExtractPrecomputeToStageInput(input, output, ctx);
+ });
+
+ PERFORM_GLOBAL_RULE("DuplicateResults", input, output, ctx,
+ [] (const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
+ return KqpDuplicateResults(input, output, ctx);
+ });
+
+ YQL_CLOG(INFO, ProviderKqp) << "FinalizingOptimized KQL query: " << KqpExprToPrettyString(*input, ctx);
+
+ return TStatus::Ok;
+ });
+}
+
+#undef PERFORM_GLOBAL_RULE
+
} // namespace NKikimr::NKqp::NOpt
diff --git a/ydb/core/kqp/opt/logical/kqp_opt_log.cpp b/ydb/core/kqp/opt/logical/kqp_opt_log.cpp
index 827582f3c5..d951c5ce30 100644
--- a/ydb/core/kqp/opt/logical/kqp_opt_log.cpp
+++ b/ydb/core/kqp/opt/logical/kqp_opt_log.cpp
@@ -40,7 +40,7 @@ public:
AddHandler(0, &TCoCalcOverWindowGroup::Match, HNDL(ExpandWindowFunctions));
AddHandler(0, &TCoTopSort::Match, HNDL(RewriteTopSortOverIndexRead));
AddHandler(0, &TCoTake::Match, HNDL(RewriteTakeOverIndexRead));
- AddHandler(0, &TCoFlatMapBase::Match, HNDL(RewriteFlatMapOverExtend));
+ AddHandler(0, &TCoFlatMapBase::Match, HNDL(RewriteFlatMapOverExtend));
AddHandler(0, &TKqlDeleteRows::Match, HNDL(DeleteOverLookup));
AddHandler(0, &TKqlUpsertRowsBase::Match, HNDL(ExcessUpsertInputColumns));
@@ -92,13 +92,13 @@ protected:
DumpAppliedRule("RewriteTakeSortToTopSort", node.Ptr(), output.Ptr(), ctx);
return output;
}
-
+
TMaybeNode<TExprBase> RewriteSqlInToEquiJoin(TExprBase node, TExprContext& ctx) {
TExprBase output = KqpRewriteSqlInToEquiJoin(node, ctx, KqpCtx, Config);
DumpAppliedRule("RewriteSqlInToEquiJoin", node.Ptr(), output.Ptr(), ctx);
return output;
}
-
+
TMaybeNode<TExprBase> RewriteSqlInCompactToJoin(TExprBase node, TExprContext& ctx) {
TExprBase output = KqpRewriteSqlInCompactToJoin(node, ctx);
DumpAppliedRule("KqpRewriteSqlInCompactToJoin", node.Ptr(), output.Ptr(), ctx);
@@ -135,12 +135,12 @@ protected:
return output;
}
- TMaybeNode<TExprBase> RewriteFlatMapOverExtend(TExprBase node, TExprContext& ctx) {
- auto output = DqFlatMapOverExtend(node, ctx);
- DumpAppliedRule("RewriteFlatMapOverExtend", node.Ptr(), output.Ptr(), ctx);
- return output;
- }
-
+ TMaybeNode<TExprBase> RewriteFlatMapOverExtend(TExprBase node, TExprContext& ctx) {
+ auto output = DqFlatMapOverExtend(node, ctx);
+ DumpAppliedRule("RewriteFlatMapOverExtend", node.Ptr(), output.Ptr(), ctx);
+ return output;
+ }
+
TMaybeNode<TExprBase> RewriteIndexRead(TExprBase node, TExprContext& ctx) {
TExprBase output = KqpRewriteIndexRead(node, ctx, KqpCtx);
DumpAppliedRule("RewriteIndexRead", node.Ptr(), output.Ptr(), ctx);
diff --git a/ydb/core/kqp/opt/logical/kqp_opt_log_join.cpp b/ydb/core/kqp/opt/logical/kqp_opt_log_join.cpp
index 4bb0f5c3cf..ac0fe0bc39 100644
--- a/ydb/core/kqp/opt/logical/kqp_opt_log_join.cpp
+++ b/ydb/core/kqp/opt/logical/kqp_opt_log_join.cpp
@@ -1,76 +1,76 @@
-#include "kqp_opt_log_rules.h"
-
+#include "kqp_opt_log_rules.h"
+
#include <ydb/core/kqp/opt/kqp_opt_impl.h>
#include <ydb/core/kqp/common/kqp_yql.h>
#include <ydb/core/kqp/provider/yql_kikimr_provider_impl.h>
#include <ydb/core/kqp/provider/yql_kikimr_opt_utils.h>
-
+
#include <ydb/library/yql/core/yql_opt_utils.h>
-
+
namespace NKikimr::NKqp::NOpt {
-
-using namespace NYql;
-using namespace NYql::NCommon;
-using namespace NYql::NDq;
-using namespace NYql::NNodes;
-
-namespace {
-
-[[maybe_unused]]
-bool IsKqlPureExpr(const TExprBase& expr) {
- auto node = FindNode(expr.Ptr(), [](const TExprNode::TPtr& node) {
- return node->IsCallable()
- && (node->Content().StartsWith("Kql")
- || node->Content().StartsWith("Kqp")
- || node->Content().StartsWith("Dq"));
- });
- return node.Get() == nullptr;
-}
-
-TDqJoin FlipLeftSemiJoin(const TDqJoin& join, TExprContext& ctx) {
- Y_VERIFY_DEBUG(join.JoinType().Value() == "LeftSemi");
-
- auto joinKeysBuilder = Build<TDqJoinKeyTupleList>(ctx, join.Pos());
- for (const auto& keys : join.JoinKeys()) {
- joinKeysBuilder.Add<TDqJoinKeyTuple>()
- .LeftLabel(keys.RightLabel())
- .LeftColumn(keys.RightColumn())
- .RightLabel(keys.LeftLabel())
- .RightColumn(keys.LeftColumn())
- .Build();
- }
-
- return Build<TDqJoin>(ctx, join.Pos())
- .LeftInput(join.RightInput())
- .LeftLabel(join.RightLabel())
- .RightInput(join.LeftInput())
- .RightLabel(join.LeftLabel())
- .JoinType().Build("RightSemi")
- .JoinKeys(joinKeysBuilder.Done())
- .Done();
-}
-
-TMaybeNode<TKqlKeyInc> GetRightTableKeyPrefix(const TKqlKeyRange& range) {
- if (!range.From().Maybe<TKqlKeyInc>() || !range.To().Maybe<TKqlKeyInc>()) {
- return {};
- }
- auto rangeFrom = range.From().Cast<TKqlKeyInc>();
- auto rangeTo = range.To().Cast<TKqlKeyInc>();
-
- if (rangeFrom.ArgCount() != rangeTo.ArgCount()) {
- return {};
- }
- for (ui32 i = 0; i < rangeFrom.ArgCount(); ++i) {
- if (rangeFrom.Arg(i).Raw() != rangeTo.Arg(i).Raw()) {
- return {};
- }
- }
-
- return rangeFrom;
-}
-
+
+using namespace NYql;
+using namespace NYql::NCommon;
+using namespace NYql::NDq;
+using namespace NYql::NNodes;
+
+namespace {
+
+[[maybe_unused]]
+bool IsKqlPureExpr(const TExprBase& expr) {
+ auto node = FindNode(expr.Ptr(), [](const TExprNode::TPtr& node) {
+ return node->IsCallable()
+ && (node->Content().StartsWith("Kql")
+ || node->Content().StartsWith("Kqp")
+ || node->Content().StartsWith("Dq"));
+ });
+ return node.Get() == nullptr;
+}
+
+TDqJoin FlipLeftSemiJoin(const TDqJoin& join, TExprContext& ctx) {
+ Y_VERIFY_DEBUG(join.JoinType().Value() == "LeftSemi");
+
+ auto joinKeysBuilder = Build<TDqJoinKeyTupleList>(ctx, join.Pos());
+ for (const auto& keys : join.JoinKeys()) {
+ joinKeysBuilder.Add<TDqJoinKeyTuple>()
+ .LeftLabel(keys.RightLabel())
+ .LeftColumn(keys.RightColumn())
+ .RightLabel(keys.LeftLabel())
+ .RightColumn(keys.LeftColumn())
+ .Build();
+ }
+
+ return Build<TDqJoin>(ctx, join.Pos())
+ .LeftInput(join.RightInput())
+ .LeftLabel(join.RightLabel())
+ .RightInput(join.LeftInput())
+ .RightLabel(join.LeftLabel())
+ .JoinType().Build("RightSemi")
+ .JoinKeys(joinKeysBuilder.Done())
+ .Done();
+}
+
+TMaybeNode<TKqlKeyInc> GetRightTableKeyPrefix(const TKqlKeyRange& range) {
+ if (!range.From().Maybe<TKqlKeyInc>() || !range.To().Maybe<TKqlKeyInc>()) {
+ return {};
+ }
+ auto rangeFrom = range.From().Cast<TKqlKeyInc>();
+ auto rangeTo = range.To().Cast<TKqlKeyInc>();
+
+ if (rangeFrom.ArgCount() != rangeTo.ArgCount()) {
+ return {};
+ }
+ for (ui32 i = 0; i < rangeFrom.ArgCount(); ++i) {
+ if (rangeFrom.Arg(i).Raw() != rangeTo.Arg(i).Raw()) {
+ return {};
+ }
+ }
+
+ return rangeFrom;
+}
+
TExprBase BuildLookupIndex(TExprContext& ctx, const TPositionHandle pos, const TKqlReadTableBase& read,
- const TExprBase& keysToLookup, const TVector<TCoAtom>& lookupNames, const TString& indexName)
+ const TExprBase& keysToLookup, const TVector<TCoAtom>& lookupNames, const TString& indexName)
{
return Build<TKqlLookupIndex>(ctx, pos)
.Table(read.Table())
@@ -87,7 +87,7 @@ TExprBase BuildLookupIndex(TExprContext& ctx, const TPositionHandle pos, const T
}
TExprBase BuildLookupTable(TExprContext& ctx, const TPositionHandle pos, const TKqlReadTableBase& read,
- const TExprBase& keysToLookup, const TVector<TCoAtom>& lookupNames)
+ const TExprBase& keysToLookup, const TVector<TCoAtom>& lookupNames)
{
return Build<TKqlLookupTable>(ctx, pos)
.Table(read.Table())
@@ -142,71 +142,71 @@ TVector<TExprBase> CreateRenames(const TMaybeNode<TCoFlatMap>& rightFlatmap, con
//#define DBG(...) YQL_CLOG(DEBUG, ProviderKqp) << __VA_ARGS__
-#define DBG(...)
-
-TMaybeNode<TExprBase> KqpJoinToIndexLookupImpl(const TDqJoin& join, TExprContext& ctx, const TKqpOptimizeContext& kqpCtx) {
- if (!join.RightLabel().Maybe<TCoAtom>()) {
- // Lookup only in tables
- return {};
- }
-
- static THashSet<TStringBuf> supportedJoinKinds = {"Inner", "Left", "LeftOnly", "LeftSemi", "RightSemi"};
- if (!supportedJoinKinds.contains(join.JoinType().Value())) {
- return {};
- }
-
+#define DBG(...)
+
+TMaybeNode<TExprBase> KqpJoinToIndexLookupImpl(const TDqJoin& join, TExprContext& ctx, const TKqpOptimizeContext& kqpCtx) {
+ if (!join.RightLabel().Maybe<TCoAtom>()) {
+ // Lookup only in tables
+ return {};
+ }
+
+ static THashSet<TStringBuf> supportedJoinKinds = {"Inner", "Left", "LeftOnly", "LeftSemi", "RightSemi"};
+ if (!supportedJoinKinds.contains(join.JoinType().Value())) {
+ return {};
+ }
+
TMaybeNode<TKqlReadTableBase> rightRead;
- TMaybeNode<TCoFlatMap> rightFlatmap;
- TMaybeNode<TCoFilterNullMembers> rightFilterNull;
- TMaybeNode<TCoSkipNullMembers> rightSkipNull;
-
+ TMaybeNode<TCoFlatMap> rightFlatmap;
+ TMaybeNode<TCoFilterNullMembers> rightFilterNull;
+ TMaybeNode<TCoSkipNullMembers> rightSkipNull;
+
if (auto readTable = join.RightInput().Maybe<TKqlReadTableBase>()) {
- rightRead = readTable;
- }
-
+ rightRead = readTable;
+ }
+
if (auto readTable = join.RightInput().Maybe<TCoFlatMap>().Input().Maybe<TKqlReadTableBase>()) {
- rightRead = readTable;
- rightFlatmap = join.RightInput().Maybe<TCoFlatMap>();
- }
-
+ rightRead = readTable;
+ rightFlatmap = join.RightInput().Maybe<TCoFlatMap>();
+ }
+
if (auto readTable = join.RightInput().Maybe<TCoFlatMap>().Input().Maybe<TCoFilterNullMembers>().Input().Maybe<TKqlReadTableBase>()) {
- rightRead = readTable;
- rightFlatmap = join.RightInput().Maybe<TCoFlatMap>();
- rightFilterNull = rightFlatmap.Input().Cast<TCoFilterNullMembers>();
- }
-
+ rightRead = readTable;
+ rightFlatmap = join.RightInput().Maybe<TCoFlatMap>();
+ rightFilterNull = rightFlatmap.Input().Cast<TCoFilterNullMembers>();
+ }
+
if (auto readTable = join.RightInput().Maybe<TCoFlatMap>().Input().Maybe<TCoSkipNullMembers>().Input().Maybe<TKqlReadTableBase>()) {
- rightRead = readTable;
- rightFlatmap = join.RightInput().Maybe<TCoFlatMap>();
- rightSkipNull = rightFlatmap.Input().Cast<TCoSkipNullMembers>();
- }
-
- if (!rightRead) {
- return {};
- }
-
+ rightRead = readTable;
+ rightFlatmap = join.RightInput().Maybe<TCoFlatMap>();
+ rightSkipNull = rightFlatmap.Input().Cast<TCoSkipNullMembers>();
+ }
+
+ if (!rightRead) {
+ return {};
+ }
+
Y_ENSURE(rightRead.Maybe<TKqlReadTable>() || rightRead.Maybe<TKqlReadTableIndex>());
const TKqlReadTableBase read = rightRead.Cast();
- if (!read.Table().SysView().Value().empty()) {
- // Can't lookup in system views
- return {};
- }
-
- if (rightFlatmap && !IsPassthroughFlatMap(rightFlatmap.Cast(), nullptr)) {
- // Can't lookup in modified table
- return {};
- }
-
- auto maybeRightTableKeyPrefix = GetRightTableKeyPrefix(read.Range());
- if (!maybeRightTableKeyPrefix) {
- return {};
- }
- auto rightTableKeyPrefix = maybeRightTableKeyPrefix.Cast();
-
+ if (!read.Table().SysView().Value().empty()) {
+ // Can't lookup in system views
+ return {};
+ }
+
+ if (rightFlatmap && !IsPassthroughFlatMap(rightFlatmap.Cast(), nullptr)) {
+ // Can't lookup in modified table
+ return {};
+ }
+
+ auto maybeRightTableKeyPrefix = GetRightTableKeyPrefix(read.Range());
+ if (!maybeRightTableKeyPrefix) {
+ return {};
+ }
+ auto rightTableKeyPrefix = maybeRightTableKeyPrefix.Cast();
+
TString lookupTable;
TString indexName;
-
+
if (auto indexRead = rightRead.Maybe<TKqlReadTableIndex>()) {
indexName = indexRead.Cast().Index().StringValue();
lookupTable = GetIndexMetadata(indexRead.Cast(), *kqpCtx.Tables, kqpCtx.Cluster)->Name;
@@ -216,242 +216,242 @@ TMaybeNode<TExprBase> KqpJoinToIndexLookupImpl(const TDqJoin& join, TExprContext
const auto& rightTableDesc = kqpCtx.Tables->ExistingTable(kqpCtx.Cluster, lookupTable);
- TMap<std::string_view, TString> rightJoinKeyToLeft;
- TVector<TCoAtom> rightKeyColumns;
- rightKeyColumns.reserve(join.JoinKeys().Size());
- TSet<TString> leftJoinKeys;
- std::map<std::string_view, std::set<TString>> equalLeftKeys;
-
- for (ui32 i = 0; i < join.JoinKeys().Size(); ++i) {
- const auto& keyTuple = join.JoinKeys().Item(i);
-
- auto leftKey = join.LeftLabel().Maybe<TCoVoid>()
- ? Join('.', keyTuple.LeftLabel().Value(), keyTuple.LeftColumn().Value())
- : keyTuple.LeftColumn().StringValue();
-
- rightKeyColumns.emplace_back(keyTuple.RightColumn()); // unique elements
-
- auto [iter, newValue] = rightJoinKeyToLeft.emplace(keyTuple.RightColumn().Value(), leftKey);
- if (!newValue) {
- equalLeftKeys[iter->second].emplace(leftKey);
- }
-
- leftJoinKeys.emplace(leftKey);
- }
-
- auto leftRowArg = Build<TCoArgument>(ctx, join.Pos())
- .Name("leftRowArg")
- .Done();
-
- TVector<TExprBase> lookupMembers;
- TVector<TCoAtom> lookupNames;
- ui32 fixedPrefix = 0;
- for (auto& rightColumnName : rightTableDesc.Metadata->KeyColumnNames) {
- TExprNode::TPtr member;
-
- auto leftColumn = rightJoinKeyToLeft.FindPtr(rightColumnName);
-
- if (fixedPrefix < rightTableKeyPrefix.ArgCount()) {
- if (leftColumn) {
- return {};
- }
-
- member = rightTableKeyPrefix.Arg(fixedPrefix).Ptr();
- fixedPrefix++;
- } else {
- if (!leftColumn) {
- break;
- }
-
- member = Build<TCoMember>(ctx, join.Pos())
- .Struct(leftRowArg)
- .Name().Build(*leftColumn)
- .Done().Ptr();
-
- const TDataExprType* leftDataType;
- const TDataExprType* rightDataType;
- if (!GetEquiJoinKeyTypes(join.LeftInput(), *leftColumn, rightTableDesc, rightColumnName, leftDataType, rightDataType)) {
- return {};
- }
-
- if (leftDataType != rightDataType) {
- bool canCast = IsDataTypeNumeric(leftDataType->GetSlot()) && IsDataTypeNumeric(rightDataType->GetSlot());
- if (!canCast) {
- canCast = leftDataType->GetName() == "Utf8" && rightDataType->GetName() == "String";
- }
- if (canCast) {
- DBG("------ cast " << leftDataType->GetName() << " to " << rightDataType->GetName());
+ TMap<std::string_view, TString> rightJoinKeyToLeft;
+ TVector<TCoAtom> rightKeyColumns;
+ rightKeyColumns.reserve(join.JoinKeys().Size());
+ TSet<TString> leftJoinKeys;
+ std::map<std::string_view, std::set<TString>> equalLeftKeys;
+
+ for (ui32 i = 0; i < join.JoinKeys().Size(); ++i) {
+ const auto& keyTuple = join.JoinKeys().Item(i);
+
+ auto leftKey = join.LeftLabel().Maybe<TCoVoid>()
+ ? Join('.', keyTuple.LeftLabel().Value(), keyTuple.LeftColumn().Value())
+ : keyTuple.LeftColumn().StringValue();
+
+ rightKeyColumns.emplace_back(keyTuple.RightColumn()); // unique elements
+
+ auto [iter, newValue] = rightJoinKeyToLeft.emplace(keyTuple.RightColumn().Value(), leftKey);
+ if (!newValue) {
+ equalLeftKeys[iter->second].emplace(leftKey);
+ }
+
+ leftJoinKeys.emplace(leftKey);
+ }
+
+ auto leftRowArg = Build<TCoArgument>(ctx, join.Pos())
+ .Name("leftRowArg")
+ .Done();
+
+ TVector<TExprBase> lookupMembers;
+ TVector<TCoAtom> lookupNames;
+ ui32 fixedPrefix = 0;
+ for (auto& rightColumnName : rightTableDesc.Metadata->KeyColumnNames) {
+ TExprNode::TPtr member;
+
+ auto leftColumn = rightJoinKeyToLeft.FindPtr(rightColumnName);
+
+ if (fixedPrefix < rightTableKeyPrefix.ArgCount()) {
+ if (leftColumn) {
+ return {};
+ }
+
+ member = rightTableKeyPrefix.Arg(fixedPrefix).Ptr();
+ fixedPrefix++;
+ } else {
+ if (!leftColumn) {
+ break;
+ }
+
+ member = Build<TCoMember>(ctx, join.Pos())
+ .Struct(leftRowArg)
+ .Name().Build(*leftColumn)
+ .Done().Ptr();
+
+ const TDataExprType* leftDataType;
+ const TDataExprType* rightDataType;
+ if (!GetEquiJoinKeyTypes(join.LeftInput(), *leftColumn, rightTableDesc, rightColumnName, leftDataType, rightDataType)) {
+ return {};
+ }
+
+ if (leftDataType != rightDataType) {
+ bool canCast = IsDataTypeNumeric(leftDataType->GetSlot()) && IsDataTypeNumeric(rightDataType->GetSlot());
+ if (!canCast) {
+ canCast = leftDataType->GetName() == "Utf8" && rightDataType->GetName() == "String";
+ }
+ if (canCast) {
+ DBG("------ cast " << leftDataType->GetName() << " to " << rightDataType->GetName());
member = Build<TCoConvert>(ctx, join.Pos())
.Input(member)
.Type().Build(rightDataType->GetName())
.Done().Ptr();
- } else {
- DBG("------ can not cast " << leftDataType->GetName() << " to " << rightDataType->GetName());
- return {};
- }
- }
- }
-
- lookupMembers.emplace_back(
- Build<TExprList>(ctx, join.Pos())
- .Add<TCoAtom>().Build(rightColumnName)
- .Add(member)
- .Done());
- lookupNames.emplace_back(ctx.NewAtom(join.Pos(), rightColumnName));
- }
-
- if (lookupMembers.size() <= fixedPrefix) {
- return {};
- }
-
+ } else {
+ DBG("------ can not cast " << leftDataType->GetName() << " to " << rightDataType->GetName());
+ return {};
+ }
+ }
+ }
+
+ lookupMembers.emplace_back(
+ Build<TExprList>(ctx, join.Pos())
+ .Add<TCoAtom>().Build(rightColumnName)
+ .Add(member)
+ .Done());
+ lookupNames.emplace_back(ctx.NewAtom(join.Pos(), rightColumnName));
+ }
+
+ if (lookupMembers.size() <= fixedPrefix) {
+ return {};
+ }
+
auto leftData = Build<TDqPrecompute>(ctx, join.Pos())
- .Input(join.LeftInput())
- .Done();
- auto leftDataDeduplicated = DeduplicateByMembers(leftData, leftJoinKeys, ctx, join.Pos());
-
- if (!equalLeftKeys.empty()) {
- auto row = Build<TCoArgument>(ctx, join.Pos())
- .Name("row")
- .Done();
-
- TVector<TExprBase> conditions;
-
- for (auto [first, others]: equalLeftKeys) {
- auto v = Build<TCoMember>(ctx, join.Pos())
- .Struct(row)
- .Name().Build(first)
- .Done();
-
- for (std::string_view other: others) {
- conditions.emplace_back(
- Build<TCoCmpEqual>(ctx, join.Pos())
- .Left(v)
- .Right<TCoMember>()
- .Struct(row)
- .Name().Build(other)
- .Build()
- .Done());
- }
- }
-
- leftDataDeduplicated = Build<TCoFilter>(ctx, join.Pos())
- .Input(leftDataDeduplicated)
- .Lambda()
- .Args({row})
- .Body<TCoCoalesce>()
- .Predicate<TCoAnd>()
- .Add(conditions)
- .Build()
- .Value<TCoBool>()
- .Literal().Build("false")
- .Build()
- .Build()
- .Build()
- .Done();
- }
-
- auto keysToLookup = Build<TCoMap>(ctx, join.Pos())
- .Input(leftDataDeduplicated)
- .Lambda()
- .Args({leftRowArg})
- .Body<TCoAsStruct>()
- .Add(lookupMembers)
- .Build()
- .Build()
- .Done();
-
+ .Input(join.LeftInput())
+ .Done();
+ auto leftDataDeduplicated = DeduplicateByMembers(leftData, leftJoinKeys, ctx, join.Pos());
+
+ if (!equalLeftKeys.empty()) {
+ auto row = Build<TCoArgument>(ctx, join.Pos())
+ .Name("row")
+ .Done();
+
+ TVector<TExprBase> conditions;
+
+ for (auto [first, others]: equalLeftKeys) {
+ auto v = Build<TCoMember>(ctx, join.Pos())
+ .Struct(row)
+ .Name().Build(first)
+ .Done();
+
+ for (std::string_view other: others) {
+ conditions.emplace_back(
+ Build<TCoCmpEqual>(ctx, join.Pos())
+ .Left(v)
+ .Right<TCoMember>()
+ .Struct(row)
+ .Name().Build(other)
+ .Build()
+ .Done());
+ }
+ }
+
+ leftDataDeduplicated = Build<TCoFilter>(ctx, join.Pos())
+ .Input(leftDataDeduplicated)
+ .Lambda()
+ .Args({row})
+ .Body<TCoCoalesce>()
+ .Predicate<TCoAnd>()
+ .Add(conditions)
+ .Build()
+ .Value<TCoBool>()
+ .Literal().Build("false")
+ .Build()
+ .Build()
+ .Build()
+ .Done();
+ }
+
+ auto keysToLookup = Build<TCoMap>(ctx, join.Pos())
+ .Input(leftDataDeduplicated)
+ .Lambda()
+ .Args({leftRowArg})
+ .Body<TCoAsStruct>()
+ .Add(lookupMembers)
+ .Build()
+ .Build()
+ .Done();
+
TExprBase lookup = indexName
? BuildLookupIndex(ctx, join.Pos(), read, keysToLookup, lookupNames, indexName)
: BuildLookupTable(ctx, join.Pos(), read, keysToLookup, lookupNames);
-
- // Skip null keys in lookup part as for equijoin semantics null != null,
- // so we can't have nulls in lookup part
- lookup = Build<TCoSkipNullMembers>(ctx, join.Pos())
- .Input(lookup)
- .Members()
- .Add(rightKeyColumns)
- .Build()
- .Done();
-
- if (rightFilterNull) {
- lookup = Build<TCoFilterNullMembers>(ctx, join.Pos())
- .Input(lookup)
- .Members(rightFilterNull.Cast().Members())
- .Done();
- }
-
- if (rightSkipNull) {
- lookup = Build<TCoSkipNullMembers>(ctx, join.Pos())
- .Input(lookup)
- .Members(rightSkipNull.Cast().Members())
- .Done();
- }
-
- if (rightFlatmap) {
- lookup = Build<TCoFlatMap>(ctx, join.Pos())
- .Input(lookup)
- .Lambda(rightFlatmap.Cast().Lambda())
- .Done();
- }
-
- if (join.JoinType().Value() == "RightSemi") {
- auto arg = TCoArgument(ctx.NewArgument(join.Pos(), "row"));
- auto rightLabel = join.RightLabel().Cast<TCoAtom>().Value();
-
+
+ // Skip null keys in lookup part as for equijoin semantics null != null,
+ // so we can't have nulls in lookup part
+ lookup = Build<TCoSkipNullMembers>(ctx, join.Pos())
+ .Input(lookup)
+ .Members()
+ .Add(rightKeyColumns)
+ .Build()
+ .Done();
+
+ if (rightFilterNull) {
+ lookup = Build<TCoFilterNullMembers>(ctx, join.Pos())
+ .Input(lookup)
+ .Members(rightFilterNull.Cast().Members())
+ .Done();
+ }
+
+ if (rightSkipNull) {
+ lookup = Build<TCoSkipNullMembers>(ctx, join.Pos())
+ .Input(lookup)
+ .Members(rightSkipNull.Cast().Members())
+ .Done();
+ }
+
+ if (rightFlatmap) {
+ lookup = Build<TCoFlatMap>(ctx, join.Pos())
+ .Input(lookup)
+ .Lambda(rightFlatmap.Cast().Lambda())
+ .Done();
+ }
+
+ if (join.JoinType().Value() == "RightSemi") {
+ auto arg = TCoArgument(ctx.NewArgument(join.Pos(), "row"));
+ auto rightLabel = join.RightLabel().Cast<TCoAtom>().Value();
+
TVector<TExprBase> renames = CreateRenames(rightFlatmap, read.Columns(), arg, rightLabel, join.Pos(), ctx);
-
- lookup = Build<TCoMap>(ctx, join.Pos())
- .Input(lookup)
- .Lambda()
- .Args({arg})
- .Body<TCoAsStruct>()
- .Add(renames)
- .Build()
- .Build()
- .Done();
-
- return lookup;
- }
-
- return Build<TDqJoin>(ctx, join.Pos())
- .LeftInput(leftData)
- .LeftLabel(join.LeftLabel())
- .RightInput(lookup)
- .RightLabel(join.RightLabel())
- .JoinType(join.JoinType())
- .JoinKeys(join.JoinKeys())
- .Done();
-}
-
-} // anonymous namespace
-
-TExprBase KqpJoinToIndexLookup(const TExprBase& node, TExprContext& ctx, const TKqpOptimizeContext& kqpCtx,
- const NYql::TKikimrConfiguration::TPtr& config)
-{
- if (!kqpCtx.IsDataQuery() || !node.Maybe<TDqJoin>()) {
- return node;
- }
- auto join = node.Cast<TDqJoin>();
-
- DBG("-- Join: " << KqpExprToPrettyString(join, ctx));
-
+
+ lookup = Build<TCoMap>(ctx, join.Pos())
+ .Input(lookup)
+ .Lambda()
+ .Args({arg})
+ .Body<TCoAsStruct>()
+ .Add(renames)
+ .Build()
+ .Build()
+ .Done();
+
+ return lookup;
+ }
+
+ return Build<TDqJoin>(ctx, join.Pos())
+ .LeftInput(leftData)
+ .LeftLabel(join.LeftLabel())
+ .RightInput(lookup)
+ .RightLabel(join.RightLabel())
+ .JoinType(join.JoinType())
+ .JoinKeys(join.JoinKeys())
+ .Done();
+}
+
+} // anonymous namespace
+
+TExprBase KqpJoinToIndexLookup(const TExprBase& node, TExprContext& ctx, const TKqpOptimizeContext& kqpCtx,
+ const NYql::TKikimrConfiguration::TPtr& config)
+{
+ if (!kqpCtx.IsDataQuery() || !node.Maybe<TDqJoin>()) {
+ return node;
+ }
+ auto join = node.Cast<TDqJoin>();
+
+ DBG("-- Join: " << KqpExprToPrettyString(join, ctx));
+
// SqlIn support (preferred lookup direction)
- if (join.JoinType().Value() == "LeftSemi" && !config->HasOptDisableJoinReverseTableLookupLeftSemi()) {
+ if (join.JoinType().Value() == "LeftSemi" && !config->HasOptDisableJoinReverseTableLookupLeftSemi()) {
auto flipJoin = FlipLeftSemiJoin(join, ctx);
- DBG("-- Flip join");
-
+ DBG("-- Flip join");
+
if (auto indexLookupJoin = KqpJoinToIndexLookupImpl(flipJoin, ctx, kqpCtx)) {
- return indexLookupJoin.Cast();
- }
- }
-
+ return indexLookupJoin.Cast();
+ }
+ }
+
if (auto indexLookupJoin = KqpJoinToIndexLookupImpl(join, ctx, kqpCtx)) {
return indexLookupJoin.Cast();
}
- return node;
-}
-
-#undef DBG
-
+ return node;
+}
+
+#undef DBG
+
} // namespace NKikimr::NKqp::NOpt
diff --git a/ydb/core/kqp/opt/logical/kqp_opt_log_rules.h b/ydb/core/kqp/opt/logical/kqp_opt_log_rules.h
index 52448355c9..9e84d83d82 100644
--- a/ydb/core/kqp/opt/logical/kqp_opt_log_rules.h
+++ b/ydb/core/kqp/opt/logical/kqp_opt_log_rules.h
@@ -11,9 +11,9 @@
namespace NKikimr::NKqp::NOpt {
-NYql::NNodes::TExprBase KqpPushPredicateToReadTable(NYql::NNodes::TExprBase node, NYql::TExprContext &ctx,
- const TKqpOptimizeContext &kqpCtx);
-
+NYql::NNodes::TExprBase KqpPushPredicateToReadTable(NYql::NNodes::TExprBase node, NYql::TExprContext &ctx,
+ const TKqpOptimizeContext &kqpCtx);
+
NYql::NNodes::TExprBase KqpPushExtractedPredicateToReadTable(NYql::NNodes::TExprBase node, NYql::TExprContext& ctx,
const TKqpOptimizeContext& kqpCtx, NYql::TTypeAnnotationContext& typesCtx);
@@ -23,12 +23,12 @@ NYql::NNodes::TExprBase KqpApplyExtractMembersToReadOlapTable(NYql::NNodes::TExp
NYql::NNodes::TExprBase KqpApplyExtractMembersToReadTableRanges(NYql::NNodes::TExprBase node, NYql::TExprContext& ctx);
-NYql::NNodes::TExprBase KqpJoinToIndexLookup(const NYql::NNodes::TExprBase& node, NYql::TExprContext& ctx,
- const TKqpOptimizeContext& kqpCtx, const NYql::TKikimrConfiguration::TPtr& config);
-
-NYql::NNodes::TExprBase KqpRewriteSqlInToEquiJoin(const NYql::NNodes::TExprBase& node, NYql::TExprContext& ctx,
- const TKqpOptimizeContext& kqpCtx, const NYql::TKikimrConfiguration::TPtr& config);
-
+NYql::NNodes::TExprBase KqpJoinToIndexLookup(const NYql::NNodes::TExprBase& node, NYql::TExprContext& ctx,
+ const TKqpOptimizeContext& kqpCtx, const NYql::TKikimrConfiguration::TPtr& config);
+
+NYql::NNodes::TExprBase KqpRewriteSqlInToEquiJoin(const NYql::NNodes::TExprBase& node, NYql::TExprContext& ctx,
+ const TKqpOptimizeContext& kqpCtx, const NYql::TKikimrConfiguration::TPtr& config);
+
NYql::NNodes::TExprBase KqpRewriteSqlInCompactToJoin(const NYql::NNodes::TExprBase& node, NYql::TExprContext& ctx);
NYql::NNodes::TExprBase KqpRewriteIndexRead(const NYql::NNodes::TExprBase& node, NYql::TExprContext& ctx,
diff --git a/ydb/core/kqp/opt/logical/kqp_opt_log_sqlin.cpp b/ydb/core/kqp/opt/logical/kqp_opt_log_sqlin.cpp
index 7bd663cb9c..fd5d213001 100644
--- a/ydb/core/kqp/opt/logical/kqp_opt_log_sqlin.cpp
+++ b/ydb/core/kqp/opt/logical/kqp_opt_log_sqlin.cpp
@@ -1,55 +1,55 @@
-#include "kqp_opt_log_rules.h"
-
+#include "kqp_opt_log_rules.h"
+
#include <ydb/core/kqp/opt/kqp_opt_impl.h>
#include <ydb/core/kqp/common/kqp_yql.h>
#include <ydb/core/kqp/provider/yql_kikimr_provider_impl.h>
#include <ydb/core/kqp/provider/yql_kikimr_opt_utils.h>
-
+
#include <ydb/library/yql/core/common_opt/yql_co_sqlin.h>
-
+
namespace NKikimr::NKqp::NOpt {
-
-using namespace NYql;
-using namespace NYql::NDq;
-using namespace NYql::NNodes;
-
-TExprBase KqpRewriteSqlInToEquiJoin(const TExprBase& node, TExprContext& ctx, const TKqpOptimizeContext& kqpCtx,
- const TKikimrConfiguration::TPtr& config)
-{
- if (!kqpCtx.IsDataQuery()) {
- return node;
- }
-
- if (config->HasOptDisableSqlInToJoin()) {
- return node;
- }
-
- if (!node.Maybe<TCoFlatMap>()) {
- return node;
- }
-
- const auto flatMap = node.Cast<TCoFlatMap>();
- const auto lambdaBody = flatMap.Lambda().Body();
-
- // SqlIn expected to be rewritten to (FlatMap <in> (OptionalIf ...)) or (FlatMap <in> (FlatListIf ...))
- if (!lambdaBody.Maybe<TCoOptionalIf>() && !lambdaBody.Maybe<TCoFlatListIf>()) {
- return node;
- }
-
- if (!FindNode(lambdaBody.Ptr(), [](const TExprNode::TPtr& x) { return TCoSqlIn::Match(x.Get()); })) {
- return node;
- }
-
+
+using namespace NYql;
+using namespace NYql::NDq;
+using namespace NYql::NNodes;
+
+TExprBase KqpRewriteSqlInToEquiJoin(const TExprBase& node, TExprContext& ctx, const TKqpOptimizeContext& kqpCtx,
+ const TKikimrConfiguration::TPtr& config)
+{
+ if (!kqpCtx.IsDataQuery()) {
+ return node;
+ }
+
+ if (config->HasOptDisableSqlInToJoin()) {
+ return node;
+ }
+
+ if (!node.Maybe<TCoFlatMap>()) {
+ return node;
+ }
+
+ const auto flatMap = node.Cast<TCoFlatMap>();
+ const auto lambdaBody = flatMap.Lambda().Body();
+
+ // SqlIn expected to be rewritten to (FlatMap <in> (OptionalIf ...)) or (FlatMap <in> (FlatListIf ...))
+ if (!lambdaBody.Maybe<TCoOptionalIf>() && !lambdaBody.Maybe<TCoFlatListIf>()) {
+ return node;
+ }
+
+ if (!FindNode(lambdaBody.Ptr(), [](const TExprNode::TPtr& x) { return TCoSqlIn::Match(x.Get()); })) {
+ return node;
+ }
+
if (!flatMap.Input().Maybe<TKqlReadTable>() && !flatMap.Input().Maybe<TKqlReadTableIndex>()) {
- return node;
- }
-
+ return node;
+ }
+
const auto readTable = flatMap.Input().Cast<TKqlReadTableBase>();
- if (!readTable.Table().SysView().Value().empty()) {
- return node;
- }
-
+ if (!readTable.Table().SysView().Value().empty()) {
+ return node;
+ }
+
TString lookupTable;
if (auto indexRead = flatMap.Input().Maybe<TKqlReadTableIndex>()) {
@@ -58,91 +58,91 @@ TExprBase KqpRewriteSqlInToEquiJoin(const TExprBase& node, TExprContext& ctx, co
lookupTable = readTable.Table().Path().StringValue();
}
- const auto& tableDesc = kqpCtx.Tables->ExistingTable(kqpCtx.Cluster, lookupTable);
- const auto& rangeFrom = readTable.Range().From();
- const auto& rangeTo = readTable.Range().To();
-
- if (!rangeFrom.Maybe<TKqlKeyInc>() || !rangeTo.Maybe<TKqlKeyInc>()) {
- return node;
- }
- if (rangeFrom.Raw() != rangeTo.Raw()) {
- // not point selection
- return node;
- }
-
- i64 keySuffixLen = (i64) tableDesc.Metadata->KeyColumnNames.size() - (i64) rangeFrom.ArgCount();
- if (keySuffixLen <= 0) {
- return node;
- }
-
- TVector<TStringBuf> keys; // remaining key parts, that can be used in SqlIn (only in asc order)
- keys.reserve(keySuffixLen);
- for (ui64 idx = rangeFrom.ArgCount(); idx < tableDesc.Metadata->KeyColumnNames.size(); ++idx) {
- keys.emplace_back(TStringBuf(tableDesc.Metadata->KeyColumnNames[idx]));
- }
-
- auto flatMapLambdaArg = flatMap.Lambda().Args().Arg(0);
-
- auto findMemberIndexInKeys = [&keys](const TCoArgument& flatMapLambdaArg, const TCoMember& member) {
- if (member.Struct().Raw() != flatMapLambdaArg.Raw()) {
- return -1;
- }
- for (size_t i = 0; i < keys.size(); ++i) {
- if (member.Name().Value() == keys[i]) {
- return (int) i;
- }
- }
- return -1;
- };
-
- auto shouldConvertSqlInToJoin = [&](const TCoSqlIn& sqlIn, bool negated) {
- if (negated) {
- // negated can't be rewritten to the index-lookup, so skip it
- return false;
- }
-
- // validate key prefix
- if (sqlIn.Lookup().Maybe<TCoMember>()) {
- if (findMemberIndexInKeys(flatMapLambdaArg, sqlIn.Lookup().Cast<TCoMember>()) != 0) {
- return false;
- }
- } else if (sqlIn.Lookup().Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Tuple) {
- auto children = sqlIn.Lookup().Ref().ChildrenList();
- TVector<int> usedKeyIndexes{Reserve(children.size())};
- for (const auto& itemPtr : children) {
- TExprBase item{itemPtr};
- if (!item.Maybe<TCoMember>()) {
- return false;
- }
- int keyIndex = findMemberIndexInKeys(flatMapLambdaArg, item.Cast<TCoMember>());
- if (keyIndex >= 0) {
- usedKeyIndexes.push_back(keyIndex);
- } else {
- return false;
- }
- }
- if (usedKeyIndexes.empty()) {
- return false;
- }
- ::Sort(usedKeyIndexes);
- for (size_t i = 0; i < usedKeyIndexes.size(); ++i) {
- if (usedKeyIndexes[i] != (int) i) {
- return false;
- }
- }
- } else {
- return false;
- }
-
- return CanRewriteSqlInToEquiJoin(sqlIn.Lookup().Ref().GetTypeAnn(), sqlIn.Collection().Ref().GetTypeAnn());
- };
-
- const bool prefixOnly = true;
- if (auto ret = TryConvertSqlInPredicatesToJoins(flatMap, shouldConvertSqlInToJoin, ctx, prefixOnly)) {
- return TExprBase(ret);
- }
-
- return node;
-}
-
+ const auto& tableDesc = kqpCtx.Tables->ExistingTable(kqpCtx.Cluster, lookupTable);
+ const auto& rangeFrom = readTable.Range().From();
+ const auto& rangeTo = readTable.Range().To();
+
+ if (!rangeFrom.Maybe<TKqlKeyInc>() || !rangeTo.Maybe<TKqlKeyInc>()) {
+ return node;
+ }
+ if (rangeFrom.Raw() != rangeTo.Raw()) {
+ // not point selection
+ return node;
+ }
+
+ i64 keySuffixLen = (i64) tableDesc.Metadata->KeyColumnNames.size() - (i64) rangeFrom.ArgCount();
+ if (keySuffixLen <= 0) {
+ return node;
+ }
+
+ TVector<TStringBuf> keys; // remaining key parts, that can be used in SqlIn (only in asc order)
+ keys.reserve(keySuffixLen);
+ for (ui64 idx = rangeFrom.ArgCount(); idx < tableDesc.Metadata->KeyColumnNames.size(); ++idx) {
+ keys.emplace_back(TStringBuf(tableDesc.Metadata->KeyColumnNames[idx]));
+ }
+
+ auto flatMapLambdaArg = flatMap.Lambda().Args().Arg(0);
+
+ auto findMemberIndexInKeys = [&keys](const TCoArgument& flatMapLambdaArg, const TCoMember& member) {
+ if (member.Struct().Raw() != flatMapLambdaArg.Raw()) {
+ return -1;
+ }
+ for (size_t i = 0; i < keys.size(); ++i) {
+ if (member.Name().Value() == keys[i]) {
+ return (int) i;
+ }
+ }
+ return -1;
+ };
+
+ auto shouldConvertSqlInToJoin = [&](const TCoSqlIn& sqlIn, bool negated) {
+ if (negated) {
+ // negated can't be rewritten to the index-lookup, so skip it
+ return false;
+ }
+
+ // validate key prefix
+ if (sqlIn.Lookup().Maybe<TCoMember>()) {
+ if (findMemberIndexInKeys(flatMapLambdaArg, sqlIn.Lookup().Cast<TCoMember>()) != 0) {
+ return false;
+ }
+ } else if (sqlIn.Lookup().Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Tuple) {
+ auto children = sqlIn.Lookup().Ref().ChildrenList();
+ TVector<int> usedKeyIndexes{Reserve(children.size())};
+ for (const auto& itemPtr : children) {
+ TExprBase item{itemPtr};
+ if (!item.Maybe<TCoMember>()) {
+ return false;
+ }
+ int keyIndex = findMemberIndexInKeys(flatMapLambdaArg, item.Cast<TCoMember>());
+ if (keyIndex >= 0) {
+ usedKeyIndexes.push_back(keyIndex);
+ } else {
+ return false;
+ }
+ }
+ if (usedKeyIndexes.empty()) {
+ return false;
+ }
+ ::Sort(usedKeyIndexes);
+ for (size_t i = 0; i < usedKeyIndexes.size(); ++i) {
+ if (usedKeyIndexes[i] != (int) i) {
+ return false;
+ }
+ }
+ } else {
+ return false;
+ }
+
+ return CanRewriteSqlInToEquiJoin(sqlIn.Lookup().Ref().GetTypeAnn(), sqlIn.Collection().Ref().GetTypeAnn());
+ };
+
+ const bool prefixOnly = true;
+ if (auto ret = TryConvertSqlInPredicatesToJoins(flatMap, shouldConvertSqlInToJoin, ctx, prefixOnly)) {
+ return TExprBase(ret);
+ }
+
+ return node;
+}
+
} // namespace NKikimr::NKqp::NOpt
diff --git a/ydb/core/kqp/opt/logical/ya.make b/ydb/core/kqp/opt/logical/ya.make
index 516d2f607f..b9811606c1 100644
--- a/ydb/core/kqp/opt/logical/ya.make
+++ b/ydb/core/kqp/opt/logical/ya.make
@@ -8,11 +8,11 @@ OWNER(
SRCS(
kqp_opt_log_effects.cpp
kqp_opt_log_extract.cpp
- kqp_opt_log_join.cpp
+ kqp_opt_log_join.cpp
kqp_opt_log_indexes.cpp
kqp_opt_log_ranges.cpp
kqp_opt_log_ranges_predext.cpp
- kqp_opt_log_sqlin.cpp
+ kqp_opt_log_sqlin.cpp
kqp_opt_log_sqlin_compact.cpp
kqp_opt_log.cpp
)
diff --git a/ydb/core/kqp/opt/physical/kqp_opt_phy.cpp b/ydb/core/kqp/opt/physical/kqp_opt_phy.cpp
index 31bb49e174..11e1af7ae8 100644
--- a/ydb/core/kqp/opt/physical/kqp_opt_phy.cpp
+++ b/ydb/core/kqp/opt/physical/kqp_opt_phy.cpp
@@ -118,7 +118,7 @@ protected:
TMaybeNode<TExprBase> PushSkipNullMembersToStage(TExprBase node, TExprContext& ctx,
IOptimizationContext& optCtx, const TGetParents& getParents)
{
- TExprBase output = DqPushSkipNullMembersToStage(node, ctx, optCtx, *getParents(), IsGlobal);
+ TExprBase output = DqPushSkipNullMembersToStage(node, ctx, optCtx, *getParents(), IsGlobal);
DumpAppliedRule("PushSkipNullMembersToStage", node.Ptr(), output.Ptr(), ctx);
return output;
}
@@ -127,7 +127,7 @@ protected:
TMaybeNode<TExprBase> PushExtractMembersToStage(TExprBase node, TExprContext& ctx,
IOptimizationContext& optCtx, const TGetParents& getParents)
{
- TExprBase output = DqPushExtractMembersToStage(node, ctx, optCtx, *getParents(), IsGlobal);
+ TExprBase output = DqPushExtractMembersToStage(node, ctx, optCtx, *getParents(), IsGlobal);
DumpAppliedRule("PushExtractMembersToStage", node.Ptr(), output.Ptr(), ctx);
return output;
}
@@ -136,7 +136,7 @@ protected:
TMaybeNode<TExprBase> BuildFlatmapStage(TExprBase node, TExprContext& ctx,
IOptimizationContext& optCtx, const TGetParents& getParents)
{
- TExprBase output = DqBuildFlatmapStage(node, ctx, optCtx, *getParents(), IsGlobal);
+ TExprBase output = DqBuildFlatmapStage(node, ctx, optCtx, *getParents(), IsGlobal);
DumpAppliedRule("BuildFlatmapStage", node.Ptr(), output.Ptr(), ctx);
return output;
}
@@ -145,7 +145,7 @@ protected:
TMaybeNode<TExprBase> PushCombineToStage(TExprBase node, TExprContext& ctx,
IOptimizationContext& optCtx, const TGetParents& getParents)
{
- TExprBase output = DqPushCombineToStage(node, ctx, optCtx, *getParents(), IsGlobal);
+ TExprBase output = DqPushCombineToStage(node, ctx, optCtx, *getParents(), IsGlobal);
DumpAppliedRule("PushCombineToStage", node.Ptr(), output.Ptr(), ctx);
return output;
}
@@ -166,7 +166,7 @@ protected:
TMaybeNode<TExprBase> BuildTopSortStage(TExprBase node, TExprContext& ctx,
IOptimizationContext& optCtx, const TGetParents& getParents)
{
- TExprBase output = DqBuildTopSortStage(node, ctx, optCtx, *getParents(), IsGlobal);
+ TExprBase output = DqBuildTopSortStage(node, ctx, optCtx, *getParents(), IsGlobal);
DumpAppliedRule("BuildTopSortStage", node.Ptr(), output.Ptr(), ctx);
return output;
}
@@ -175,7 +175,7 @@ protected:
TMaybeNode<TExprBase> BuildTakeSkipStage(TExprBase node, TExprContext& ctx,
IOptimizationContext& optCtx, const TGetParents& getParents)
{
- TExprBase output = DqBuildTakeSkipStage(node, ctx, optCtx, *getParents(), IsGlobal);
+ TExprBase output = DqBuildTakeSkipStage(node, ctx, optCtx, *getParents(), IsGlobal);
DumpAppliedRule("BuildTakeSkipStage", node.Ptr(), output.Ptr(), ctx);
return output;
}
@@ -184,7 +184,7 @@ protected:
TMaybeNode<TExprBase> BuildSortStage(TExprBase node, TExprContext& ctx,
IOptimizationContext& optCtx, const TGetParents& getParents)
{
- TExprBase output = DqBuildSortStage(node, ctx, optCtx, *getParents(), IsGlobal);
+ TExprBase output = DqBuildSortStage(node, ctx, optCtx, *getParents(), IsGlobal);
DumpAppliedRule("BuildSortStage", node.Ptr(), output.Ptr(), ctx);
return output;
}
@@ -193,7 +193,7 @@ protected:
TMaybeNode<TExprBase> BuildTakeStage(TExprBase node, TExprContext& ctx,
IOptimizationContext& optCtx, const TGetParents& getParents)
{
- TExprBase output = DqBuildTakeStage(node, ctx, optCtx, *getParents(), IsGlobal);
+ TExprBase output = DqBuildTakeStage(node, ctx, optCtx, *getParents(), IsGlobal);
DumpAppliedRule("BuildTakeStage", node.Ptr(), output.Ptr(), ctx);
return output;
}
@@ -220,7 +220,7 @@ protected:
TMaybeNode<TExprBase> PushJoinToStage(TExprBase node, TExprContext& ctx,
IOptimizationContext& optCtx, const TGetParents& getParents)
{
- TExprBase output = DqPushJoinToStage(node, ctx, optCtx, *getParents(), IsGlobal);
+ TExprBase output = DqPushJoinToStage(node, ctx, optCtx, *getParents(), IsGlobal);
DumpAppliedRule("PushJoinToStage", node.Ptr(), output.Ptr(), ctx);
return output;
}
@@ -244,7 +244,7 @@ protected:
TMaybeNode<TExprBase> PushLMapToStage(TExprBase node, TExprContext& ctx,
IOptimizationContext& optCtx, const TGetParents& getParents)
{
- TExprBase output = DqPushLMapToStage(node, ctx, optCtx, *getParents(), IsGlobal);
+ TExprBase output = DqPushLMapToStage(node, ctx, optCtx, *getParents(), IsGlobal);
DumpAppliedRule("PushLMapToStage", node.Ptr(), output.Ptr(), ctx);
return output;
}
@@ -253,7 +253,7 @@ protected:
TMaybeNode<TExprBase> PushOrderedLMapToStage(TExprBase node, TExprContext& ctx,
IOptimizationContext& optCtx, const TGetParents& getParents)
{
- TExprBase output = DqPushOrderedLMapToStage(node, ctx, optCtx, *getParents(), IsGlobal);
+ TExprBase output = DqPushOrderedLMapToStage(node, ctx, optCtx, *getParents(), IsGlobal);
DumpAppliedRule("PushOrderedLMapToStage", node.Ptr(), output.Ptr(), ctx);
return output;
}
@@ -300,14 +300,14 @@ protected:
return output;
}
- TMaybeNode<TExprBase> BuildHasItems(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx) {
- TExprBase output = DqBuildHasItems(node, ctx, optCtx);
+ TMaybeNode<TExprBase> BuildHasItems(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx) {
+ TExprBase output = DqBuildHasItems(node, ctx, optCtx);
DumpAppliedRule("DqBuildHasItems", node.Ptr(), output.Ptr(), ctx);
return output;
}
- TMaybeNode<TExprBase> BuildScalarPrecompute(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx) {
- TExprBase output = DqBuildScalarPrecompute(node, ctx, optCtx);
+ TMaybeNode<TExprBase> BuildScalarPrecompute(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx) {
+ TExprBase output = DqBuildScalarPrecompute(node, ctx, optCtx);
DumpAppliedRule("BuildScalarPrecompute", node.Ptr(), output.Ptr(), ctx);
return output;
}
diff --git a/ydb/core/kqp/opt/physical/kqp_opt_phy_build_stage.cpp b/ydb/core/kqp/opt/physical/kqp_opt_phy_build_stage.cpp
index 99fb89da95..f49aff031f 100644
--- a/ydb/core/kqp/opt/physical/kqp_opt_phy_build_stage.cpp
+++ b/ydb/core/kqp/opt/physical/kqp_opt_phy_build_stage.cpp
@@ -7,7 +7,7 @@
#include <ydb/core/tx/schemeshard/schemeshard_utils.h>
#include <ydb/public/lib/scheme_types/scheme_type_id.h>
-
+
#include <ydb/library/yql/dq/opt/dq_opt.h>
#include <ydb/library/yql/core/yql_opt_utils.h>
@@ -95,7 +95,7 @@ TExprBase KqpBuildReadTableStage(TExprBase node, TExprContext& ctx, const TKqpOp
.Build()
.Program()
.Args({})
- .Body<TCoToStream>()
+ .Body<TCoToStream>()
.Input<TCoJust>()
.Input<TExprList>()
.Add(values)
@@ -195,7 +195,7 @@ TExprBase KqpBuildReadTableRangesStage(TExprBase node, TExprContext& ctx,
.Build()
.Program()
.Args({})
- .Body<TCoToStream>()
+ .Body<TCoToStream>()
.Input<TCoJust>()
.Input<TExprList>()
.Add(ranges)
@@ -278,39 +278,39 @@ TExprBase KqpBuildReadTableRangesStage(TExprBase node, TExprContext& ctx,
.Done();
}
-bool RequireLookupPrecomputeStage(const TKqlLookupTable& lookup) {
- if (!lookup.LookupKeys().Maybe<TCoAsList>()) {
- return true;
- }
- auto asList = lookup.LookupKeys().Cast<TCoAsList>();
-
- for (auto row : asList) {
- if (auto maybeAsStruct = row.Maybe<TCoAsStruct>()) {
- auto asStruct = maybeAsStruct.Cast();
- for (auto item : asStruct) {
- auto tuple = item.Cast<TCoNameValueTuple>();
- if (tuple.Value().Maybe<TCoParameter>()) {
- // pass
- } else if (tuple.Value().Maybe<TCoDataCtor>()) {
- auto slot = tuple.Value().Ref().GetTypeAnn()->Cast<TDataExprType>()->GetSlot();
- auto typeId = NUdf::GetDataTypeInfo(slot).TypeId;
- if (NScheme::NTypeIds::IsYqlType(typeId) && NSchemeShard::IsAllowedKeyType(typeId)) {
- // pass
- } else {
- return true;
- }
- } else {
- return true;
- }
- }
- } else {
- return true;
- }
- }
-
- return false;
-}
-
+bool RequireLookupPrecomputeStage(const TKqlLookupTable& lookup) {
+ if (!lookup.LookupKeys().Maybe<TCoAsList>()) {
+ return true;
+ }
+ auto asList = lookup.LookupKeys().Cast<TCoAsList>();
+
+ for (auto row : asList) {
+ if (auto maybeAsStruct = row.Maybe<TCoAsStruct>()) {
+ auto asStruct = maybeAsStruct.Cast();
+ for (auto item : asStruct) {
+ auto tuple = item.Cast<TCoNameValueTuple>();
+ if (tuple.Value().Maybe<TCoParameter>()) {
+ // pass
+ } else if (tuple.Value().Maybe<TCoDataCtor>()) {
+ auto slot = tuple.Value().Ref().GetTypeAnn()->Cast<TDataExprType>()->GetSlot();
+ auto typeId = NUdf::GetDataTypeInfo(slot).TypeId;
+ if (NScheme::NTypeIds::IsYqlType(typeId) && NSchemeShard::IsAllowedKeyType(typeId)) {
+ // pass
+ } else {
+ return true;
+ }
+ } else {
+ return true;
+ }
+ }
+ } else {
+ return true;
+ }
+ }
+
+ return false;
+}
+
TExprBase KqpBuildLookupTableStage(TExprBase node, TExprContext& ctx) {
if (!node.Maybe<TKqlLookupTable>()) {
return node;
@@ -319,54 +319,54 @@ TExprBase KqpBuildLookupTableStage(TExprBase node, TExprContext& ctx) {
YQL_ENSURE(lookup.CallableName() == TKqlLookupTable::CallableName());
- TMaybeNode<TDqStage> stage;
-
- if (!RequireLookupPrecomputeStage(lookup)) {
- stage = Build<TDqStage>(ctx, lookup.Pos())
- .Inputs()
- .Build()
- .Program()
- .Args({})
- .Body<TKqpLookupTable>()
- .Table(lookup.Table())
- .LookupKeys<TCoIterator>()
- .List(lookup.LookupKeys())
- .Build()
- .Columns(lookup.Columns())
- .Build()
+ TMaybeNode<TDqStage> stage;
+
+ if (!RequireLookupPrecomputeStage(lookup)) {
+ stage = Build<TDqStage>(ctx, lookup.Pos())
+ .Inputs()
+ .Build()
+ .Program()
+ .Args({})
+ .Body<TKqpLookupTable>()
+ .Table(lookup.Table())
+ .LookupKeys<TCoIterator>()
+ .List(lookup.LookupKeys())
+ .Build()
+ .Columns(lookup.Columns())
+ .Build()
+ .Build()
+ .Settings().Build()
+ .Done();
+ } else {
+ auto precompute = BuildLookupKeysPrecompute(lookup.LookupKeys(), ctx);
+ if (!precompute) {
+ return node;
+ }
+
+ stage = Build<TDqStage>(ctx, lookup.Pos())
+ .Inputs()
+ .Add(precompute.Cast())
.Build()
- .Settings().Build()
- .Done();
- } else {
- auto precompute = BuildLookupKeysPrecompute(lookup.LookupKeys(), ctx);
- if (!precompute) {
- return node;
- }
-
- stage = Build<TDqStage>(ctx, lookup.Pos())
- .Inputs()
- .Add(precompute.Cast())
- .Build()
- .Program()
- .Args({"keys_arg"})
- .Body<TKqpLookupTable>()
- .Table(lookup.Table())
- .LookupKeys<TCoIterator>()
- .List("keys_arg")
- .Build()
- .Columns(lookup.Columns())
- .Build()
- .Build()
- .Settings().Build()
- .Done();
- }
-
+ .Program()
+ .Args({"keys_arg"})
+ .Body<TKqpLookupTable>()
+ .Table(lookup.Table())
+ .LookupKeys<TCoIterator>()
+ .List("keys_arg")
+ .Build()
+ .Columns(lookup.Columns())
+ .Build()
+ .Build()
+ .Settings().Build()
+ .Done();
+ }
+
return Build<TDqCnUnionAll>(ctx, lookup.Pos())
.Output()
- .Stage(stage.Cast())
+ .Stage(stage.Cast())
.Index().Build("0")
.Build()
.Done();
}
-
+
} // namespace NKikimr::NKqp::NOpt
diff --git a/ydb/core/kqp/opt/ya.make b/ydb/core/kqp/opt/ya.make
index 382a8acbd5..6287365b19 100644
--- a/ydb/core/kqp/opt/ya.make
+++ b/ydb/core/kqp/opt/ya.make
@@ -6,14 +6,14 @@ OWNER(
)
SRCS(
- kqp_opt.cpp
- kqp_opt_build_txs.cpp
+ kqp_opt.cpp
+ kqp_opt_build_txs.cpp
kqp_opt_effects.cpp
kqp_opt_join.cpp
kqp_opt_kql.cpp
kqp_opt_phase.cpp
- kqp_opt_phy_check.cpp
- kqp_opt_phy_finalize.cpp
+ kqp_opt_phy_check.cpp
+ kqp_opt_phy_finalize.cpp
)
PEERDIR(
diff --git a/ydb/core/kqp/prepare/kqp_prepare.h b/ydb/core/kqp/prepare/kqp_prepare.h
index fff13470f1..d114a44945 100644
--- a/ydb/core/kqp/prepare/kqp_prepare.h
+++ b/ydb/core/kqp/prepare/kqp_prepare.h
@@ -88,8 +88,8 @@ struct TKqlTransformContext : TThrRefBase {
NActors::TActorId ReplyTarget;
TKqpAnalyzeResults AnalyzeResults;
- NKikimrKqp::TPreparedKql* PreparingKql = nullptr;
- const NKikimrKqp::TPreparedKql* PreparedKql;
+ NKikimrKqp::TPreparedKql* PreparingKql = nullptr;
+ const NKikimrKqp::TPreparedKql* PreparedKql;
NKqpProto::TKqpStatsQuery QueryStats;
const NKqpProto::TKqpPhyQuery* PhysicalQuery;
@@ -109,12 +109,12 @@ struct TKqlTransformContext : TThrRefBase {
YQL_ENSURE(false, "Unexpected value for CommitSafety.");
}
- NKikimrKqp::TPreparedKql& GetPreparingKql() {
- YQL_ENSURE(PreparingKql);
- return *PreparingKql;
- }
-
- const NKikimrKqp::TPreparedKql& GetPreparedKql() {
+ NKikimrKqp::TPreparedKql& GetPreparingKql() {
+ YQL_ENSURE(PreparingKql);
+ return *PreparingKql;
+ }
+
+ const NKikimrKqp::TPreparedKql& GetPreparedKql() {
YQL_ENSURE(PreparedKql);
return *PreparedKql;
}
@@ -128,7 +128,7 @@ struct TKqlTransformContext : TThrRefBase {
AnalyzeResults = {};
MkqlResults.clear();
QueryStats = {};
- PreparingKql = nullptr;
+ PreparingKql = nullptr;
PreparedKql = nullptr;
PhysicalQuery = nullptr;
PhysicalQueryResults.clear();
@@ -189,5 +189,5 @@ TAutoPtr<NYql::IGraphTransformer> CreateKqpTypeAnnotationTransformer(const TStri
TAutoPtr<NYql::IGraphTransformer> CreateKqpCheckKiProgramTransformer();
TAutoPtr<NYql::IGraphTransformer> CreateKqpCheckQueryTransformer();
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/prepare/kqp_prepare_impl.h b/ydb/core/kqp/prepare/kqp_prepare_impl.h
index 2fb9f2ff13..1d6e4c5064 100644
--- a/ydb/core/kqp/prepare/kqp_prepare_impl.h
+++ b/ydb/core/kqp/prepare/kqp_prepare_impl.h
@@ -32,5 +32,5 @@ TMkqlExecuteResult ExecuteMkql(NYql::NNodes::TKiProgram program, TIntrusivePtr<I
void LogMkqlResult(const NKikimrMiniKQL::TResult& result, NYql::TExprContext& ctx);
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/prepare/kqp_query_analyze.cpp b/ydb/core/kqp/prepare/kqp_query_analyze.cpp
index 2a24ce18f1..b37e4eb058 100644
--- a/ydb/core/kqp/prepare/kqp_query_analyze.cpp
+++ b/ydb/core/kqp/prepare/kqp_query_analyze.cpp
@@ -562,5 +562,5 @@ TAutoPtr<IGraphTransformer> CreateKqpAnalyzeTransformer(TIntrusivePtr<TKqlTransf
return new TKqpAnalyzeTransformer(transformCtx);
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/prepare/kqp_query_exec.cpp b/ydb/core/kqp/prepare/kqp_query_exec.cpp
index 30cb7b048a..bc2d4e94ce 100644
--- a/ydb/core/kqp/prepare/kqp_query_exec.cpp
+++ b/ydb/core/kqp/prepare/kqp_query_exec.cpp
@@ -272,7 +272,7 @@ public:
}
if (TransformCtx->Settings.GetCommitTx() && !TransformCtx->QueryCtx->PrepareOnly) {
- if (TxState->Tx().DeferredEffects.Empty()) {
+ if (TxState->Tx().DeferredEffects.Empty()) {
// Merge read-only query with commit tx
return TStatus::Ok;
}
@@ -346,8 +346,8 @@ public:
TransformCtx->MkqlResults.push_back(mkqlResult);
if (TransformCtx->QueryCtx->PrepareOnly) {
- YQL_ENSURE(!TransformCtx->GetPreparingKql().GetMkqls().empty());
- auto& mkql = *TransformCtx->GetPreparingKql().MutableMkqls()->rbegin();
+ YQL_ENSURE(!TransformCtx->GetPreparingKql().GetMkqls().empty());
+ auto& mkql = *TransformCtx->GetPreparingKql().MutableMkqls()->rbegin();
mkql.SetProgram(result.CompiledProgram);
mkql.SetProgramText(MkqlExecuteResult.Program);
} else {
@@ -386,12 +386,12 @@ private:
void ExtractQueryStats(NKqpProto::TKqpStatsQuery& dst, const NKikimrQueryStats::TTxStats& txStats) {
auto& dstExec = *dst.AddExecutions();
- NKqpProto::TKqpExecutionExtraStats executionExtraStats;
+ NKqpProto::TKqpExecutionExtraStats executionExtraStats;
- dstExec.SetDurationUs(txStats.GetDurationUs());
- dstExec.SetCpuTimeUs(txStats.GetComputeCpuTimeUsec());
-
- auto& dstComputeTime = *executionExtraStats.MutableComputeCpuTimeUs();
+ dstExec.SetDurationUs(txStats.GetDurationUs());
+ dstExec.SetCpuTimeUs(txStats.GetComputeCpuTimeUsec());
+
+ auto& dstComputeTime = *executionExtraStats.MutableComputeCpuTimeUs();
dstComputeTime.SetMin(txStats.GetComputeCpuTimeUsec());
dstComputeTime.SetMax(txStats.GetComputeCpuTimeUsec());
dstComputeTime.SetSum(txStats.GetComputeCpuTimeUsec());
@@ -416,7 +416,7 @@ void ExtractQueryStats(NKqpProto::TKqpStatsQuery& dst, const NKikimrQueryStats::
++cnt;
}
if (cnt) {
- auto& dstShardTime = *executionExtraStats.MutableShardsCpuTimeUs();
+ auto& dstShardTime = *executionExtraStats.MutableShardsCpuTimeUs();
dstShardTime.SetMin(minCpu);
dstShardTime.SetMax(maxCpu);
dstShardTime.SetSum(sumCpu);
@@ -425,8 +425,8 @@ void ExtractQueryStats(NKqpProto::TKqpStatsQuery& dst, const NKikimrQueryStats::
dst.SetReadSetsCount(dst.GetReadSetsCount() + sumReadSets);
dst.SetMaxShardProgramSize(Max(dst.GetMaxShardProgramSize(), maxProgramSize));
dst.SetMaxShardReplySize(Max(dst.GetMaxShardReplySize(), maxReplySize));
-
- dstExec.SetCpuTimeUs(dstExec.GetCpuTimeUs() + sumCpu);
+
+ dstExec.SetCpuTimeUs(dstExec.GetCpuTimeUs() + sumCpu);
}
}
@@ -439,16 +439,16 @@ void ExtractQueryStats(NKqpProto::TKqpStatsQuery& dst, const NKikimrQueryStats::
dstTable.SetWriteRows(table.GetUpdateRow().GetRows());
dstTable.SetWriteBytes(table.GetUpdateRow().GetBytes());
dstTable.SetEraseRows(table.GetEraseRow().GetRows());
- dstTable.SetAffectedPartitions(table.GetShardCount());
+ dstTable.SetAffectedPartitions(table.GetShardCount());
// NOTE: This might be incorrect in case when single shard has several
// tables, i.e. collocated tables.
affectedShards += table.GetShardCount();
}
- executionExtraStats.SetAffectedShards(affectedShards);
-
- dstExec.MutableExtra()->PackFrom(executionExtraStats);
+ executionExtraStats.SetAffectedShards(affectedShards);
+
+ dstExec.MutableExtra()->PackFrom(executionExtraStats);
}
} // namespace
@@ -499,7 +499,7 @@ TMkqlExecuteResult ExecuteMkql(TKiProgram program, TIntrusivePtr<IKqpGateway> ga
if (transformCtx->QueryCtx->PrepareOnly) {
YQL_CLOG(INFO, ProviderKqp) << "Preparing MiniKQL program:" << Endl << mkqlProgramText;
- auto& mkql = *transformCtx->GetPreparingKql().AddMkqls();
+ auto& mkql = *transformCtx->GetPreparingKql().AddMkqls();
for (auto& binding : paramBindings) {
mkql.AddBindings()->CopyFrom(binding);
}
@@ -536,7 +536,7 @@ bool AddDeferredEffect(NNodes::TExprBase effect, const TVector<NKikimrKqp::TPara
TExprContext& ctx, TKqpTransactionState& txState, TKqlTransformContext& transformCtx, bool preserveParamValues)
{
if (transformCtx.QueryCtx->PrepareOnly) {
- auto& newEffect = *transformCtx.GetPreparingKql().AddEffects();
+ auto& newEffect = *transformCtx.GetPreparingKql().AddEffects();
newEffect.SetNodeAst(NCommon::SerializeExpr(ctx, effect.Ref()));
for (auto& binding : bindings) {
newEffect.AddBindings()->CopyFrom(binding);
@@ -558,10 +558,10 @@ bool AddDeferredEffect(NNodes::TExprBase effect, const TVector<NKikimrKqp::TPara
effect = PreserveParams(effect, bindingsMap, ctx, txState, transformCtx, preserveParamValues);
}
- bool added = txState.Tx().AddDeferredEffect(effect);
- YQL_ENSURE(added, "Cannot execute new- and old- execution engine queries in the same transaction");
+ bool added = txState.Tx().AddDeferredEffect(effect);
+ YQL_ENSURE(added, "Cannot execute new- and old- execution engine queries in the same transaction");
- YQL_CLOG(INFO, ProviderKqp) << "Adding deferred effect, total " << txState.Tx().DeferredEffects.Size() << ": "
+ YQL_CLOG(INFO, ProviderKqp) << "Adding deferred effect, total " << txState.Tx().DeferredEffects.Size() << ": "
<< Endl << KqpExprToPrettyString(effect, ctx);
return true;
}
@@ -840,5 +840,5 @@ TAutoPtr<IGraphTransformer> CreateKqpExecTransformer(TIntrusivePtr<IKqpGateway>
return new TKqpExecTransformer(gateway, cluster, txState, transformCtx);
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/prepare/kqp_query_finalize.cpp b/ydb/core/kqp/prepare/kqp_query_finalize.cpp
index 64e29a9c47..5e442780f7 100644
--- a/ydb/core/kqp/prepare/kqp_query_finalize.cpp
+++ b/ydb/core/kqp/prepare/kqp_query_finalize.cpp
@@ -23,14 +23,14 @@ const TStringBuf LocksTableVersion = "0";
const TString LocksTablePathId = TKikimrPathId(TSysTables::SysSchemeShard, TSysTables::SysTableLocks2).ToString();
const ui64 LocksInvalidatedCount = 1;
-TExprBase GetDeferredEffectsList(const TDeferredEffects& effects, TPositionHandle pos, TExprContext& ctx) {
- if (effects.Empty()) {
+TExprBase GetDeferredEffectsList(const TDeferredEffects& effects, TPositionHandle pos, TExprContext& ctx) {
+ if (effects.Empty()) {
return GetEmptyEffectsList(pos, ctx);
}
TVector<TExprBase> effectNodes;
- effectNodes.reserve(effects.Size());
- for (const auto& effect : effects) {
+ effectNodes.reserve(effects.Size());
+ for (const auto& effect : effects) {
YQL_ENSURE(effect.Params.empty());
YQL_ENSURE(effect.Node);
effectNodes.push_back(effect.Node.Cast());
@@ -128,7 +128,7 @@ public:
YQL_ENSURE(!settings.GetCommitTx());
YQL_CLOG(INFO, ProviderKqp) << "Rollback Tx"
- << ", deferred effects count: " << TxState->Tx().DeferredEffects.Size()
+ << ", deferred effects count: " << TxState->Tx().DeferredEffects.Size()
<< ", locks count: " << TxState->Tx().Locks.Size();
effectsNode = GetRollbackEffects(input->Pos(), ctx);
@@ -154,7 +154,7 @@ public:
}
YQL_CLOG(INFO, ProviderKqp) << "Commit Tx"
- << ", deferred effects count: " << TxState->Tx().DeferredEffects.Size()
+ << ", deferred effects count: " << TxState->Tx().DeferredEffects.Size()
<< ", locks count: " << TxState->Tx().Locks.Size();
State = EFinalizeState::CommitInProgress;
@@ -290,7 +290,7 @@ private:
YQL_ENSURE(State == EFinalizeState::Initial);
YQL_CLOG(INFO, ProviderKqp) << "Rollback Tx On Error"
- << ", deferred effects count: " << TxState->Tx().DeferredEffects.Size()
+ << ", deferred effects count: " << TxState->Tx().DeferredEffects.Size()
<< ", locks count: " << TxState->Tx().Locks.Size();
auto program = Build<TKiProgram>(ctx, TPositionHandle())
@@ -399,12 +399,12 @@ private:
return true;
}
- void ResetTxState(bool committed) {
- if (!committed) {
+ void ResetTxState(bool committed) {
+ if (!committed) {
TxState->Tx().Invalidate();
}
- TxState->Tx().ClearDeferredEffects();
+ TxState->Tx().ClearDeferredEffects();
TxState->Tx().Locks.Clear();
TxState->Tx().Finish();
}
@@ -427,7 +427,7 @@ private:
}
TExprBase GetCommitEffects(TPositionHandle pos, TExprContext& ctx, bool& hasDataEffects) {
- hasDataEffects = !TxState->Tx().DeferredEffects.Empty();
+ hasDataEffects = !TxState->Tx().DeferredEffects.Empty();
Y_VERIFY_DEBUG(!hasDataEffects || !TxState->Tx().Locks.Broken());
@@ -634,5 +634,5 @@ TAutoPtr<IGraphTransformer> CreateKqpFinalizeTransformer(TIntrusivePtr<IKqpGatew
return new TKqpFinalizeTransformer(gateway, cluster, txState, transformCtx);
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/prepare/kqp_query_plan.cpp b/ydb/core/kqp/prepare/kqp_query_plan.cpp
index e732395bab..346a8b15cb 100644
--- a/ydb/core/kqp/prepare/kqp_query_plan.cpp
+++ b/ydb/core/kqp/prepare/kqp_query_plan.cpp
@@ -3,7 +3,7 @@
#include <ydb/core/kqp/common/kqp_yql.h>
#include <ydb/core/kqp/provider/yql_kikimr_provider_impl.h>
#include <ydb/public/lib/value/value.h>
-
+
#include <ydb/library/yql/ast/yql_ast_escaping.h>
#include <ydb/library/yql/core/yql_expr_optimize.h>
#include <ydb/library/yql/core/yql_opt_utils.h>
@@ -33,9 +33,9 @@ struct TTableRead {
ETableReadType Type = ETableReadType::Unspecified;
TVector<TString> LookupBy;
TVector<TString> ScanBy;
- TVector<TString> Columns;
+ TVector<TString> Columns;
TMaybe<TString> Limit;
- bool Reverse = false;
+ bool Reverse = false;
};
struct TTableWrite {
@@ -195,11 +195,11 @@ void FillTablesInfo(const TExprNode::TPtr& query, TMap<TString, TTableInfo>& tab
read.Limit = GetExprStr(TCoNameValueTuple(limitSetting).Value().Cast());
}
- auto reverseSettings = GetSetting(selectRange.Settings().Ref(), "Reverse");
- if (reverseSettings) {
- read.Reverse = true;
- }
-
+ auto reverseSettings = GetSetting(selectRange.Settings().Ref(), "Reverse");
+ if (reverseSettings) {
+ read.Reverse = true;
+ }
+
for (const auto& column : selectRange.Select()) {
read.Columns.emplace_back(column);
}
@@ -658,15 +658,15 @@ private:
planNode.CteRefName = *cteNode.CteName;
return;
- }
-
+ }
+
auto& stagePlanNode = AddPlanNode(planNode);
stagePlanNode.Guid = stageGuid;
SerializerCtx.StageGuidToId[stageGuid] = SerializerCtx.PlanNodeId;
VisitedStages.insert(expr.Raw());
auto node = expr.Cast<TDqStageBase>().Program().Body().Ptr();
Visit(node, stagePlanNode);
-
+
/* is that collect stage? */
if (stagePlanNode.TypeName.Empty()) {
if (expr.Cast<TDqStageBase>().Program().Body().Maybe<TCoArgument>()) {
@@ -683,11 +683,11 @@ private:
inputPlanNode.Type = EPlanNodeType::Connection;
if (inputCn.Maybe<TDqCnUnionAll>()) {
- inputPlanNode.TypeName = "UnionAll";
+ inputPlanNode.TypeName = "UnionAll";
} else if (inputCn.Maybe<TDqCnBroadcast>()) {
- inputPlanNode.TypeName = "Broadcast";
+ inputPlanNode.TypeName = "Broadcast";
} else if (auto hashShuffle = inputCn.Maybe<TDqCnHashShuffle>()) {
- inputPlanNode.TypeName = "HashShuffle";
+ inputPlanNode.TypeName = "HashShuffle";
auto& keyColumns = inputPlanNode.NodeInfo["KeyColumns"];
for (const auto& column : hashShuffle.Cast().KeyColumns()) {
keyColumns.AppendValue(TString(column.Value()));
@@ -1292,9 +1292,9 @@ void WriteCommonTablesInfo(NJsonWriter::TBuf& writer, TMap<TString, TTableInfo>&
if (read.Limit) {
writer.WriteKey("limit").WriteString(*read.Limit);
}
- if (read.Reverse) {
- writer.WriteKey("reverse").WriteBool(true);
- }
+ if (read.Reverse) {
+ writer.WriteKey("reverse").WriteBool(true);
+ }
if (!read.Columns.empty()) {
writer.WriteKey("columns");
@@ -1423,13 +1423,13 @@ void PhyQuerySetTxPlans(NKqpProto::TKqpPhyQuery& queryProto, const TKqpPhysicalQ
queryProto.SetQueryPlan(writer.Str());
}
-TString AddExecStatsToTxPlan(const TString& txPlanJson, const NYql::NDqProto::TDqExecutionStats& stats) {
+TString AddExecStatsToTxPlan(const TString& txPlanJson, const NYql::NDqProto::TDqExecutionStats& stats) {
if (txPlanJson.empty()) {
return {};
}
- THashMap<TProtoStringType, const NYql::NDqProto::TDqStageStats*> stages;
- for (const auto& stage : stats.GetStages()) {
+ THashMap<TProtoStringType, const NYql::NDqProto::TDqStageStats*> stages;
+ for (const auto& stage : stats.GetStages()) {
stages[stage.GetStageGuid()] = &stage;
}
@@ -1511,9 +1511,9 @@ TString SerializeTxPlans(const TVector<const TString>& txPlans, const TString co
TString SerializeExplainPlan(const NKqpProto::TKqpPhyQuery& phyQuery) {
TVector<const TString> txPlans;
- txPlans.reserve(phyQuery.GetTransactions().size());
+ txPlans.reserve(phyQuery.GetTransactions().size());
for (const auto& phyTx: phyQuery.GetTransactions()) {
- txPlans.emplace_back(phyTx.GetPlan());
+ txPlans.emplace_back(phyTx.GetPlan());
}
return SerializeTxPlans(txPlans, phyQuery.GetQueryPlan());
}
@@ -1569,5 +1569,5 @@ TString SerializeScriptPlan(const TVector<const TString>& queryPlans) {
return writer.Str();
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/prepare/kqp_query_plan.h b/ydb/core/kqp/prepare/kqp_query_plan.h
index 4ef1566754..4e299e1490 100644
--- a/ydb/core/kqp/prepare/kqp_query_plan.h
+++ b/ydb/core/kqp/prepare/kqp_query_plan.h
@@ -26,7 +26,7 @@ void PhyQuerySetTxPlans(NKqpProto::TKqpPhyQuery& queryProto, const NYql::NNodes:
* Fill stages in given txPlan with ExecutionStats fields. Each plan stage stores StageGuid which is
* used to find corresponding TKqpStatsExecution object.
*/
-TString AddExecStatsToTxPlan(const TString& txPlan, const NYql::NDqProto::TDqExecutionStats& stats);
+TString AddExecStatsToTxPlan(const TString& txPlan, const NYql::NDqProto::TDqExecutionStats& stats);
TString SerializeExplainPlan(const NKqpProto::TKqpPhyQuery& phyQuery);
diff --git a/ydb/core/kqp/prepare/kqp_query_rewrite.cpp b/ydb/core/kqp/prepare/kqp_query_rewrite.cpp
index 03d5ae7a71..cb24a1b8a1 100644
--- a/ydb/core/kqp/prepare/kqp_query_rewrite.cpp
+++ b/ydb/core/kqp/prepare/kqp_query_rewrite.cpp
@@ -394,5 +394,5 @@ TAutoPtr<IGraphTransformer> CreateKqpRewriteTransformer(TIntrusivePtr<TKqlTransf
return new TKqpRewriteTransformer(transformCtx);
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/prepare/kqp_query_simplify.cpp b/ydb/core/kqp/prepare/kqp_query_simplify.cpp
index 352fdbb2c7..d8fefeb939 100644
--- a/ydb/core/kqp/prepare/kqp_query_simplify.cpp
+++ b/ydb/core/kqp/prepare/kqp_query_simplify.cpp
@@ -261,33 +261,33 @@ TExprNode::TPtr MergeMapsWithSameLambda(TExprBase node, TExprContext& ctx) {
return ret.Ptr();
}
-TExprNode::TPtr RewritePresentIfToFlatMap(TExprBase node, TExprContext& ctx) {
- if (!node.Maybe<TCoIfPresent>()) {
- return node.Ptr();
- }
- auto ifPresent = node.Cast<TCoIfPresent>();
-
- if (IsKqlPureLambda(ifPresent.PresentHandler())) {
- return node.Ptr();
- }
-
- if (!ifPresent.MissingValue().Maybe<TCoNothing>()) {
- return node.Ptr();
- }
-
- return Build<TCoFlatMap>(ctx, node.Pos())
- .Input(ifPresent.Optional())
- .Lambda()
- .Args({"item"})
- .Body<TExprApplier>()
- .Apply(ifPresent.PresentHandler())
- .With(ifPresent.PresentHandler().Args().Arg(0), "item")
- .Build()
- .Build()
- .Done()
- .Ptr();
-}
-
+TExprNode::TPtr RewritePresentIfToFlatMap(TExprBase node, TExprContext& ctx) {
+ if (!node.Maybe<TCoIfPresent>()) {
+ return node.Ptr();
+ }
+ auto ifPresent = node.Cast<TCoIfPresent>();
+
+ if (IsKqlPureLambda(ifPresent.PresentHandler())) {
+ return node.Ptr();
+ }
+
+ if (!ifPresent.MissingValue().Maybe<TCoNothing>()) {
+ return node.Ptr();
+ }
+
+ return Build<TCoFlatMap>(ctx, node.Pos())
+ .Input(ifPresent.Optional())
+ .Lambda()
+ .Args({"item"})
+ .Body<TExprApplier>()
+ .Apply(ifPresent.PresentHandler())
+ .With(ifPresent.PresentHandler().Args().Arg(0), "item")
+ .Build()
+ .Build()
+ .Done()
+ .Ptr();
+}
+
class TKqpSimplifyTransformer : public TSyncTransformerBase {
public:
TKqpSimplifyTransformer()
@@ -335,11 +335,11 @@ public:
return ret;
}
- ret = RewritePresentIfToFlatMap(node, ctx);
- if (ret != input) {
- return ret;
- }
-
+ ret = RewritePresentIfToFlatMap(node, ctx);
+ if (ret != input) {
+ return ret;
+ }
+
return ret;
}, ctx, optSettings);
@@ -368,5 +368,5 @@ TAutoPtr<IGraphTransformer> CreateKqpSimplifyTransformer() {
return new TKqpSimplifyTransformer();
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/prepare/kqp_query_substitute.cpp b/ydb/core/kqp/prepare/kqp_query_substitute.cpp
index ab6d73bbf4..6d289bf78b 100644
--- a/ydb/core/kqp/prepare/kqp_query_substitute.cpp
+++ b/ydb/core/kqp/prepare/kqp_query_substitute.cpp
@@ -69,5 +69,5 @@ TAutoPtr<IGraphTransformer> CreateKqpSubstituteTransformer(TIntrusivePtr<TKqpTra
return new TKqpSubstituteTransformer(txState, transformCtx);
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/prepare/kqp_type_ann.cpp b/ydb/core/kqp/prepare/kqp_type_ann.cpp
index 5b46706d20..ee51c7349c 100644
--- a/ydb/core/kqp/prepare/kqp_type_ann.cpp
+++ b/ydb/core/kqp/prepare/kqp_type_ann.cpp
@@ -59,24 +59,24 @@ bool CheckKeyTuple(const TKqlKeyTuple& tuple, const TKikimrTableDescription& tab
return true;
}
-TStatus AnnotateTable(const TExprNode::TPtr& node, TExprContext& ctx, const TString& cluster,
+TStatus AnnotateTable(const TExprNode::TPtr& node, TExprContext& ctx, const TString& cluster,
const TKikimrTablesData& tablesData)
{
- if (!EnsureArgsCount(*node, 4, ctx)) {
- return TStatus::Error;
- }
-
- auto* path = node->Child(TKqpTable::idx_Path);
- auto* pathId = node->Child(TKqpTable::idx_PathId);
- auto* sysView = node->Child(TKqpTable::idx_SysView);
- auto* version = node->Child(TKqpTable::idx_Version);
-
- if (!EnsureAtom(*path, ctx) || !EnsureAtom(*pathId, ctx) || !EnsureAtom(*sysView, ctx) || !EnsureAtom(*version, ctx)) {
- return TStatus::Error;
- }
-
- TString tablePath(path->Content());
- auto tableDesc = tablesData.EnsureTableExists(cluster, tablePath, node->Pos(), ctx);
+ if (!EnsureArgsCount(*node, 4, ctx)) {
+ return TStatus::Error;
+ }
+
+ auto* path = node->Child(TKqpTable::idx_Path);
+ auto* pathId = node->Child(TKqpTable::idx_PathId);
+ auto* sysView = node->Child(TKqpTable::idx_SysView);
+ auto* version = node->Child(TKqpTable::idx_Version);
+
+ if (!EnsureAtom(*path, ctx) || !EnsureAtom(*pathId, ctx) || !EnsureAtom(*sysView, ctx) || !EnsureAtom(*version, ctx)) {
+ return TStatus::Error;
+ }
+
+ TString tablePath(path->Content());
+ auto tableDesc = tablesData.EnsureTableExists(cluster, tablePath, node->Pos(), ctx);
if (!tableDesc) {
return TStatus::Error;
}
@@ -84,47 +84,47 @@ TStatus AnnotateTable(const TExprNode::TPtr& node, TExprContext& ctx, const TStr
YQL_ENSURE(tableDesc->Metadata);
auto& meta = *tableDesc->Metadata;
- if (meta.PathId.ToString() != pathId->Content()) {
- ctx.AddError(YqlIssue(ctx.GetPosition(node->Pos()), TIssuesIds::UNEXPECTED, TStringBuilder()
+ if (meta.PathId.ToString() != pathId->Content()) {
+ ctx.AddError(YqlIssue(ctx.GetPosition(node->Pos()), TIssuesIds::UNEXPECTED, TStringBuilder()
<< "Query compilation, unexpected table id"
<< ", table: " << meta.Name
<< ", expected: " << meta.PathId.ToString()
- << ", actual: " << pathId->Content()));
+ << ", actual: " << pathId->Content()));
return TStatus::Error;
}
- if (meta.SchemaVersion != FromString<ui64>(version->Content())) {
- ctx.AddError(YqlIssue(ctx.GetPosition(node->Pos()), TIssuesIds::UNEXPECTED, TStringBuilder()
+ if (meta.SchemaVersion != FromString<ui64>(version->Content())) {
+ ctx.AddError(YqlIssue(ctx.GetPosition(node->Pos()), TIssuesIds::UNEXPECTED, TStringBuilder()
<< "Query compilation, unexpected table version"
<< ", table: " << meta.Name
<< ", expected: " << meta.SchemaVersion
- << ", actual: " << version->Content()));
+ << ", actual: " << version->Content()));
return TStatus::Error;
}
- node->SetTypeAnn(ctx.MakeType<TVoidExprType>());
+ node->SetTypeAnn(ctx.MakeType<TVoidExprType>());
return TStatus::Ok;
}
-std::pair<TString, const TKikimrTableDescription*> ResolveTable(const TExprNode* kqpTableNode, TExprContext& ctx,
- const TString& cluster, const TKikimrTablesData& tablesData)
-{
- if (!EnsureCallable(*kqpTableNode, ctx)) {
- return {"", nullptr};
- }
-
- if (!TKqpTable::Match(kqpTableNode)) {
- ctx.AddError(TIssue(ctx.GetPosition(kqpTableNode->Pos()), TStringBuilder()
- << "Expected " << TKqpTable::CallableName()));
- return {"", nullptr};
- }
-
- TString tableName{kqpTableNode->Child(TKqpTable::idx_Path)->Content()};
-
- auto tableDesc = tablesData.EnsureTableExists(cluster, tableName, kqpTableNode->Pos(), ctx);
- return {std::move(tableName), tableDesc};
-}
-
+std::pair<TString, const TKikimrTableDescription*> ResolveTable(const TExprNode* kqpTableNode, TExprContext& ctx,
+ const TString& cluster, const TKikimrTablesData& tablesData)
+{
+ if (!EnsureCallable(*kqpTableNode, ctx)) {
+ return {"", nullptr};
+ }
+
+ if (!TKqpTable::Match(kqpTableNode)) {
+ ctx.AddError(TIssue(ctx.GetPosition(kqpTableNode->Pos()), TStringBuilder()
+ << "Expected " << TKqpTable::CallableName()));
+ return {"", nullptr};
+ }
+
+ TString tableName{kqpTableNode->Child(TKqpTable::idx_Path)->Content()};
+
+ auto tableDesc = tablesData.EnsureTableExists(cluster, tableName, kqpTableNode->Pos(), ctx);
+ return {std::move(tableName), tableDesc};
+}
+
const TFlowExprType* GetWideRowsType(TExprContext& ctx, const TStructExprType* rowType) {
YQL_ENSURE(rowType);
@@ -166,23 +166,23 @@ bool CalcKeyColumnsCount(TExprContext& ctx, const TPositionHandle pos, const TSt
return true;
}
-TStatus AnnotateReadTable(const TExprNode::TPtr& node, TExprContext& ctx, const TString& cluster,
+TStatus AnnotateReadTable(const TExprNode::TPtr& node, TExprContext& ctx, const TString& cluster,
const TKikimrTablesData& tablesData, bool withSystemColumns)
{
const bool readIndex = TKqlReadTableIndex::Match(node.Get());
if (readIndex && !EnsureArgsCount(*node, 5, ctx)) {
- return TStatus::Error;
- }
-
+ return TStatus::Error;
+ }
+
if (!readIndex && !EnsureArgsCount(*node, 4, ctx)) {
return TStatus::Error;
}
- auto table = ResolveTable(node->Child(TKqlReadTableBase::idx_Table), ctx, cluster, tablesData);
- if (!table.second) {
- return TStatus::Error;
- }
-
+ auto table = ResolveTable(node->Child(TKqlReadTableBase::idx_Table), ctx, cluster, tablesData);
+ if (!table.second) {
+ return TStatus::Error;
+ }
+
YQL_ENSURE(table.second->Metadata, "Expected loaded metadata");
TKikimrTableMetadataPtr meta;
@@ -196,23 +196,23 @@ TStatus AnnotateReadTable(const TExprNode::TPtr& node, TExprContext& ctx, const
meta = table.second->Metadata;
}
- const auto& columns = node->ChildPtr(TKqlReadTableBase::idx_Columns);
- if (!EnsureTupleOfAtoms(*columns, ctx)) {
- return TStatus::Error;
- }
-
- auto rowType = GetReadTableRowType(ctx, tablesData, cluster, table.first, TCoAtomList(columns), withSystemColumns);
+ const auto& columns = node->ChildPtr(TKqlReadTableBase::idx_Columns);
+ if (!EnsureTupleOfAtoms(*columns, ctx)) {
+ return TStatus::Error;
+ }
+
+ auto rowType = GetReadTableRowType(ctx, tablesData, cluster, table.first, TCoAtomList(columns), withSystemColumns);
if (!rowType) {
return TStatus::Error;
}
- if (!TKqlKeyRange::Match(node->Child(TKqlReadTableBase::idx_Range))) {
- ctx.AddError(TIssue(ctx.GetPosition(node->Child(TKqlReadTableBase::idx_Range)->Pos()), "Expected TKqlKeyRange tuple"));
+ if (!TKqlKeyRange::Match(node->Child(TKqlReadTableBase::idx_Range))) {
+ ctx.AddError(TIssue(ctx.GetPosition(node->Child(TKqlReadTableBase::idx_Range)->Pos()), "Expected TKqlKeyRange tuple"));
return TStatus::Error;
}
- TKqlKeyRange range{node->ChildPtr(TKqlReadTableBase::idx_Range)};
-
+ TKqlKeyRange range{node->ChildPtr(TKqlReadTableBase::idx_Range)};
+
if (!CheckKeyTuple(range.From(), *table.second, meta, ctx)) {
return TStatus::Error;
}
@@ -222,14 +222,14 @@ TStatus AnnotateReadTable(const TExprNode::TPtr& node, TExprContext& ctx, const
}
if (TKqlReadTable::Match(node.Get()) || TKqlReadTableIndex::Match(node.Get())) {
- node->SetTypeAnn(ctx.MakeType<TListExprType>(rowType));
- } else if (TKqpReadTable::Match(node.Get())) {
- node->SetTypeAnn(ctx.MakeType<TFlowExprType>(rowType));
- } else if (TKqpWideReadTable::Match(node.Get())) {
+ node->SetTypeAnn(ctx.MakeType<TListExprType>(rowType));
+ } else if (TKqpReadTable::Match(node.Get())) {
+ node->SetTypeAnn(ctx.MakeType<TFlowExprType>(rowType));
+ } else if (TKqpWideReadTable::Match(node.Get())) {
node->SetTypeAnn(GetWideRowsType(ctx, rowType->Cast<TStructExprType>()));
} else {
- YQL_ENSURE(false, "Unexpected ReadTable callable: " << node->Content());
- }
+ YQL_ENSURE(false, "Unexpected ReadTable callable: " << node->Content());
+ }
return TStatus::Ok;
}
@@ -317,52 +317,52 @@ TStatus AnnotateReadTableRanges(const TExprNode::TPtr& node, TExprContext& ctx,
return TStatus::Ok;
}
-TStatus AnnotateLookupTable(const TExprNode::TPtr& node, TExprContext& ctx, const TString& cluster,
+TStatus AnnotateLookupTable(const TExprNode::TPtr& node, TExprContext& ctx, const TString& cluster,
const TKikimrTablesData& tablesData, bool withSystemColumns)
{
if (!EnsureArgsCount(*node, TKqlLookupIndex::Match(node.Get()) ? 4 : 3, ctx)) {
- return TStatus::Error;
- }
+ return TStatus::Error;
+ }
- auto table = ResolveTable(node->Child(TKqlLookupTableBase::idx_Table), ctx, cluster, tablesData);
- if (!table.second) {
+ auto table = ResolveTable(node->Child(TKqlLookupTableBase::idx_Table), ctx, cluster, tablesData);
+ if (!table.second) {
return TStatus::Error;
}
- if (!EnsureTupleOfAtoms(*node->Child(TKqlLookupTableBase::idx_Columns), ctx)) {
+ if (!EnsureTupleOfAtoms(*node->Child(TKqlLookupTableBase::idx_Columns), ctx)) {
return TStatus::Error;
}
- TCoAtomList columns{node->ChildPtr(TKqlLookupTableBase::idx_Columns)};
+ TCoAtomList columns{node->ChildPtr(TKqlLookupTableBase::idx_Columns)};
- auto rowType = GetReadTableRowType(ctx, tablesData, cluster, table.first, columns, withSystemColumns);
- if (!rowType) {
- return TStatus::Error;
- }
+ auto rowType = GetReadTableRowType(ctx, tablesData, cluster, table.first, columns, withSystemColumns);
+ if (!rowType) {
+ return TStatus::Error;
+ }
+
+ bool isPhysical = TKqpLookupTable::Match(node.Get());
- bool isPhysical = TKqpLookupTable::Match(node.Get());
-
const TTypeAnnotationNode* lookupType;
if (isPhysical) {
- if (!EnsureNewSeqType<false, false, true>(*node->Child(TKqlLookupTableBase::idx_LookupKeys), ctx, &lookupType)) {
+ if (!EnsureNewSeqType<false, false, true>(*node->Child(TKqlLookupTableBase::idx_LookupKeys), ctx, &lookupType)) {
return TStatus::Error;
}
- if (node->Child(TKqlLookupTableBase::idx_LookupKeys)->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Flow) {
- auto streamLookupKeys = Build<TCoFromFlow>(ctx, node->Pos())
- .Input(node->ChildRef(TKqlLookupTableBase::idx_LookupKeys))
+ if (node->Child(TKqlLookupTableBase::idx_LookupKeys)->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Flow) {
+ auto streamLookupKeys = Build<TCoFromFlow>(ctx, node->Pos())
+ .Input(node->ChildRef(TKqlLookupTableBase::idx_LookupKeys))
.Done();
- node->ChildRef(TKqpLookupTable::idx_LookupKeys) = streamLookupKeys.Ptr();
+ node->ChildRef(TKqpLookupTable::idx_LookupKeys) = streamLookupKeys.Ptr();
return TStatus::Repeat;
}
} else {
- if (!EnsureNewSeqType<false, true, false>(*node->Child(TKqlLookupTableBase::idx_LookupKeys), ctx, &lookupType)) {
+ if (!EnsureNewSeqType<false, true, false>(*node->Child(TKqlLookupTableBase::idx_LookupKeys), ctx, &lookupType)) {
return TStatus::Error;
}
}
YQL_ENSURE(lookupType);
- if (!EnsureStructType(node->Pos(), *lookupType, ctx)) {
+ if (!EnsureStructType(node->Pos(), *lookupType, ctx)) {
return TStatus::Error;
}
auto structType = lookupType->Cast<TStructExprType>();
@@ -386,81 +386,81 @@ TStatus AnnotateLookupTable(const TExprNode::TPtr& node, TExprContext& ctx, cons
}
if (!keyColumnsCount) {
- ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "Table lookup has no key columns."));
+ ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "Table lookup has no key columns."));
return TStatus::Error;
}
if (structType->GetSize() != keyColumnsCount) {
- ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "Table lookup contains non-key columns."));
+ ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "Table lookup contains non-key columns."));
return TStatus::Error;
}
if (isPhysical) {
- node->SetTypeAnn(ctx.MakeType<TFlowExprType>(rowType));
+ node->SetTypeAnn(ctx.MakeType<TFlowExprType>(rowType));
} else {
- node->SetTypeAnn(ctx.MakeType<TListExprType>(rowType));
+ node->SetTypeAnn(ctx.MakeType<TListExprType>(rowType));
}
return TStatus::Ok;
}
-TStatus AnnotateKeyTuple(const TExprNode::TPtr& node, TExprContext& ctx) {
+TStatus AnnotateKeyTuple(const TExprNode::TPtr& node, TExprContext& ctx) {
TVector<const TTypeAnnotationNode*> keyTypes;
- for (const auto& arg : node->ChildrenList()) {
- keyTypes.push_back(arg->GetTypeAnn());
+ for (const auto& arg : node->ChildrenList()) {
+ keyTypes.push_back(arg->GetTypeAnn());
}
auto tupleType = ctx.MakeType<TTupleExprType>(keyTypes);
- node->SetTypeAnn(tupleType);
+ node->SetTypeAnn(tupleType);
return TStatus::Ok;
}
-TStatus AnnotateUpsertRows(const TExprNode::TPtr& node, TExprContext& ctx, const TString& cluster,
+TStatus AnnotateUpsertRows(const TExprNode::TPtr& node, TExprContext& ctx, const TString& cluster,
const TKikimrTablesData& tablesData)
{
if (!EnsureMinArgsCount(*node, 3, ctx)) {
return TStatus::Error;
}
- auto table = ResolveTable(node->Child(TKqlUpsertRowsBase::idx_Table), ctx, cluster, tablesData);
- if (!table.second) {
- return TStatus::Error;
- }
-
- if (!EnsureTupleOfAtoms(*node->Child(TKqlUpsertRowsBase::idx_Columns), ctx)) {
- return TStatus::Error;
- }
- TCoAtomList columns{node->ChildPtr(TKqlUpsertRowsBase::idx_Columns)};
-
- const TTypeAnnotationNode* itemType = nullptr;
- bool isStream;
-
- auto* input = node->Child(TKqlUpsertRowsBase::idx_Input);
-
- if (TKqpUpsertRows::Match(node.Get())) {
- if (!EnsureStreamType(*input, ctx)) {
- return TStatus::Error;
- }
- itemType = input->GetTypeAnn()->Cast<TStreamExprType>()->GetItemType();
- isStream = true;
- } else {
+ auto table = ResolveTable(node->Child(TKqlUpsertRowsBase::idx_Table), ctx, cluster, tablesData);
+ if (!table.second) {
+ return TStatus::Error;
+ }
+
+ if (!EnsureTupleOfAtoms(*node->Child(TKqlUpsertRowsBase::idx_Columns), ctx)) {
+ return TStatus::Error;
+ }
+ TCoAtomList columns{node->ChildPtr(TKqlUpsertRowsBase::idx_Columns)};
+
+ const TTypeAnnotationNode* itemType = nullptr;
+ bool isStream;
+
+ auto* input = node->Child(TKqlUpsertRowsBase::idx_Input);
+
+ if (TKqpUpsertRows::Match(node.Get())) {
+ if (!EnsureStreamType(*input, ctx)) {
+ return TStatus::Error;
+ }
+ itemType = input->GetTypeAnn()->Cast<TStreamExprType>()->GetItemType();
+ isStream = true;
+ } else {
YQL_ENSURE(TKqlUpsertRows::Match(node.Get()) || TKqlUpsertRowsIndex::Match(node.Get()));
- if (!EnsureListType(*input, ctx)) {
- return TStatus::Error;
- }
- itemType = input->GetTypeAnn()->Cast<TListExprType>()->GetItemType();
- isStream = false;
+ if (!EnsureListType(*input, ctx)) {
+ return TStatus::Error;
+ }
+ itemType = input->GetTypeAnn()->Cast<TListExprType>()->GetItemType();
+ isStream = false;
}
- if (!EnsureStructType(input->Pos(), *itemType, ctx)) {
+ if (!EnsureStructType(input->Pos(), *itemType, ctx)) {
return TStatus::Error;
}
auto rowType = itemType->Cast<TStructExprType>();
- for (const auto& column : columns) {
- if (!rowType->FindItem(column.Value())) {
- ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder()
- << "Missing column in input type: " << column.Value()));
+ for (const auto& column : columns) {
+ if (!rowType->FindItem(column.Value())) {
+ ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder()
+ << "Missing column in input type: " << column.Value()));
return TStatus::Error;
}
}
@@ -471,10 +471,10 @@ TStatus AnnotateUpsertRows(const TExprNode::TPtr& node, TExprContext& ctx, const
return TStatus::Error;
}
- for (auto& keyColumnName : table.second->Metadata->KeyColumnNames) {
+ for (auto& keyColumnName : table.second->Metadata->KeyColumnNames) {
if (!rowType->FindItem(keyColumnName)) {
ctx.AddError(YqlIssue(ctx.GetPosition(node->Pos()), TIssuesIds::KIKIMR_PRECONDITION_FAILED, TStringBuilder()
- << "Missing key column in input type: " << keyColumnName));
+ << "Missing key column in input type: " << keyColumnName));
return TStatus::Error;
}
}
@@ -500,58 +500,58 @@ TStatus AnnotateUpsertRows(const TExprNode::TPtr& node, TExprContext& ctx, const
}
auto effectType = MakeKqpEffectType(ctx);
- if (isStream) {
- node->SetTypeAnn(ctx.MakeType<TStreamExprType>(effectType));
- } else {
- node->SetTypeAnn(ctx.MakeType<TListExprType>(effectType));
- }
+ if (isStream) {
+ node->SetTypeAnn(ctx.MakeType<TStreamExprType>(effectType));
+ } else {
+ node->SetTypeAnn(ctx.MakeType<TListExprType>(effectType));
+ }
return TStatus::Ok;
}
-TStatus AnnotateInsertRows(const TExprNode::TPtr& node, TExprContext& ctx, const TString& cluster,
- const TKikimrTablesData& tablesData)
-{
- if (!EnsureArgsCount(*node, 4, ctx)) {
- return TStatus::Error;
- }
-
- auto table = ResolveTable(node->Child(TKqlInsertRows::idx_Table), ctx, cluster, tablesData);
- if (!table.second) {
- return TStatus::Error;
- }
-
- if (!EnsureTupleOfAtoms(*node->Child(TKqlInsertRows::idx_Columns), ctx)) {
- return TStatus::Error;
- }
- TCoAtomList columns{node->ChildPtr(TKqlInsertRows::idx_Columns)};
-
- auto* input = node->Child(TKqlInsertRows::idx_Input);
- if (!EnsureListType(*input, ctx)) {
- return TStatus::Error;
- }
-
- auto itemType = input->GetTypeAnn()->Cast<TListExprType>()->GetItemType();
- if (!EnsureStructType(input->Pos(), *itemType, ctx)) {
- return TStatus::Error;
- }
-
- auto rowType = itemType->Cast<TStructExprType>();
- for (const auto& column : columns) {
- if (!rowType->FindItem(column.Value())) {
- ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder()
- << "Missing column in input type: " << column.Value()));
- return TStatus::Error;
- }
- }
-
- for (auto& keyColumnName : table.second->Metadata->KeyColumnNames) {
- if (!rowType->FindItem(keyColumnName)) {
+TStatus AnnotateInsertRows(const TExprNode::TPtr& node, TExprContext& ctx, const TString& cluster,
+ const TKikimrTablesData& tablesData)
+{
+ if (!EnsureArgsCount(*node, 4, ctx)) {
+ return TStatus::Error;
+ }
+
+ auto table = ResolveTable(node->Child(TKqlInsertRows::idx_Table), ctx, cluster, tablesData);
+ if (!table.second) {
+ return TStatus::Error;
+ }
+
+ if (!EnsureTupleOfAtoms(*node->Child(TKqlInsertRows::idx_Columns), ctx)) {
+ return TStatus::Error;
+ }
+ TCoAtomList columns{node->ChildPtr(TKqlInsertRows::idx_Columns)};
+
+ auto* input = node->Child(TKqlInsertRows::idx_Input);
+ if (!EnsureListType(*input, ctx)) {
+ return TStatus::Error;
+ }
+
+ auto itemType = input->GetTypeAnn()->Cast<TListExprType>()->GetItemType();
+ if (!EnsureStructType(input->Pos(), *itemType, ctx)) {
+ return TStatus::Error;
+ }
+
+ auto rowType = itemType->Cast<TStructExprType>();
+ for (const auto& column : columns) {
+ if (!rowType->FindItem(column.Value())) {
+ ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder()
+ << "Missing column in input type: " << column.Value()));
+ return TStatus::Error;
+ }
+ }
+
+ for (auto& keyColumnName : table.second->Metadata->KeyColumnNames) {
+ if (!rowType->FindItem(keyColumnName)) {
ctx.AddError(YqlIssue(ctx.GetPosition(node->Pos()), TIssuesIds::KIKIMR_PRECONDITION_FAILED, TStringBuilder()
- << "Missing key column in input type: " << keyColumnName));
- return TStatus::Error;
- }
- }
-
+ << "Missing key column in input type: " << keyColumnName));
+ return TStatus::Error;
+ }
+ }
+
for (auto& [name, meta] : table.second->Metadata->Columns) {
if (meta.NotNull && !rowType->FindItem(name)) {
ctx.AddError(YqlIssue(ctx.GetPosition(node->Pos()), TIssuesIds::KIKIMR_NO_COLUMN_DEFAULT_VALUE, TStringBuilder()
@@ -568,61 +568,61 @@ TStatus AnnotateInsertRows(const TExprNode::TPtr& node, TExprContext& ctx, const
}
}
- if (!EnsureAtom(*node->Child(TKqlInsertRows::idx_OnConflict), ctx)) {
- return TStatus::Error;
- }
-
- TStringBuf onConflict = node->Child(TKqlInsertRows::idx_OnConflict)->Content();
+ if (!EnsureAtom(*node->Child(TKqlInsertRows::idx_OnConflict), ctx)) {
+ return TStatus::Error;
+ }
+
+ TStringBuf onConflict = node->Child(TKqlInsertRows::idx_OnConflict)->Content();
if (onConflict != "abort"sv && onConflict != "revert"sv) {
- ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder()
- << "Unsupported insert on-conflict policy `" << onConflict << "`."));
- return TStatus::Error;
- }
-
- node->SetTypeAnn(ctx.MakeType<TListExprType>(MakeKqpEffectType(ctx)));
- return TStatus::Ok;
-}
-
-TStatus AnnotateUpdateRows(const TExprNode::TPtr& node, TExprContext& ctx, const TString& cluster,
- const TKikimrTablesData& tablesData)
-{
- if (!EnsureArgsCount(*node, 3, ctx)) {
- return TStatus::Error;
- }
-
- auto table = ResolveTable(node->Child(TKqlUpdateRows::idx_Table), ctx, cluster, tablesData);
- if (!table.second) {
- return TStatus::Error;
- }
-
- if (!EnsureTupleOfAtoms(*node->Child(TKqlUpdateRows::idx_Columns), ctx)) {
- return TStatus::Error;
- }
- TCoAtomList columns{node->ChildPtr(TKqlUpdateRows::idx_Columns)};
-
- auto* input = node->Child(TKqlUpdateRows::idx_Input);
- if (!EnsureListType(*input, ctx)) {
- return TStatus::Error;
- }
-
- auto itemType = input->GetTypeAnn()->Cast<TListExprType>()->GetItemType();
- if (!EnsureStructType(input->Pos(), *itemType, ctx)) {
- return TStatus::Error;
- }
-
- auto rowType = itemType->Cast<TStructExprType>();
- for (const auto& column : columns) {
- YQL_ENSURE(rowType->FindItem(column.Value()), "Missing column in input type: " << column.Value());
- }
-
- for (auto& keyColumnName : table.second->Metadata->KeyColumnNames) {
+ ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder()
+ << "Unsupported insert on-conflict policy `" << onConflict << "`."));
+ return TStatus::Error;
+ }
+
+ node->SetTypeAnn(ctx.MakeType<TListExprType>(MakeKqpEffectType(ctx)));
+ return TStatus::Ok;
+}
+
+TStatus AnnotateUpdateRows(const TExprNode::TPtr& node, TExprContext& ctx, const TString& cluster,
+ const TKikimrTablesData& tablesData)
+{
+ if (!EnsureArgsCount(*node, 3, ctx)) {
+ return TStatus::Error;
+ }
+
+ auto table = ResolveTable(node->Child(TKqlUpdateRows::idx_Table), ctx, cluster, tablesData);
+ if (!table.second) {
+ return TStatus::Error;
+ }
+
+ if (!EnsureTupleOfAtoms(*node->Child(TKqlUpdateRows::idx_Columns), ctx)) {
+ return TStatus::Error;
+ }
+ TCoAtomList columns{node->ChildPtr(TKqlUpdateRows::idx_Columns)};
+
+ auto* input = node->Child(TKqlUpdateRows::idx_Input);
+ if (!EnsureListType(*input, ctx)) {
+ return TStatus::Error;
+ }
+
+ auto itemType = input->GetTypeAnn()->Cast<TListExprType>()->GetItemType();
+ if (!EnsureStructType(input->Pos(), *itemType, ctx)) {
+ return TStatus::Error;
+ }
+
+ auto rowType = itemType->Cast<TStructExprType>();
+ for (const auto& column : columns) {
+ YQL_ENSURE(rowType->FindItem(column.Value()), "Missing column in input type: " << column.Value());
+ }
+
+ for (auto& keyColumnName : table.second->Metadata->KeyColumnNames) {
if (!rowType->FindItem(keyColumnName)) {
ctx.AddError(YqlIssue(ctx.GetPosition(node->Pos()), TIssuesIds::KIKIMR_NO_COLUMN_DEFAULT_VALUE, TStringBuilder()
<< "Missing key column in input type: " << keyColumnName));
return TStatus::Error;
}
- }
-
+ }
+
for (const auto& item : rowType->GetItems()) {
auto column = table.second->Metadata->Columns.FindPtr(TString(item->GetName()));
YQL_ENSURE(column);
@@ -633,65 +633,65 @@ TStatus AnnotateUpdateRows(const TExprNode::TPtr& node, TExprContext& ctx, const
}
}
- node->SetTypeAnn(ctx.MakeType<TListExprType>(MakeKqpEffectType(ctx)));
- return TStatus::Ok;
-}
-
-TStatus AnnotateDeleteRows(const TExprNode::TPtr& node, TExprContext& ctx, const TString& cluster,
+ node->SetTypeAnn(ctx.MakeType<TListExprType>(MakeKqpEffectType(ctx)));
+ return TStatus::Ok;
+}
+
+TStatus AnnotateDeleteRows(const TExprNode::TPtr& node, TExprContext& ctx, const TString& cluster,
const TKikimrTablesData& tablesData)
{
- if (!EnsureArgsCount(*node, 2, ctx)) {
+ if (!EnsureArgsCount(*node, 2, ctx)) {
+ return TStatus::Error;
+ }
+
+ auto table = ResolveTable(node->Child(TKqlDeleteRowsBase::idx_Table), ctx, cluster, tablesData);
+ if (!table.second) {
return TStatus::Error;
}
- auto table = ResolveTable(node->Child(TKqlDeleteRowsBase::idx_Table), ctx, cluster, tablesData);
- if (!table.second) {
- return TStatus::Error;
- }
-
const TTypeAnnotationNode* itemType = nullptr;
bool isStream = false;
- auto* input = node->Child(TKqlDeleteRowsBase::idx_Input);
-
- if (TKqpDeleteRows::Match(node.Get())) {
- if (!EnsureStreamType(*input, ctx)) {
+ auto* input = node->Child(TKqlDeleteRowsBase::idx_Input);
+
+ if (TKqpDeleteRows::Match(node.Get())) {
+ if (!EnsureStreamType(*input, ctx)) {
return TStatus::Error;
}
- itemType = input->GetTypeAnn()->Cast<TStreamExprType>()->GetItemType();
+ itemType = input->GetTypeAnn()->Cast<TStreamExprType>()->GetItemType();
isStream = true;
} else {
YQL_ENSURE(TKqlDeleteRows::Match(node.Get()) || TKqlDeleteRowsIndex::Match(node.Get()));
- if (!EnsureListType(*input, ctx)) {
+ if (!EnsureListType(*input, ctx)) {
return TStatus::Error;
}
- itemType = input->GetTypeAnn()->Cast<TListExprType>()->GetItemType();
+ itemType = input->GetTypeAnn()->Cast<TListExprType>()->GetItemType();
isStream = false;
}
- if (!EnsureStructType(input->Pos(), *itemType, ctx)) {
+ if (!EnsureStructType(input->Pos(), *itemType, ctx)) {
return TStatus::Error;
}
auto rowType = itemType->Cast<TStructExprType>();
- for (auto& keyColumnName : table.second->Metadata->KeyColumnNames) {
+ for (auto& keyColumnName : table.second->Metadata->KeyColumnNames) {
if (!rowType->FindItem(keyColumnName)) {
ctx.AddError(YqlIssue(ctx.GetPosition(node->Pos()), TIssuesIds::KIKIMR_PRECONDITION_FAILED, TStringBuilder()
- << "Missing key column in input type: " << keyColumnName));
+ << "Missing key column in input type: " << keyColumnName));
return TStatus::Error;
}
}
- if (rowType->GetItems().size() != table.second->Metadata->KeyColumnNames.size()) {
- ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "Input type contains non-key columns"));
- return TStatus::Error;
+ if (rowType->GetItems().size() != table.second->Metadata->KeyColumnNames.size()) {
+ ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "Input type contains non-key columns"));
+ return TStatus::Error;
}
auto effectType = MakeKqpEffectType(ctx);
if (isStream) {
- node->SetTypeAnn(ctx.MakeType<TStreamExprType>(effectType));
+ node->SetTypeAnn(ctx.MakeType<TStreamExprType>(effectType));
} else {
- node->SetTypeAnn(ctx.MakeType<TListExprType>(effectType));
+ node->SetTypeAnn(ctx.MakeType<TListExprType>(effectType));
}
return TStatus::Ok;
}
@@ -721,19 +721,19 @@ TStatus AnnotateOlapFilter(const TExprNode::TPtr& node, TExprContext& ctx) {
}
TStatus AnnotateOlapFilterCompare(const TExprNode::TPtr& node, TExprContext& ctx) {
- if (!EnsureArgsCount(*node, 3, ctx)) {
- return TStatus::Error;
- }
-
+ if (!EnsureArgsCount(*node, 3, ctx)) {
+ return TStatus::Error;
+ }
+
auto* input = node->Child(TKqpOlapFilterCompare::idx_Input);
-
+
const TTypeAnnotationNode* itemType;
- if (!EnsureNewSeqType<false, false, true>(*input, ctx, &itemType)) {
+ if (!EnsureNewSeqType<false, false, true>(*input, ctx, &itemType)) {
return TStatus::Error;
}
- if (!EnsureStructType(input->Pos(), *itemType, ctx)) {
+ if (!EnsureStructType(input->Pos(), *itemType, ctx)) {
return TStatus::Error;
}
@@ -741,11 +741,11 @@ TStatus AnnotateOlapFilterCompare(const TExprNode::TPtr& node, TExprContext& ctx
// Column name, validate that it is present in Input node
if (TCoAtom::Match(node)) {
auto rowType = itemType->Cast<TStructExprType>();
-
+
if (rowType->FindItem(node->Content())) {
return true;
}
-
+
ctx.AddError(TIssue(
ctx.GetPosition(node->Pos()),
TStringBuilder() << "Missing column in input type: " << node->Content()
@@ -860,170 +860,170 @@ TStatus AnnotateKqpTxInternalBinding(const TExprNode::TPtr& node, TExprContext&
return TStatus::Ok;
}
-TStatus AnnotateKqpTxResultBinding(const TExprNode::TPtr& node, TExprContext& ctx) {
- if (!EnsureArgsCount(*node, 3, ctx)) {
- return TStatus::Error;
- }
-
- auto* txIndex = node->Child(TKqpTxResultBinding::idx_TxIndex);
- auto* txResultIndex = node->Child(TKqpTxResultBinding::idx_ResultIndex);
- if (!EnsureAtom(*txIndex, ctx) || !EnsureAtom(*txResultIndex, ctx)) {
- return TStatus::Error;
- }
-
- auto* type = node->Child(TKqpTxResultBinding::idx_Type);
- if (!EnsureType(*type, ctx)) {
- return TStatus::Error;
- }
-
- node->SetTypeAnn(type->GetTypeAnn()->Cast<TTypeExprType>()->GetType());
+TStatus AnnotateKqpTxResultBinding(const TExprNode::TPtr& node, TExprContext& ctx) {
+ if (!EnsureArgsCount(*node, 3, ctx)) {
+ return TStatus::Error;
+ }
+
+ auto* txIndex = node->Child(TKqpTxResultBinding::idx_TxIndex);
+ auto* txResultIndex = node->Child(TKqpTxResultBinding::idx_ResultIndex);
+ if (!EnsureAtom(*txIndex, ctx) || !EnsureAtom(*txResultIndex, ctx)) {
+ return TStatus::Error;
+ }
+
+ auto* type = node->Child(TKqpTxResultBinding::idx_Type);
+ if (!EnsureType(*type, ctx)) {
+ return TStatus::Error;
+ }
+
+ node->SetTypeAnn(type->GetTypeAnn()->Cast<TTypeExprType>()->GetType());
return TStatus::Ok;
}
-TStatus AnnotateKqpPhysicalTx(const TExprNode::TPtr& node, TExprContext& ctx) {
- if (!EnsureArgsCount(*node, 4, ctx)) {
- return TStatus::Error;
- }
-
- // TODO: ???
-
- node->SetTypeAnn(ctx.MakeType<TVoidExprType>());
+TStatus AnnotateKqpPhysicalTx(const TExprNode::TPtr& node, TExprContext& ctx) {
+ if (!EnsureArgsCount(*node, 4, ctx)) {
+ return TStatus::Error;
+ }
+
+ // TODO: ???
+
+ node->SetTypeAnn(ctx.MakeType<TVoidExprType>());
return TStatus::Ok;
}
-TStatus AnnotateKqpPhysicalQuery(const TExprNode::TPtr& node, TExprContext& ctx) {
- if (!EnsureArgsCount(*node, 3, ctx)) {
- return TStatus::Error;
- }
-
- // TODO: ???
-
- node->SetTypeAnn(ctx.MakeType<TVoidExprType>());
+TStatus AnnotateKqpPhysicalQuery(const TExprNode::TPtr& node, TExprContext& ctx) {
+ if (!EnsureArgsCount(*node, 3, ctx)) {
+ return TStatus::Error;
+ }
+
+ // TODO: ???
+
+ node->SetTypeAnn(ctx.MakeType<TVoidExprType>());
return TStatus::Ok;
}
-TStatus AnnotateKqpEffects(const TExprNode::TPtr& node, TExprContext& ctx) {
+TStatus AnnotateKqpEffects(const TExprNode::TPtr& node, TExprContext& ctx) {
auto kqpEffectType = MakeKqpEffectType(ctx);
- for (const auto& arg : node->ChildrenList()) {
- if (!EnsureCallable(*arg, ctx)) {
+ for (const auto& arg : node->ChildrenList()) {
+ if (!EnsureCallable(*arg, ctx)) {
return TStatus::Error;
}
- if (!TKqpUpsertRows::Match(arg.Get()) && !TKqpDeleteRows::Match(arg.Get())) {
- ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder()
- << "Unexpected effect: " << arg->Content()));
+ if (!TKqpUpsertRows::Match(arg.Get()) && !TKqpDeleteRows::Match(arg.Get())) {
+ ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder()
+ << "Unexpected effect: " << arg->Content()));
return TStatus::Error;
}
- if (!EnsureStreamType(*arg, ctx)) {
+ if (!EnsureStreamType(*arg, ctx)) {
return TStatus::Error;
}
- auto itemType = arg->GetTypeAnn()->Cast<TStreamExprType>()->GetItemType();
+ auto itemType = arg->GetTypeAnn()->Cast<TStreamExprType>()->GetItemType();
if (!IsSameAnnotation(*kqpEffectType, *itemType)) {
- ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder()
+ ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder()
<< "Invalid YDB effect type, expected: " << FormatType(kqpEffectType)
<< ", actual: " << FormatType(itemType)));
return TStatus::Error;
}
}
- node->SetTypeAnn(ctx.MakeType<TStreamExprType>(kqpEffectType));
+ node->SetTypeAnn(ctx.MakeType<TStreamExprType>(kqpEffectType));
+ return TStatus::Ok;
+}
+
+TStatus AnnotateKqpProgram(const TExprNode::TPtr& node, TExprContext& ctx) {
+ if (!EnsureArgsCount(*node, 2, ctx)) {
+ return TStatus::Error;
+ }
+
+ auto* argsType = node->Child(TKqpProgram::idx_ArgsType);
+
+ if (!EnsureType(*argsType, ctx)) {
+ return TStatus::Error;
+ }
+ auto argTypesTupleRaw = argsType->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+
+ if (!EnsureTupleType(node->Pos(), *argTypesTupleRaw, ctx)) {
+ return TStatus::Error;
+ }
+ auto argTypesTuple = argTypesTupleRaw->Cast<TTupleExprType>();
+
+ std::vector<const TTypeAnnotationNode*> argTypes;
+ argTypes.reserve(argTypesTuple->GetSize());
+
+ for (const auto& argTypeRaw : argTypesTuple->GetItems()) {
+ if (!EnsureStreamType(node->Pos(), *argTypeRaw, ctx)) {
+ return TStatus::Error;
+ }
+ argTypes.push_back(argTypeRaw);
+ }
+
+ auto& lambda = node->ChildRef(TKqpProgram::idx_Lambda);
+ if (!EnsureLambda(*lambda, ctx)) {
+ return TStatus::Error;
+ }
+
+ if (!UpdateLambdaAllArgumentsTypes(lambda, argTypes, ctx)) {
+ return TStatus::Error;
+ }
+
+ if (!lambda->GetTypeAnn()) {
+ return TStatus::Repeat;
+ }
+
+ node->SetTypeAnn(ctx.MakeType<TVoidExprType>());
+ return TStatus::Ok;
+}
+
+TStatus AnnotateKqpEnsure(const TExprNode::TPtr& node, TExprContext& ctx) {
+ if (!EnsureArgsCount(*node, 4, ctx)) {
+ return TStatus::Error;
+ }
+
+ if (!EnsureComputable(node->Head(), ctx)) {
+ return TStatus::Error;
+ }
+
+ const TDataExprType* dataType;
+ bool isOptional;
+ if (!EnsureDataOrOptionalOfData(*node->Child(TKqpEnsure::idx_Predicate), isOptional, dataType, ctx)) {
+ return TStatus::Error;
+ }
+
+ if (!EnsureSpecificDataType(node->Child(TKqpEnsure::idx_Predicate)->Pos(), *dataType, EDataSlot::Bool, ctx)) {
+ return TStatus::Error;
+ }
+
+ if (!EnsureAtom(*node->Child(TKqpEnsure::idx_IssueCode), ctx)) {
+ return TStatus::Error;
+ }
+
+ ui64 code;
+ if (!TryFromString(node->Child(TKqpEnsure::idx_IssueCode)->Content(), code)) {
+ ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder()
+ << "Issue code expected to be an integer: " << node->Child(TKqpEnsure::idx_IssueCode)->Content()));
+ return TStatus::Error;
+ }
+
+ const auto* message = node->Child(TKqpEnsure::idx_Message);
+ if (message->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Data) {
+ ctx.AddError(TIssue(ctx.GetPosition(message->Pos()), TStringBuilder()
+ << "Expected Utf8, but got: " << *message->GetTypeAnn()));
+ return TStatus::Error;
+ }
+
+ if (const auto dataSlot = message->GetTypeAnn()->Cast<TDataExprType>()->GetSlot(); dataSlot != EDataSlot::Utf8) {
+ ctx.AddError(TIssue(ctx.GetPosition(message->Pos()), TStringBuilder()
+ << "Expected Utf8, but got: " << *message->GetTypeAnn()));
+ return TStatus::Error;
+ }
+
+ node->SetTypeAnn(node->Head().GetTypeAnn());
return TStatus::Ok;
}
-TStatus AnnotateKqpProgram(const TExprNode::TPtr& node, TExprContext& ctx) {
- if (!EnsureArgsCount(*node, 2, ctx)) {
- return TStatus::Error;
- }
-
- auto* argsType = node->Child(TKqpProgram::idx_ArgsType);
-
- if (!EnsureType(*argsType, ctx)) {
- return TStatus::Error;
- }
- auto argTypesTupleRaw = argsType->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
-
- if (!EnsureTupleType(node->Pos(), *argTypesTupleRaw, ctx)) {
- return TStatus::Error;
- }
- auto argTypesTuple = argTypesTupleRaw->Cast<TTupleExprType>();
-
- std::vector<const TTypeAnnotationNode*> argTypes;
- argTypes.reserve(argTypesTuple->GetSize());
-
- for (const auto& argTypeRaw : argTypesTuple->GetItems()) {
- if (!EnsureStreamType(node->Pos(), *argTypeRaw, ctx)) {
- return TStatus::Error;
- }
- argTypes.push_back(argTypeRaw);
- }
-
- auto& lambda = node->ChildRef(TKqpProgram::idx_Lambda);
- if (!EnsureLambda(*lambda, ctx)) {
- return TStatus::Error;
- }
-
- if (!UpdateLambdaAllArgumentsTypes(lambda, argTypes, ctx)) {
- return TStatus::Error;
- }
-
- if (!lambda->GetTypeAnn()) {
- return TStatus::Repeat;
- }
-
- node->SetTypeAnn(ctx.MakeType<TVoidExprType>());
- return TStatus::Ok;
-}
-
-TStatus AnnotateKqpEnsure(const TExprNode::TPtr& node, TExprContext& ctx) {
- if (!EnsureArgsCount(*node, 4, ctx)) {
- return TStatus::Error;
- }
-
- if (!EnsureComputable(node->Head(), ctx)) {
- return TStatus::Error;
- }
-
- const TDataExprType* dataType;
- bool isOptional;
- if (!EnsureDataOrOptionalOfData(*node->Child(TKqpEnsure::idx_Predicate), isOptional, dataType, ctx)) {
- return TStatus::Error;
- }
-
- if (!EnsureSpecificDataType(node->Child(TKqpEnsure::idx_Predicate)->Pos(), *dataType, EDataSlot::Bool, ctx)) {
- return TStatus::Error;
- }
-
- if (!EnsureAtom(*node->Child(TKqpEnsure::idx_IssueCode), ctx)) {
- return TStatus::Error;
- }
-
- ui64 code;
- if (!TryFromString(node->Child(TKqpEnsure::idx_IssueCode)->Content(), code)) {
- ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder()
- << "Issue code expected to be an integer: " << node->Child(TKqpEnsure::idx_IssueCode)->Content()));
- return TStatus::Error;
- }
-
- const auto* message = node->Child(TKqpEnsure::idx_Message);
- if (message->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Data) {
- ctx.AddError(TIssue(ctx.GetPosition(message->Pos()), TStringBuilder()
- << "Expected Utf8, but got: " << *message->GetTypeAnn()));
- return TStatus::Error;
- }
-
- if (const auto dataSlot = message->GetTypeAnn()->Cast<TDataExprType>()->GetSlot(); dataSlot != EDataSlot::Utf8) {
- ctx.AddError(TIssue(ctx.GetPosition(message->Pos()), TStringBuilder()
- << "Expected Utf8, but got: " << *message->GetTypeAnn()));
- return TStatus::Error;
- }
-
- node->SetTypeAnn(node->Head().GetTypeAnn());
- return TStatus::Ok;
-}
-
} // namespace
TAutoPtr<IGraphTransformer> CreateKqpTypeAnnotationTransformer(const TString& cluster,
@@ -1036,46 +1036,46 @@ TAutoPtr<IGraphTransformer> CreateKqpTypeAnnotationTransformer(const TString& cl
TExprContext& ctx) -> TStatus
{
output = input;
-
- TIssueScopeGuard issueScope(ctx.IssueManager, [&input, &ctx] {
- return MakeIntrusive<TIssue>(ctx.GetPosition(input->Pos()),
- TStringBuilder() << "At function: " << input->Content());
- });
-
- if (TKqpTable::Match(input.Get())) {
- return AnnotateTable(input, ctx, cluster, *tablesData);
+
+ TIssueScopeGuard issueScope(ctx.IssueManager, [&input, &ctx] {
+ return MakeIntrusive<TIssue>(ctx.GetPosition(input->Pos()),
+ TStringBuilder() << "At function: " << input->Content());
+ });
+
+ if (TKqpTable::Match(input.Get())) {
+ return AnnotateTable(input, ctx, cluster, *tablesData);
}
- if (TKqlReadTableBase::Match(input.Get())) {
- return AnnotateReadTable(input, ctx, cluster, *tablesData, config->SystemColumnsEnabled());
+ if (TKqlReadTableBase::Match(input.Get())) {
+ return AnnotateReadTable(input, ctx, cluster, *tablesData, config->SystemColumnsEnabled());
}
if (TKqlReadTableRangesBase::Match(input.Get())) {
return AnnotateReadTableRanges(input, ctx, cluster, *tablesData, config->SystemColumnsEnabled());
}
- if (TKqlLookupTableBase::Match(input.Get())) {
- return AnnotateLookupTable(input, ctx, cluster, *tablesData, config->SystemColumnsEnabled());
+ if (TKqlLookupTableBase::Match(input.Get())) {
+ return AnnotateLookupTable(input, ctx, cluster, *tablesData, config->SystemColumnsEnabled());
}
- if (TKqlKeyInc::Match(input.Get()) || TKqlKeyExc::Match(input.Get())) {
- return AnnotateKeyTuple(input, ctx);
+ if (TKqlKeyInc::Match(input.Get()) || TKqlKeyExc::Match(input.Get())) {
+ return AnnotateKeyTuple(input, ctx);
}
- if (TKqlUpsertRowsBase::Match(input.Get())) {
- return AnnotateUpsertRows(input, ctx, cluster, *tablesData);
+ if (TKqlUpsertRowsBase::Match(input.Get())) {
+ return AnnotateUpsertRows(input, ctx, cluster, *tablesData);
}
if (TKqlInsertRowsBase::Match(input.Get())) {
- return AnnotateInsertRows(input, ctx, cluster, *tablesData);
- }
-
+ return AnnotateInsertRows(input, ctx, cluster, *tablesData);
+ }
+
if (TKqlUpdateRowsBase::Match(input.Get())) {
- return AnnotateUpdateRows(input, ctx, cluster, *tablesData);
- }
-
- if (TKqlDeleteRowsBase::Match(input.Get())) {
- return AnnotateDeleteRows(input, ctx, cluster, *tablesData);
+ return AnnotateUpdateRows(input, ctx, cluster, *tablesData);
+ }
+
+ if (TKqlDeleteRowsBase::Match(input.Get())) {
+ return AnnotateDeleteRows(input, ctx, cluster, *tablesData);
}
if (TKqpOlapFilter::Match(input.Get())) {
@@ -1090,38 +1090,38 @@ TAutoPtr<IGraphTransformer> CreateKqpTypeAnnotationTransformer(const TString& cl
return AnnotateOlapFilterExists(input, ctx);
}
- if (TKqpCnMapShard::Match(input.Get()) || TKqpCnShuffleShard::Match(input.Get())) {
- return AnnotateDqConnection(input, ctx);
+ if (TKqpCnMapShard::Match(input.Get()) || TKqpCnShuffleShard::Match(input.Get())) {
+ return AnnotateDqConnection(input, ctx);
+ }
+
+ if (TKqpTxResultBinding::Match(input.Get())) {
+ return AnnotateKqpTxResultBinding(input, ctx);
}
- if (TKqpTxResultBinding::Match(input.Get())) {
- return AnnotateKqpTxResultBinding(input, ctx);
- }
-
if (TKqpTxInternalBinding::Match(input.Get())) {
return AnnotateKqpTxInternalBinding(input, ctx);
}
- if (TKqpPhysicalTx::Match(input.Get())) {
- return AnnotateKqpPhysicalTx(input, ctx);
+ if (TKqpPhysicalTx::Match(input.Get())) {
+ return AnnotateKqpPhysicalTx(input, ctx);
}
- if (TKqpPhysicalQuery::Match(input.Get())) {
- return AnnotateKqpPhysicalQuery(input, ctx);
+ if (TKqpPhysicalQuery::Match(input.Get())) {
+ return AnnotateKqpPhysicalQuery(input, ctx);
}
- if (TKqpEffects::Match(input.Get())) {
- return AnnotateKqpEffects(input, ctx);
+ if (TKqpEffects::Match(input.Get())) {
+ return AnnotateKqpEffects(input, ctx);
}
- if (TKqpProgram::Match(input.Get())) {
- return AnnotateKqpProgram(input, ctx);
+ if (TKqpProgram::Match(input.Get())) {
+ return AnnotateKqpProgram(input, ctx);
+ }
+
+ if (TKqpEnsure::Match(input.Get())) {
+ return AnnotateKqpEnsure(input, ctx);
}
- if (TKqpEnsure::Match(input.Get())) {
- return AnnotateKqpEnsure(input, ctx);
- }
-
return dqTransformer->Transform(input, output, ctx);
});
}
@@ -1135,12 +1135,12 @@ TAutoPtr<IGraphTransformer> CreateKqpCheckQueryTransformer() {
auto query = TKqlQuery(input);
for (const auto& result : query.Results()) {
- if (!EnsureTupleSize(result.Ref(), 2, ctx)) {
+ if (!EnsureTupleSize(result.Ref(), 2, ctx)) {
+ return TStatus::Error;
+ }
+ if (!EnsureListType(result.Value().Ref(), ctx)) {
return TStatus::Error;
}
- if (!EnsureListType(result.Value().Ref(), ctx)) {
- return TStatus::Error;
- }
}
return TStatus::Ok;
@@ -1171,5 +1171,5 @@ TAutoPtr<IGraphTransformer> CreateKqpCheckKiProgramTransformer() {
});
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/provider/ya.make b/ydb/core/kqp/provider/ya.make
index 558341ba41..19dd0db44b 100644
--- a/ydb/core/kqp/provider/ya.make
+++ b/ydb/core/kqp/provider/ya.make
@@ -3,7 +3,7 @@ LIBRARY()
OWNER(spuchin)
SRCS(
- kqp_opt_helpers.cpp
+ kqp_opt_helpers.cpp
yql_kikimr_datasink.cpp
yql_kikimr_datasource.cpp
yql_kikimr_exec.cpp
@@ -16,12 +16,12 @@ SRCS(
yql_kikimr_opt_build.cpp
yql_kikimr_opt_join.cpp
yql_kikimr_opt_range.cpp
- yql_kikimr_opt_utils.cpp
+ yql_kikimr_opt_utils.cpp
yql_kikimr_opt.cpp
yql_kikimr_provider.h
yql_kikimr_provider.cpp
yql_kikimr_provider_impl.h
- yql_kikimr_query_traits.cpp
+ yql_kikimr_query_traits.cpp
yql_kikimr_results.cpp
yql_kikimr_results.h
yql_kikimr_settings.cpp
diff --git a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp
index 1d2416a8d9..035d6b7701 100644
--- a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp
@@ -315,7 +315,7 @@ public:
, TypeAnnotationTransformer(CreateKiSinkTypeAnnotationTransformer(gateway, sessionCtx))
, LogicalOptProposalTransformer(CreateKiLogicalOptProposalTransformer(sessionCtx))
, PhysicalOptProposalTransformer(CreateKiPhysicalOptProposalTransformer(sessionCtx))
- , CallableExecutionTransformer(CreateKiSinkCallableExecutionTransformer(gateway, sessionCtx, queryExecutor))
+ , CallableExecutionTransformer(CreateKiSinkCallableExecutionTransformer(gateway, sessionCtx, queryExecutor))
, PlanInfoTransformer(CreateKiSinkPlanInfoTransformer(queryExecutor))
{
Y_UNUSED(FunctionRegistry);
@@ -418,7 +418,7 @@ public:
}
bool CanExecute(const TExprNode& node) override {
- if (node.IsCallable(TKiExecDataQuery::CallableName())) {
+ if (node.IsCallable(TKiExecDataQuery::CallableName())) {
return true;
}
@@ -438,7 +438,7 @@ public:
}
if (auto maybeRight = TMaybeNode<TCoNth>(&node).Tuple().Maybe<TCoRight>()) {
- if (maybeRight.Input().Maybe<TKiExecDataQuery>()) {
+ if (maybeRight.Input().Maybe<TKiExecDataQuery>()) {
return true;
}
}
@@ -661,16 +661,16 @@ public:
return false;
}
- void GetInputs(const TExprNode&, TVector<TPinInfo>&) override {
+ void GetInputs(const TExprNode&, TVector<TPinInfo>&) override {
}
- void GetOutputs(const TExprNode&, TVector<TPinInfo>&) override {
+ void GetOutputs(const TExprNode&, TVector<TPinInfo>&) override {
}
- void WritePlanDetails(const TExprNode&, NYson::TYsonWriter&) override {
+ void WritePlanDetails(const TExprNode&, NYson::TYsonWriter&) override {
}
- TString GetProviderPath(const TExprNode&) override {
+ TString GetProviderPath(const TExprNode&) override {
return TString(KikimrProviderName);
}
diff --git a/ydb/core/kqp/provider/yql_kikimr_datasource.cpp b/ydb/core/kqp/provider/yql_kikimr_datasource.cpp
index 497df7ed3a..65ef7eb932 100644
--- a/ydb/core/kqp/provider/yql_kikimr_datasource.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_datasource.cpp
@@ -121,7 +121,7 @@ public:
YQL_ENSURE(!future.HasException());
const auto& value = future.GetValue();
switch (queryType) {
- case EKikimrQueryType::Unspecified: {
+ case EKikimrQueryType::Unspecified: {
if (value.Metadata) {
if (!value.Metadata->Indexes.empty()) {
result->AddIssue(TIssue({}, TStringBuilder()
@@ -220,7 +220,7 @@ protected:
bool HandleAttr(TPositionHandle pos, const TString& cluster, const TString& name, const TMaybe<TString>& value,
TExprContext& ctx) final
{
- YQL_ENSURE(SessionCtx->Query().Type != EKikimrQueryType::Unspecified);
+ YQL_ENSURE(SessionCtx->Query().Type != EKikimrQueryType::Unspecified);
bool applied = Dispatcher->Dispatch(cluster, name, value, NCommon::TSettingDispatcher::EStage::STATIC);
@@ -252,9 +252,9 @@ protected:
}
bool HandleAuth(TPositionHandle pos, const TString& cluster, const TString& alias, TExprContext& ctx) final {
- YQL_ENSURE(SessionCtx->Query().Type != EKikimrQueryType::Unspecified);
-
- if (SessionCtx->Query().Type != EKikimrQueryType::YqlInternal) {
+ YQL_ENSURE(SessionCtx->Query().Type != EKikimrQueryType::Unspecified);
+
+ if (SessionCtx->Query().Type != EKikimrQueryType::YqlInternal) {
ctx.AddError(YqlIssue(ctx.GetPosition(pos), TIssuesIds::KIKIMR_PRAGMA_NOT_SUPPORTED, TStringBuilder()
<< "Pragma auth not supported inside Kikimr query."));
return false;
@@ -419,7 +419,7 @@ public:
}
if (auto maybeRight = TMaybeNode<TCoNth>(&node).Tuple().Maybe<TCoRight>()) {
- if (maybeRight.Input().Maybe<TKiExecDataQuery>()) {
+ if (maybeRight.Input().Maybe<TKiExecDataQuery>()) {
return true;
}
}
@@ -498,11 +498,11 @@ public:
TExprNode::TPtr OptimizePull(const TExprNode::TPtr& source, const TFillSettings& fillSettings, TExprContext& ctx,
IOptimizationContext& optCtx) override
{
- auto queryType = SessionCtx->Query().Type;
+ auto queryType = SessionCtx->Query().Type;
if (queryType == EKikimrQueryType::Scan) {
- return source;
- }
-
+ return source;
+ }
+
if (auto execQuery = TMaybeNode<TCoNth>(source).Tuple().Maybe<TCoRight>().Input().Maybe<TKiExecDataQuery>()) {
auto nth = TCoNth(source);
ui32 index = ::FromString<ui32>(nth.Index());
diff --git a/ydb/core/kqp/provider/yql_kikimr_exec.cpp b/ydb/core/kqp/provider/yql_kikimr_exec.cpp
index 5fe05dbf14..ad360fcaeb 100644
--- a/ydb/core/kqp/provider/yql_kikimr_exec.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_exec.cpp
@@ -128,30 +128,30 @@ public:
TKiSinkPlanInfoTransformer(TIntrusivePtr<IKikimrQueryExecutor> queryExecutor)
: QueryExecutor(queryExecutor) {}
- TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ) final {
+ TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ) final {
output = input;
- VisitExpr(input, [](const TExprNode::TPtr& node) {
+ VisitExpr(input, [](const TExprNode::TPtr& node) {
if (auto maybeExec = TMaybeNode<TKiExecDataQuery>(node)) {
auto exec = maybeExec.Cast();
if (exec.Ast().Maybe<TCoVoid>()) {
- YQL_ENSURE(false);
+ YQL_ENSURE(false);
}
}
return true;
});
- return TStatus::Ok;
+ return TStatus::Ok;
}
TFuture<void> DoGetAsyncFuture(const TExprNode& input) final {
Y_UNUSED(input);
- return MakeFuture();
+ return MakeFuture();
}
- TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext&) final {
+ TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext&) final {
output = input;
- return TStatus::Ok;
+ return TStatus::Ok;
}
private:
@@ -247,7 +247,7 @@ public:
if (input->Content() == "Result") {
auto resultInput = TExprBase(input->ChildPtr(0));
auto exec = resultInput.Maybe<TCoNth>().Tuple().Maybe<TCoRight>().Input();
- YQL_ENSURE(exec.Maybe<TKiExecDataQuery>());
+ YQL_ENSURE(exec.Maybe<TKiExecDataQuery>());
ui32 index = FromString<ui32>(resultInput.Cast<TCoNth>().Index().Value());
YQL_ENSURE(index == 0);
@@ -407,7 +407,7 @@ public:
TIntrusivePtr<IKikimrGateway> gateway,
TIntrusivePtr<TKikimrSessionContext> sessionCtx,
TIntrusivePtr<IKikimrQueryExecutor> queryExecutor)
- : Gateway(gateway)
+ : Gateway(gateway)
, SessionCtx(sessionCtx)
, QueryExecutor(queryExecutor) {}
@@ -1043,7 +1043,7 @@ private:
return std::make_pair(IGraphTransformer::TStatus::Error, TAsyncTransformCallbackFuture());
}
- auto insertResult = SessionCtx->Query().Results.emplace(resultId, std::move(result));
+ auto insertResult = SessionCtx->Query().Results.emplace(resultId, std::move(result));
YQL_ENSURE(insertResult.second);
SessionCtx->Query().ExecutionOrder.push_back(resultId);
@@ -1179,7 +1179,7 @@ TAutoPtr<IGraphTransformer> CreateKiSinkCallableExecutionTransformer(
TIntrusivePtr<TKikimrSessionContext> sessionCtx,
TIntrusivePtr<IKikimrQueryExecutor> queryExecutor)
{
- return new TKiSinkCallableExecutionTransformer(gateway, sessionCtx, queryExecutor);
+ return new TKiSinkCallableExecutionTransformer(gateway, sessionCtx, queryExecutor);
}
TAutoPtr<IGraphTransformer> CreateKiSinkPlanInfoTransformer(TIntrusivePtr<IKikimrQueryExecutor> queryExecutor) {
diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.h b/ydb/core/kqp/provider/yql_kikimr_gateway.h
index b8a558e3d8..7598494a2b 100644
--- a/ydb/core/kqp/provider/yql_kikimr_gateway.h
+++ b/ydb/core/kqp/provider/yql_kikimr_gateway.h
@@ -1,7 +1,7 @@
#pragma once
-#include "yql_kikimr_query_traits.h"
-
+#include "yql_kikimr_query_traits.h"
+
#include <ydb/library/yql/public/udf/udf_data_type.h>
#include <ydb/library/yql/utils/resetable_setting.h>
#include <ydb/library/yql/utils/yql_panic.h>
@@ -57,8 +57,8 @@ struct TIndexDescription {
enum class EIndexState : ui32 {
Invalid = 0, // this state should not be used
Ready = 1, // index is ready to use
- NotReady = 2, // index is visible but not ready to use
- WriteOnly = 3 // index is visible only write operations to index are allowed
+ NotReady = 2, // index is visible but not ready to use
+ WriteOnly = 3 // index is visible only write operations to index are allowed
};
const TString Name;
@@ -490,7 +490,7 @@ template<typename TResult>
class IKikimrAsyncResult : public TThrRefBase {
public:
virtual bool HasResult() const = 0;
- virtual TResult GetResult() = 0;
+ virtual TResult GetResult() = 0;
virtual NThreading::TFuture<bool> Continue() = 0;
virtual ~IKikimrAsyncResult() {}
@@ -499,16 +499,16 @@ public:
template<typename TResult>
class TKikimrResultHolder : public IKikimrAsyncResult<TResult> {
public:
- TKikimrResultHolder(TResult&& result)
- : Result(std::move(result)) {}
+ TKikimrResultHolder(TResult&& result)
+ : Result(std::move(result)) {}
bool HasResult() const override {
- return Full;
+ return Full;
}
- TResult GetResult() override {
- Full = false;
- return std::move(Result);
+ TResult GetResult() override {
+ Full = false;
+ return std::move(Result);
}
NThreading::TFuture<bool> Continue() override {
@@ -517,12 +517,12 @@ public:
private:
TResult Result;
- bool Full = true;
+ bool Full = true;
};
template<typename TResult>
-static TIntrusivePtr<TKikimrResultHolder<TResult>> MakeKikimrResultHolder(TResult&& result) {
- return MakeIntrusive<TKikimrResultHolder<TResult>>(std::move(result));
+static TIntrusivePtr<TKikimrResultHolder<TResult>> MakeKikimrResultHolder(TResult&& result) {
+ return MakeIntrusive<TKikimrResultHolder<TResult>>(std::move(result));
}
class IKikimrGateway : public TThrRefBase {
@@ -546,9 +546,9 @@ public:
TVector<NKikimrMiniKQL::TResult*> Results;
TMaybe<NKikimrKqp::TQueryProfile> Profile; // TODO: Deprecate.
NKqpProto::TKqpStatsQuery QueryStats;
- std::unique_ptr<NKikimrKqp::TPreparedQuery> PreparingQuery;
- std::shared_ptr<const NKikimrKqp::TPreparedQuery> PreparedQuery;
- std::optional<NKikimr::NKqp::TQueryTraits> QueryTraits;
+ std::unique_ptr<NKikimrKqp::TPreparedQuery> PreparingQuery;
+ std::shared_ptr<const NKikimrKqp::TPreparedQuery> PreparedQuery;
+ std::optional<NKikimr::NKqp::TQueryTraits> QueryTraits;
TString QueryAst;
TString QueryPlan;
std::shared_ptr<google::protobuf::Arena> ProtobufArenaPtr;
diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp b/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp
index 7fe9c66331..791a617c08 100644
--- a/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp
@@ -68,10 +68,10 @@ void CreateSampleTables(TKikimrRunner& runner) {
}
TIntrusivePtr<IKqpGateway> GetIcGateway(Tests::TServer& server) {
- auto counters = MakeIntrusive<TKqpRequestCounters>();
- counters->Counters = new TKqpCounters(server.GetRuntime()->GetAppData(0).Counters);
- counters->TxProxyMon = new NTxProxy::TTxProxyMon(server.GetRuntime()->GetAppData(0).Counters);
-
+ auto counters = MakeIntrusive<TKqpRequestCounters>();
+ counters->Counters = new TKqpCounters(server.GetRuntime()->GetAppData(0).Counters);
+ counters->TxProxyMon = new NTxProxy::TTxProxyMon(server.GetRuntime()->GetAppData(0).Counters);
+
std::shared_ptr<NYql::IKikimrGateway::IKqpTableMetadataLoader> loader = std::make_shared<TKqpTableMetadataLoader>(server.GetRuntime()->GetAnyNodeActorSystem(), false);
return CreateKikimrIcGateway(TestCluster, "/Root", std::move(loader), server.GetRuntime()->GetAnyNodeActorSystem(),
server.GetRuntime()->GetNodeId(0), counters, MakeMiniKQLCompileServiceID());
diff --git a/ydb/core/kqp/provider/yql_kikimr_opt.cpp b/ydb/core/kqp/provider/yql_kikimr_opt.cpp
index 1ae0f4bbaf..cdbb6e600d 100644
--- a/ydb/core/kqp/provider/yql_kikimr_opt.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_opt.cpp
@@ -593,7 +593,7 @@ TExprNode::TPtr KiRewritePartialTakeSortOverSelectIndexRange(TExprBase node, con
if (!CanPushPartialSort(maybePartialSort.Cast(), indexDesc, &sortByColumns)) {
return node.Ptr();
}
-
+
auto filter = [&ctx, &node, &partialSort, &partialTake](const TExprBase& in) mutable {
auto out = Build<TKiPartialTake>(ctx, node.Pos())
.Input<TKiPartialSort>()
@@ -670,11 +670,11 @@ TAutoPtr<IGraphTransformer> CreateKiLogicalOptProposalTransformer(TIntrusivePtr<
auto ret = inputNode;
TExprBase node(inputNode);
- ret = KiSqlInToEquiJoin(node, sessionCtx->Tables(), sessionCtx->Config(), ctx);
- if (ret != inputNode) {
- return ret;
- }
-
+ ret = KiSqlInToEquiJoin(node, sessionCtx->Tables(), sessionCtx->Config(), ctx);
+ if (ret != inputNode) {
+ return ret;
+ }
+
ret = KiApplyExtractMembersToSelectRange(node, ctx);
if (ret != inputNode) {
return ret;
@@ -755,11 +755,11 @@ TAutoPtr<IGraphTransformer> CreateKiPhysicalOptProposalTransformer(TIntrusivePtr
if (auto maybeDatasink = node.Maybe<TCoCommit>().DataSink().Maybe<TKiDataSink>()) {
auto cluster = TString(maybeDatasink.Cast().Cluster());
auto useNewEngine = sessionCtx->Config().UseNewEngine.Get();
- if (!useNewEngine.Defined() && sessionCtx->Config().HasKqpForceNewEngine()) {
- useNewEngine = true;
- }
+ if (!useNewEngine.Defined() && sessionCtx->Config().HasKqpForceNewEngine()) {
+ useNewEngine = true;
+ }
- ret = KiBuildQuery(node, useNewEngine, ctx);
+ ret = KiBuildQuery(node, useNewEngine, ctx);
if (ret != inputNode) {
return ret;
@@ -768,7 +768,7 @@ TAutoPtr<IGraphTransformer> CreateKiPhysicalOptProposalTransformer(TIntrusivePtr
if (sessionCtx->Config().HasDefaultCluster()) {
auto defaultCluster = sessionCtx->Config()._DefaultCluster.Get().GetRef();
- ret = KiBuildResult(node, defaultCluster, ctx);
+ ret = KiBuildResult(node, defaultCluster, ctx);
if (ret != inputNode) {
return ret;
}
diff --git a/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp b/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp
index 6c08591093..0ba0b34952 100644
--- a/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp
@@ -570,56 +570,56 @@ TExprNode::TPtr KiBuildQuery(TExprBase node, const TMaybe<bool>& useNewEngine, T
return ret;
}
-TExprNode::TPtr KiBuildResult(TExprBase node, const TString& cluster, TExprContext& ctx) {
- if (!node.Maybe<TResFill>()) {
+TExprNode::TPtr KiBuildResult(TExprBase node, const TString& cluster, TExprContext& ctx) {
+ if (!node.Maybe<TResFill>()) {
return node.Ptr();
}
- auto resFill = node.Cast<TResFill>();
+ auto resFill = node.Cast<TResFill>();
- if (resFill.DelegatedSource().Value() != KikimrProviderName) {
+ if (resFill.DelegatedSource().Value() != KikimrProviderName) {
return node.Ptr();
}
- if (resFill.Data().Maybe<TCoNth>().Tuple().Maybe<TCoRight>().Input().Maybe<TKiExecDataQuery>()) {
+ if (resFill.Data().Maybe<TCoNth>().Tuple().Maybe<TCoRight>().Input().Maybe<TKiExecDataQuery>()) {
return node.Ptr();
}
- auto dataQuery = Build<TKiDataQuery>(ctx, node.Pos())
- .Operations()
- .Build()
- .Results()
- .Add()
- .Value(resFill.Data())
- .Columns(GetResultColumns(resFill, ctx))
- .RowsLimit().Build(GetResultRowsLimit(resFill))
+ auto dataQuery = Build<TKiDataQuery>(ctx, node.Pos())
+ .Operations()
+ .Build()
+ .Results()
+ .Add()
+ .Value(resFill.Data())
+ .Columns(GetResultColumns(resFill, ctx))
+ .RowsLimit().Build(GetResultRowsLimit(resFill))
.Build()
.Build()
- .Effects()
+ .Effects()
.Build()
- .Done();
-
- auto exec = Build<TKiExecDataQuery>(ctx, node.Pos())
- .World(resFill.World())
- .DataSink<TKiDataSink>()
- .Category().Build(KikimrProviderName)
- .Cluster().Build(cluster)
+ .Done();
+
+ auto exec = Build<TKiExecDataQuery>(ctx, node.Pos())
+ .World(resFill.World())
+ .DataSink<TKiDataSink>()
+ .Category().Build(KikimrProviderName)
+ .Cluster().Build(cluster)
.Build()
- .Query(dataQuery)
+ .Query(dataQuery)
.Settings()
.Build()
- .Ast<TCoVoid>().Build()
+ .Ast<TCoVoid>().Build()
.Done();
auto data = Build<TCoNth>(ctx, node.Pos())
.Tuple<TCoRight>()
- .Input(exec)
+ .Input(exec)
.Build()
.Index().Build(0)
.Done();
auto world = Build<TCoLeft>(ctx, node.Pos())
- .Input(exec)
+ .Input(exec)
.Done();
return ctx.ChangeChild(*ctx.ChangeChild(resFill.Ref(), 0, world.Ptr()), 3, data.Ptr());
diff --git a/ydb/core/kqp/provider/yql_kikimr_opt_join.cpp b/ydb/core/kqp/provider/yql_kikimr_opt_join.cpp
index 7af578b760..662082c543 100644
--- a/ydb/core/kqp/provider/yql_kikimr_opt_join.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_opt_join.cpp
@@ -1,5 +1,5 @@
#include "yql_kikimr_provider_impl.h"
-#include "yql_kikimr_opt_utils.h"
+#include "yql_kikimr_opt_utils.h"
#include <ydb/library/yql/core/yql_join.h>
#include <ydb/library/yql/core/yql_opt_utils.h>
@@ -25,7 +25,7 @@ TExprBase GetEquiJoinInputList(TCoAtom scope, const TJoinLabels& joinLabels,
void GatherEquiJoinLables(TExprBase joinScope, TVector<TString>& labels) {
if (joinScope.Maybe<TCoAtom>()) {
- labels.emplace_back(TString(joinScope.Cast<TCoAtom>()));
+ labels.emplace_back(TString(joinScope.Cast<TCoAtom>()));
return;
}
@@ -40,36 +40,36 @@ TSet<TString> GetEquiJoinLabelReps(TExprBase joinScope) {
return TSet<TString>(labels.begin(), labels.end());
}
-TExprBase ConvertToTuples(const TSet<TString>& columns, const TCoArgument& structArg, TExprContext& ctx,
- TPositionHandle pos)
-{
- TVector<TExprBase> tuples{Reserve(columns.size())};
-
- for (const auto& key : columns) {
- tuples.emplace_back(Build<TCoMember>(ctx, pos)
- .Struct(structArg)
- .Name().Build(key)
- .Done());
- }
-
- if (tuples.size() == 1) {
- return tuples[0];
- }
-
- return Build<TExprList>(ctx, pos)
- .Add(tuples)
- .Done();
-}
-
-TVector<TExprBase> ConvertToAtoms(const TSet<TString>& columns, TExprContext& ctx, TPositionHandle pos) {
- TVector<TExprBase> list{Reserve(columns.size())};
- for (const auto& column : columns) {
- list.emplace_back(TCoAtom(ctx.NewAtom(pos, column)));
- }
- return list;
-};
-
-
+TExprBase ConvertToTuples(const TSet<TString>& columns, const TCoArgument& structArg, TExprContext& ctx,
+ TPositionHandle pos)
+{
+ TVector<TExprBase> tuples{Reserve(columns.size())};
+
+ for (const auto& key : columns) {
+ tuples.emplace_back(Build<TCoMember>(ctx, pos)
+ .Struct(structArg)
+ .Name().Build(key)
+ .Done());
+ }
+
+ if (tuples.size() == 1) {
+ return tuples[0];
+ }
+
+ return Build<TExprList>(ctx, pos)
+ .Add(tuples)
+ .Done();
+}
+
+TVector<TExprBase> ConvertToAtoms(const TSet<TString>& columns, TExprContext& ctx, TPositionHandle pos) {
+ TVector<TExprBase> list{Reserve(columns.size())};
+ for (const auto& column : columns) {
+ list.emplace_back(TCoAtom(ctx.NewAtom(pos, column)));
+ }
+ return list;
+};
+
+
TMaybeNode<TExprBase> EquiJoinGetIdxLookupValue(const TStringBuf& leftDataName, const TStringBuf& rightDataName,
TExprBase leftRow, const TString& leftMemberName, TPositionHandle pos, TExprContext& ctx)
{
@@ -129,8 +129,8 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c
joinNames.RightSemi
};
- const TStringBuf joinType = joinTuple.Type().Value();
- if (Find(allowedJoins, joinType) == allowedJoins.cend()) {
+ const TStringBuf joinType = joinTuple.Type().Value();
+ if (Find(allowedJoins, joinType) == allowedJoins.cend()) {
return false;
}
@@ -185,10 +185,10 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c
return false;
}
- const auto selectRange = rightSelect.Cast();
- const TStringBuf cluster = selectRange.Cluster().Value();
+ const auto selectRange = rightSelect.Cast();
+ const TStringBuf cluster = selectRange.Cluster().Value();
const TStringBuf lookupTable = indexTable ? indexTable.Cast().Value() : selectRange.Table().Path().Value();
- const TKikimrTableDescription& lookupTableDesc = tablesData.ExistingTable(cluster, lookupTable);
+ const TKikimrTableDescription& lookupTableDesc = tablesData.ExistingTable(cluster, lookupTable);
auto rightKeyRange = TKikimrKeyRange::GetPointKeyRange(ctx, lookupTableDesc, selectRange.Range());
if (!rightKeyRange) {
@@ -240,12 +240,12 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c
return false;
}
- TString leftColumnName = ToString(joinLeftColumn.Value());
+ TString leftColumnName = ToString(joinLeftColumn.Value());
TString leftMemberName = prefixLeftMembers
? FullColumnName(joinLeftLabel.Value(), joinLeftColumn.Value())
: leftColumnName;
- TString rightColumnName = ToString(joinRightColumn.Value());
+ TString rightColumnName = ToString(joinRightColumn.Value());
auto keyColumnIdx = lookupTableDesc.GetKeyColumnIndex(rightColumnName);
if (!keyColumnIdx) {
@@ -264,7 +264,7 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c
auto leftInput = GetEquiJoinInputList(joinLeftLabel, joinLabels, joinInputs);
const TDataExprType* leftData;
const TDataExprType* rightData;
- if (!GetEquiJoinKeyTypes(leftInput, leftColumnName, lookupTableDesc, rightColumnName, leftData, rightData)) {
+ if (!GetEquiJoinKeyTypes(leftInput, leftColumnName, lookupTableDesc, rightColumnName, leftData, rightData)) {
return false;
}
@@ -278,11 +278,11 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c
}
for (size_t i = 0; i < keyColumnRanges.size(); ++i) {
- bool leftColumnDefined = keyColumnRanges[i].IsDefined();
- if (leftColumnDefined && rightKeyRange->GetColumnRange(i).IsDefined()) {
+ bool leftColumnDefined = keyColumnRanges[i].IsDefined();
+ if (leftColumnDefined && rightKeyRange->GetColumnRange(i).IsDefined()) {
return false;
- }
- if (!leftColumnDefined) {
+ }
+ if (!leftColumnDefined) {
keyColumnRanges[i] = rightKeyRange->GetColumnRange(i);
}
@@ -298,12 +298,12 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c
}
TCoAtomList lookupColumns = selectRange.Select();
- bool requireIndexValues = false; // 'true' means that some requested columns are not presented in the index-table,
- // so read from data-table is required
+ bool requireIndexValues = false; // 'true' means that some requested columns are not presented in the index-table,
+ // so read from data-table is required
if (indexTable) {
- // In this case lookupTableDesc == indexTable,
- // so check whether index-table contains all lookup-columns
- for (const auto& lookupColumn : lookupColumns) {
+ // In this case lookupTableDesc == indexTable,
+ // so check whether index-table contains all lookup-columns
+ for (const auto& lookupColumn : lookupColumns) {
if (!lookupTableDesc.Metadata->Columns.contains(TString(lookupColumn.Value()))) {
requireIndexValues = true;
break;
@@ -311,19 +311,19 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c
}
}
- auto selectedColumns = (indexTable && requireIndexValues)
+ auto selectedColumns = (indexTable && requireIndexValues)
? BuildKeyColumnsList(lookupTableDesc, selectRange.Pos(), ctx)
- : lookupColumns;
+ : lookupColumns;
auto lookup = TKikimrKeyRange::BuildReadRangeExpr(lookupTableDesc, TKeyRange(ctx, keyColumnRanges, {}),
- selectedColumns, false /* allowNulls */, ctx);
+ selectedColumns, false /* allowNulls */, ctx);
// Skip null keys in lookup part as for equijoin semantics null != null,
// so we can't have nulls in lookup part
lookup = Build<TCoSkipNullMembers>(ctx, joinTuple.Pos())
.Input(lookup)
.Members()
- .Add(ConvertToAtoms(rightKeyColumnsSet, ctx, joinTuple.Pos()))
+ .Add(ConvertToAtoms(rightKeyColumnsSet, ctx, joinTuple.Pos()))
.Build()
.Done();
@@ -353,7 +353,7 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c
auto addJoinResults = [&joinLabels, &ctx]
(const TExprBase& joinTuple, const TExprBase& rowArg, TVector<TExprBase>& resultTuples,
TVector<TString>* resultColumns)
- {
+ {
TVector<std::pair<TString, TString>> joinColumns;
if (auto maybeAtom = joinTuple.Maybe<TCoAtom>()) {
auto maybeInput = joinLabels.FindInput(maybeAtom.Cast().Value());
@@ -381,7 +381,7 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c
for (auto& pair : joinColumns) {
auto& member = pair.first;
auto& column = pair.second;
-
+
if (resultColumns) {
resultColumns->push_back(column);
}
@@ -395,69 +395,69 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c
.Done();
resultTuples.emplace_back(std::move(tuple));
- }
- };
+ }
+ };
- auto injectRightDataKey = [&tablesData, &selectRange, &ctx]
- (TMaybeNode<TExprBase>& rightRow, TVector<TExprBase>& joinResultTuples)
- {
- const TStringBuf cluster = selectRange.Cluster();
+ auto injectRightDataKey = [&tablesData, &selectRange, &ctx]
+ (TMaybeNode<TExprBase>& rightRow, TVector<TExprBase>& joinResultTuples)
+ {
+ const TStringBuf cluster = selectRange.Cluster();
const TStringBuf table = selectRange.Table().Path();
- const auto& desc = tablesData.ExistingTable(cluster, table);
- for (const auto& col : desc.Metadata->KeyColumnNames) {
- auto tuple = Build<TCoNameValueTuple>(ctx, selectRange.Pos())
- .Name().Build(FullColumnName(desc.Metadata->Name, col))
- .Value<TCoMember>()
- .Struct(rightRow.Cast())
- .Name().Build(col)
- .Build()
- .Done();
- joinResultTuples.push_back(tuple);
- }
- };
+ const auto& desc = tablesData.ExistingTable(cluster, table);
+ for (const auto& col : desc.Metadata->KeyColumnNames) {
+ auto tuple = Build<TCoNameValueTuple>(ctx, selectRange.Pos())
+ .Name().Build(FullColumnName(desc.Metadata->Name, col))
+ .Value<TCoMember>()
+ .Struct(rightRow.Cast())
+ .Name().Build(col)
+ .Build()
+ .Done();
+ joinResultTuples.push_back(tuple);
+ }
+ };
auto getJoinResultExpr = [requireIndexValues, &indexTable, &addJoinResults,
&injectRightDataKey, &ctx, &joinTuple]
- (TMaybeNode<TExprBase> leftRowArg, TMaybeNode<TExprBase> rightRowArg)
- {
- TVector<TString> resultColumns;
- TVector<TExprBase> joinResultTuples;
-
- if (leftRowArg.IsValid()) {
+ (TMaybeNode<TExprBase> leftRowArg, TMaybeNode<TExprBase> rightRowArg)
+ {
+ TVector<TString> resultColumns;
+ TVector<TExprBase> joinResultTuples;
+
+ if (leftRowArg.IsValid()) {
addJoinResults(joinTuple.LeftScope(), leftRowArg.Cast(), joinResultTuples, &resultColumns);
- }
-
- if (rightRowArg.IsValid()) {
- if (!requireIndexValues) {
+ }
+
+ if (rightRowArg.IsValid()) {
+ if (!requireIndexValues) {
addJoinResults(joinTuple.RightScope(), rightRowArg.Cast(), joinResultTuples, &resultColumns);
- } else {
- // we need data from the right table that not contains in the lookup-table (not indexed columns),
- // so we add reads from the right table
- YQL_ENSURE(indexTable.IsValid());
- //TODO: May be it'a a good idea to get some collumns data from the index-read?
- injectRightDataKey(rightRowArg, joinResultTuples);
+ } else {
+ // we need data from the right table that not contains in the lookup-table (not indexed columns),
+ // so we add reads from the right table
+ YQL_ENSURE(indexTable.IsValid());
+ //TODO: May be it'a a good idea to get some collumns data from the index-read?
+ injectRightDataKey(rightRowArg, joinResultTuples);
}
- } else {
- YQL_ENSURE(!requireIndexValues);
- }
+ } else {
+ YQL_ENSURE(!requireIndexValues);
+ }
- auto expr = Build<TCoAsStruct>(ctx, joinTuple.Pos())
+ auto expr = Build<TCoAsStruct>(ctx, joinTuple.Pos())
.Add(joinResultTuples)
.Done();
- return std::make_pair(expr, resultColumns);
- };
+ return std::make_pair(expr, resultColumns);
+ };
auto finalizeJoinResultExpr = [&joinTuple, &tablesData, &ctx,
&selectRange, &addJoinResults, &rightFlatmap, &joinType]
(const TExprBase& input, const TVector<TString>& finishedColumns, bool needExtraRead)
-> NNodes::TExprBase
- {
- if (!needExtraRead) {
- return input;
- }
-
- TCoArgument joinResult = Build<TCoArgument>(ctx, joinTuple.Pos())
+ {
+ if (!needExtraRead) {
+ return input;
+ }
+
+ TCoArgument joinResult = Build<TCoArgument>(ctx, joinTuple.Pos())
.Name("joinResult")
.Done();
@@ -466,22 +466,22 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c
auto tuple = Build<TCoNameValueTuple>(ctx, joinTuple.Pos())
.Name().Build(col)
.Value<TCoMember>()
- .Struct(joinResult)
+ .Struct(joinResult)
.Name().Build(col)
.Build()
.Done();
joinResultTuples.push_back(tuple);
}
- const TStringBuf cluster = selectRange.Cluster();
+ const TStringBuf cluster = selectRange.Cluster();
const TStringBuf table = selectRange.Table().Path();
- const auto& tableDesc = tablesData.ExistingTable(cluster, table);
+ const auto& tableDesc = tablesData.ExistingTable(cluster, table);
TExprBase select = Build<TKiSelectRow>(ctx, joinTuple.Pos())
- .Cluster(selectRange.Cluster())
- .Table(selectRange.Table())
- .Key(ExtractNamedKeyTuples(joinResult, tableDesc, ctx, tableDesc.Metadata->Name))
- .Select(selectRange.Select())
- .Done();
+ .Cluster(selectRange.Cluster())
+ .Table(selectRange.Table())
+ .Key(ExtractNamedKeyTuples(joinResult, tableDesc, ctx, tableDesc.Metadata->Name))
+ .Select(selectRange.Select())
+ .Done();
if (rightFlatmap) {
select = Build<TCoFlatMap>(ctx, joinTuple.Pos())
@@ -538,7 +538,7 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c
.Build()
.Done();
- auto joinResultExpr = getJoinResultExpr(leftRowArg, rightRowArg);
+ auto joinResultExpr = getJoinResultExpr(leftRowArg, rightRowArg);
auto joinMap = Build<TCoFlatMap>(ctx, joinTuple.Pos())
.Input(leftExpr)
@@ -562,17 +562,17 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c
.Build()
.Done();
idxLookupExpr = finalizeJoinResultExpr(joinMap, joinResultExpr.second, requireIndexValues);
- return true;
- }
+ return true;
+ }
if (joinType == joinNames.Inner) {
- const auto joinResultExpr = getJoinResultExpr(leftRowArg, rightRowArg);
-
+ const auto joinResultExpr = getJoinResultExpr(leftRowArg, rightRowArg);
+
auto joinMap = Build<TCoFlatMap>(ctx, joinTuple.Pos())
.Input<TCoSkipNullMembers>()
.Input(leftExpr)
.Members()
- .Add(ConvertToAtoms(leftKeyColumnsSet, ctx, joinTuple.Pos()))
+ .Add(ConvertToAtoms(leftKeyColumnsSet, ctx, joinTuple.Pos()))
.Build()
.Build()
.Lambda()
@@ -587,15 +587,15 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c
.Build()
.Done();
idxLookupExpr = finalizeJoinResultExpr(joinMap, joinResultExpr.second, requireIndexValues);
- return true;
- }
-
+ return true;
+ }
+
if (joinType == joinNames.LeftSemi) {
idxLookupExpr = Build<TCoFlatMap>(ctx, joinTuple.Pos())
.Input<TCoSkipNullMembers>()
.Input(leftExpr)
.Members()
- .Add(ConvertToAtoms(leftKeyColumnsSet, ctx, joinTuple.Pos()))
+ .Add(ConvertToAtoms(leftKeyColumnsSet, ctx, joinTuple.Pos()))
.Build()
.Build()
.Lambda()
@@ -608,9 +608,9 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c
.Build()
.Build()
.Done();
- return true;
- }
-
+ return true;
+ }
+
if (joinType == joinNames.LeftOnly) {
idxLookupExpr = Build<TCoFlatMap>(ctx, joinTuple.Pos())
.Input(leftExpr)
@@ -626,47 +626,47 @@ bool EquiJoinToIdxLookup(TGetExprFunc getLeftExpr, TCoEquiJoinTuple joinTuple, c
.Build()
.Build()
.Done();
- return true;
+ return true;
}
if (joinType == joinNames.RightSemi) {
- // In this case we iterate over left table (with deduplication)
- // and do index-lookup in the right one.
-
- auto joinResultExpr = getJoinResultExpr({}, rightRowArg);
-
- // drop nulls
- leftExpr = Build<TCoSkipNullMembers>(ctx, joinTuple.Pos())
- .Input(leftExpr)
- .Members()
- .Add(ConvertToAtoms(leftKeyColumnsSet, ctx, joinTuple.Pos()))
- .Build()
- .Done();
-
- // deduplicate keys in the left table
- leftExpr = DeduplicateByMembers(leftExpr, leftKeyColumnsSet, ctx, joinTuple.Pos());
-
- auto joinMap = Build<TCoFlatMap>(ctx, joinTuple.Pos())
- .Input(leftExpr)
- .Lambda()
- .Args({leftRowArg})
- .Body<TCoMap>()
- .Input(lookup)
- .Lambda()
- .Args({rightRowArg})
- .Body(joinResultExpr.first)
- .Build()
- .Build()
- .Build()
- .Done();
-
- // add extra reads if required
- idxLookupExpr = finalizeJoinResultExpr(joinMap, joinResultExpr.second, requireIndexValues);
- return true;
- }
-
- YQL_ENSURE(false, "Unexpected join type " << joinType);
- return false;
+ // In this case we iterate over left table (with deduplication)
+ // and do index-lookup in the right one.
+
+ auto joinResultExpr = getJoinResultExpr({}, rightRowArg);
+
+ // drop nulls
+ leftExpr = Build<TCoSkipNullMembers>(ctx, joinTuple.Pos())
+ .Input(leftExpr)
+ .Members()
+ .Add(ConvertToAtoms(leftKeyColumnsSet, ctx, joinTuple.Pos()))
+ .Build()
+ .Done();
+
+ // deduplicate keys in the left table
+ leftExpr = DeduplicateByMembers(leftExpr, leftKeyColumnsSet, ctx, joinTuple.Pos());
+
+ auto joinMap = Build<TCoFlatMap>(ctx, joinTuple.Pos())
+ .Input(leftExpr)
+ .Lambda()
+ .Args({leftRowArg})
+ .Body<TCoMap>()
+ .Input(lookup)
+ .Lambda()
+ .Args({rightRowArg})
+ .Body(joinResultExpr.first)
+ .Build()
+ .Build()
+ .Build()
+ .Done();
+
+ // add extra reads if required
+ idxLookupExpr = finalizeJoinResultExpr(joinMap, joinResultExpr.second, requireIndexValues);
+ return true;
+ }
+
+ YQL_ENSURE(false, "Unexpected join type " << joinType);
+ return false;
}
TExprBase GetEquiJoinLabelsNode(const TVector<TString>& labels, TPositionHandle pos, TExprContext& ctx) {
@@ -721,7 +721,7 @@ TCoEquiJoin BuildPairEquiJoin(TCoEquiJoinTuple joinTuple, TExprBase leftList, TE
return join;
}
-TExprBase GetEquiJoinTreeExpr(const TExprBase& joinScope, const TVector<TCoEquiJoinInput>& joinInputs,
+TExprBase GetEquiJoinTreeExpr(const TExprBase& joinScope, const TVector<TCoEquiJoinInput>& joinInputs,
const TJoinLabels& joinLabels, TExprContext& ctx)
{
if (joinScope.Maybe<TCoAtom>()) {
@@ -755,41 +755,41 @@ TExprBase GetEquiJoinTreeExpr(const TExprBase& joinScope, const TVector<TCoEquiJ
return join;
}
-TMaybe<TStringBuf> TryFlipJoinType(TStringBuf joinType) {
+TMaybe<TStringBuf> TryFlipJoinType(TStringBuf joinType) {
if (joinType == TStringBuf("Inner")) {
return TStringBuf("Inner");
- }
+ }
if (joinType == TStringBuf("LeftSemi")) {
return TStringBuf("RightSemi");
- }
+ }
if (joinType == TStringBuf("RightSemi")) {
return TStringBuf("LeftSemi");
- }
+ }
if (joinType == TStringBuf("Right")) {
return TStringBuf("Left");
- }
+ }
if (joinType == TStringBuf("RightOnly")) {
return TStringBuf("LeftOnly");
- }
- return Nothing();
-}
-
-bool RewriteEquiJoinInternal(const TCoEquiJoinTuple& joinTree, const TVector<TCoEquiJoinInput>& joinInputs,
+ }
+ return Nothing();
+}
+
+bool RewriteEquiJoinInternal(const TCoEquiJoinTuple& joinTree, const TVector<TCoEquiJoinInput>& joinInputs,
const TJoinLabels& joinLabels, const TKikimrTablesData& tablesData, TExprContext& ctx,
- const TKikimrConfiguration& config, TMaybeNode<TExprBase>& rewrittenExpr)
+ const TKikimrConfiguration& config, TMaybeNode<TExprBase>& rewrittenExpr)
{
bool leftRewritten = false;
TMaybeNode<TExprBase> leftExpr;
if (!joinTree.LeftScope().Maybe<TCoAtom>()) {
leftRewritten = RewriteEquiJoinInternal(joinTree.LeftScope().Cast<TCoEquiJoinTuple>(), joinInputs,
- joinLabels, tablesData, ctx, config, leftExpr);
+ joinLabels, tablesData, ctx, config, leftExpr);
}
bool rightRewritten = false;
TMaybeNode<TExprBase> rightExpr;
if (!joinTree.RightScope().Maybe<TCoAtom>()) {
rightRewritten = RewriteEquiJoinInternal(joinTree.RightScope().Cast<TCoEquiJoinTuple>(), joinInputs,
- joinLabels, tablesData, ctx, config, rightExpr);
+ joinLabels, tablesData, ctx, config, rightExpr);
}
auto getLeftExpr = [leftRewritten, leftExpr, joinTree, &joinInputs, &joinLabels, &ctx] () {
@@ -805,33 +805,33 @@ bool RewriteEquiJoinInternal(const TCoEquiJoinTuple& joinTree, const TVector<TCo
};
if (!config.HasOptDisableJoinTableLookup()) {
- if (EquiJoinToIdxLookup(getLeftExpr, joinTree, joinLabels, joinInputs, tablesData, ctx, rewrittenExpr)) {
- return true;
- }
-
+ if (EquiJoinToIdxLookup(getLeftExpr, joinTree, joinLabels, joinInputs, tablesData, ctx, rewrittenExpr)) {
+ return true;
+ }
+
bool tryFlip = joinTree.Type().Value() == TStringBuf("LeftSemi")
? !config.HasOptDisableJoinReverseTableLookupLeftSemi()
: !config.HasOptDisableJoinReverseTableLookup();
if (tryFlip) {
- // try to switch left and right subtrees and do rewrite one more time
- if (auto flipJoinType = TryFlipJoinType(joinTree.Type().Value())) {
- auto flipJoinTree = Build<TCoEquiJoinTuple>(ctx, joinTree.Pos())
- .Type().Build(*flipJoinType)
- .LeftScope(joinTree.RightScope())
- .LeftKeys(joinTree.RightKeys())
- .RightScope(joinTree.LeftScope())
- .RightKeys(joinTree.LeftKeys())
- .Options(joinTree.Options())
- .Done();
-
- if (EquiJoinToIdxLookup(getRightExpr, flipJoinTree, joinLabels, joinInputs, tablesData, ctx,
- rewrittenExpr))
- {
- return true;
- }
- }
- }
+ // try to switch left and right subtrees and do rewrite one more time
+ if (auto flipJoinType = TryFlipJoinType(joinTree.Type().Value())) {
+ auto flipJoinTree = Build<TCoEquiJoinTuple>(ctx, joinTree.Pos())
+ .Type().Build(*flipJoinType)
+ .LeftScope(joinTree.RightScope())
+ .LeftKeys(joinTree.RightKeys())
+ .RightScope(joinTree.LeftScope())
+ .RightKeys(joinTree.LeftKeys())
+ .Options(joinTree.Options())
+ .Done();
+
+ if (EquiJoinToIdxLookup(getRightExpr, flipJoinTree, joinLabels, joinInputs, tablesData, ctx,
+ rewrittenExpr))
+ {
+ return true;
+ }
+ }
+ }
}
if (leftRewritten || rightRewritten) {
@@ -907,62 +907,62 @@ TExprNode::TPtr KiRewriteEquiJoin(TExprBase node, const TKikimrTablesData& table
}
TMaybeNode<TExprBase> rewrittenJoin;
- if (!RewriteEquiJoinInternal(joinTree, joinInputs, joinLabels, tablesData, ctx, config, rewrittenJoin)) {
- return node.Ptr();
- }
-
- YQL_ENSURE(rewrittenJoin.IsValid());
- YQL_CLOG(INFO, ProviderKikimr) << "KiRewriteEquiJoin";
-
- auto joinOptions = join.Arg(join.ArgCount() - 1).Cast<TExprList>();
-
- if (rewrittenJoin.Maybe<TCoEquiJoin>()) {
- auto equiJoin = rewrittenJoin.Cast<TCoEquiJoin>();
- return ctx.ChangeChild(*equiJoin.Ptr(), equiJoin.ArgCount() - 1, joinOptions.Ptr());
- }
-
- TExprBase joinExpr = rewrittenJoin.Cast();
- auto resultType = node.Ptr()->GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>();
- auto renameMap = LoadJoinRenameMap(*joinOptions.Ptr());
-
- THashMap<TStringBuf, TStringBuf> reverseRenameMap;
- for (auto& pair : renameMap) {
- for (TStringBuf newName : pair.second) {
- reverseRenameMap[newName] = pair.first;
- }
- }
-
- TCoArgument rowArg = Build<TCoArgument>(ctx, joinExpr.Pos())
- .Name("row")
- .Done();
-
- TVector<TExprBase> renameTuples;
- for (auto& item : resultType->GetItems()) {
- TStringBuf newName = item->GetName();
- auto renamedFrom = reverseRenameMap.FindPtr(newName);
- TStringBuf oldName = renamedFrom ? *renamedFrom : newName;
-
- auto tuple = Build<TCoNameValueTuple>(ctx, joinExpr.Pos())
- .Name().Build(newName)
- .Value<TCoMember>()
- .Struct(rowArg)
- .Name().Build(oldName)
- .Build()
- .Done();
-
- renameTuples.push_back(tuple);
- }
-
- return Build<TCoMap>(ctx, joinExpr.Pos())
- .Input(joinExpr)
- .Lambda()
- .Args({rowArg})
- .Body<TCoAsStruct>()
- .Add(renameTuples)
- .Build()
- .Build()
- .Done()
- .Ptr();
+ if (!RewriteEquiJoinInternal(joinTree, joinInputs, joinLabels, tablesData, ctx, config, rewrittenJoin)) {
+ return node.Ptr();
+ }
+
+ YQL_ENSURE(rewrittenJoin.IsValid());
+ YQL_CLOG(INFO, ProviderKikimr) << "KiRewriteEquiJoin";
+
+ auto joinOptions = join.Arg(join.ArgCount() - 1).Cast<TExprList>();
+
+ if (rewrittenJoin.Maybe<TCoEquiJoin>()) {
+ auto equiJoin = rewrittenJoin.Cast<TCoEquiJoin>();
+ return ctx.ChangeChild(*equiJoin.Ptr(), equiJoin.ArgCount() - 1, joinOptions.Ptr());
+ }
+
+ TExprBase joinExpr = rewrittenJoin.Cast();
+ auto resultType = node.Ptr()->GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>();
+ auto renameMap = LoadJoinRenameMap(*joinOptions.Ptr());
+
+ THashMap<TStringBuf, TStringBuf> reverseRenameMap;
+ for (auto& pair : renameMap) {
+ for (TStringBuf newName : pair.second) {
+ reverseRenameMap[newName] = pair.first;
+ }
+ }
+
+ TCoArgument rowArg = Build<TCoArgument>(ctx, joinExpr.Pos())
+ .Name("row")
+ .Done();
+
+ TVector<TExprBase> renameTuples;
+ for (auto& item : resultType->GetItems()) {
+ TStringBuf newName = item->GetName();
+ auto renamedFrom = reverseRenameMap.FindPtr(newName);
+ TStringBuf oldName = renamedFrom ? *renamedFrom : newName;
+
+ auto tuple = Build<TCoNameValueTuple>(ctx, joinExpr.Pos())
+ .Name().Build(newName)
+ .Value<TCoMember>()
+ .Struct(rowArg)
+ .Name().Build(oldName)
+ .Build()
+ .Done();
+
+ renameTuples.push_back(tuple);
+ }
+
+ return Build<TCoMap>(ctx, joinExpr.Pos())
+ .Input(joinExpr)
+ .Lambda()
+ .Args({rowArg})
+ .Body<TCoAsStruct>()
+ .Add(renameTuples)
+ .Build()
+ .Build()
+ .Done()
+ .Ptr();
}
} // namespace NYql
diff --git a/ydb/core/kqp/provider/yql_kikimr_opt_range.cpp b/ydb/core/kqp/provider/yql_kikimr_opt_range.cpp
index a6e8f9327b..cce81efc42 100644
--- a/ydb/core/kqp/provider/yql_kikimr_opt_range.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_opt_range.cpp
@@ -1,5 +1,5 @@
#include "yql_kikimr_provider_impl.h"
-#include "yql_kikimr_opt_utils.h"
+#include "yql_kikimr_opt_utils.h"
#include <ydb/library/yql/core/common_opt/yql_co_sqlin.h>
#include <ydb/library/yql/core/yql_opt_utils.h>
@@ -52,11 +52,11 @@ TTableLookup::TCompareResult CompareIntegralNodes(TCoAtom left, TCoAtom right, N
case TTableLookup::TCompareResult::Greater:
adjacent = leftValue == rightValue + 1;
- break;
+ break;
case TTableLookup::TCompareResult::Less:
adjacent = rightValue == leftValue + 1;
- break;
+ break;
}
return TTableLookup::TCompareResult(compareResult, adjacent);
@@ -924,123 +924,123 @@ NNodes::TExprBase TKikimrKeyRange::BuildIndexReadRangeExpr(const TKikimrTableDes
.Done();
}
-TExprNode::TPtr KiSqlInToEquiJoin(NNodes::TExprBase node, const TKikimrTablesData& tablesData,
- const TKikimrConfiguration& config, TExprContext& ctx)
-{
- if (config.HasOptDisableSqlInToJoin()) {
- return node.Ptr();
- }
-
- if (!node.Maybe<TCoFlatMap>()) {
- return node.Ptr();
- }
- auto flatMap = node.Cast<TCoFlatMap>();
-
- // SqlIn expected to be rewritten to (FlatMap <collection> (OptionalIf ...))
+TExprNode::TPtr KiSqlInToEquiJoin(NNodes::TExprBase node, const TKikimrTablesData& tablesData,
+ const TKikimrConfiguration& config, TExprContext& ctx)
+{
+ if (config.HasOptDisableSqlInToJoin()) {
+ return node.Ptr();
+ }
+
+ if (!node.Maybe<TCoFlatMap>()) {
+ return node.Ptr();
+ }
+ auto flatMap = node.Cast<TCoFlatMap>();
+
+ // SqlIn expected to be rewritten to (FlatMap <collection> (OptionalIf ...))
// or (FlatMap <collection> (FlatListIf ...))
if (!flatMap.Lambda().Body().Maybe<TCoOptionalIf>() && !flatMap.Lambda().Body().Maybe<TCoFlatListIf>()) {
- return node.Ptr();
- }
-
- if (!flatMap.Input().Maybe<TKiSelectRangeBase>()) {
- return node.Ptr();
- }
-
- auto selectRange = flatMap.Input().Cast<TKiSelectRangeBase>();
-
- TMaybeNode<TCoAtom> indexTable;
- if (auto indexSelect = selectRange.Maybe<TKiSelectIndexRange>()) {
- indexTable = indexSelect.Cast().IndexName();
- }
-
- // retrieve selected ranges
+ return node.Ptr();
+ }
+
+ if (!flatMap.Input().Maybe<TKiSelectRangeBase>()) {
+ return node.Ptr();
+ }
+
+ auto selectRange = flatMap.Input().Cast<TKiSelectRangeBase>();
+
+ TMaybeNode<TCoAtom> indexTable;
+ if (auto indexSelect = selectRange.Maybe<TKiSelectIndexRange>()) {
+ indexTable = indexSelect.Cast().IndexName();
+ }
+
+ // retrieve selected ranges
const TStringBuf lookupTable = indexTable ? indexTable.Cast().Value() : selectRange.Table().Path().Value();
- const TKikimrTableDescription& tableDesc = tablesData.ExistingTable(selectRange.Cluster().Value(), lookupTable);
- auto selectKeyRange = TKikimrKeyRange::GetPointKeyRange(ctx, tableDesc, selectRange.Range());
- if (!selectKeyRange) {
- return node.Ptr();
- }
-
- // check which key prefixes are used (and only with points)
- TVector<TStringBuf> keys; // remaining key parts, that can be used in SqlIn (only in asc order)
- for (size_t idx = 0; idx < selectKeyRange->GetColumnRanges().size(); ++idx) {
- const auto& columnRange = selectKeyRange->GetColumnRange(idx);
- if (columnRange.IsDefined()) {
- if (!keys.empty()) {
- return node.Ptr();
- }
- if (columnRange.IsPoint()) {
- continue;
- }
- return node.Ptr();
- }
- keys.emplace_back(tableDesc.Metadata->KeyColumnNames[idx]);
- }
- if (keys.empty()) {
- return node.Ptr();
- }
-
- auto flatMapLambdaArg = flatMap.Lambda().Args().Arg(0);
-
- auto findMemberIndexInKeys = [&keys](const TCoArgument& flatMapLambdaArg, const TCoMember& member) {
- if (member.Struct().Raw() != flatMapLambdaArg.Raw()) {
- return -1;
- }
- for (size_t i = 0; i < keys.size(); ++i) {
- if (member.Name().Value() == keys[i]) {
- return (int) i;
- }
- }
- return -1;
- };
-
- auto shouldConvertSqlInToJoin = [&flatMapLambdaArg, &findMemberIndexInKeys](const TCoSqlIn& sqlIn, bool negated) {
- if (negated) {
- // negated can't be rewritten to the index-lookup, so skip it
- return false;
- }
-
- // validate key prefix
- if (sqlIn.Lookup().Maybe<TCoMember>()) {
- if (findMemberIndexInKeys(flatMapLambdaArg, sqlIn.Lookup().Cast<TCoMember>()) != 0) {
- return false;
- }
- } else if (sqlIn.Lookup().Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Tuple) {
- auto children = sqlIn.Lookup().Ref().ChildrenList();
- TVector<int> usedKeyIndexes{Reserve(children.size())};
- for (const auto& itemPtr : children) {
- TExprBase item{itemPtr};
- if (!item.Maybe<TCoMember>()) {
- return false;
- }
- int keyIndex = findMemberIndexInKeys(flatMapLambdaArg, item.Cast<TCoMember>());
- if (keyIndex >= 0) {
- usedKeyIndexes.push_back(keyIndex);
- }
- }
- if (usedKeyIndexes.empty()) {
- return false;
- }
- ::Sort(usedKeyIndexes);
- for (size_t i = 0; i < usedKeyIndexes.size(); ++i) {
- if (usedKeyIndexes[i] != (int) i) {
- return false;
- }
- }
- } else {
- return false;
- }
-
- return CanRewriteSqlInToEquiJoin(sqlIn.Lookup().Ref().GetTypeAnn(), sqlIn.Collection().Ref().GetTypeAnn());
- };
-
+ const TKikimrTableDescription& tableDesc = tablesData.ExistingTable(selectRange.Cluster().Value(), lookupTable);
+ auto selectKeyRange = TKikimrKeyRange::GetPointKeyRange(ctx, tableDesc, selectRange.Range());
+ if (!selectKeyRange) {
+ return node.Ptr();
+ }
+
+ // check which key prefixes are used (and only with points)
+ TVector<TStringBuf> keys; // remaining key parts, that can be used in SqlIn (only in asc order)
+ for (size_t idx = 0; idx < selectKeyRange->GetColumnRanges().size(); ++idx) {
+ const auto& columnRange = selectKeyRange->GetColumnRange(idx);
+ if (columnRange.IsDefined()) {
+ if (!keys.empty()) {
+ return node.Ptr();
+ }
+ if (columnRange.IsPoint()) {
+ continue;
+ }
+ return node.Ptr();
+ }
+ keys.emplace_back(tableDesc.Metadata->KeyColumnNames[idx]);
+ }
+ if (keys.empty()) {
+ return node.Ptr();
+ }
+
+ auto flatMapLambdaArg = flatMap.Lambda().Args().Arg(0);
+
+ auto findMemberIndexInKeys = [&keys](const TCoArgument& flatMapLambdaArg, const TCoMember& member) {
+ if (member.Struct().Raw() != flatMapLambdaArg.Raw()) {
+ return -1;
+ }
+ for (size_t i = 0; i < keys.size(); ++i) {
+ if (member.Name().Value() == keys[i]) {
+ return (int) i;
+ }
+ }
+ return -1;
+ };
+
+ auto shouldConvertSqlInToJoin = [&flatMapLambdaArg, &findMemberIndexInKeys](const TCoSqlIn& sqlIn, bool negated) {
+ if (negated) {
+ // negated can't be rewritten to the index-lookup, so skip it
+ return false;
+ }
+
+ // validate key prefix
+ if (sqlIn.Lookup().Maybe<TCoMember>()) {
+ if (findMemberIndexInKeys(flatMapLambdaArg, sqlIn.Lookup().Cast<TCoMember>()) != 0) {
+ return false;
+ }
+ } else if (sqlIn.Lookup().Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Tuple) {
+ auto children = sqlIn.Lookup().Ref().ChildrenList();
+ TVector<int> usedKeyIndexes{Reserve(children.size())};
+ for (const auto& itemPtr : children) {
+ TExprBase item{itemPtr};
+ if (!item.Maybe<TCoMember>()) {
+ return false;
+ }
+ int keyIndex = findMemberIndexInKeys(flatMapLambdaArg, item.Cast<TCoMember>());
+ if (keyIndex >= 0) {
+ usedKeyIndexes.push_back(keyIndex);
+ }
+ }
+ if (usedKeyIndexes.empty()) {
+ return false;
+ }
+ ::Sort(usedKeyIndexes);
+ for (size_t i = 0; i < usedKeyIndexes.size(); ++i) {
+ if (usedKeyIndexes[i] != (int) i) {
+ return false;
+ }
+ }
+ } else {
+ return false;
+ }
+
+ return CanRewriteSqlInToEquiJoin(sqlIn.Lookup().Ref().GetTypeAnn(), sqlIn.Collection().Ref().GetTypeAnn());
+ };
+
const bool prefixOnly = true;
if (auto ret = TryConvertSqlInPredicatesToJoins(flatMap, shouldConvertSqlInToJoin, ctx, prefixOnly)) {
- YQL_CLOG(INFO, ProviderKikimr) << "KiSqlInToEquiJoin";
- return ret;
- }
-
- return node.Ptr();
-}
-
+ YQL_CLOG(INFO, ProviderKikimr) << "KiSqlInToEquiJoin";
+ return ret;
+ }
+
+ return node.Ptr();
+}
+
} // namespace NYql
diff --git a/ydb/core/kqp/provider/yql_kikimr_opt_utils.cpp b/ydb/core/kqp/provider/yql_kikimr_opt_utils.cpp
index 3948df7cfa..9f5938bccb 100644
--- a/ydb/core/kqp/provider/yql_kikimr_opt_utils.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_opt_utils.cpp
@@ -1,70 +1,70 @@
-#include "yql_kikimr_opt_utils.h"
-
-namespace NYql {
-
-using namespace NNodes;
-using namespace NKikimr;
-using namespace NKikimr::NUdf;
-
-bool GetEquiJoinKeyTypes(TExprBase leftInput, const TString& leftColumnName,
- const TKikimrTableDescription& rightTable, const TString& rightColumnName,
- const TDataExprType*& leftData, const TDataExprType*& rightData)
-{
- auto rightType = rightTable.GetColumnType(rightColumnName);
- YQL_ENSURE(rightType);
- YQL_ENSURE(rightType->GetKind() == ETypeAnnotationKind::Optional);
- rightType = rightType->Cast<TOptionalExprType>()->GetItemType();
- YQL_ENSURE(rightType->GetKind() == ETypeAnnotationKind::Data);
- rightData = rightType->Cast<TDataExprType>();
-
- auto leftInputType = leftInput.Ref().GetTypeAnn();
- YQL_ENSURE(leftInputType);
- YQL_ENSURE(leftInputType->GetKind() == ETypeAnnotationKind::List);
- auto itemType = leftInputType->Cast<TListExprType>()->GetItemType();
- YQL_ENSURE(itemType->GetKind() == ETypeAnnotationKind::Struct);
- auto structType = itemType->Cast<TStructExprType>();
- auto memberIndex = structType->FindItem(leftColumnName);
- YQL_ENSURE(memberIndex);
-
- auto leftType = structType->GetItems()[*memberIndex]->GetItemType();
- if (leftType->GetKind() == ETypeAnnotationKind::Optional) {
- leftType = leftType->Cast<TOptionalExprType>()->GetItemType();
- }
-
- if (leftType->GetKind() != ETypeAnnotationKind::Data) {
- return false;
- }
-
- leftData = leftType->Cast<TDataExprType>();
- return true;
-}
-
-bool CanRewriteSqlInToEquiJoin(const TTypeAnnotationNode* lookupType, const TTypeAnnotationNode* collectionType) {
- // SqlIn in Dict
- if (collectionType->GetKind() == ETypeAnnotationKind::Dict) {
- return IsDataOrOptionalOfData(lookupType);
- }
-
- // SqlIn in List<DataType> or List<Tuple<DataType...>>
- if (collectionType->GetKind() == ETypeAnnotationKind::List) {
- auto collectionItemType = collectionType->Cast<TListExprType>()->GetItemType();
-
- if (collectionItemType->GetKind() == ETypeAnnotationKind::Tuple) {
- if (lookupType->GetKind() != ETypeAnnotationKind::Tuple) {
- return false;
- }
- auto lookupItems = lookupType->Cast<TTupleExprType>()->GetItems();
- auto collectionItems = collectionItemType->Cast<TTupleExprType>()->GetItems();
- if (lookupItems.size() != collectionItems.size()) {
- return false;
- }
- return AllOf(collectionItems, [](const auto& item) { return IsDataOrOptionalOfData(item); });
- }
-
- return IsDataOrOptionalOfData(collectionItemType);
- }
-
- return false;
-}
-
-} // namespace NYql
+#include "yql_kikimr_opt_utils.h"
+
+namespace NYql {
+
+using namespace NNodes;
+using namespace NKikimr;
+using namespace NKikimr::NUdf;
+
+bool GetEquiJoinKeyTypes(TExprBase leftInput, const TString& leftColumnName,
+ const TKikimrTableDescription& rightTable, const TString& rightColumnName,
+ const TDataExprType*& leftData, const TDataExprType*& rightData)
+{
+ auto rightType = rightTable.GetColumnType(rightColumnName);
+ YQL_ENSURE(rightType);
+ YQL_ENSURE(rightType->GetKind() == ETypeAnnotationKind::Optional);
+ rightType = rightType->Cast<TOptionalExprType>()->GetItemType();
+ YQL_ENSURE(rightType->GetKind() == ETypeAnnotationKind::Data);
+ rightData = rightType->Cast<TDataExprType>();
+
+ auto leftInputType = leftInput.Ref().GetTypeAnn();
+ YQL_ENSURE(leftInputType);
+ YQL_ENSURE(leftInputType->GetKind() == ETypeAnnotationKind::List);
+ auto itemType = leftInputType->Cast<TListExprType>()->GetItemType();
+ YQL_ENSURE(itemType->GetKind() == ETypeAnnotationKind::Struct);
+ auto structType = itemType->Cast<TStructExprType>();
+ auto memberIndex = structType->FindItem(leftColumnName);
+ YQL_ENSURE(memberIndex);
+
+ auto leftType = structType->GetItems()[*memberIndex]->GetItemType();
+ if (leftType->GetKind() == ETypeAnnotationKind::Optional) {
+ leftType = leftType->Cast<TOptionalExprType>()->GetItemType();
+ }
+
+ if (leftType->GetKind() != ETypeAnnotationKind::Data) {
+ return false;
+ }
+
+ leftData = leftType->Cast<TDataExprType>();
+ return true;
+}
+
+bool CanRewriteSqlInToEquiJoin(const TTypeAnnotationNode* lookupType, const TTypeAnnotationNode* collectionType) {
+ // SqlIn in Dict
+ if (collectionType->GetKind() == ETypeAnnotationKind::Dict) {
+ return IsDataOrOptionalOfData(lookupType);
+ }
+
+ // SqlIn in List<DataType> or List<Tuple<DataType...>>
+ if (collectionType->GetKind() == ETypeAnnotationKind::List) {
+ auto collectionItemType = collectionType->Cast<TListExprType>()->GetItemType();
+
+ if (collectionItemType->GetKind() == ETypeAnnotationKind::Tuple) {
+ if (lookupType->GetKind() != ETypeAnnotationKind::Tuple) {
+ return false;
+ }
+ auto lookupItems = lookupType->Cast<TTupleExprType>()->GetItems();
+ auto collectionItems = collectionItemType->Cast<TTupleExprType>()->GetItems();
+ if (lookupItems.size() != collectionItems.size()) {
+ return false;
+ }
+ return AllOf(collectionItems, [](const auto& item) { return IsDataOrOptionalOfData(item); });
+ }
+
+ return IsDataOrOptionalOfData(collectionItemType);
+ }
+
+ return false;
+}
+
+} // namespace NYql
diff --git a/ydb/core/kqp/provider/yql_kikimr_opt_utils.h b/ydb/core/kqp/provider/yql_kikimr_opt_utils.h
index 053d9a73c3..219a36f5e8 100644
--- a/ydb/core/kqp/provider/yql_kikimr_opt_utils.h
+++ b/ydb/core/kqp/provider/yql_kikimr_opt_utils.h
@@ -1,13 +1,13 @@
-#pragma once
-
-#include "yql_kikimr_provider.h"
-
-namespace NYql {
-
-bool GetEquiJoinKeyTypes(NNodes::TExprBase leftInput, const TString& leftColumnName,
- const TKikimrTableDescription& rightTable, const TString& rightColumnName,
- const TDataExprType*& leftData, const TDataExprType*& rightData);
-
-bool CanRewriteSqlInToEquiJoin(const TTypeAnnotationNode* lookupType, const TTypeAnnotationNode* collectionType);
-
-} // namespace NYql
+#pragma once
+
+#include "yql_kikimr_provider.h"
+
+namespace NYql {
+
+bool GetEquiJoinKeyTypes(NNodes::TExprBase leftInput, const TString& leftColumnName,
+ const TKikimrTableDescription& rightTable, const TString& rightColumnName,
+ const TDataExprType*& leftData, const TDataExprType*& rightData);
+
+bool CanRewriteSqlInToEquiJoin(const TTypeAnnotationNode* lookupType, const TTypeAnnotationNode* collectionType);
+
+} // namespace NYql
diff --git a/ydb/core/kqp/provider/yql_kikimr_provider.cpp b/ydb/core/kqp/provider/yql_kikimr_provider.cpp
index 841192a740..635d164827 100644
--- a/ydb/core/kqp/provider/yql_kikimr_provider.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_provider.cpp
@@ -638,7 +638,7 @@ bool TKikimrTransactionContextBase::ApplyTableOperations(const TVector<NKqpProto
if ((KikimrRequireUnmodifiedOps() & newOp) && isolationLevel != NKikimrKqp::ISOLATION_LEVEL_SERIALIZABLE) {
TString message = TStringBuilder()
- << "Operation '" << newOp << "' is only supported with SERIALIZABLE isolation level";
+ << "Operation '" << newOp << "' is only supported with SERIALIZABLE isolation level";
ctx.AddError(YqlIssue(pos, TIssuesIds::KIKIMR_BAD_OPERATION, message));
return false;
}
diff --git a/ydb/core/kqp/provider/yql_kikimr_provider.h b/ydb/core/kqp/provider/yql_kikimr_provider.h
index fb830c1a35..6903e6f193 100644
--- a/ydb/core/kqp/provider/yql_kikimr_provider.h
+++ b/ydb/core/kqp/provider/yql_kikimr_provider.h
@@ -1,9 +1,9 @@
#pragma once
-#include "yql_kikimr_gateway.h"
-#include "yql_kikimr_settings.h"
-#include "yql_kikimr_query_traits.h"
-
+#include "yql_kikimr_gateway.h"
+#include "yql_kikimr_settings.h"
+#include "yql_kikimr_query_traits.h"
+
#include <ydb/library/yql/ast/yql_gc_nodes.h>
#include <ydb/library/yql/core/yql_type_annotation.h>
#include <ydb/library/yql/minikql/mkql_function_registry.h>
@@ -85,7 +85,7 @@ struct TKikimrQueryContext : TThrRefBase {
/*
* Defuse DDL-prohibiting checks when PrepareOnly = true. Used in scripting query explain.
*/
- bool SuppressDdlChecks = false;
+ bool SuppressDdlChecks = false;
EKikimrStatsMode StatsMode = EKikimrStatsMode::None;
EKikimrQueryType Type = EKikimrQueryType::Unspecified;
@@ -96,12 +96,12 @@ struct TKikimrQueryContext : TThrRefBase {
// full mode can be enabled explicitly.
bool DocumentApiRestricted = true;
- // Force NewEngine stuff
- // remove it after enabling NewEngine
- std::optional<NKikimr::NKqp::TQueryTraits> QueryTraits;
-
- std::unique_ptr<NKikimrKqp::TPreparedQuery> PreparingQuery;
- std::shared_ptr<const NKikimrKqp::TPreparedQuery> PreparedQuery;
+ // Force NewEngine stuff
+ // remove it after enabling NewEngine
+ std::optional<NKikimr::NKqp::TQueryTraits> QueryTraits;
+
+ std::unique_ptr<NKikimrKqp::TPreparedQuery> PreparingQuery;
+ std::shared_ptr<const NKikimrKqp::TPreparedQuery> PreparedQuery;
TKikimrParamsMap Parameters;
THashMap<ui64, IKikimrQueryExecutor::TQueryResult> Results;
@@ -152,21 +152,21 @@ struct TKikimrQueryContext : TThrRefBase {
void Reset() {
PrepareOnly = false;
- SuppressDdlChecks = false;
+ SuppressDdlChecks = false;
StatsMode = EKikimrStatsMode::None;
Type = EKikimrQueryType::Unspecified;
Deadlines = {};
Limits = {};
- QueryTraits.reset();
- PreparingQuery.reset();
- PreparedQuery.reset();
+ QueryTraits.reset();
+ PreparingQuery.reset();
+ PreparedQuery.reset();
Parameters.clear();
Results.clear();
InProgress.clear();
ExecutionOrder.clear();
-
+
RlPath.Clear();
CachedNow.reset();
diff --git a/ydb/core/kqp/provider/yql_kikimr_provider_impl.h b/ydb/core/kqp/provider/yql_kikimr_provider_impl.h
index 7adf204e5e..e1aa3cda2b 100644
--- a/ydb/core/kqp/provider/yql_kikimr_provider_impl.h
+++ b/ydb/core/kqp/provider/yql_kikimr_provider_impl.h
@@ -137,18 +137,18 @@ public:
, Completed(false) {}
bool HasResult() const override {
- if (Completed) {
- YQL_ENSURE(ExtractedResult.has_value());
- }
+ if (Completed) {
+ YQL_ENSURE(ExtractedResult.has_value());
+ }
return Completed;
}
- TResult GetResult() override {
+ TResult GetResult() override {
YQL_ENSURE(Completed);
- if (ExtractedResult) {
- return std::move(*ExtractedResult);
- }
- return std::move(Future.ExtractValue());
+ if (ExtractedResult) {
+ return std::move(*ExtractedResult);
+ }
+ return std::move(Future.ExtractValue());
}
NThreading::TFuture<bool> Continue() override {
@@ -157,8 +157,8 @@ public:
}
if (Future.HasValue()) {
- ExtractedResult.emplace(std::move(Future.ExtractValue()));
- ExtractedResult->ReportIssues(ExprCtx.IssueManager);
+ ExtractedResult.emplace(std::move(Future.ExtractValue()));
+ ExtractedResult->ReportIssues(ExprCtx.IssueManager);
Completed = true;
return NThreading::MakeFuture(true);
@@ -172,7 +172,7 @@ public:
private:
NThreading::TFuture<TResult> Future;
- std::optional<TResult> ExtractedResult;
+ std::optional<TResult> ExtractedResult;
TExprContext& ExprCtx;
bool Completed;
};
@@ -242,15 +242,15 @@ NNodes::TExprBase DeduplicateByMembers(const NNodes::TExprBase& expr, const TSet
// Optimizer rules
TExprNode::TPtr KiBuildQuery(NNodes::TExprBase node, const TMaybe<bool>& useNewEngine, TExprContext& ctx);
-TExprNode::TPtr KiBuildResult(NNodes::TExprBase node, const TString& cluster, TExprContext& ctx);
+TExprNode::TPtr KiBuildResult(NNodes::TExprBase node, const TString& cluster, TExprContext& ctx);
TExprNode::TPtr KiApplyLimitToSelectRange(NNodes::TExprBase node, TExprContext& ctx);
TExprNode::TPtr KiPushPredicateToSelectRange(NNodes::TExprBase node, TExprContext& ctx,
const TKikimrTablesData& tablesData, const TKikimrConfiguration& config);
TExprNode::TPtr KiApplyExtractMembersToSelectRow(NNodes::TExprBase node, TExprContext& ctx);
TExprNode::TPtr KiRewriteEquiJoin(NNodes::TExprBase node, const TKikimrTablesData& tablesData,
const TKikimrConfiguration& config, TExprContext& ctx);
-TExprNode::TPtr KiSqlInToEquiJoin(NNodes::TExprBase node, const TKikimrTablesData& tablesData,
- const TKikimrConfiguration& config, TExprContext& ctx);
+TExprNode::TPtr KiSqlInToEquiJoin(NNodes::TExprBase node, const TKikimrTablesData& tablesData,
+ const TKikimrConfiguration& config, TExprContext& ctx);
bool KiTableLookupCanCompare(NNodes::TExprBase node);
NNodes::TMaybeNode<NNodes::TExprBase> KiTableLookupGetValue(NNodes::TExprBase node, const TTypeAnnotationNode* type,
diff --git a/ydb/core/kqp/provider/yql_kikimr_query_traits.cpp b/ydb/core/kqp/provider/yql_kikimr_query_traits.cpp
index ccef45fbd7..8a6056d549 100644
--- a/ydb/core/kqp/provider/yql_kikimr_query_traits.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_query_traits.cpp
@@ -1,44 +1,44 @@
-#include "yql_kikimr_query_traits.h"
-#include "yql_kikimr_expr_nodes.h"
-
+#include "yql_kikimr_query_traits.h"
+#include "yql_kikimr_expr_nodes.h"
+
#include <ydb/library/yql/ast/yql_expr.h>
#include <ydb/library/yql/core/yql_expr_optimize.h>
#include <ydb/library/yql/utils/log/log.h>
-
-namespace NKikimr {
-namespace NKqp {
-
-using namespace NYql;
-
-TQueryTraits CollectQueryTraits(const NNodes::TExprBase& program, TExprContext&) {
- using namespace NNodes;
-
- TQueryTraits traits;
-
- VisitExpr(program.Ptr(), [&traits](const TExprNode::TPtr& node) {
- const auto* x = node.Get();
-
- if (TKiUpdateRow::Match(x) || TKiEraseRow::Match(x) || TKiDeleteTable::Match(x) || TKiEffects::Match(x)) {
- traits.ReadOnly = 0;
- } else if (TKiSelectIndexRange::Match(x)) {
- traits.WithIndex = 1;
- } else if (TCoEquiJoin::Match(x) || TCoJoin::Match(x) || TCoMapJoinCore::Match(x) || TCoJoinDict::Match(x)) {
- traits.WithJoin = 1;
- } else if (TCoSqlIn::Match(x)) {
- traits.WithSqlIn = 1;
- } else if (TCoUdf::Match(x)) {
- traits.WithUdf = 1;
- } else if (auto selectRange = TMaybeNode<TKiSelectRange>(node)) {
- if (selectRange.Cast().Table().Path().Value().ends_with("/indexImplTable"sv)) {
- traits.WithIndex = 1;
- }
- }
-
- return true;
- });
-
- return traits;
-}
-
-} // namespace NKqp
-} // namespace NKikimr
+
+namespace NKikimr {
+namespace NKqp {
+
+using namespace NYql;
+
+TQueryTraits CollectQueryTraits(const NNodes::TExprBase& program, TExprContext&) {
+ using namespace NNodes;
+
+ TQueryTraits traits;
+
+ VisitExpr(program.Ptr(), [&traits](const TExprNode::TPtr& node) {
+ const auto* x = node.Get();
+
+ if (TKiUpdateRow::Match(x) || TKiEraseRow::Match(x) || TKiDeleteTable::Match(x) || TKiEffects::Match(x)) {
+ traits.ReadOnly = 0;
+ } else if (TKiSelectIndexRange::Match(x)) {
+ traits.WithIndex = 1;
+ } else if (TCoEquiJoin::Match(x) || TCoJoin::Match(x) || TCoMapJoinCore::Match(x) || TCoJoinDict::Match(x)) {
+ traits.WithJoin = 1;
+ } else if (TCoSqlIn::Match(x)) {
+ traits.WithSqlIn = 1;
+ } else if (TCoUdf::Match(x)) {
+ traits.WithUdf = 1;
+ } else if (auto selectRange = TMaybeNode<TKiSelectRange>(node)) {
+ if (selectRange.Cast().Table().Path().Value().ends_with("/indexImplTable"sv)) {
+ traits.WithIndex = 1;
+ }
+ }
+
+ return true;
+ });
+
+ return traits;
+}
+
+} // namespace NKqp
+} // namespace NKikimr
diff --git a/ydb/core/kqp/provider/yql_kikimr_query_traits.h b/ydb/core/kqp/provider/yql_kikimr_query_traits.h
index 201fe81322..26f83cdb0b 100644
--- a/ydb/core/kqp/provider/yql_kikimr_query_traits.h
+++ b/ydb/core/kqp/provider/yql_kikimr_query_traits.h
@@ -1,31 +1,31 @@
-#pragma once
-
-#include <util/string/builder.h>
-
-namespace NYql {
-struct TExprContext;
-namespace NNodes {
-class TExprBase;
-} // namespace NNodes
-} // namespace NYql
-
-namespace NKikimr {
-namespace NKqp {
-
-struct TQueryTraits {
- ui32 ReadOnly:1 = 1;
- ui32 WithJoin:1 = 0;
- ui32 WithSqlIn:1 = 0;
- ui32 WithIndex:1 = 0;
- ui32 WithUdf:1 = 0;
-
- TString ToString() const {
- return TStringBuilder() << "{ro: " << ReadOnly << ", join: " << WithJoin << ", sqlIn: " << WithSqlIn
- << ", index: " << WithIndex << ", udf: " << WithUdf << '}';
- }
-};
-
-TQueryTraits CollectQueryTraits(const NYql::NNodes::TExprBase& program, NYql::TExprContext& ctx);
-
-} // namespace NKqp
-} // namespace NKikimr
+#pragma once
+
+#include <util/string/builder.h>
+
+namespace NYql {
+struct TExprContext;
+namespace NNodes {
+class TExprBase;
+} // namespace NNodes
+} // namespace NYql
+
+namespace NKikimr {
+namespace NKqp {
+
+struct TQueryTraits {
+ ui32 ReadOnly:1 = 1;
+ ui32 WithJoin:1 = 0;
+ ui32 WithSqlIn:1 = 0;
+ ui32 WithIndex:1 = 0;
+ ui32 WithUdf:1 = 0;
+
+ TString ToString() const {
+ return TStringBuilder() << "{ro: " << ReadOnly << ", join: " << WithJoin << ", sqlIn: " << WithSqlIn
+ << ", index: " << WithIndex << ", udf: " << WithUdf << '}';
+ }
+};
+
+TQueryTraits CollectQueryTraits(const NYql::NNodes::TExprBase& program, NYql::TExprContext& ctx);
+
+} // namespace NKqp
+} // namespace NKikimr
diff --git a/ydb/core/kqp/provider/yql_kikimr_settings.cpp b/ydb/core/kqp/provider/yql_kikimr_settings.cpp
index 63e82bb61f..ebd392e16c 100644
--- a/ydb/core/kqp/provider/yql_kikimr_settings.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_settings.cpp
@@ -40,11 +40,11 @@ TKikimrConfiguration::TKikimrConfiguration() {
REGISTER_SETTING(*this, _KqpSlowLogTraceThresholdMs);
REGISTER_SETTING(*this, _KqpYqlSyntaxVersion);
REGISTER_SETTING(*this, _KqpAllowNewEngine);
- REGISTER_SETTING(*this, _KqpForceNewEngine);
+ REGISTER_SETTING(*this, _KqpForceNewEngine);
REGISTER_SETTING(*this, _KqpAllowUnsafeCommit);
- REGISTER_SETTING(*this, _KqpMaxComputeActors);
- REGISTER_SETTING(*this, _KqpEnableSpilling);
- REGISTER_SETTING(*this, _KqpDisableLlvmForUdfStages);
+ REGISTER_SETTING(*this, _KqpMaxComputeActors);
+ REGISTER_SETTING(*this, _KqpEnableSpilling);
+ REGISTER_SETTING(*this, _KqpDisableLlvmForUdfStages);
REGISTER_SETTING(*this, _KqpPushOlapProcess);
REGISTER_SETTING(*this, KqpPushOlapProcess);
@@ -66,8 +66,8 @@ TKikimrConfiguration::TKikimrConfiguration() {
REGISTER_SETTING(*this, OptDisableJoinReverseTableLookup);
REGISTER_SETTING(*this, OptDisableJoinReverseTableLookupLeftSemi);
REGISTER_SETTING(*this, OptDisableTopSort);
- REGISTER_SETTING(*this, OptDisableSqlInToJoin);
- REGISTER_SETTING(*this, OptEnableInplaceUpdate);
+ REGISTER_SETTING(*this, OptDisableSqlInToJoin);
+ REGISTER_SETTING(*this, OptEnableInplaceUpdate);
REGISTER_SETTING(*this, OptEnablePredicateExtract);
/* Runtime */
@@ -91,14 +91,14 @@ bool TKikimrSettings::HasAllowKqpNewEngine() const {
return GetFlagValue(_KqpAllowNewEngine.Get());
}
-bool TKikimrSettings::HasKqpForceNewEngine() const {
- return GetFlagValue(_KqpForceNewEngine.Get());
-}
-
-bool TKikimrSettings::HasUseNewEngine() const {
- return GetFlagValue(UseNewEngine.Get());
-}
-
+bool TKikimrSettings::HasKqpForceNewEngine() const {
+ return GetFlagValue(_KqpForceNewEngine.Get());
+}
+
+bool TKikimrSettings::HasUseNewEngine() const {
+ return GetFlagValue(UseNewEngine.Get());
+}
+
bool TKikimrSettings::HasAllowKqpUnsafeCommit() const {
return GetFlagValue(_KqpAllowUnsafeCommit.Get());
}
@@ -115,14 +115,14 @@ bool TKikimrSettings::SystemColumnsEnabled() const {
return GetFlagValue(EnableSystemColumns.Get());
}
-bool TKikimrSettings::SpillingEnabled() const {
- return GetFlagValue(_KqpEnableSpilling.Get());
-}
-
-bool TKikimrSettings::DisableLlvmForUdfStages() const {
- return GetFlagValue(_KqpDisableLlvmForUdfStages.Get());
-}
-
+bool TKikimrSettings::SpillingEnabled() const {
+ return GetFlagValue(_KqpEnableSpilling.Get());
+}
+
+bool TKikimrSettings::DisableLlvmForUdfStages() const {
+ return GetFlagValue(_KqpDisableLlvmForUdfStages.Get());
+}
+
bool TKikimrSettings::PushOlapProcess() const {
auto settingsFlag = GetFlagValue(_KqpPushOlapProcess.Get());
auto runtimeFlag = GetFlagValue(KqpPushOlapProcess.Get());
@@ -161,14 +161,14 @@ bool TKikimrSettings::HasOptDisableTopSort() const {
return GetFlagValue(OptDisableTopSort.Get());
}
-bool TKikimrSettings::HasOptDisableSqlInToJoin() const {
- return GetFlagValue(OptDisableSqlInToJoin.Get());
-}
-
-bool TKikimrSettings::HasOptEnableInplaceUpdate() const {
- return GetFlagValue(OptEnableInplaceUpdate.Get());
-}
-
+bool TKikimrSettings::HasOptDisableSqlInToJoin() const {
+ return GetFlagValue(OptDisableSqlInToJoin.Get());
+}
+
+bool TKikimrSettings::HasOptEnableInplaceUpdate() const {
+ return GetFlagValue(OptEnableInplaceUpdate.Get());
+}
+
EOptionalFlag TKikimrSettings::GetOptPredicateExtract() const {
return GetOptionalFlagValue(OptEnablePredicateExtract.Get());
}
diff --git a/ydb/core/kqp/provider/yql_kikimr_settings.h b/ydb/core/kqp/provider/yql_kikimr_settings.h
index c161bd6cbd..530cfb2855 100644
--- a/ydb/core/kqp/provider/yql_kikimr_settings.h
+++ b/ydb/core/kqp/provider/yql_kikimr_settings.h
@@ -29,11 +29,11 @@ struct TKikimrSettings {
NCommon::TConfSetting<ui32, false> _KqpSlowLogTraceThresholdMs;
NCommon::TConfSetting<ui32, false> _KqpYqlSyntaxVersion;
NCommon::TConfSetting<bool, false> _KqpAllowNewEngine;
- NCommon::TConfSetting<bool, false> _KqpForceNewEngine;
+ NCommon::TConfSetting<bool, false> _KqpForceNewEngine;
NCommon::TConfSetting<bool, false> _KqpAllowUnsafeCommit;
- NCommon::TConfSetting<ui32, false> _KqpMaxComputeActors;
- NCommon::TConfSetting<bool, false> _KqpEnableSpilling;
- NCommon::TConfSetting<bool, false> _KqpDisableLlvmForUdfStages;
+ NCommon::TConfSetting<ui32, false> _KqpMaxComputeActors;
+ NCommon::TConfSetting<bool, false> _KqpEnableSpilling;
+ NCommon::TConfSetting<bool, false> _KqpDisableLlvmForUdfStages;
/*
* Both settings for predicates push are needed.
*/
@@ -59,8 +59,8 @@ struct TKikimrSettings {
NCommon::TConfSetting<bool, false> OptDisableJoinReverseTableLookup;
NCommon::TConfSetting<bool, false> OptDisableJoinReverseTableLookupLeftSemi;
NCommon::TConfSetting<bool, false> OptDisableTopSort;
- NCommon::TConfSetting<bool, false> OptDisableSqlInToJoin;
- NCommon::TConfSetting<bool, false> OptEnableInplaceUpdate;
+ NCommon::TConfSetting<bool, false> OptDisableSqlInToJoin;
+ NCommon::TConfSetting<bool, false> OptEnableInplaceUpdate;
NCommon::TConfSetting<bool, false> OptEnablePredicateExtract;
/* Runtime */
@@ -75,14 +75,14 @@ struct TKikimrSettings {
bool HasAllowNullCompareInIndex() const;
bool HasUnwrapReadTableValues() const;
bool HasAllowKqpNewEngine() const;
- bool HasKqpForceNewEngine() const;
- bool HasUseNewEngine() const;
+ bool HasKqpForceNewEngine() const;
+ bool HasUseNewEngine() const;
bool AllowReverseRange() const;
bool HasDefaultCluster() const;
bool HasAllowKqpUnsafeCommit() const;
bool SystemColumnsEnabled() const;
- bool SpillingEnabled() const;
- bool DisableLlvmForUdfStages() const;
+ bool SpillingEnabled() const;
+ bool DisableLlvmForUdfStages() const;
bool PushOlapProcess() const;
bool HasOptDisableJoinRewrite() const;
@@ -90,12 +90,12 @@ struct TKikimrSettings {
bool HasOptDisableJoinReverseTableLookup() const;
bool HasOptDisableJoinReverseTableLookupLeftSemi() const;
bool HasOptDisableTopSort() const;
- bool HasOptDisableSqlInToJoin() const;
+ bool HasOptDisableSqlInToJoin() const;
EOptionalFlag GetOptPredicateExtract() const;
EOptionalFlag GetEnableLlvm() const;
-
+
// WARNING: For testing purposes only, inplace update is not ready for production usage.
- bool HasOptEnableInplaceUpdate() const;
+ bool HasOptEnableInplaceUpdate() const;
};
struct TKikimrConfiguration : public TKikimrSettings, public NCommon::TSettingDispatcher {
diff --git a/ydb/core/kqp/proxy/kqp_proxy_service.cpp b/ydb/core/kqp/proxy/kqp_proxy_service.cpp
index 5215047cc9..decd2cb301 100644
--- a/ydb/core/kqp/proxy/kqp_proxy_service.cpp
+++ b/ydb/core/kqp/proxy/kqp_proxy_service.cpp
@@ -285,12 +285,12 @@ public:
// Subscribe for tenant changes
Send(MakeTenantPoolRootID(), new TEvents::TEvSubscribe());
- if (auto& cfg = TableServiceConfig.GetSpillingServiceConfig().GetLocalFileConfig(); cfg.GetEnable()) {
+ if (auto& cfg = TableServiceConfig.GetSpillingServiceConfig().GetLocalFileConfig(); cfg.GetEnable()) {
SpillingService = TlsActivationContext->ExecutorThread.RegisterActor(CreateKqpLocalFileSpillingService(cfg, Counters));
TlsActivationContext->ExecutorThread.ActorSystem->RegisterLocalService(
MakeKqpLocalFileSpillingServiceID(SelfId().NodeId()), SpillingService);
- }
-
+ }
+
// Create compile service
CompileService = TlsActivationContext->ExecutorThread.RegisterActor(CreateKqpCompileService(TableServiceConfig,
KqpSettings, ModuleResolverState, Counters, std::move(QueryReplayFactory)));
@@ -300,7 +300,7 @@ public:
KqpNodeService = TlsActivationContext->ExecutorThread.RegisterActor(CreateKqpNodeService(TableServiceConfig, Counters));
TlsActivationContext->ExecutorThread.ActorSystem->RegisterLocalService(
MakeKqpNodeServiceID(SelfId().NodeId()), KqpNodeService);
-
+
NActors::TMon* mon = AppData()->Mon;
if (mon) {
NMonitoring::TIndexMonPage* actorsMonPage = mon->RegisterIndexPage("actors", "Actors");
@@ -1291,7 +1291,7 @@ private:
TActorId BoardLookupActor;
TActorId BoardPublishActor;
TActorId CompileService;
- TActorId KqpNodeService;
+ TActorId KqpNodeService;
TActorId SpillingService;
TActorId WhiteBoardService;
NKikimrKqp::TKqpProxyNodeResources NodeResources;
diff --git a/ydb/core/kqp/rm/kqp_resource_estimation.cpp b/ydb/core/kqp/rm/kqp_resource_estimation.cpp
index 101ae2a702..aeeef80177 100644
--- a/ydb/core/kqp/rm/kqp_resource_estimation.cpp
+++ b/ydb/core/kqp/rm/kqp_resource_estimation.cpp
@@ -1,69 +1,69 @@
-#include "kqp_resource_estimation.h"
-
-namespace NKikimr::NKqp {
-
-using namespace NYql::NDqProto;
-using namespace NKikimrConfig;
-
+#include "kqp_resource_estimation.h"
+
+namespace NKikimr::NKqp {
+
+using namespace NYql::NDqProto;
+using namespace NKikimrConfig;
+
TTaskResourceEstimation EstimateTaskResources(const TDqTask& task, int nScans, ui32 dsOnNodeCount,
const TTableServiceConfig::TResourceManager& config)
-{
- TTaskResourceEstimation ret;
- EstimateTaskResources(task, nScans, dsOnNodeCount, config, ret);
- return ret;
-}
-
-void EstimateTaskResources(const TDqTask& task, int nScans, ui32 dsOnNodeCount,
- const TTableServiceConfig::TResourceManager& config, TTaskResourceEstimation& ret)
-{
- ret.TaskId = task.GetId();
-
+{
+ TTaskResourceEstimation ret;
+ EstimateTaskResources(task, nScans, dsOnNodeCount, config, ret);
+ return ret;
+}
+
+void EstimateTaskResources(const TDqTask& task, int nScans, ui32 dsOnNodeCount,
+ const TTableServiceConfig::TResourceManager& config, TTaskResourceEstimation& ret)
+{
+ ret.TaskId = task.GetId();
+
if (nScans > 0) {
ret.ScanBuffersCount = nScans;
- ret.ScanBufferMemoryLimit = config.GetScanBufferSize();
-
- if (dsOnNodeCount && ret.ScanBufferMemoryLimit * dsOnNodeCount > config.GetMaxTotalScanBuffersSize()) {
- ret.ScanBufferMemoryLimit = std::max(config.GetMinScanBufferSize(),
- config.GetMaxTotalScanBuffersSize() / dsOnNodeCount);
- }
- }
-
- for (const auto& input : task.GetInputs()) {
- ret.ChannelBuffersCount += input.ChannelsSize();
- }
- for (const auto& output : task.GetOutputs()) {
- ret.ChannelBuffersCount += output.ChannelsSize();
- }
-
- ui64 channelBuffersSize = ret.ChannelBuffersCount * config.GetChannelBufferSize();
- if (channelBuffersSize > config.GetMaxTotalChannelBuffersSize()) {
- ret.ChannelBufferMemoryLimit = std::max(config.GetMinChannelBufferSize(),
- config.GetMaxTotalChannelBuffersSize() / ret.ChannelBuffersCount);
- } else {
- ret.ChannelBufferMemoryLimit = config.GetChannelBufferSize();
- }
-
- const auto& opts = task.GetProgram().GetSettings();
- if (/* opts.GetHasSort() || */opts.GetHasMapJoin()) {
- ret.MkqlProgramMemoryLimit = config.GetMkqlHeavyProgramMemoryLimit();
- } else {
- ret.MkqlProgramMemoryLimit = config.GetMkqlLightProgramMemoryLimit();
- }
-
- ret.TotalMemoryLimit = ret.ScanBuffersCount * ret.ScanBufferMemoryLimit
- + ret.ChannelBuffersCount * ret.ChannelBufferMemoryLimit
- + ret.MkqlProgramMemoryLimit;
-}
-
-TVector<TTaskResourceEstimation> EstimateTasksResources(const TVector<NYql::NDqProto::TDqTask>& tasks, int nScans,
- ui32 dsOnNodeCount, const TTableServiceConfig::TResourceManager& config)
-{
- TVector<TTaskResourceEstimation> ret;
- ret.resize(tasks.size());
- for (ui64 i = 0; i < tasks.size(); ++i) {
- EstimateTaskResources(tasks[i], nScans, dsOnNodeCount, config, ret[i]);
- }
- return ret;
-}
-
-} // namespace NKikimr::NKqp
+ ret.ScanBufferMemoryLimit = config.GetScanBufferSize();
+
+ if (dsOnNodeCount && ret.ScanBufferMemoryLimit * dsOnNodeCount > config.GetMaxTotalScanBuffersSize()) {
+ ret.ScanBufferMemoryLimit = std::max(config.GetMinScanBufferSize(),
+ config.GetMaxTotalScanBuffersSize() / dsOnNodeCount);
+ }
+ }
+
+ for (const auto& input : task.GetInputs()) {
+ ret.ChannelBuffersCount += input.ChannelsSize();
+ }
+ for (const auto& output : task.GetOutputs()) {
+ ret.ChannelBuffersCount += output.ChannelsSize();
+ }
+
+ ui64 channelBuffersSize = ret.ChannelBuffersCount * config.GetChannelBufferSize();
+ if (channelBuffersSize > config.GetMaxTotalChannelBuffersSize()) {
+ ret.ChannelBufferMemoryLimit = std::max(config.GetMinChannelBufferSize(),
+ config.GetMaxTotalChannelBuffersSize() / ret.ChannelBuffersCount);
+ } else {
+ ret.ChannelBufferMemoryLimit = config.GetChannelBufferSize();
+ }
+
+ const auto& opts = task.GetProgram().GetSettings();
+ if (/* opts.GetHasSort() || */opts.GetHasMapJoin()) {
+ ret.MkqlProgramMemoryLimit = config.GetMkqlHeavyProgramMemoryLimit();
+ } else {
+ ret.MkqlProgramMemoryLimit = config.GetMkqlLightProgramMemoryLimit();
+ }
+
+ ret.TotalMemoryLimit = ret.ScanBuffersCount * ret.ScanBufferMemoryLimit
+ + ret.ChannelBuffersCount * ret.ChannelBufferMemoryLimit
+ + ret.MkqlProgramMemoryLimit;
+}
+
+TVector<TTaskResourceEstimation> EstimateTasksResources(const TVector<NYql::NDqProto::TDqTask>& tasks, int nScans,
+ ui32 dsOnNodeCount, const TTableServiceConfig::TResourceManager& config)
+{
+ TVector<TTaskResourceEstimation> ret;
+ ret.resize(tasks.size());
+ for (ui64 i = 0; i < tasks.size(); ++i) {
+ EstimateTaskResources(tasks[i], nScans, dsOnNodeCount, config, ret[i]);
+ }
+ return ret;
+}
+
+} // namespace NKikimr::NKqp
diff --git a/ydb/core/kqp/rm/kqp_resource_estimation.h b/ydb/core/kqp/rm/kqp_resource_estimation.h
index 41be7ee08d..b69ac20351 100644
--- a/ydb/core/kqp/rm/kqp_resource_estimation.h
+++ b/ydb/core/kqp/rm/kqp_resource_estimation.h
@@ -1,42 +1,42 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/protos/config.pb.h>
#include <ydb/core/kqp/runtime/kqp_scan_data.h>
-
+
#include <ydb/library/yql/dq/proto/dq_tasks.pb.h>
-
-
-namespace NKikimr::NKqp {
-
-struct TTaskResourceEstimation {
- ui64 TaskId = 0;
- ui32 ScanBuffersCount = 0;
- ui32 ChannelBuffersCount = 0;
- ui64 ScanBufferMemoryLimit = 0;
- ui64 ChannelBufferMemoryLimit = 0;
- ui64 MkqlProgramMemoryLimit = 0;
- ui64 TotalMemoryLimit = 0;
-
- TString ToString() const {
- return TStringBuilder() << "TaskResourceEstimation{"
- << " TaskId: " << TaskId
- << ", ScanBuffersCount: " << ScanBuffersCount
- << ", ChannelBuffersCount: " << ChannelBuffersCount
- << ", ScanBufferMemoryLimit: " << ScanBufferMemoryLimit
- << ", ChannelBufferMemoryLimit: " << ChannelBufferMemoryLimit
- << ", MkqlProgramMemoryLimit: " << MkqlProgramMemoryLimit
- << ", TotalMemoryLimit: " << TotalMemoryLimit
- << " }";
- }
-};
-
+
+
+namespace NKikimr::NKqp {
+
+struct TTaskResourceEstimation {
+ ui64 TaskId = 0;
+ ui32 ScanBuffersCount = 0;
+ ui32 ChannelBuffersCount = 0;
+ ui64 ScanBufferMemoryLimit = 0;
+ ui64 ChannelBufferMemoryLimit = 0;
+ ui64 MkqlProgramMemoryLimit = 0;
+ ui64 TotalMemoryLimit = 0;
+
+ TString ToString() const {
+ return TStringBuilder() << "TaskResourceEstimation{"
+ << " TaskId: " << TaskId
+ << ", ScanBuffersCount: " << ScanBuffersCount
+ << ", ChannelBuffersCount: " << ChannelBuffersCount
+ << ", ScanBufferMemoryLimit: " << ScanBufferMemoryLimit
+ << ", ChannelBufferMemoryLimit: " << ChannelBufferMemoryLimit
+ << ", MkqlProgramMemoryLimit: " << MkqlProgramMemoryLimit
+ << ", TotalMemoryLimit: " << TotalMemoryLimit
+ << " }";
+ }
+};
+
TTaskResourceEstimation EstimateTaskResources(const NYql::NDqProto::TDqTask& task, int nScans, ui32 dsOnNodeCount,
- const NKikimrConfig::TTableServiceConfig::TResourceManager& config);
-
-void EstimateTaskResources(const NYql::NDqProto::TDqTask& task, int nScans, ui32 dsOnNodeCount,
- const NKikimrConfig::TTableServiceConfig::TResourceManager& config, TTaskResourceEstimation& result);
-
-TVector<TTaskResourceEstimation> EstimateTasksResources(const TVector<NYql::NDqProto::TDqTask>& tasks, int nScans,
- ui32 dsOnNodeCount, const NKikimrConfig::TTableServiceConfig::TResourceManager& config);
-
-} // namespace NKikimr::NKqp
+ const NKikimrConfig::TTableServiceConfig::TResourceManager& config);
+
+void EstimateTaskResources(const NYql::NDqProto::TDqTask& task, int nScans, ui32 dsOnNodeCount,
+ const NKikimrConfig::TTableServiceConfig::TResourceManager& config, TTaskResourceEstimation& result);
+
+TVector<TTaskResourceEstimation> EstimateTasksResources(const TVector<NYql::NDqProto::TDqTask>& tasks, int nScans,
+ ui32 dsOnNodeCount, const NKikimrConfig::TTableServiceConfig::TResourceManager& config);
+
+} // namespace NKikimr::NKqp
diff --git a/ydb/core/kqp/rm/kqp_resource_estimation_ut.cpp b/ydb/core/kqp/rm/kqp_resource_estimation_ut.cpp
index 1f43f4c788..689c0ed86b 100644
--- a/ydb/core/kqp/rm/kqp_resource_estimation_ut.cpp
+++ b/ydb/core/kqp/rm/kqp_resource_estimation_ut.cpp
@@ -1,85 +1,85 @@
#include <ydb/core/kqp/rm/kqp_resource_estimation.h>
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <util/generic/size_literals.h>
-
-namespace NKikimr::NKqp {
-
-Y_UNIT_TEST_SUITE(KqpResourceEstimation) {
-
-Y_UNIT_TEST(TestChannelSize) {
- NKikimrConfig::TTableServiceConfig::TResourceManager config;
- config.SetChannelBufferSize(8_MB);
- config.SetMinChannelBufferSize(2_MB);
- config.SetMaxTotalChannelBuffersSize(2_GB);
- config.SetMkqlLightProgramMemoryLimit(100);
-
- NYql::NDqProto::TDqTask task;
-
- // 100 input channels
- auto* input = task.MutableInputs()->Add();
- for (int i = 0; i < 100; ++i) {
- input->MutableChannels()->Add();
- }
-
- // 100 input channels
- input = task.MutableInputs()->Add();
- for (int i = 0; i < 100; ++i) {
- input->MutableChannels()->Add();
- }
-
- auto* output = task.MutableOutputs()->Add();
- output->MutableChannels()->Add();
-
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/generic/size_literals.h>
+
+namespace NKikimr::NKqp {
+
+Y_UNIT_TEST_SUITE(KqpResourceEstimation) {
+
+Y_UNIT_TEST(TestChannelSize) {
+ NKikimrConfig::TTableServiceConfig::TResourceManager config;
+ config.SetChannelBufferSize(8_MB);
+ config.SetMinChannelBufferSize(2_MB);
+ config.SetMaxTotalChannelBuffersSize(2_GB);
+ config.SetMkqlLightProgramMemoryLimit(100);
+
+ NYql::NDqProto::TDqTask task;
+
+ // 100 input channels
+ auto* input = task.MutableInputs()->Add();
+ for (int i = 0; i < 100; ++i) {
+ input->MutableChannels()->Add();
+ }
+
+ // 100 input channels
+ input = task.MutableInputs()->Add();
+ for (int i = 0; i < 100; ++i) {
+ input->MutableChannels()->Add();
+ }
+
+ auto* output = task.MutableOutputs()->Add();
+ output->MutableChannels()->Add();
+
auto est = EstimateTaskResources(task, 0, 0, config);
- UNIT_ASSERT_EQUAL(201, est.ChannelBuffersCount);
- UNIT_ASSERT_EQUAL(est.ChannelBufferMemoryLimit, config.GetChannelBufferSize());
-
- // add more channels, to be more then 256
- input = task.MutableInputs()->Add();
- for (int i = 0; i < 100; ++i) {
- input->MutableChannels()->Add();
- }
-
+ UNIT_ASSERT_EQUAL(201, est.ChannelBuffersCount);
+ UNIT_ASSERT_EQUAL(est.ChannelBufferMemoryLimit, config.GetChannelBufferSize());
+
+ // add more channels, to be more then 256
+ input = task.MutableInputs()->Add();
+ for (int i = 0; i < 100; ++i) {
+ input->MutableChannels()->Add();
+ }
+
est = EstimateTaskResources(task, 0, 0, config);
- UNIT_ASSERT_EQUAL(301, est.ChannelBuffersCount);
-
- UNIT_ASSERT(est.ChannelBufferMemoryLimit < config.GetChannelBufferSize());
- UNIT_ASSERT(est.ChannelBufferMemoryLimit >= config.GetMinChannelBufferSize());
-}
-
-Y_UNIT_TEST(TestScanBufferSize) {
- NKikimrConfig::TTableServiceConfig::TResourceManager config;
- config.SetScanBufferSize(10_MB);
- config.SetMaxTotalScanBuffersSize(50_MB);
- config.SetMinScanBufferSize(2_MB);
- config.SetChannelBufferSize(100);
- config.SetMinChannelBufferSize(100);
- config.SetMaxTotalChannelBuffersSize(200_GB);
- config.SetMkqlLightProgramMemoryLimit(100);
-
- NYql::NDqProto::TDqTask task;
-
- for (int i = 1; i <= 5; ++i) {
- auto est = EstimateTaskResources(task, i, i, config);
- UNIT_ASSERT_VALUES_EQUAL(10_MB, est.ScanBufferMemoryLimit);
- UNIT_ASSERT_VALUES_EQUAL(i, est.ScanBuffersCount);
- }
-
- for (int i = 6; i <= 24; ++i) {
- auto est = EstimateTaskResources(task, i, i, config);
- UNIT_ASSERT(10_MB >= est.ScanBufferMemoryLimit);
- UNIT_ASSERT(2_MB < est.ScanBufferMemoryLimit);
- UNIT_ASSERT_VALUES_EQUAL(i, est.ScanBuffersCount);
- }
-
- for (int i = 25; i <= 30; ++i) {
- auto est = EstimateTaskResources(task, i, i, config);
- UNIT_ASSERT_VALUES_EQUAL(2_MB, est.ScanBufferMemoryLimit);
- UNIT_ASSERT_VALUES_EQUAL(i, est.ScanBuffersCount);
- }
-}
-
-} // suite KqpResourceEstimation
-
-} // namespace NKikimr::NKqp
+ UNIT_ASSERT_EQUAL(301, est.ChannelBuffersCount);
+
+ UNIT_ASSERT(est.ChannelBufferMemoryLimit < config.GetChannelBufferSize());
+ UNIT_ASSERT(est.ChannelBufferMemoryLimit >= config.GetMinChannelBufferSize());
+}
+
+Y_UNIT_TEST(TestScanBufferSize) {
+ NKikimrConfig::TTableServiceConfig::TResourceManager config;
+ config.SetScanBufferSize(10_MB);
+ config.SetMaxTotalScanBuffersSize(50_MB);
+ config.SetMinScanBufferSize(2_MB);
+ config.SetChannelBufferSize(100);
+ config.SetMinChannelBufferSize(100);
+ config.SetMaxTotalChannelBuffersSize(200_GB);
+ config.SetMkqlLightProgramMemoryLimit(100);
+
+ NYql::NDqProto::TDqTask task;
+
+ for (int i = 1; i <= 5; ++i) {
+ auto est = EstimateTaskResources(task, i, i, config);
+ UNIT_ASSERT_VALUES_EQUAL(10_MB, est.ScanBufferMemoryLimit);
+ UNIT_ASSERT_VALUES_EQUAL(i, est.ScanBuffersCount);
+ }
+
+ for (int i = 6; i <= 24; ++i) {
+ auto est = EstimateTaskResources(task, i, i, config);
+ UNIT_ASSERT(10_MB >= est.ScanBufferMemoryLimit);
+ UNIT_ASSERT(2_MB < est.ScanBufferMemoryLimit);
+ UNIT_ASSERT_VALUES_EQUAL(i, est.ScanBuffersCount);
+ }
+
+ for (int i = 25; i <= 30; ++i) {
+ auto est = EstimateTaskResources(task, i, i, config);
+ UNIT_ASSERT_VALUES_EQUAL(2_MB, est.ScanBufferMemoryLimit);
+ UNIT_ASSERT_VALUES_EQUAL(i, est.ScanBuffersCount);
+ }
+}
+
+} // suite KqpResourceEstimation
+
+} // namespace NKikimr::NKqp
diff --git a/ydb/core/kqp/rm/kqp_rm.cpp b/ydb/core/kqp/rm/kqp_rm.cpp
index 8ba46d0f19..30fc08ff5f 100644
--- a/ydb/core/kqp/rm/kqp_rm.cpp
+++ b/ydb/core/kqp/rm/kqp_rm.cpp
@@ -1,5 +1,5 @@
-#include "kqp_rm.h"
-
+#include "kqp_rm.h"
+
#include <ydb/core/base/statestorage.h>
#include <ydb/core/cms/console/configs_dispatcher.h>
#include <ydb/core/cms/console/console.h>
@@ -7,929 +7,929 @@
#include <ydb/core/mon/mon.h>
#include <ydb/core/tablet/resource_broker.h>
#include <ydb/core/kqp/kqp.h>
-
-#include <library/cpp/actors/core/actor_bootstrapped.h>
-#include <library/cpp/monlib/service/pages/templates.h>
-
+
+#include <library/cpp/actors/core/actor_bootstrapped.h>
+#include <library/cpp/monlib/service/pages/templates.h>
+
#include <ydb/library/yql/utils/yql_panic.h>
-
-namespace NKikimr {
-namespace NKqp {
-namespace NRm {
-
-using namespace NActors;
-using namespace NResourceBroker;
-
-#define LOG_C(stream) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream)
-#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream)
-#define LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream)
-#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream)
-#define LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream)
+
+namespace NKikimr {
+namespace NKqp {
+namespace NRm {
+
+using namespace NActors;
+using namespace NResourceBroker;
+
+#define LOG_C(stream) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream)
+#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream)
+#define LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream)
+#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream)
+#define LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream)
#define LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream)
-
-namespace {
-
-template <typename T>
-class TLimitedResource {
-public:
- explicit TLimitedResource(T limit)
- : Limit(limit)
- , Used(0) {}
-
- T Available() const {
- return Limit > Used ? Limit - Used : 0;
- }
-
- bool Has(T amount) const {
- return Available() >= amount;
- }
-
- bool Acquire(T value) {
- if (Available() >= value) {
- Used += value;
- return true;
- }
- return false;
- }
-
- void Release(T value) {
- if (Used > value) {
- Used -= value;
- } else {
- Used = 0;
- }
- }
-
- void SetNewLimit(T limit) {
- Limit = limit;
- }
-
- T GetLimit() const {
- return Limit;
- }
-
- TString ToString() const {
- return TStringBuilder() << Used << '/' << Limit;
- }
-
-private:
- T Limit;
- T Used;
-};
-
-struct TTaskState {
- ui64 ScanQueryMemory = 0;
- ui64 ExternalDataQueryMemory = 0;
- ui32 ExecutionUnits = 0;
- ui64 ResourceBrokerTaskId = 0;
- TInstant CreatedAt;
-};
-
-struct TTxState {
- std::unordered_map<ui64, TTaskState> Tasks;
- ui64 TxScanQueryMemory = 0;
- ui64 TxExternalDataQueryMemory = 0;
- ui32 TxExecutionUnits = 0;
- TInstant CreatedAt;
-};
-
-struct TTxStatesBucket {
- std::unordered_map<ui64, TTxState> Txs; // TxId -> TxState
- TMutex Lock;
-};
-
-constexpr ui64 BucketsCount = 64;
-
-class TTakeResourcesSnapshotActor : public TActorBootstrapped<TTakeResourcesSnapshotActor> {
-public:
- TTakeResourcesSnapshotActor(const TString& boardPath, ui32 stateStorageGroupId,
- std::function<void(TVector<NKikimrKqp::TKqpNodeResources>&&)>&& callback)
- : BoardPath(boardPath)
- , StateStorageGroupId(stateStorageGroupId)
- , Callback(std::move(callback)) {}
-
- void Bootstrap() {
- auto boardLookup = CreateBoardLookupActor(BoardPath, SelfId(), StateStorageGroupId, EBoardLookupMode::Majority,
- false, false);
- BoardLookupId = Register(boardLookup);
-
- Become(&TTakeResourcesSnapshotActor::WorkState);
- }
-
- STATEFN(WorkState) {
- switch (ev->GetTypeRewrite()) {
- hFunc(TEvStateStorage::TEvBoardInfo, HandleWait);
- cFunc(TEvents::TSystem::Poison, PassAway);
- default:
- LOG_C("Unexpected event type: " << ev->GetTypeRewrite()
- << ", event: " << (ev->HasEvent() ? ev->GetBase()->ToString().data() : "<serialized>"));
- }
- }
-
- void HandleWait(TEvStateStorage::TEvBoardInfo::TPtr& ev) {
- BoardLookupId = {};
-
- TEvStateStorage::TEvBoardInfo* event = ev->Get();
- TVector<NKikimrKqp::TKqpNodeResources> resources;
-
- if (event->Status == TEvStateStorage::TEvBoardInfo::EStatus::Ok) {
- LOG_I("WhiteBoard entries: " << event->InfoEntries.size());
- resources.resize(event->InfoEntries.size());
-
- int i = 0;
- for (auto& [_, entry] : event->InfoEntries) {
+
+namespace {
+
+template <typename T>
+class TLimitedResource {
+public:
+ explicit TLimitedResource(T limit)
+ : Limit(limit)
+ , Used(0) {}
+
+ T Available() const {
+ return Limit > Used ? Limit - Used : 0;
+ }
+
+ bool Has(T amount) const {
+ return Available() >= amount;
+ }
+
+ bool Acquire(T value) {
+ if (Available() >= value) {
+ Used += value;
+ return true;
+ }
+ return false;
+ }
+
+ void Release(T value) {
+ if (Used > value) {
+ Used -= value;
+ } else {
+ Used = 0;
+ }
+ }
+
+ void SetNewLimit(T limit) {
+ Limit = limit;
+ }
+
+ T GetLimit() const {
+ return Limit;
+ }
+
+ TString ToString() const {
+ return TStringBuilder() << Used << '/' << Limit;
+ }
+
+private:
+ T Limit;
+ T Used;
+};
+
+struct TTaskState {
+ ui64 ScanQueryMemory = 0;
+ ui64 ExternalDataQueryMemory = 0;
+ ui32 ExecutionUnits = 0;
+ ui64 ResourceBrokerTaskId = 0;
+ TInstant CreatedAt;
+};
+
+struct TTxState {
+ std::unordered_map<ui64, TTaskState> Tasks;
+ ui64 TxScanQueryMemory = 0;
+ ui64 TxExternalDataQueryMemory = 0;
+ ui32 TxExecutionUnits = 0;
+ TInstant CreatedAt;
+};
+
+struct TTxStatesBucket {
+ std::unordered_map<ui64, TTxState> Txs; // TxId -> TxState
+ TMutex Lock;
+};
+
+constexpr ui64 BucketsCount = 64;
+
+class TTakeResourcesSnapshotActor : public TActorBootstrapped<TTakeResourcesSnapshotActor> {
+public:
+ TTakeResourcesSnapshotActor(const TString& boardPath, ui32 stateStorageGroupId,
+ std::function<void(TVector<NKikimrKqp::TKqpNodeResources>&&)>&& callback)
+ : BoardPath(boardPath)
+ , StateStorageGroupId(stateStorageGroupId)
+ , Callback(std::move(callback)) {}
+
+ void Bootstrap() {
+ auto boardLookup = CreateBoardLookupActor(BoardPath, SelfId(), StateStorageGroupId, EBoardLookupMode::Majority,
+ false, false);
+ BoardLookupId = Register(boardLookup);
+
+ Become(&TTakeResourcesSnapshotActor::WorkState);
+ }
+
+ STATEFN(WorkState) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvStateStorage::TEvBoardInfo, HandleWait);
+ cFunc(TEvents::TSystem::Poison, PassAway);
+ default:
+ LOG_C("Unexpected event type: " << ev->GetTypeRewrite()
+ << ", event: " << (ev->HasEvent() ? ev->GetBase()->ToString().data() : "<serialized>"));
+ }
+ }
+
+ void HandleWait(TEvStateStorage::TEvBoardInfo::TPtr& ev) {
+ BoardLookupId = {};
+
+ TEvStateStorage::TEvBoardInfo* event = ev->Get();
+ TVector<NKikimrKqp::TKqpNodeResources> resources;
+
+ if (event->Status == TEvStateStorage::TEvBoardInfo::EStatus::Ok) {
+ LOG_I("WhiteBoard entries: " << event->InfoEntries.size());
+ resources.resize(event->InfoEntries.size());
+
+ int i = 0;
+ for (auto& [_, entry] : event->InfoEntries) {
Y_PROTOBUF_SUPPRESS_NODISCARD resources[i].ParseFromString(entry.Payload);
- LOG_D("WhiteBoard [" << i << "]: " << resources[i].ShortDebugString());
- i++;
- }
- } else {
- LOG_E("WhiteBoard error: " << (int) event->Status << ", path: " << event->Path);
- }
-
- Callback(std::move(resources));
-
- PassAway();
- }
-
- void PassAway() {
- if (BoardLookupId) {
- Send(BoardLookupId, new TEvents::TEvPoison);
- }
- IActor::PassAway();
- }
-
-private:
- const TString BoardPath;
- const ui32 StateStorageGroupId;
- std::function<void(TVector<NKikimrKqp::TKqpNodeResources>&&)> Callback;
- TActorId BoardLookupId;
-};
-
-struct TKqpNodeResourceManager {
- ui32 NodeId;
- IKqpResourceManager* Instance;
-
- explicit TKqpNodeResourceManager(ui32 nodeId, IKqpResourceManager* instance)
- : NodeId(nodeId)
- , Instance(instance) {}
-};
-
-struct TResourceManagers {
- std::atomic<TKqpNodeResourceManager*> Default = nullptr;
-
- TMutex Lock;
- std::unordered_map<ui32, TKqpNodeResourceManager*> ByNodeId;
-
- ~TResourceManagers() {
- with_lock(Lock) {
- for (auto [nodeId, rm] : ByNodeId) {
- delete rm;
- }
- }
- }
-};
-
-TResourceManagers ResourceManagers;
-
-} // namespace
-
-
-class TKqpResourceManagerActor : public IKqpResourceManager, public TActorBootstrapped<TKqpResourceManagerActor> {
- using TBase = TActorBootstrapped<TKqpResourceManagerActor>;
-
- struct TEvPrivate {
- enum EEv {
- EvPublishResources = EventSpaceBegin(TEvents::ES_PRIVATE),
- EvSchedulePublishResources,
- EvTakeResourcesSnapshot,
- };
-
- struct TEvPublishResources : public TEventLocal<TEvPublishResources, EEv::EvPublishResources> {
- };
-
- struct TEvSchedulePublishResources : public TEventLocal<TEvSchedulePublishResources, EEv::EvSchedulePublishResources> {
- };
-
- struct TEvTakeResourcesSnapshot : public TEventLocal<TEvTakeResourcesSnapshot, EEv::EvTakeResourcesSnapshot> {
- std::function<void(TVector<NKikimrKqp::TKqpNodeResources>&&)> Callback;
- };
- };
-
-public:
- static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
- return NKikimrServices::TActivity::KQP_RESOURCE_MANAGER;
- }
-
- TKqpResourceManagerActor(const NKikimrConfig::TTableServiceConfig::TResourceManager& config,
- TIntrusivePtr<TKqpCounters> counters, const TActorId& resourceBrokerId)
- : Config(config)
- , Counters(counters)
- , ResourceBrokerId(resourceBrokerId ? resourceBrokerId : MakeResourceBrokerID())
- , ExecutionUnitsResource(Config.GetComputeActorsCount())
- , ScanQueryMemoryResource(Config.GetQueryMemoryLimit()) {}
-
- void Bootstrap() {
- ActorSystem = TlsActivationContext->ActorSystem();
- if (!Counters) {
+ LOG_D("WhiteBoard [" << i << "]: " << resources[i].ShortDebugString());
+ i++;
+ }
+ } else {
+ LOG_E("WhiteBoard error: " << (int) event->Status << ", path: " << event->Path);
+ }
+
+ Callback(std::move(resources));
+
+ PassAway();
+ }
+
+ void PassAway() {
+ if (BoardLookupId) {
+ Send(BoardLookupId, new TEvents::TEvPoison);
+ }
+ IActor::PassAway();
+ }
+
+private:
+ const TString BoardPath;
+ const ui32 StateStorageGroupId;
+ std::function<void(TVector<NKikimrKqp::TKqpNodeResources>&&)> Callback;
+ TActorId BoardLookupId;
+};
+
+struct TKqpNodeResourceManager {
+ ui32 NodeId;
+ IKqpResourceManager* Instance;
+
+ explicit TKqpNodeResourceManager(ui32 nodeId, IKqpResourceManager* instance)
+ : NodeId(nodeId)
+ , Instance(instance) {}
+};
+
+struct TResourceManagers {
+ std::atomic<TKqpNodeResourceManager*> Default = nullptr;
+
+ TMutex Lock;
+ std::unordered_map<ui32, TKqpNodeResourceManager*> ByNodeId;
+
+ ~TResourceManagers() {
+ with_lock(Lock) {
+ for (auto [nodeId, rm] : ByNodeId) {
+ delete rm;
+ }
+ }
+ }
+};
+
+TResourceManagers ResourceManagers;
+
+} // namespace
+
+
+class TKqpResourceManagerActor : public IKqpResourceManager, public TActorBootstrapped<TKqpResourceManagerActor> {
+ using TBase = TActorBootstrapped<TKqpResourceManagerActor>;
+
+ struct TEvPrivate {
+ enum EEv {
+ EvPublishResources = EventSpaceBegin(TEvents::ES_PRIVATE),
+ EvSchedulePublishResources,
+ EvTakeResourcesSnapshot,
+ };
+
+ struct TEvPublishResources : public TEventLocal<TEvPublishResources, EEv::EvPublishResources> {
+ };
+
+ struct TEvSchedulePublishResources : public TEventLocal<TEvSchedulePublishResources, EEv::EvSchedulePublishResources> {
+ };
+
+ struct TEvTakeResourcesSnapshot : public TEventLocal<TEvTakeResourcesSnapshot, EEv::EvTakeResourcesSnapshot> {
+ std::function<void(TVector<NKikimrKqp::TKqpNodeResources>&&)> Callback;
+ };
+ };
+
+public:
+ static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
+ return NKikimrServices::TActivity::KQP_RESOURCE_MANAGER;
+ }
+
+ TKqpResourceManagerActor(const NKikimrConfig::TTableServiceConfig::TResourceManager& config,
+ TIntrusivePtr<TKqpCounters> counters, const TActorId& resourceBrokerId)
+ : Config(config)
+ , Counters(counters)
+ , ResourceBrokerId(resourceBrokerId ? resourceBrokerId : MakeResourceBrokerID())
+ , ExecutionUnitsResource(Config.GetComputeActorsCount())
+ , ScanQueryMemoryResource(Config.GetQueryMemoryLimit()) {}
+
+ void Bootstrap() {
+ ActorSystem = TlsActivationContext->ActorSystem();
+ if (!Counters) {
Counters = MakeIntrusive<TKqpCounters>(AppData()->Counters);
- }
-
- LOG_D("Start KqpResourceManagerActor at " << SelfId() << " with ResourceBroker at " << ResourceBrokerId);
-
- // Subscribe for tenant changes
- Send(MakeTenantPoolRootID(), new TEvents::TEvSubscribe);
-
- // Subscribe for TableService config changes
- ui32 tableServiceConfigKind = (ui32) NKikimrConsole::TConfigItem::TableServiceConfigItem;
-
- Send(NConsole::MakeConfigsDispatcherID(SelfId().NodeId()),
- new NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest({tableServiceConfigKind}),
- IEventHandle::FlagTrackDelivery);
-
- ToBroker(new TEvResourceBroker::TEvResourceBrokerRequest);
- ToBroker(new TEvResourceBroker::TEvConfigRequest(NLocalDb::KqpResourceManagerQueue));
-
- if (auto* mon = AppData()->Mon) {
- NMonitoring::TIndexMonPage* actorsMonPage = mon->RegisterIndexPage("actors", "Actors");
- mon->RegisterActorPage(actorsMonPage, "kqp_resource_manager", "KQP Resource Manager", false,
- ActorSystem, SelfId());
- }
-
- Become(&TKqpResourceManagerActor::WorkState);
-
- auto rm = new TKqpNodeResourceManager(SelfId().NodeId(), this);
- with_lock (ResourceManagers.Lock) {
- ResourceManagers.ByNodeId[SelfId().NodeId()] = rm;
- }
- ResourceManagers.Default.store(rm, std::memory_order_release);
- }
-
-public:
- bool AllocateResources(ui64 txId, ui64 taskId, const TKqpResourcesRequest& resources,
- TKqpNotEnoughResources* details) override
- {
- Y_VERIFY(resources.MemoryPool == EKqpMemoryPool::ScanQuery);
- if (Y_UNLIKELY(resources.Memory == 0 && resources.ExecutionUnits == 0)) {
- return true;
- }
-
- auto now = ActorSystem->Timestamp();
- bool hasScanQueryMemory = true;
- bool hasExecutionUnits = true;
- ui64 queryMemoryLimit = 0;
-
- with_lock (Lock) {
- if (Y_UNLIKELY(!ResourceBroker)) {
- LOG_E("AllocateResources: not ready yet. TxId: " << txId << ", taskId: " << taskId);
- if (details) {
- details->SetNotReady();
- }
- return false;
- }
-
- hasScanQueryMemory = ScanQueryMemoryResource.Has(resources.Memory);
- hasExecutionUnits = ExecutionUnitsResource.Has(resources.ExecutionUnits);
-
- if (hasScanQueryMemory && hasExecutionUnits) {
- ScanQueryMemoryResource.Acquire(resources.Memory);
- ExecutionUnitsResource.Acquire(resources.ExecutionUnits);
- queryMemoryLimit = Config.GetQueryMemoryLimit();
- }
- } // with_lock (Lock)
-
- if (!hasScanQueryMemory) {
- Counters->RmNotEnoughMemory->Inc();
+ }
+
+ LOG_D("Start KqpResourceManagerActor at " << SelfId() << " with ResourceBroker at " << ResourceBrokerId);
+
+ // Subscribe for tenant changes
+ Send(MakeTenantPoolRootID(), new TEvents::TEvSubscribe);
+
+ // Subscribe for TableService config changes
+ ui32 tableServiceConfigKind = (ui32) NKikimrConsole::TConfigItem::TableServiceConfigItem;
+
+ Send(NConsole::MakeConfigsDispatcherID(SelfId().NodeId()),
+ new NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest({tableServiceConfigKind}),
+ IEventHandle::FlagTrackDelivery);
+
+ ToBroker(new TEvResourceBroker::TEvResourceBrokerRequest);
+ ToBroker(new TEvResourceBroker::TEvConfigRequest(NLocalDb::KqpResourceManagerQueue));
+
+ if (auto* mon = AppData()->Mon) {
+ NMonitoring::TIndexMonPage* actorsMonPage = mon->RegisterIndexPage("actors", "Actors");
+ mon->RegisterActorPage(actorsMonPage, "kqp_resource_manager", "KQP Resource Manager", false,
+ ActorSystem, SelfId());
+ }
+
+ Become(&TKqpResourceManagerActor::WorkState);
+
+ auto rm = new TKqpNodeResourceManager(SelfId().NodeId(), this);
+ with_lock (ResourceManagers.Lock) {
+ ResourceManagers.ByNodeId[SelfId().NodeId()] = rm;
+ }
+ ResourceManagers.Default.store(rm, std::memory_order_release);
+ }
+
+public:
+ bool AllocateResources(ui64 txId, ui64 taskId, const TKqpResourcesRequest& resources,
+ TKqpNotEnoughResources* details) override
+ {
+ Y_VERIFY(resources.MemoryPool == EKqpMemoryPool::ScanQuery);
+ if (Y_UNLIKELY(resources.Memory == 0 && resources.ExecutionUnits == 0)) {
+ return true;
+ }
+
+ auto now = ActorSystem->Timestamp();
+ bool hasScanQueryMemory = true;
+ bool hasExecutionUnits = true;
+ ui64 queryMemoryLimit = 0;
+
+ with_lock (Lock) {
+ if (Y_UNLIKELY(!ResourceBroker)) {
+ LOG_E("AllocateResources: not ready yet. TxId: " << txId << ", taskId: " << taskId);
+ if (details) {
+ details->SetNotReady();
+ }
+ return false;
+ }
+
+ hasScanQueryMemory = ScanQueryMemoryResource.Has(resources.Memory);
+ hasExecutionUnits = ExecutionUnitsResource.Has(resources.ExecutionUnits);
+
+ if (hasScanQueryMemory && hasExecutionUnits) {
+ ScanQueryMemoryResource.Acquire(resources.Memory);
+ ExecutionUnitsResource.Acquire(resources.ExecutionUnits);
+ queryMemoryLimit = Config.GetQueryMemoryLimit();
+ }
+ } // with_lock (Lock)
+
+ if (!hasScanQueryMemory) {
+ Counters->RmNotEnoughMemory->Inc();
LOG_N("TxId: " << txId << ", taskId: " << taskId << ". Not enough ScanQueryMemory, requested: " << resources.Memory);
- if (details) {
- details->SetScanQueryMemory();
- }
- return false;
- }
-
- if (!hasExecutionUnits) {
- Counters->RmNotEnoughComputeActors->Inc();
+ if (details) {
+ details->SetScanQueryMemory();
+ }
+ return false;
+ }
+
+ if (!hasExecutionUnits) {
+ Counters->RmNotEnoughComputeActors->Inc();
LOG_N("TxId: " << txId << ", taskId: " << taskId << ". Not enough ExecutionUnits, requested: " << resources.ExecutionUnits);
- if (details) {
- details->SetExecutionUnits();
- }
- return false;
- }
-
- ui64 rbTaskId = LastResourceBrokerTaskId.fetch_add(1) + 1;
- TString rbTaskName = TStringBuilder() << "kqp-" << txId << '-' << taskId << '-' << rbTaskId;
- bool extraAlloc = false;
-
- auto& txBucket = TxBucket(txId);
- with_lock (txBucket.Lock) {
- if (auto it = txBucket.Txs.find(txId); it != txBucket.Txs.end()) {
- if (it->second.TxScanQueryMemory + resources.Memory > queryMemoryLimit) {
- auto unguard = ::Unguard(txBucket.Lock);
-
- with_lock (Lock) {
- ScanQueryMemoryResource.Release(resources.Memory);
- ExecutionUnitsResource.Release(resources.ExecutionUnits);
- } // with_lock (Lock)
-
- Counters->RmNotEnoughMemory->Inc();
+ if (details) {
+ details->SetExecutionUnits();
+ }
+ return false;
+ }
+
+ ui64 rbTaskId = LastResourceBrokerTaskId.fetch_add(1) + 1;
+ TString rbTaskName = TStringBuilder() << "kqp-" << txId << '-' << taskId << '-' << rbTaskId;
+ bool extraAlloc = false;
+
+ auto& txBucket = TxBucket(txId);
+ with_lock (txBucket.Lock) {
+ if (auto it = txBucket.Txs.find(txId); it != txBucket.Txs.end()) {
+ if (it->second.TxScanQueryMemory + resources.Memory > queryMemoryLimit) {
+ auto unguard = ::Unguard(txBucket.Lock);
+
+ with_lock (Lock) {
+ ScanQueryMemoryResource.Release(resources.Memory);
+ ExecutionUnitsResource.Release(resources.ExecutionUnits);
+ } // with_lock (Lock)
+
+ Counters->RmNotEnoughMemory->Inc();
LOG_N("TxId: " << txId << ", taskId: " << taskId << ". Query memory limit exceeded: "
- << "requested " << (it->second.TxScanQueryMemory + resources.Memory));
- if (details) {
- details->SetQueryMemoryLimit();
- }
- return false;
- }
- }
-
- bool allocated = ResourceBroker->SubmitTaskInstant(
- TEvResourceBroker::TEvSubmitTask(rbTaskId, rbTaskName, {0, resources.Memory}, "kqp_query", 0, {}),
- SelfId());
-
- if (!allocated) {
- auto unguard = ::Unguard(txBucket.Lock);
-
- with_lock (Lock) {
- ScanQueryMemoryResource.Release(resources.Memory);
- ExecutionUnitsResource.Release(resources.ExecutionUnits);
- } // with_lock (Lock)
-
- Counters->RmNotEnoughMemory->Inc();
+ << "requested " << (it->second.TxScanQueryMemory + resources.Memory));
+ if (details) {
+ details->SetQueryMemoryLimit();
+ }
+ return false;
+ }
+ }
+
+ bool allocated = ResourceBroker->SubmitTaskInstant(
+ TEvResourceBroker::TEvSubmitTask(rbTaskId, rbTaskName, {0, resources.Memory}, "kqp_query", 0, {}),
+ SelfId());
+
+ if (!allocated) {
+ auto unguard = ::Unguard(txBucket.Lock);
+
+ with_lock (Lock) {
+ ScanQueryMemoryResource.Release(resources.Memory);
+ ExecutionUnitsResource.Release(resources.ExecutionUnits);
+ } // with_lock (Lock)
+
+ Counters->RmNotEnoughMemory->Inc();
LOG_N("TxId: " << txId << ", taskId: " << taskId << ". Not enough ScanQueryMemory: "
- << "requested " << resources.Memory);
- if (details) {
- details->SetScanQueryMemory();
- }
- return false;
- }
-
- auto& txState = txBucket.Txs[txId];
-
- txState.TxScanQueryMemory += resources.Memory;
- txState.TxExecutionUnits += resources.ExecutionUnits;
- if (!txState.CreatedAt) {
- txState.CreatedAt = now;
- }
-
- auto& taskState = txState.Tasks[taskId];
- taskState.ScanQueryMemory += resources.Memory;
- taskState.ExecutionUnits += resources.ExecutionUnits;
- if (!taskState.CreatedAt) {
- taskState.CreatedAt = now;
- }
-
- if (!taskState.ResourceBrokerTaskId) {
- taskState.ResourceBrokerTaskId = rbTaskId;
- } else {
- extraAlloc = true;
- bool merged = ResourceBroker->MergeTasksInstant(taskState.ResourceBrokerTaskId, rbTaskId, SelfId());
- Y_VERIFY(merged);
- }
- } // with_lock (txBucket.Lock)
-
- LOG_D("TxId: " << txId << ", taskId: " << taskId << ". Allocated " << resources.ToString());
-
- Counters->RmComputeActors->Add(resources.ExecutionUnits);
- Counters->RmMemory->Add(resources.Memory);
- if (extraAlloc) {
- Counters->RmExtraMemAllocs->Inc();
- }
-
- FireResourcesPublishing();
- return true;
- }
-
- bool AllocateResources(ui64 txId, ui64 taskId, const TKqpResourcesRequest& resources,
- TResourcesAllocatedCallback&& onSuccess, TNotEnoughtResourcesCallback&& onFail, TDuration timeout) override
- {
- Y_UNUSED(txId, taskId, resources, onSuccess, onFail, timeout);
-
- // TODO: for DataQuery resources only
- return false;
- }
-
- void FreeResources(ui64 txId, ui64 taskId) override {
- ui64 releaseScanQueryMemory = 0;
- ui32 releaseExecutionUnits = 0;
- ui32 remainsTasks = 0;
-
- auto& txBucket = TxBucket(txId);
-
- with_lock (txBucket.Lock) {
- auto txIt = txBucket.Txs.find(txId);
- if (txIt == txBucket.Txs.end()) {
- return;
- }
-
- auto taskIt = txIt->second.Tasks.find(taskId);
- if (taskIt == txIt->second.Tasks.end()) {
- return;
- }
-
- releaseScanQueryMemory = taskIt->second.ScanQueryMemory;
- releaseExecutionUnits = taskIt->second.ExecutionUnits;
-
- bool finished = ResourceBroker->FinishTaskInstant(
- TEvResourceBroker::TEvFinishTask(taskIt->second.ResourceBrokerTaskId), SelfId());
- Y_VERIFY_DEBUG(finished);
-
- remainsTasks = txIt->second.Tasks.size() - 1;
-
- if (remainsTasks == 0) {
- txBucket.Txs.erase(txIt);
- } else {
- txIt->second.Tasks.erase(taskIt);
- txIt->second.TxScanQueryMemory -= releaseScanQueryMemory;
- txIt->second.TxExecutionUnits -= releaseExecutionUnits;
- }
- } // with_lock (txBucket.Lock)
-
- with_lock (Lock) {
- ScanQueryMemoryResource.Release(releaseScanQueryMemory);
- ExecutionUnitsResource.Release(releaseExecutionUnits);
- } // with_lock (Lock)
-
- LOG_D("TxId: " << txId << ", taskId: " << taskId << ". Released resources, "
- << "ScanQueryMemory: " << releaseScanQueryMemory << ", ExecutionUnits: " << releaseExecutionUnits << ". "
- << "Remains " << remainsTasks << " tasks in this tx.");
-
- Counters->RmComputeActors->Sub(releaseExecutionUnits);
- Counters->RmMemory->Sub(releaseScanQueryMemory);
-
- Y_VERIFY_DEBUG(Counters->RmComputeActors->Val() >= 0);
- Y_VERIFY_DEBUG(Counters->RmMemory->Val() >= 0);
-
- FireResourcesPublishing();
- }
-
- void FreeResources(ui64 txId) override {
- ui64 releaseScanQueryMemory = 0;
- ui32 releaseExecutionUnits = 0;
-
- auto& txBucket = TxBucket(txId);
-
- with_lock (txBucket.Lock) {
- auto txIt = txBucket.Txs.find(txId);
- if (txIt == txBucket.Txs.end()) {
- return;
- }
-
- for (auto& [taskId, taskState] : txIt->second.Tasks) {
- bool finished = ResourceBroker->FinishTaskInstant(
- TEvResourceBroker::TEvFinishTask(taskState.ResourceBrokerTaskId), SelfId());
- Y_VERIFY_DEBUG(finished);
- }
-
- releaseScanQueryMemory = txIt->second.TxScanQueryMemory;
- releaseExecutionUnits = txIt->second.TxExecutionUnits;
-
- txBucket.Txs.erase(txIt);
- } // with_lock (txBucket.Lock)
-
- with_lock (Lock) {
- ScanQueryMemoryResource.Release(releaseScanQueryMemory);
- ExecutionUnitsResource.Release(releaseExecutionUnits);
- } // with_lock (Lock)
-
- LOG_D("TxId: " << txId << ". Released resources, "
- << "ScanQueryMemory: " << releaseScanQueryMemory << ", ExecutionUnits: " << releaseExecutionUnits << ". "
- << "Tx completed.");
-
- Counters->RmComputeActors->Sub(releaseExecutionUnits);
- Counters->RmMemory->Sub(releaseScanQueryMemory);
-
- Y_VERIFY_DEBUG(Counters->RmComputeActors->Val() >= 0);
- Y_VERIFY_DEBUG(Counters->RmMemory->Val() >= 0);
-
- FireResourcesPublishing();
- }
-
- void NotifyExternalResourcesAllocated(ui64 txId, ui64 taskId, const TKqpResourcesRequest& resources) override {
- LOG_D("TxId: " << txId << ", taskId: " << taskId << ". External allocation: " << resources.ToString());
-
- YQL_ENSURE(resources.ExecutionUnits == 0);
- YQL_ENSURE(resources.MemoryPool == EKqpMemoryPool::DataQuery);
-
- auto& txBucket = TxBucket(txId);
- with_lock (txBucket.Lock) {
- auto& tx = txBucket.Txs[txId];
- auto& task = tx.Tasks[taskId];
-
- task.ExternalDataQueryMemory = resources.Memory;
- tx.TxExternalDataQueryMemory += resources.Memory;
- } // with_lock (txBucket.Lock)
-
- with_lock (Lock) {
- ExternalDataQueryMemory += resources.Memory;
- } // with_lock (Lock)
-
- Counters->RmExternalMemory->Add(resources.Memory);
-
- FireResourcesPublishing();
- }
-
- void NotifyExternalResourcesFreed(ui64 txId, ui64 taskId) override {
- LOG_D("TxId: " << txId << ", taskId: " << taskId << ". External free.");
-
- ui64 releaseMemory = 0;
-
- auto& txBucket = TxBucket(txId);
- with_lock (txBucket.Lock) {
- auto txIt = txBucket.Txs.find(txId);
- if (txIt == txBucket.Txs.end()) {
- return;
- }
-
- auto taskIt = txIt->second.Tasks.find(taskId);
- if (taskIt == txIt->second.Tasks.end()) {
- return;
- }
-
- releaseMemory = taskIt->second.ExternalDataQueryMemory;
-
- if (txIt->second.Tasks.size() == 1) {
- txBucket.Txs.erase(txId);
- } else {
- txIt->second.Tasks.erase(taskIt);
- txIt->second.TxExternalDataQueryMemory -= releaseMemory;
- }
- } // with_lock (txBucket.Lock)
-
- with_lock (Lock) {
- Y_VERIFY_DEBUG(ExternalDataQueryMemory >= releaseMemory);
- ExternalDataQueryMemory -= releaseMemory;
- } // with_lock (Lock)
-
- Counters->RmExternalMemory->Sub(releaseMemory);
- Y_VERIFY_DEBUG(Counters->RmExternalMemory->Val() >= 0);
-
- FireResourcesPublishing();
- }
-
- void RequestClusterResourcesInfo(std::function<void(TVector<NKikimrKqp::TKqpNodeResources>&&)>&& callback) override {
- LOG_DEBUG_S(*ActorSystem, NKikimrServices::KQP_RESOURCE_MANAGER, "Schedule Snapshot request");
- auto ev = MakeHolder<TEvPrivate::TEvTakeResourcesSnapshot>();
- ev->Callback = std::move(callback);
- TAutoPtr<IEventHandle> handle = new IEventHandle(SelfId(), SelfId(), ev.Release());
- ActorSystem->Send(handle);
- }
-
- TKqpLocalNodeResources GetLocalResources() const override {
- TKqpLocalNodeResources result;
- result.Memory.fill(0);
-
- with_lock (Lock) {
- result.ExecutionUnits = ExecutionUnitsResource.Available();
- result.Memory[EKqpMemoryPool::ScanQuery] = ScanQueryMemoryResource.Available();
- }
-
- return result;
- }
-
- NKikimrConfig::TTableServiceConfig::TResourceManager GetConfig() override {
- with_lock (Lock) {
- return Config;
- }
- }
-
-private:
- STATEFN(WorkState) {
- switch (ev->GetTypeRewrite()) {
- hFunc(TEvPrivate::TEvPublishResources, HandleWork);
- hFunc(TEvPrivate::TEvSchedulePublishResources, HandleWork);
- hFunc(TEvPrivate::TEvTakeResourcesSnapshot, HandleWork);
- hFunc(TEvResourceBroker::TEvConfigResponse, HandleWork);
- hFunc(TEvResourceBroker::TEvResourceBrokerResponse, HandleWork);
- hFunc(TEvTenantPool::TEvTenantPoolStatus, HandleWork);
- hFunc(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse, HandleWork);
- hFunc(NConsole::TEvConsole::TEvConfigNotificationRequest, HandleWork);
- hFunc(TEvents::TEvUndelivered, HandleWork);
- hFunc(TEvents::TEvPoison, HandleWork);
- hFunc(NMon::TEvHttpInfo, HandleWork);
- default: {
- Y_FAIL("Unexpected event 0x%x at TKqpResourceManagerActor::WorkState", ev->GetTypeRewrite());
- }
- }
- }
-
- void HandleWork(TEvPrivate::TEvPublishResources::TPtr&) {
- WbState.PublishScheduledAt.reset();
- PublishResourceUsage("batching");
- }
-
- void HandleWork(TEvPrivate::TEvSchedulePublishResources::TPtr&) {
- PublishResourceUsage("alloc");
- }
-
- void HandleWork(TEvPrivate::TEvTakeResourcesSnapshot::TPtr& ev) {
- if (WbState.StateStorageGroupId == std::numeric_limits<ui32>::max()) {
- LOG_E("Can not take resources snapshot, ssGroupId not set. Tenant: " << WbState.Tenant
- << ", Board: " << WbState.BoardPath << ", ssGroupId: " << WbState.StateStorageGroupId);
- ev->Get()->Callback({});
- return;
- }
-
- LOG_D("Create Snapshot actor, board: " << WbState.BoardPath << ", ssGroupId: " << WbState.StateStorageGroupId);
-
- Register(new TTakeResourcesSnapshotActor(
- WbState.BoardPath, WbState.StateStorageGroupId, std::move(ev->Get()->Callback)));
- }
-
- void HandleWork(TEvResourceBroker::TEvConfigResponse::TPtr& ev) {
- if (!ev->Get()->QueueConfig) {
- LOG_E(NLocalDb::KqpResourceManagerQueue << " not configured!");
- return;
- }
- auto& queueConfig = *ev->Get()->QueueConfig;
-
- if (queueConfig.GetLimit().GetMemory() > 0) {
- with_lock (Lock) {
- ScanQueryMemoryResource.SetNewLimit(queueConfig.GetLimit().GetMemory());
- }
- LOG_I("Total node memory for scan queries: " << queueConfig.GetLimit().GetMemory() << " bytes");
- }
- }
-
- void HandleWork(TEvResourceBroker::TEvResourceBrokerResponse::TPtr& ev) {
- with_lock (Lock) {
- ResourceBroker = ev->Get()->ResourceBroker;
- }
- }
-
- void HandleWork(TEvTenantPool::TEvTenantPoolStatus::TPtr& ev) {
- TString tenant;
- for (auto &slot : ev->Get()->Record.GetSlots()) {
- if (slot.HasAssignedTenant()) {
- if (tenant.empty()) {
- tenant = slot.GetAssignedTenant();
- } else {
- LOG_E("Multiple tenants are served by the node: " << ev->Get()->Record.ShortDebugString());
- }
- }
- }
-
- WbState.Tenant = tenant;
- WbState.BoardPath = MakeKqpRmBoardPath(tenant);
-
- if (auto* domainInfo = AppData()->DomainsInfo->GetDomainByName(ExtractDomain(tenant))) {
- WbState.StateStorageGroupId = domainInfo->DefaultStateStorageGroup;
- } else {
- WbState.StateStorageGroupId = std::numeric_limits<ui32>::max();
- }
-
- LOG_I("Received tenant pool status, serving tenant: " << tenant << ", board: " << WbState.BoardPath
- << ", ssGroupId: " << WbState.StateStorageGroupId);
-
- PublishResourceUsage("tenant updated");
- }
-
- static void HandleWork(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse::TPtr&) {
- LOG_D("Subscribed for config changes");
- }
-
- void HandleWork(NConsole::TEvConsole::TEvConfigNotificationRequest::TPtr& ev) {
- auto& event = ev->Get()->Record;
- Send(ev->Sender, new NConsole::TEvConsole::TEvConfigNotificationResponse(event), IEventHandle::FlagTrackDelivery, ev->Cookie);
-
- auto& config = *event.MutableConfig()->MutableTableServiceConfig()->MutableResourceManager();
-
-#define FORCE_VALUE(name) if (!config.Has ## name ()) config.Set ## name(config.Get ## name());
- FORCE_VALUE(ComputeActorsCount)
- FORCE_VALUE(ChannelBufferSize)
- FORCE_VALUE(ScanBufferSize)
- FORCE_VALUE(MkqlLightProgramMemoryLimit)
- FORCE_VALUE(MkqlHeavyProgramMemoryLimit)
- FORCE_VALUE(QueryMemoryLimit)
- FORCE_VALUE(PublishStatisticsIntervalSec);
- FORCE_VALUE(EnableInstantMkqlMemoryAlloc);
- FORCE_VALUE(MaxTotalChannelBuffersSize);
- FORCE_VALUE(MinChannelBufferSize);
- FORCE_VALUE(MaxTotalScanBuffersSize);
- FORCE_VALUE(MinScanBufferSize);
-#undef FORCE_VALUE
-
- LOG_I("Updated table service config: " << config.DebugString());
-
- with_lock (Lock) {
- ExecutionUnitsResource.SetNewLimit(config.GetComputeActorsCount());
- Config.Swap(&config);
- }
- }
-
- static void HandleWork(TEvents::TEvUndelivered::TPtr& ev) {
- switch (ev->Get()->SourceType) {
- case NConsole::TEvConfigsDispatcher::EvSetConfigSubscriptionRequest:
- LOG_C("Failed to deliver subscription request to config dispatcher");
- break;
-
- case NConsole::TEvConsole::EvConfigNotificationResponse:
- LOG_E("Failed to deliver config notification response");
- break;
-
- default:
- LOG_C("Undelivered event with unexpected source type: " << ev->Get()->SourceType);
- break;
- }
- }
-
- void HandleWork(TEvents::TEvPoison::TPtr&) {
- PassAway();
- }
-
- void HandleWork(NMon::TEvHttpInfo::TPtr& ev) {
- TStringStream str;
- str.Reserve(8 * 1024);
- HTML(str) {
- PRE() {
- str << "Current config:" << Endl;
- with_lock (Lock) {
- str << Config.DebugString() << Endl;
- }
-
- str << "State storage key: " << WbState.Tenant << Endl;
- with_lock (Lock) {
- str << "ScanQuery memory resource: " << ScanQueryMemoryResource.ToString() << Endl;
- str << "External DataQuery memory: " << ExternalDataQueryMemory << Endl;
- str << "ExecutionUnits resource: " << ExecutionUnitsResource.ToString() << Endl;
- }
- str << "Last resource broker task id: " << LastResourceBrokerTaskId.load() << Endl;
- if (WbState.LastPublishTime) {
- str << "Last publish time: " << *WbState.LastPublishTime << Endl;
- }
- if (WbState.PublishScheduledAt) {
- str << "Next publish time: " << *WbState.PublishScheduledAt << Endl;
- }
-
- str << Endl << "Transactions:" << Endl;
- for (auto& bucket : Buckets) {
- with_lock (bucket.Lock) {
- for (auto& [txId, txState] : bucket.Txs) {
- str << " TxId: " << txId << Endl;
- str << " ScanQuery memory: " << txState.TxScanQueryMemory << Endl;
- str << " External DataQuery memory: " << txState.TxExternalDataQueryMemory << Endl;
- str << " Execution units: " << txState.TxExecutionUnits << Endl;
- str << " Create at: " << txState.CreatedAt << Endl;
- str << " Tasks:" << Endl;
- for (auto& [taskId, taskState] : txState.Tasks) {
- str << " TaskId: " << taskId << Endl;
- str << " ScanQuery memory: " << taskState.ScanQueryMemory << Endl;
- str << " External DataQuery memory: " << taskState.ExternalDataQueryMemory << Endl;
- str << " Execution units: " << taskState.ExecutionUnits << Endl;
- str << " ResourceBroker TaskId: " << taskState.ResourceBrokerTaskId << Endl;
- str << " Created at: " << taskState.CreatedAt << Endl;
- }
- }
- } // with_lock (bucket.Lock)
- }
- } // PRE()
- }
-
- Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str()));
- }
-
-private:
- TTxStatesBucket& TxBucket(ui64 txId) {
- return Buckets[txId % Buckets.size()];
- }
-
- void PassAway() override {
- ToBroker(new TEvResourceBroker::TEvNotifyActorDied);
- TActor::PassAway();
- }
-
- void ToBroker(IEventBase* ev) {
- Send(ResourceBrokerId, ev);
- }
-
- static TString MakeKqpRmBoardPath(TStringBuf database) {
- return TStringBuilder() << "kqprm+" << database;
- }
-
- void FireResourcesPublishing() {
- ActorSystem->Send(SelfId(), new TEvPrivate::TEvSchedulePublishResources);
- }
-
- void PublishResourceUsage(TStringBuf reason) {
- if (WbState.PublishScheduledAt) {
- return;
- }
-
- TDuration publishInterval;
- with_lock (Lock) {
- publishInterval = TDuration::Seconds(Config.GetPublishStatisticsIntervalSec());
- }
-
- auto now = ActorSystem->Timestamp();
- if (publishInterval && WbState.LastPublishTime && now - *WbState.LastPublishTime < publishInterval) {
- WbState.PublishScheduledAt = *WbState.LastPublishTime + publishInterval;
- Schedule(*WbState.PublishScheduledAt - now, new TEvPrivate::TEvPublishResources);
- LOG_D("Schedule publish at " << *WbState.PublishScheduledAt << ", after " << (*WbState.PublishScheduledAt - now));
- return;
- }
-
- if (WbState.BoardPublisherActorId) {
- LOG_I("Kill previous board publisher for '" << WbState.BoardPath
- << "' at " << WbState.BoardPublisherActorId << ", reason: " << reason);
- Send(WbState.BoardPublisherActorId, new TEvents::TEvPoison);
- }
-
- WbState.BoardPublisherActorId = TActorId();
-
- if (WbState.StateStorageGroupId == std::numeric_limits<ui32>::max()) {
- LOG_E("Can not find default state storage group for database " << WbState.Tenant);
- return;
- }
-
- NKikimrKqp::TKqpNodeResources payload;
- payload.SetNodeId(SelfId().NodeId());
- payload.SetTimestamp(now.Seconds());
- ActorIdToProto(MakeKqpResourceManagerServiceID(SelfId().NodeId()), payload.MutableResourceManagerActorId()); // legacy
- with_lock (Lock) {
- payload.SetAvailableComputeActors(ExecutionUnitsResource.Available()); // legacy
- payload.SetTotalMemory(ScanQueryMemoryResource.GetLimit()); // legacy
- payload.SetUsedMemory(ScanQueryMemoryResource.GetLimit() - ScanQueryMemoryResource.Available()); // legacy
-
- payload.SetExecutionUnits(ExecutionUnitsResource.Available());
- auto* pool = payload.MutableMemory()->Add();
- pool->SetPool(EKqpMemoryPool::ScanQuery);
- pool->SetAvailable(ScanQueryMemoryResource.Available());
- }
-
- auto boardPublisher = CreateBoardPublishActor(WbState.BoardPath, payload.SerializeAsString(), SelfId(),
- WbState.StateStorageGroupId, /* ttlMs */ 0, /* reg */ true);
- WbState.BoardPublisherActorId = Register(boardPublisher);
-
- WbState.LastPublishTime = now;
-
- LOG_I("Publish resource usage for '" << WbState.BoardPath << "' at " << WbState.BoardPublisherActorId
- << ", reason: " << reason << ", groupId: " << WbState.StateStorageGroupId
- << ", payload: " << payload.ShortDebugString());
- }
-
-private:
- NKikimrConfig::TTableServiceConfig::TResourceManager Config; // guarded by Lock
- TIntrusivePtr<TKqpCounters> Counters;
- const TActorId ResourceBrokerId;
- TIntrusivePtr<NResourceBroker::IResourceBroker> ResourceBroker;
- TActorSystem* ActorSystem = nullptr;
-
- // common guard
- TAdaptiveLock Lock;
-
- // limits (guarded by Lock)
- TLimitedResource<ui32> ExecutionUnitsResource;
- TLimitedResource<ui64> ScanQueryMemoryResource;
- ui64 ExternalDataQueryMemory = 0;
-
- // current state
- std::array<TTxStatesBucket, BucketsCount> Buckets;
- std::atomic<ui64> LastResourceBrokerTaskId = 0;
-
- // Whiteboard specific fields
- struct TWhiteBoardState {
- TString Tenant;
- TString BoardPath;
- ui32 StateStorageGroupId = std::numeric_limits<ui32>::max();
- TActorId BoardPublisherActorId;
- std::optional<TInstant> LastPublishTime;
- std::optional<TInstant> PublishScheduledAt;
- };
- TWhiteBoardState WbState;
-};
-
-} // namespace NRm
-
-
-NActors::IActor* CreateKqpResourceManagerActor(const NKikimrConfig::TTableServiceConfig::TResourceManager& config,
- TIntrusivePtr<TKqpCounters> counters, NActors::TActorId resourceBroker)
-{
- return new NRm::TKqpResourceManagerActor(config, counters, resourceBroker);
-}
-
-NRm::IKqpResourceManager* GetKqpResourceManager(TMaybe<ui32> _nodeId) {
- if (auto* rm = TryGetKqpResourceManager(_nodeId)) {
- return rm;
- }
-
- ui32 nodeId = _nodeId ? *_nodeId : TActivationContext::ActorSystem()->NodeId;
- Y_FAIL("KqpResourceManager not ready yet, node #%" PRIu32, nodeId);
-}
-
-NRm::IKqpResourceManager* TryGetKqpResourceManager(TMaybe<ui32> _nodeId) {
- ui32 nodeId = _nodeId ? *_nodeId : TActivationContext::ActorSystem()->NodeId;
- auto rm = NRm::ResourceManagers.Default.load(std::memory_order_acquire);
- if (Y_LIKELY(rm && rm->NodeId == nodeId)) {
- return rm->Instance;
- }
-
- // for tests only
- with_lock (NRm::ResourceManagers.Lock) {
- auto it = NRm::ResourceManagers.ByNodeId.find(nodeId);
- if (it != NRm::ResourceManagers.ByNodeId.end()) {
- return it->second->Instance;
- }
- }
-
- return nullptr;
-}
-
-} // namespace NKqp
-} // namespace NKikimr
+ << "requested " << resources.Memory);
+ if (details) {
+ details->SetScanQueryMemory();
+ }
+ return false;
+ }
+
+ auto& txState = txBucket.Txs[txId];
+
+ txState.TxScanQueryMemory += resources.Memory;
+ txState.TxExecutionUnits += resources.ExecutionUnits;
+ if (!txState.CreatedAt) {
+ txState.CreatedAt = now;
+ }
+
+ auto& taskState = txState.Tasks[taskId];
+ taskState.ScanQueryMemory += resources.Memory;
+ taskState.ExecutionUnits += resources.ExecutionUnits;
+ if (!taskState.CreatedAt) {
+ taskState.CreatedAt = now;
+ }
+
+ if (!taskState.ResourceBrokerTaskId) {
+ taskState.ResourceBrokerTaskId = rbTaskId;
+ } else {
+ extraAlloc = true;
+ bool merged = ResourceBroker->MergeTasksInstant(taskState.ResourceBrokerTaskId, rbTaskId, SelfId());
+ Y_VERIFY(merged);
+ }
+ } // with_lock (txBucket.Lock)
+
+ LOG_D("TxId: " << txId << ", taskId: " << taskId << ". Allocated " << resources.ToString());
+
+ Counters->RmComputeActors->Add(resources.ExecutionUnits);
+ Counters->RmMemory->Add(resources.Memory);
+ if (extraAlloc) {
+ Counters->RmExtraMemAllocs->Inc();
+ }
+
+ FireResourcesPublishing();
+ return true;
+ }
+
+ bool AllocateResources(ui64 txId, ui64 taskId, const TKqpResourcesRequest& resources,
+ TResourcesAllocatedCallback&& onSuccess, TNotEnoughtResourcesCallback&& onFail, TDuration timeout) override
+ {
+ Y_UNUSED(txId, taskId, resources, onSuccess, onFail, timeout);
+
+ // TODO: for DataQuery resources only
+ return false;
+ }
+
+ void FreeResources(ui64 txId, ui64 taskId) override {
+ ui64 releaseScanQueryMemory = 0;
+ ui32 releaseExecutionUnits = 0;
+ ui32 remainsTasks = 0;
+
+ auto& txBucket = TxBucket(txId);
+
+ with_lock (txBucket.Lock) {
+ auto txIt = txBucket.Txs.find(txId);
+ if (txIt == txBucket.Txs.end()) {
+ return;
+ }
+
+ auto taskIt = txIt->second.Tasks.find(taskId);
+ if (taskIt == txIt->second.Tasks.end()) {
+ return;
+ }
+
+ releaseScanQueryMemory = taskIt->second.ScanQueryMemory;
+ releaseExecutionUnits = taskIt->second.ExecutionUnits;
+
+ bool finished = ResourceBroker->FinishTaskInstant(
+ TEvResourceBroker::TEvFinishTask(taskIt->second.ResourceBrokerTaskId), SelfId());
+ Y_VERIFY_DEBUG(finished);
+
+ remainsTasks = txIt->second.Tasks.size() - 1;
+
+ if (remainsTasks == 0) {
+ txBucket.Txs.erase(txIt);
+ } else {
+ txIt->second.Tasks.erase(taskIt);
+ txIt->second.TxScanQueryMemory -= releaseScanQueryMemory;
+ txIt->second.TxExecutionUnits -= releaseExecutionUnits;
+ }
+ } // with_lock (txBucket.Lock)
+
+ with_lock (Lock) {
+ ScanQueryMemoryResource.Release(releaseScanQueryMemory);
+ ExecutionUnitsResource.Release(releaseExecutionUnits);
+ } // with_lock (Lock)
+
+ LOG_D("TxId: " << txId << ", taskId: " << taskId << ". Released resources, "
+ << "ScanQueryMemory: " << releaseScanQueryMemory << ", ExecutionUnits: " << releaseExecutionUnits << ". "
+ << "Remains " << remainsTasks << " tasks in this tx.");
+
+ Counters->RmComputeActors->Sub(releaseExecutionUnits);
+ Counters->RmMemory->Sub(releaseScanQueryMemory);
+
+ Y_VERIFY_DEBUG(Counters->RmComputeActors->Val() >= 0);
+ Y_VERIFY_DEBUG(Counters->RmMemory->Val() >= 0);
+
+ FireResourcesPublishing();
+ }
+
+ void FreeResources(ui64 txId) override {
+ ui64 releaseScanQueryMemory = 0;
+ ui32 releaseExecutionUnits = 0;
+
+ auto& txBucket = TxBucket(txId);
+
+ with_lock (txBucket.Lock) {
+ auto txIt = txBucket.Txs.find(txId);
+ if (txIt == txBucket.Txs.end()) {
+ return;
+ }
+
+ for (auto& [taskId, taskState] : txIt->second.Tasks) {
+ bool finished = ResourceBroker->FinishTaskInstant(
+ TEvResourceBroker::TEvFinishTask(taskState.ResourceBrokerTaskId), SelfId());
+ Y_VERIFY_DEBUG(finished);
+ }
+
+ releaseScanQueryMemory = txIt->second.TxScanQueryMemory;
+ releaseExecutionUnits = txIt->second.TxExecutionUnits;
+
+ txBucket.Txs.erase(txIt);
+ } // with_lock (txBucket.Lock)
+
+ with_lock (Lock) {
+ ScanQueryMemoryResource.Release(releaseScanQueryMemory);
+ ExecutionUnitsResource.Release(releaseExecutionUnits);
+ } // with_lock (Lock)
+
+ LOG_D("TxId: " << txId << ". Released resources, "
+ << "ScanQueryMemory: " << releaseScanQueryMemory << ", ExecutionUnits: " << releaseExecutionUnits << ". "
+ << "Tx completed.");
+
+ Counters->RmComputeActors->Sub(releaseExecutionUnits);
+ Counters->RmMemory->Sub(releaseScanQueryMemory);
+
+ Y_VERIFY_DEBUG(Counters->RmComputeActors->Val() >= 0);
+ Y_VERIFY_DEBUG(Counters->RmMemory->Val() >= 0);
+
+ FireResourcesPublishing();
+ }
+
+ void NotifyExternalResourcesAllocated(ui64 txId, ui64 taskId, const TKqpResourcesRequest& resources) override {
+ LOG_D("TxId: " << txId << ", taskId: " << taskId << ". External allocation: " << resources.ToString());
+
+ YQL_ENSURE(resources.ExecutionUnits == 0);
+ YQL_ENSURE(resources.MemoryPool == EKqpMemoryPool::DataQuery);
+
+ auto& txBucket = TxBucket(txId);
+ with_lock (txBucket.Lock) {
+ auto& tx = txBucket.Txs[txId];
+ auto& task = tx.Tasks[taskId];
+
+ task.ExternalDataQueryMemory = resources.Memory;
+ tx.TxExternalDataQueryMemory += resources.Memory;
+ } // with_lock (txBucket.Lock)
+
+ with_lock (Lock) {
+ ExternalDataQueryMemory += resources.Memory;
+ } // with_lock (Lock)
+
+ Counters->RmExternalMemory->Add(resources.Memory);
+
+ FireResourcesPublishing();
+ }
+
+ void NotifyExternalResourcesFreed(ui64 txId, ui64 taskId) override {
+ LOG_D("TxId: " << txId << ", taskId: " << taskId << ". External free.");
+
+ ui64 releaseMemory = 0;
+
+ auto& txBucket = TxBucket(txId);
+ with_lock (txBucket.Lock) {
+ auto txIt = txBucket.Txs.find(txId);
+ if (txIt == txBucket.Txs.end()) {
+ return;
+ }
+
+ auto taskIt = txIt->second.Tasks.find(taskId);
+ if (taskIt == txIt->second.Tasks.end()) {
+ return;
+ }
+
+ releaseMemory = taskIt->second.ExternalDataQueryMemory;
+
+ if (txIt->second.Tasks.size() == 1) {
+ txBucket.Txs.erase(txId);
+ } else {
+ txIt->second.Tasks.erase(taskIt);
+ txIt->second.TxExternalDataQueryMemory -= releaseMemory;
+ }
+ } // with_lock (txBucket.Lock)
+
+ with_lock (Lock) {
+ Y_VERIFY_DEBUG(ExternalDataQueryMemory >= releaseMemory);
+ ExternalDataQueryMemory -= releaseMemory;
+ } // with_lock (Lock)
+
+ Counters->RmExternalMemory->Sub(releaseMemory);
+ Y_VERIFY_DEBUG(Counters->RmExternalMemory->Val() >= 0);
+
+ FireResourcesPublishing();
+ }
+
+ void RequestClusterResourcesInfo(std::function<void(TVector<NKikimrKqp::TKqpNodeResources>&&)>&& callback) override {
+ LOG_DEBUG_S(*ActorSystem, NKikimrServices::KQP_RESOURCE_MANAGER, "Schedule Snapshot request");
+ auto ev = MakeHolder<TEvPrivate::TEvTakeResourcesSnapshot>();
+ ev->Callback = std::move(callback);
+ TAutoPtr<IEventHandle> handle = new IEventHandle(SelfId(), SelfId(), ev.Release());
+ ActorSystem->Send(handle);
+ }
+
+ TKqpLocalNodeResources GetLocalResources() const override {
+ TKqpLocalNodeResources result;
+ result.Memory.fill(0);
+
+ with_lock (Lock) {
+ result.ExecutionUnits = ExecutionUnitsResource.Available();
+ result.Memory[EKqpMemoryPool::ScanQuery] = ScanQueryMemoryResource.Available();
+ }
+
+ return result;
+ }
+
+ NKikimrConfig::TTableServiceConfig::TResourceManager GetConfig() override {
+ with_lock (Lock) {
+ return Config;
+ }
+ }
+
+private:
+ STATEFN(WorkState) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvPrivate::TEvPublishResources, HandleWork);
+ hFunc(TEvPrivate::TEvSchedulePublishResources, HandleWork);
+ hFunc(TEvPrivate::TEvTakeResourcesSnapshot, HandleWork);
+ hFunc(TEvResourceBroker::TEvConfigResponse, HandleWork);
+ hFunc(TEvResourceBroker::TEvResourceBrokerResponse, HandleWork);
+ hFunc(TEvTenantPool::TEvTenantPoolStatus, HandleWork);
+ hFunc(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse, HandleWork);
+ hFunc(NConsole::TEvConsole::TEvConfigNotificationRequest, HandleWork);
+ hFunc(TEvents::TEvUndelivered, HandleWork);
+ hFunc(TEvents::TEvPoison, HandleWork);
+ hFunc(NMon::TEvHttpInfo, HandleWork);
+ default: {
+ Y_FAIL("Unexpected event 0x%x at TKqpResourceManagerActor::WorkState", ev->GetTypeRewrite());
+ }
+ }
+ }
+
+ void HandleWork(TEvPrivate::TEvPublishResources::TPtr&) {
+ WbState.PublishScheduledAt.reset();
+ PublishResourceUsage("batching");
+ }
+
+ void HandleWork(TEvPrivate::TEvSchedulePublishResources::TPtr&) {
+ PublishResourceUsage("alloc");
+ }
+
+ void HandleWork(TEvPrivate::TEvTakeResourcesSnapshot::TPtr& ev) {
+ if (WbState.StateStorageGroupId == std::numeric_limits<ui32>::max()) {
+ LOG_E("Can not take resources snapshot, ssGroupId not set. Tenant: " << WbState.Tenant
+ << ", Board: " << WbState.BoardPath << ", ssGroupId: " << WbState.StateStorageGroupId);
+ ev->Get()->Callback({});
+ return;
+ }
+
+ LOG_D("Create Snapshot actor, board: " << WbState.BoardPath << ", ssGroupId: " << WbState.StateStorageGroupId);
+
+ Register(new TTakeResourcesSnapshotActor(
+ WbState.BoardPath, WbState.StateStorageGroupId, std::move(ev->Get()->Callback)));
+ }
+
+ void HandleWork(TEvResourceBroker::TEvConfigResponse::TPtr& ev) {
+ if (!ev->Get()->QueueConfig) {
+ LOG_E(NLocalDb::KqpResourceManagerQueue << " not configured!");
+ return;
+ }
+ auto& queueConfig = *ev->Get()->QueueConfig;
+
+ if (queueConfig.GetLimit().GetMemory() > 0) {
+ with_lock (Lock) {
+ ScanQueryMemoryResource.SetNewLimit(queueConfig.GetLimit().GetMemory());
+ }
+ LOG_I("Total node memory for scan queries: " << queueConfig.GetLimit().GetMemory() << " bytes");
+ }
+ }
+
+ void HandleWork(TEvResourceBroker::TEvResourceBrokerResponse::TPtr& ev) {
+ with_lock (Lock) {
+ ResourceBroker = ev->Get()->ResourceBroker;
+ }
+ }
+
+ void HandleWork(TEvTenantPool::TEvTenantPoolStatus::TPtr& ev) {
+ TString tenant;
+ for (auto &slot : ev->Get()->Record.GetSlots()) {
+ if (slot.HasAssignedTenant()) {
+ if (tenant.empty()) {
+ tenant = slot.GetAssignedTenant();
+ } else {
+ LOG_E("Multiple tenants are served by the node: " << ev->Get()->Record.ShortDebugString());
+ }
+ }
+ }
+
+ WbState.Tenant = tenant;
+ WbState.BoardPath = MakeKqpRmBoardPath(tenant);
+
+ if (auto* domainInfo = AppData()->DomainsInfo->GetDomainByName(ExtractDomain(tenant))) {
+ WbState.StateStorageGroupId = domainInfo->DefaultStateStorageGroup;
+ } else {
+ WbState.StateStorageGroupId = std::numeric_limits<ui32>::max();
+ }
+
+ LOG_I("Received tenant pool status, serving tenant: " << tenant << ", board: " << WbState.BoardPath
+ << ", ssGroupId: " << WbState.StateStorageGroupId);
+
+ PublishResourceUsage("tenant updated");
+ }
+
+ static void HandleWork(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse::TPtr&) {
+ LOG_D("Subscribed for config changes");
+ }
+
+ void HandleWork(NConsole::TEvConsole::TEvConfigNotificationRequest::TPtr& ev) {
+ auto& event = ev->Get()->Record;
+ Send(ev->Sender, new NConsole::TEvConsole::TEvConfigNotificationResponse(event), IEventHandle::FlagTrackDelivery, ev->Cookie);
+
+ auto& config = *event.MutableConfig()->MutableTableServiceConfig()->MutableResourceManager();
+
+#define FORCE_VALUE(name) if (!config.Has ## name ()) config.Set ## name(config.Get ## name());
+ FORCE_VALUE(ComputeActorsCount)
+ FORCE_VALUE(ChannelBufferSize)
+ FORCE_VALUE(ScanBufferSize)
+ FORCE_VALUE(MkqlLightProgramMemoryLimit)
+ FORCE_VALUE(MkqlHeavyProgramMemoryLimit)
+ FORCE_VALUE(QueryMemoryLimit)
+ FORCE_VALUE(PublishStatisticsIntervalSec);
+ FORCE_VALUE(EnableInstantMkqlMemoryAlloc);
+ FORCE_VALUE(MaxTotalChannelBuffersSize);
+ FORCE_VALUE(MinChannelBufferSize);
+ FORCE_VALUE(MaxTotalScanBuffersSize);
+ FORCE_VALUE(MinScanBufferSize);
+#undef FORCE_VALUE
+
+ LOG_I("Updated table service config: " << config.DebugString());
+
+ with_lock (Lock) {
+ ExecutionUnitsResource.SetNewLimit(config.GetComputeActorsCount());
+ Config.Swap(&config);
+ }
+ }
+
+ static void HandleWork(TEvents::TEvUndelivered::TPtr& ev) {
+ switch (ev->Get()->SourceType) {
+ case NConsole::TEvConfigsDispatcher::EvSetConfigSubscriptionRequest:
+ LOG_C("Failed to deliver subscription request to config dispatcher");
+ break;
+
+ case NConsole::TEvConsole::EvConfigNotificationResponse:
+ LOG_E("Failed to deliver config notification response");
+ break;
+
+ default:
+ LOG_C("Undelivered event with unexpected source type: " << ev->Get()->SourceType);
+ break;
+ }
+ }
+
+ void HandleWork(TEvents::TEvPoison::TPtr&) {
+ PassAway();
+ }
+
+ void HandleWork(NMon::TEvHttpInfo::TPtr& ev) {
+ TStringStream str;
+ str.Reserve(8 * 1024);
+ HTML(str) {
+ PRE() {
+ str << "Current config:" << Endl;
+ with_lock (Lock) {
+ str << Config.DebugString() << Endl;
+ }
+
+ str << "State storage key: " << WbState.Tenant << Endl;
+ with_lock (Lock) {
+ str << "ScanQuery memory resource: " << ScanQueryMemoryResource.ToString() << Endl;
+ str << "External DataQuery memory: " << ExternalDataQueryMemory << Endl;
+ str << "ExecutionUnits resource: " << ExecutionUnitsResource.ToString() << Endl;
+ }
+ str << "Last resource broker task id: " << LastResourceBrokerTaskId.load() << Endl;
+ if (WbState.LastPublishTime) {
+ str << "Last publish time: " << *WbState.LastPublishTime << Endl;
+ }
+ if (WbState.PublishScheduledAt) {
+ str << "Next publish time: " << *WbState.PublishScheduledAt << Endl;
+ }
+
+ str << Endl << "Transactions:" << Endl;
+ for (auto& bucket : Buckets) {
+ with_lock (bucket.Lock) {
+ for (auto& [txId, txState] : bucket.Txs) {
+ str << " TxId: " << txId << Endl;
+ str << " ScanQuery memory: " << txState.TxScanQueryMemory << Endl;
+ str << " External DataQuery memory: " << txState.TxExternalDataQueryMemory << Endl;
+ str << " Execution units: " << txState.TxExecutionUnits << Endl;
+ str << " Create at: " << txState.CreatedAt << Endl;
+ str << " Tasks:" << Endl;
+ for (auto& [taskId, taskState] : txState.Tasks) {
+ str << " TaskId: " << taskId << Endl;
+ str << " ScanQuery memory: " << taskState.ScanQueryMemory << Endl;
+ str << " External DataQuery memory: " << taskState.ExternalDataQueryMemory << Endl;
+ str << " Execution units: " << taskState.ExecutionUnits << Endl;
+ str << " ResourceBroker TaskId: " << taskState.ResourceBrokerTaskId << Endl;
+ str << " Created at: " << taskState.CreatedAt << Endl;
+ }
+ }
+ } // with_lock (bucket.Lock)
+ }
+ } // PRE()
+ }
+
+ Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str()));
+ }
+
+private:
+ TTxStatesBucket& TxBucket(ui64 txId) {
+ return Buckets[txId % Buckets.size()];
+ }
+
+ void PassAway() override {
+ ToBroker(new TEvResourceBroker::TEvNotifyActorDied);
+ TActor::PassAway();
+ }
+
+ void ToBroker(IEventBase* ev) {
+ Send(ResourceBrokerId, ev);
+ }
+
+ static TString MakeKqpRmBoardPath(TStringBuf database) {
+ return TStringBuilder() << "kqprm+" << database;
+ }
+
+ void FireResourcesPublishing() {
+ ActorSystem->Send(SelfId(), new TEvPrivate::TEvSchedulePublishResources);
+ }
+
+ void PublishResourceUsage(TStringBuf reason) {
+ if (WbState.PublishScheduledAt) {
+ return;
+ }
+
+ TDuration publishInterval;
+ with_lock (Lock) {
+ publishInterval = TDuration::Seconds(Config.GetPublishStatisticsIntervalSec());
+ }
+
+ auto now = ActorSystem->Timestamp();
+ if (publishInterval && WbState.LastPublishTime && now - *WbState.LastPublishTime < publishInterval) {
+ WbState.PublishScheduledAt = *WbState.LastPublishTime + publishInterval;
+ Schedule(*WbState.PublishScheduledAt - now, new TEvPrivate::TEvPublishResources);
+ LOG_D("Schedule publish at " << *WbState.PublishScheduledAt << ", after " << (*WbState.PublishScheduledAt - now));
+ return;
+ }
+
+ if (WbState.BoardPublisherActorId) {
+ LOG_I("Kill previous board publisher for '" << WbState.BoardPath
+ << "' at " << WbState.BoardPublisherActorId << ", reason: " << reason);
+ Send(WbState.BoardPublisherActorId, new TEvents::TEvPoison);
+ }
+
+ WbState.BoardPublisherActorId = TActorId();
+
+ if (WbState.StateStorageGroupId == std::numeric_limits<ui32>::max()) {
+ LOG_E("Can not find default state storage group for database " << WbState.Tenant);
+ return;
+ }
+
+ NKikimrKqp::TKqpNodeResources payload;
+ payload.SetNodeId(SelfId().NodeId());
+ payload.SetTimestamp(now.Seconds());
+ ActorIdToProto(MakeKqpResourceManagerServiceID(SelfId().NodeId()), payload.MutableResourceManagerActorId()); // legacy
+ with_lock (Lock) {
+ payload.SetAvailableComputeActors(ExecutionUnitsResource.Available()); // legacy
+ payload.SetTotalMemory(ScanQueryMemoryResource.GetLimit()); // legacy
+ payload.SetUsedMemory(ScanQueryMemoryResource.GetLimit() - ScanQueryMemoryResource.Available()); // legacy
+
+ payload.SetExecutionUnits(ExecutionUnitsResource.Available());
+ auto* pool = payload.MutableMemory()->Add();
+ pool->SetPool(EKqpMemoryPool::ScanQuery);
+ pool->SetAvailable(ScanQueryMemoryResource.Available());
+ }
+
+ auto boardPublisher = CreateBoardPublishActor(WbState.BoardPath, payload.SerializeAsString(), SelfId(),
+ WbState.StateStorageGroupId, /* ttlMs */ 0, /* reg */ true);
+ WbState.BoardPublisherActorId = Register(boardPublisher);
+
+ WbState.LastPublishTime = now;
+
+ LOG_I("Publish resource usage for '" << WbState.BoardPath << "' at " << WbState.BoardPublisherActorId
+ << ", reason: " << reason << ", groupId: " << WbState.StateStorageGroupId
+ << ", payload: " << payload.ShortDebugString());
+ }
+
+private:
+ NKikimrConfig::TTableServiceConfig::TResourceManager Config; // guarded by Lock
+ TIntrusivePtr<TKqpCounters> Counters;
+ const TActorId ResourceBrokerId;
+ TIntrusivePtr<NResourceBroker::IResourceBroker> ResourceBroker;
+ TActorSystem* ActorSystem = nullptr;
+
+ // common guard
+ TAdaptiveLock Lock;
+
+ // limits (guarded by Lock)
+ TLimitedResource<ui32> ExecutionUnitsResource;
+ TLimitedResource<ui64> ScanQueryMemoryResource;
+ ui64 ExternalDataQueryMemory = 0;
+
+ // current state
+ std::array<TTxStatesBucket, BucketsCount> Buckets;
+ std::atomic<ui64> LastResourceBrokerTaskId = 0;
+
+ // Whiteboard specific fields
+ struct TWhiteBoardState {
+ TString Tenant;
+ TString BoardPath;
+ ui32 StateStorageGroupId = std::numeric_limits<ui32>::max();
+ TActorId BoardPublisherActorId;
+ std::optional<TInstant> LastPublishTime;
+ std::optional<TInstant> PublishScheduledAt;
+ };
+ TWhiteBoardState WbState;
+};
+
+} // namespace NRm
+
+
+NActors::IActor* CreateKqpResourceManagerActor(const NKikimrConfig::TTableServiceConfig::TResourceManager& config,
+ TIntrusivePtr<TKqpCounters> counters, NActors::TActorId resourceBroker)
+{
+ return new NRm::TKqpResourceManagerActor(config, counters, resourceBroker);
+}
+
+NRm::IKqpResourceManager* GetKqpResourceManager(TMaybe<ui32> _nodeId) {
+ if (auto* rm = TryGetKqpResourceManager(_nodeId)) {
+ return rm;
+ }
+
+ ui32 nodeId = _nodeId ? *_nodeId : TActivationContext::ActorSystem()->NodeId;
+ Y_FAIL("KqpResourceManager not ready yet, node #%" PRIu32, nodeId);
+}
+
+NRm::IKqpResourceManager* TryGetKqpResourceManager(TMaybe<ui32> _nodeId) {
+ ui32 nodeId = _nodeId ? *_nodeId : TActivationContext::ActorSystem()->NodeId;
+ auto rm = NRm::ResourceManagers.Default.load(std::memory_order_acquire);
+ if (Y_LIKELY(rm && rm->NodeId == nodeId)) {
+ return rm->Instance;
+ }
+
+ // for tests only
+ with_lock (NRm::ResourceManagers.Lock) {
+ auto it = NRm::ResourceManagers.ByNodeId.find(nodeId);
+ if (it != NRm::ResourceManagers.ByNodeId.end()) {
+ return it->second->Instance;
+ }
+ }
+
+ return nullptr;
+}
+
+} // namespace NKqp
+} // namespace NKikimr
diff --git a/ydb/core/kqp/rm/kqp_rm.h b/ydb/core/kqp/rm/kqp_rm.h
index faea319af8..568e945d80 100644
--- a/ydb/core/kqp/rm/kqp_rm.h
+++ b/ydb/core/kqp/rm/kqp_rm.h
@@ -1,100 +1,100 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/protos/config.pb.h>
#include <ydb/core/kqp/counters/kqp_counters.h>
-
-#include <library/cpp/actors/core/actor.h>
-
-#include <util/datetime/base.h>
-#include <util/string/builder.h>
-
-#include <array>
-#include <bitset>
-#include <functional>
-
-
-namespace NKikimr {
-namespace NKqp {
-
-namespace NRm {
-
-/// memory pools
-enum EKqpMemoryPool : ui32 {
- Unspecified = 0,
- ScanQuery = 1, // slow allocations via ResourceBroker
- DataQuery = 2, // fast allocations via memory-arena
-
- Count = 3
-};
-
-/// resources request
-struct TKqpResourcesRequest {
- ui32 ExecutionUnits = 0;
- EKqpMemoryPool MemoryPool = EKqpMemoryPool::Unspecified;
- ui64 Memory = 0;
-
- TString ToString() const {
- return TStringBuilder() << "TKqpResourcesRequest{ MemoryPool: " << (ui32) MemoryPool << ", Memory: " << Memory
- << ", ExecutionUnits: " << ExecutionUnits << " }";
- }
-};
-
-/// detailed information on allocation failure
-struct TKqpNotEnoughResources {
- std::bitset<32> State;
-
- bool NotReady() const { return State.test(0); }
- bool ExecutionUnits() const { return State.test(1); }
- bool QueryMemoryLimit() const { return State.test(2); }
- bool ScanQueryMemory() const { return State.test(3); }
- bool DataQueryMemory() const { return State.test(4); }
-
- void SetNotReady() { State.set(0); }
- void SetExecutionUnits() { State.set(1); }
- void SetQueryMemoryLimit() { State.set(2); }
- void SetScanQueryMemory() { State.set(3); }
- void SetDataQueryMemory() { State.set(4); }
-};
-
-/// local resources snapshot
-struct TKqpLocalNodeResources {
- ui32 ExecutionUnits = 0;
- std::array<ui64, EKqpMemoryPool::Count> Memory;
-};
-
-/// per node singleton with instant API
-class IKqpResourceManager : private TNonCopyable {
-public:
- virtual ~IKqpResourceManager() = default;
-
- virtual bool AllocateResources(ui64 queryId, ui64 taskId, const TKqpResourcesRequest& resources,
- TKqpNotEnoughResources* details = nullptr) = 0;
-
- using TResourcesAllocatedCallback = std::function<void(NActors::TActorSystem* as)>;
- using TNotEnoughtResourcesCallback = std::function<void(NActors::TActorSystem* as, const TString& reason, bool byTimeout)>;
-
- virtual bool AllocateResources(ui64 queryId, ui64 taskId, const TKqpResourcesRequest& resources,
- TResourcesAllocatedCallback&& onSuccess, TNotEnoughtResourcesCallback&& onFail, TDuration timeout = {}) = 0;
-
- virtual void FreeResources(ui64 queryId, ui64 taskId) = 0;
- virtual void FreeResources(ui64 queryId) = 0;
-
- virtual void NotifyExternalResourcesAllocated(ui64 queryId, ui64 taskId, const TKqpResourcesRequest& resources) = 0;
- virtual void NotifyExternalResourcesFreed(ui64 queryId, ui64 taskId) = 0;
-
- virtual void RequestClusterResourcesInfo(std::function<void(TVector<NKikimrKqp::TKqpNodeResources>&& resources)>&& callback) = 0;
-
- virtual TKqpLocalNodeResources GetLocalResources() const = 0;
- virtual NKikimrConfig::TTableServiceConfig::TResourceManager GetConfig() = 0;
-};
-
-} // namespace NRm
-
-NActors::IActor* CreateKqpResourceManagerActor(const NKikimrConfig::TTableServiceConfig::TResourceManager& config,
- TIntrusivePtr<TKqpCounters> counters, NActors::TActorId resourceBroker = {});
-
-NRm::IKqpResourceManager* GetKqpResourceManager(TMaybe<ui32> nodeId = Nothing());
-NRm::IKqpResourceManager* TryGetKqpResourceManager(TMaybe<ui32> nodeId = Nothing());
-
-} // namespace NKqp
-} // namespace NKikimr
+
+#include <library/cpp/actors/core/actor.h>
+
+#include <util/datetime/base.h>
+#include <util/string/builder.h>
+
+#include <array>
+#include <bitset>
+#include <functional>
+
+
+namespace NKikimr {
+namespace NKqp {
+
+namespace NRm {
+
+/// memory pools
+enum EKqpMemoryPool : ui32 {
+ Unspecified = 0,
+ ScanQuery = 1, // slow allocations via ResourceBroker
+ DataQuery = 2, // fast allocations via memory-arena
+
+ Count = 3
+};
+
+/// resources request
+struct TKqpResourcesRequest {
+ ui32 ExecutionUnits = 0;
+ EKqpMemoryPool MemoryPool = EKqpMemoryPool::Unspecified;
+ ui64 Memory = 0;
+
+ TString ToString() const {
+ return TStringBuilder() << "TKqpResourcesRequest{ MemoryPool: " << (ui32) MemoryPool << ", Memory: " << Memory
+ << ", ExecutionUnits: " << ExecutionUnits << " }";
+ }
+};
+
+/// detailed information on allocation failure
+struct TKqpNotEnoughResources {
+ std::bitset<32> State;
+
+ bool NotReady() const { return State.test(0); }
+ bool ExecutionUnits() const { return State.test(1); }
+ bool QueryMemoryLimit() const { return State.test(2); }
+ bool ScanQueryMemory() const { return State.test(3); }
+ bool DataQueryMemory() const { return State.test(4); }
+
+ void SetNotReady() { State.set(0); }
+ void SetExecutionUnits() { State.set(1); }
+ void SetQueryMemoryLimit() { State.set(2); }
+ void SetScanQueryMemory() { State.set(3); }
+ void SetDataQueryMemory() { State.set(4); }
+};
+
+/// local resources snapshot
+struct TKqpLocalNodeResources {
+ ui32 ExecutionUnits = 0;
+ std::array<ui64, EKqpMemoryPool::Count> Memory;
+};
+
+/// per node singleton with instant API
+class IKqpResourceManager : private TNonCopyable {
+public:
+ virtual ~IKqpResourceManager() = default;
+
+ virtual bool AllocateResources(ui64 queryId, ui64 taskId, const TKqpResourcesRequest& resources,
+ TKqpNotEnoughResources* details = nullptr) = 0;
+
+ using TResourcesAllocatedCallback = std::function<void(NActors::TActorSystem* as)>;
+ using TNotEnoughtResourcesCallback = std::function<void(NActors::TActorSystem* as, const TString& reason, bool byTimeout)>;
+
+ virtual bool AllocateResources(ui64 queryId, ui64 taskId, const TKqpResourcesRequest& resources,
+ TResourcesAllocatedCallback&& onSuccess, TNotEnoughtResourcesCallback&& onFail, TDuration timeout = {}) = 0;
+
+ virtual void FreeResources(ui64 queryId, ui64 taskId) = 0;
+ virtual void FreeResources(ui64 queryId) = 0;
+
+ virtual void NotifyExternalResourcesAllocated(ui64 queryId, ui64 taskId, const TKqpResourcesRequest& resources) = 0;
+ virtual void NotifyExternalResourcesFreed(ui64 queryId, ui64 taskId) = 0;
+
+ virtual void RequestClusterResourcesInfo(std::function<void(TVector<NKikimrKqp::TKqpNodeResources>&& resources)>&& callback) = 0;
+
+ virtual TKqpLocalNodeResources GetLocalResources() const = 0;
+ virtual NKikimrConfig::TTableServiceConfig::TResourceManager GetConfig() = 0;
+};
+
+} // namespace NRm
+
+NActors::IActor* CreateKqpResourceManagerActor(const NKikimrConfig::TTableServiceConfig::TResourceManager& config,
+ TIntrusivePtr<TKqpCounters> counters, NActors::TActorId resourceBroker = {});
+
+NRm::IKqpResourceManager* GetKqpResourceManager(TMaybe<ui32> nodeId = Nothing());
+NRm::IKqpResourceManager* TryGetKqpResourceManager(TMaybe<ui32> nodeId = Nothing());
+
+} // namespace NKqp
+} // namespace NKikimr
diff --git a/ydb/core/kqp/rm/kqp_rm_ut.cpp b/ydb/core/kqp/rm/kqp_rm_ut.cpp
index db3f37e362..3ea0e1beb5 100644
--- a/ydb/core/kqp/rm/kqp_rm_ut.cpp
+++ b/ydb/core/kqp/rm/kqp_rm_ut.cpp
@@ -1,377 +1,377 @@
#include <ydb/core/kqp/rm/kqp_rm.h>
#include <ydb/core/tablet/resource_broker_impl.h>
-
+
#include <ydb/core/testlib/actor_helpers.h>
#include <ydb/core/testlib/tablet_helpers.h>
#include <ydb/core/testlib/tenant_runtime.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-#ifndef NDEBUG
-const bool DETAILED_LOG = false;
-#else
-const bool DETAILED_LOG = true;
-#endif
-
-namespace NKikimr {
-namespace NKqp {
-
-using namespace NKikimrResourceBroker;
-using namespace NResourceBroker;
-
-namespace {
-
-TTenantTestConfig MakeTenantTestConfig() {
- TTenantTestConfig cfg = {
- // Domains {name, schemeshard {{ subdomain_names }}}
- {{{DOMAIN1_NAME, SCHEME_SHARD1_ID, {{TENANT1_1_NAME, TENANT1_2_NAME}}}}},
- // HiveId
- HIVE_ID,
- // FakeTenantSlotBroker
- true,
- // FakeSchemeShard
- true,
- // CreateConsole
- false,
- // Nodes
- {{
- // Node0
- {
- // TenantPoolConfig
- {
- // Static slots {tenant, {cpu, memory, network}}
- {{{DOMAIN1_NAME, {1, 1, 1}}}},
- // Dynamic slots {id, type, domain, tenant, {cpu, memory, network}}
- {{{DOMAIN1_SLOT1, SLOT1_TYPE, DOMAIN1_NAME, "", {1, 1, 1}},
- {DOMAIN1_SLOT2, SLOT2_TYPE, DOMAIN1_NAME, "", {2, 2, 2}},
- {DOMAIN1_SLOT3, SLOT3_TYPE, DOMAIN1_NAME, "", {3, 3, 3}}}},
- "node-type"
- }
- }
- }},
- // DataCenterCount
- 1
- };
- return cfg;
-}
-
-TResourceBrokerConfig MakeResourceBrokerTestConfig() {
- TResourceBrokerConfig config;
-
- auto queue = config.AddQueues();
- queue->SetName("queue_default");
- queue->SetWeight(5);
- queue->MutableLimit()->AddResource(4);
-
- queue = config.AddQueues();
- queue->SetName("queue_kqp_resource_manager");
- queue->SetWeight(20);
- queue->MutableLimit()->AddResource(4);
- queue->MutableLimit()->AddResource(50'000);
-
- auto task = config.AddTasks();
- task->SetName("unknown");
- task->SetQueueName("queue_default");
- task->SetDefaultDuration(TDuration::Seconds(5).GetValue());
-
- task = config.AddTasks();
- task->SetName(NLocalDb::KqpResourceManagerTaskName);
- task->SetQueueName("queue_kqp_resource_manager");
- task->SetDefaultDuration(TDuration::Seconds(5).GetValue());
-
- config.MutableResourceLimit()->AddResource(10);
- config.MutableResourceLimit()->AddResource(100'000);
-
- return config;
-}
-
-NKikimrConfig::TTableServiceConfig::TResourceManager MakeKqpResourceManagerConfig() {
- NKikimrConfig::TTableServiceConfig::TResourceManager config;
-
- config.SetComputeActorsCount(100);
- config.SetPublishStatisticsIntervalSec(0);
- config.SetQueryMemoryLimit(1000);
-
- return config;
-}
-
-}
-
-class KqpRm : public TTestBase {
-public:
- void SetUp() override {
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#ifndef NDEBUG
+const bool DETAILED_LOG = false;
+#else
+const bool DETAILED_LOG = true;
+#endif
+
+namespace NKikimr {
+namespace NKqp {
+
+using namespace NKikimrResourceBroker;
+using namespace NResourceBroker;
+
+namespace {
+
+TTenantTestConfig MakeTenantTestConfig() {
+ TTenantTestConfig cfg = {
+ // Domains {name, schemeshard {{ subdomain_names }}}
+ {{{DOMAIN1_NAME, SCHEME_SHARD1_ID, {{TENANT1_1_NAME, TENANT1_2_NAME}}}}},
+ // HiveId
+ HIVE_ID,
+ // FakeTenantSlotBroker
+ true,
+ // FakeSchemeShard
+ true,
+ // CreateConsole
+ false,
+ // Nodes
+ {{
+ // Node0
+ {
+ // TenantPoolConfig
+ {
+ // Static slots {tenant, {cpu, memory, network}}
+ {{{DOMAIN1_NAME, {1, 1, 1}}}},
+ // Dynamic slots {id, type, domain, tenant, {cpu, memory, network}}
+ {{{DOMAIN1_SLOT1, SLOT1_TYPE, DOMAIN1_NAME, "", {1, 1, 1}},
+ {DOMAIN1_SLOT2, SLOT2_TYPE, DOMAIN1_NAME, "", {2, 2, 2}},
+ {DOMAIN1_SLOT3, SLOT3_TYPE, DOMAIN1_NAME, "", {3, 3, 3}}}},
+ "node-type"
+ }
+ }
+ }},
+ // DataCenterCount
+ 1
+ };
+ return cfg;
+}
+
+TResourceBrokerConfig MakeResourceBrokerTestConfig() {
+ TResourceBrokerConfig config;
+
+ auto queue = config.AddQueues();
+ queue->SetName("queue_default");
+ queue->SetWeight(5);
+ queue->MutableLimit()->AddResource(4);
+
+ queue = config.AddQueues();
+ queue->SetName("queue_kqp_resource_manager");
+ queue->SetWeight(20);
+ queue->MutableLimit()->AddResource(4);
+ queue->MutableLimit()->AddResource(50'000);
+
+ auto task = config.AddTasks();
+ task->SetName("unknown");
+ task->SetQueueName("queue_default");
+ task->SetDefaultDuration(TDuration::Seconds(5).GetValue());
+
+ task = config.AddTasks();
+ task->SetName(NLocalDb::KqpResourceManagerTaskName);
+ task->SetQueueName("queue_kqp_resource_manager");
+ task->SetDefaultDuration(TDuration::Seconds(5).GetValue());
+
+ config.MutableResourceLimit()->AddResource(10);
+ config.MutableResourceLimit()->AddResource(100'000);
+
+ return config;
+}
+
+NKikimrConfig::TTableServiceConfig::TResourceManager MakeKqpResourceManagerConfig() {
+ NKikimrConfig::TTableServiceConfig::TResourceManager config;
+
+ config.SetComputeActorsCount(100);
+ config.SetPublishStatisticsIntervalSec(0);
+ config.SetQueryMemoryLimit(1000);
+
+ return config;
+}
+
+}
+
+class KqpRm : public TTestBase {
+public:
+ void SetUp() override {
Runtime = MakeHolder<TTenantTestRuntime>(MakeTenantTestConfig());
-
- NActors::NLog::EPriority priority = DETAILED_LOG ? NLog::PRI_DEBUG : NLog::PRI_ERROR;
- Runtime->SetLogPriority(NKikimrServices::RESOURCE_BROKER, priority);
- Runtime->SetLogPriority(NKikimrServices::KQP_RESOURCE_MANAGER, priority);
-
- auto now = Now();
- Runtime->UpdateCurrentTime(now);
-
- Counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
-
- auto resourceBrokerConfig = MakeResourceBrokerTestConfig();
- auto broker = CreateResourceBrokerActor(resourceBrokerConfig, Counters);
- ResourceBrokerActorId = Runtime->Register(broker);
- WaitForBootstrap();
- }
-
- void WaitForBootstrap() {
- TDispatchOptions options;
- options.FinalEvents.emplace_back(TEvents::TSystem::Bootstrap, 1);
- UNIT_ASSERT(Runtime->DispatchEvents(options));
- }
-
- IActor* CreateKqpResourceManager(const NKikimrConfig::TTableServiceConfig::TResourceManager& config = {}) {
- auto kqpCounters = MakeIntrusive<TKqpCounters>(Counters);
- auto resman = CreateKqpResourceManagerActor(config, kqpCounters, ResourceBrokerActorId);
- ResourceManagerActorId = Runtime->Register(resman);
- Runtime->EnableScheduleForActor(ResourceManagerActorId, true);
- WaitForBootstrap();
- return resman;
- }
-
- void AssertResourceBrokerSensors(i64 cpu, i64 mem, i64 enqueued, i64 finished, i64 infly) {
- auto q = Counters->GetSubgroup("queue", "queue_kqp_resource_manager");
-// Cerr << "-- queue_kqp_resource_manager\n";
-// q->OutputPlainText(Cerr, " ");
- UNIT_ASSERT_VALUES_EQUAL(q->GetCounter("CPUConsumption")->Val(), cpu);
- UNIT_ASSERT_VALUES_EQUAL(q->GetCounter("MemoryConsumption")->Val(), mem);
- UNIT_ASSERT_VALUES_EQUAL(q->GetCounter("EnqueuedTasks")->Val(), enqueued);
- UNIT_ASSERT_VALUES_EQUAL(q->GetCounter("FinishedTasks")->Val(), finished);
- UNIT_ASSERT_VALUES_EQUAL(q->GetCounter("InFlyTasks")->Val(), infly);
-
- auto t = Counters->GetSubgroup("task", "kqp_query");
-// Cerr << "-- kqp_query\n";
-// t->OutputPlainText(Cerr, " ");
- UNIT_ASSERT_VALUES_EQUAL(t->GetCounter("CPUConsumption")->Val(), cpu);
- UNIT_ASSERT_VALUES_EQUAL(t->GetCounter("MemoryConsumption")->Val(), mem);
- UNIT_ASSERT_VALUES_EQUAL(t->GetCounter("EnqueuedTasks")->Val(), enqueued);
- UNIT_ASSERT_VALUES_EQUAL(t->GetCounter("FinishedTasks")->Val(), finished);
- UNIT_ASSERT_VALUES_EQUAL(t->GetCounter("InFlyTasks")->Val(), infly);
- }
-
- void AssertResourceManagerStats(const NRm::IKqpResourceManager* rm, ui64 scanQueryMemory, ui32 executionUnits) {
- auto stats = rm->GetLocalResources();
- UNIT_ASSERT_VALUES_EQUAL(scanQueryMemory, stats.Memory[NRm::EKqpMemoryPool::ScanQuery]);
- UNIT_ASSERT_VALUES_EQUAL(executionUnits, stats.ExecutionUnits);
- }
-
- UNIT_TEST_SUITE(KqpRm);
- UNIT_TEST(SingleTask);
- UNIT_TEST(ManyTasks);
- UNIT_TEST(NotEnoughMemory);
- UNIT_TEST(NotEnoughExecutionUnits);
- UNIT_TEST(ResourceBrokerNotEnoughResources);
- UNIT_TEST(Snapshot);
- UNIT_TEST_SUITE_END();
-
- void SingleTask();
- void ManyTasks();
- void NotEnoughMemory();
- void NotEnoughExecutionUnits();
- void ResourceBrokerNotEnoughResources();
- void Snapshot();
-
-private:
- THolder<TTestBasicRuntime> Runtime;
- TIntrusivePtr<NMonitoring::TDynamicCounters> Counters;
- // TIntrusivePtr<TKqpCounters> KqpCounters;
- TActorId ResourceBrokerActorId;
- TActorId ResourceManagerActorId;
-};
-UNIT_TEST_SUITE_REGISTRATION(KqpRm);
-
-
-void KqpRm::SingleTask() {
- CreateKqpResourceManager(MakeKqpResourceManagerConfig());
- NKikimr::TActorSystemStub stub;
-
- auto* rm = GetKqpResourceManager(ResourceManagerActorId.NodeId());
-
- NRm::TKqpResourcesRequest request;
- request.ExecutionUnits = 10;
- request.MemoryPool = NRm::EKqpMemoryPool::ScanQuery;
- request.Memory = 100;
-
- bool allocated = rm->AllocateResources(1, 2, request);
- UNIT_ASSERT(allocated);
-
- AssertResourceManagerStats(rm, 900, 90);
- AssertResourceBrokerSensors(0, 100, 0, 0, 1);
-
- rm->FreeResources(1, 2);
- AssertResourceManagerStats(rm, 1000, 100);
- AssertResourceBrokerSensors(0, 0, 0, 1, 0);
-}
-
-void KqpRm::ManyTasks() {
- CreateKqpResourceManager(MakeKqpResourceManagerConfig());
- NKikimr::TActorSystemStub stub;
-
- auto* rm = GetKqpResourceManager(ResourceManagerActorId.NodeId());
-
- NRm::TKqpResourcesRequest request;
- request.ExecutionUnits = 10;
- request.MemoryPool = NRm::EKqpMemoryPool::ScanQuery;
- request.Memory = 100;
-
- for (ui32 i = 1; i < 10; ++i) {
- bool allocated = rm->AllocateResources(1, i, request);
- UNIT_ASSERT(allocated);
-
- AssertResourceManagerStats(rm, 1000 - 100 * i, 100 - 10 * i);
- AssertResourceBrokerSensors(0, 100 * i, 0, 0, i);
- }
-
- // invalid taskId
- rm->FreeResources(1, 0);
- AssertResourceManagerStats(rm, 100, 10);
- AssertResourceBrokerSensors(0, 900, 0, 0, 9);
-
- // invalid txId
- rm->FreeResources(10, 1);
- AssertResourceManagerStats(rm, 100, 10);
- AssertResourceBrokerSensors(0, 900, 0, 0, 9);
-
- rm->FreeResources(1, 1);
- AssertResourceManagerStats(rm, 200, 20);
- AssertResourceBrokerSensors(0, 800, 0, 1, 8);
-
- rm->FreeResources(1);
- AssertResourceManagerStats(rm, 1000, 100);
- AssertResourceBrokerSensors(0, 0, 0, 9, 0);
-}
-
-void KqpRm::NotEnoughMemory() {
- CreateKqpResourceManager(MakeKqpResourceManagerConfig());
- NKikimr::TActorSystemStub stub;
-
- auto* rm = GetKqpResourceManager(ResourceManagerActorId.NodeId());
-
- NRm::TKqpResourcesRequest request;
- request.ExecutionUnits = 10;
- request.MemoryPool = NRm::EKqpMemoryPool::ScanQuery;
- request.Memory = 10'000;
-
- bool allocated = rm->AllocateResources(1, 2, request);
- UNIT_ASSERT(!allocated);
-
- AssertResourceManagerStats(rm, 1000, 100);
- AssertResourceBrokerSensors(0, 0, 0, 0, 0);
-}
-
-void KqpRm::NotEnoughExecutionUnits() {
- CreateKqpResourceManager(MakeKqpResourceManagerConfig());
- NKikimr::TActorSystemStub stub;
-
- auto* rm = GetKqpResourceManager(ResourceManagerActorId.NodeId());
-
- NRm::TKqpResourcesRequest request;
- request.ExecutionUnits = 1000;
- request.MemoryPool = NRm::EKqpMemoryPool::ScanQuery;
- request.Memory = 100;
-
- bool allocated = rm->AllocateResources(1, 2, request);
- UNIT_ASSERT(!allocated);
-
- AssertResourceManagerStats(rm, 1000, 100);
- AssertResourceBrokerSensors(0, 0, 0, 0, 0);
-}
-
-void KqpRm::ResourceBrokerNotEnoughResources() {
- auto config = MakeKqpResourceManagerConfig();
- config.SetQueryMemoryLimit(100000000);
- CreateKqpResourceManager(config);
- NKikimr::TActorSystemStub stub;
-
- auto* rm = GetKqpResourceManager(ResourceManagerActorId.NodeId());
-
- NRm::TKqpResourcesRequest request;
- request.ExecutionUnits = 10;
- request.MemoryPool = NRm::EKqpMemoryPool::ScanQuery;
- request.Memory = 1'000;
-
- bool allocated = rm->AllocateResources(1, 2, request);
- UNIT_ASSERT(allocated);
-
- request.MemoryPool = NRm::EKqpMemoryPool::ScanQuery;
- request.Memory = 100'000;
- allocated = rm->AllocateResources(1, 2, request);
- UNIT_ASSERT(!allocated);
-
- AssertResourceManagerStats(rm, config.GetQueryMemoryLimit() - 1000, 90);
- AssertResourceBrokerSensors(0, 1000, 0, 0, 1);
-}
-
-void KqpRm::Snapshot() {
- CreateKqpResourceManager(MakeKqpResourceManagerConfig());
- NKikimr::TActorSystemStub stub;
-
- auto* rm = GetKqpResourceManager(ResourceManagerActorId.NodeId());
-
- NRm::TKqpResourcesRequest request;
- request.ExecutionUnits = 10;
- request.MemoryPool = NRm::EKqpMemoryPool::ScanQuery;
- request.Memory = 100;
-
- bool allocated = rm->AllocateResources(1, 2, request);
- UNIT_ASSERT(allocated);
-
- allocated = rm->AllocateResources(2, 1, request);
- UNIT_ASSERT(allocated);
-
- AssertResourceManagerStats(rm, 800, 80);
- AssertResourceBrokerSensors(0, 200, 0, 0, 2);
-
- Runtime->DispatchEvents(TDispatchOptions(), TDuration::MilliSeconds(500));
-
- {
- TVector<NKikimrKqp::TKqpNodeResources> snapshot;
- std::atomic<int> ready = 0;
- rm->RequestClusterResourcesInfo([&](TVector<NKikimrKqp::TKqpNodeResources>&& resources) {
- snapshot = std::move(resources);
- ready = 1;
- });
-
- while (ready.load() != 1) {
- Runtime->DispatchEvents(TDispatchOptions(), TDuration::MilliSeconds(100));
- }
-
- UNIT_ASSERT_VALUES_EQUAL(1, snapshot.size());
- UNIT_ASSERT_VALUES_EQUAL(80, snapshot[0].GetExecutionUnits());
- UNIT_ASSERT_VALUES_EQUAL(1, snapshot[0].GetMemory().size());
- UNIT_ASSERT_VALUES_EQUAL(1, snapshot[0].GetMemory()[0].GetPool());
- UNIT_ASSERT_VALUES_EQUAL(800, snapshot[0].GetMemory()[0].GetAvailable());
- }
-
- rm->FreeResources(1);
- rm->FreeResources(2);
- AssertResourceManagerStats(rm, 1000, 100);
- AssertResourceBrokerSensors(0, 0, 0, 2, 0);
-
- Runtime->DispatchEvents(TDispatchOptions(), TDuration::MilliSeconds(500));
-
- {
- TVector<NKikimrKqp::TKqpNodeResources> snapshot;
- std::atomic<int> ready = 0;
- rm->RequestClusterResourcesInfo([&](TVector<NKikimrKqp::TKqpNodeResources>&& resources) {
- snapshot = std::move(resources);
- ready = 1;
- });
-
- while (ready.load() != 1) {
- Runtime->DispatchEvents(TDispatchOptions(), TDuration::MilliSeconds(100));
- }
-
- UNIT_ASSERT_VALUES_EQUAL(1, snapshot.size());
- UNIT_ASSERT_VALUES_EQUAL(100, snapshot[0].GetExecutionUnits());
- UNIT_ASSERT_VALUES_EQUAL(1, snapshot[0].GetMemory().size());
- UNIT_ASSERT_VALUES_EQUAL(1, snapshot[0].GetMemory()[0].GetPool());
- UNIT_ASSERT_VALUES_EQUAL(1000, snapshot[0].GetMemory()[0].GetAvailable());
- }
-}
-
-} // namespace NKqp
-} // namespace NKikimr
+
+ NActors::NLog::EPriority priority = DETAILED_LOG ? NLog::PRI_DEBUG : NLog::PRI_ERROR;
+ Runtime->SetLogPriority(NKikimrServices::RESOURCE_BROKER, priority);
+ Runtime->SetLogPriority(NKikimrServices::KQP_RESOURCE_MANAGER, priority);
+
+ auto now = Now();
+ Runtime->UpdateCurrentTime(now);
+
+ Counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
+
+ auto resourceBrokerConfig = MakeResourceBrokerTestConfig();
+ auto broker = CreateResourceBrokerActor(resourceBrokerConfig, Counters);
+ ResourceBrokerActorId = Runtime->Register(broker);
+ WaitForBootstrap();
+ }
+
+ void WaitForBootstrap() {
+ TDispatchOptions options;
+ options.FinalEvents.emplace_back(TEvents::TSystem::Bootstrap, 1);
+ UNIT_ASSERT(Runtime->DispatchEvents(options));
+ }
+
+ IActor* CreateKqpResourceManager(const NKikimrConfig::TTableServiceConfig::TResourceManager& config = {}) {
+ auto kqpCounters = MakeIntrusive<TKqpCounters>(Counters);
+ auto resman = CreateKqpResourceManagerActor(config, kqpCounters, ResourceBrokerActorId);
+ ResourceManagerActorId = Runtime->Register(resman);
+ Runtime->EnableScheduleForActor(ResourceManagerActorId, true);
+ WaitForBootstrap();
+ return resman;
+ }
+
+ void AssertResourceBrokerSensors(i64 cpu, i64 mem, i64 enqueued, i64 finished, i64 infly) {
+ auto q = Counters->GetSubgroup("queue", "queue_kqp_resource_manager");
+// Cerr << "-- queue_kqp_resource_manager\n";
+// q->OutputPlainText(Cerr, " ");
+ UNIT_ASSERT_VALUES_EQUAL(q->GetCounter("CPUConsumption")->Val(), cpu);
+ UNIT_ASSERT_VALUES_EQUAL(q->GetCounter("MemoryConsumption")->Val(), mem);
+ UNIT_ASSERT_VALUES_EQUAL(q->GetCounter("EnqueuedTasks")->Val(), enqueued);
+ UNIT_ASSERT_VALUES_EQUAL(q->GetCounter("FinishedTasks")->Val(), finished);
+ UNIT_ASSERT_VALUES_EQUAL(q->GetCounter("InFlyTasks")->Val(), infly);
+
+ auto t = Counters->GetSubgroup("task", "kqp_query");
+// Cerr << "-- kqp_query\n";
+// t->OutputPlainText(Cerr, " ");
+ UNIT_ASSERT_VALUES_EQUAL(t->GetCounter("CPUConsumption")->Val(), cpu);
+ UNIT_ASSERT_VALUES_EQUAL(t->GetCounter("MemoryConsumption")->Val(), mem);
+ UNIT_ASSERT_VALUES_EQUAL(t->GetCounter("EnqueuedTasks")->Val(), enqueued);
+ UNIT_ASSERT_VALUES_EQUAL(t->GetCounter("FinishedTasks")->Val(), finished);
+ UNIT_ASSERT_VALUES_EQUAL(t->GetCounter("InFlyTasks")->Val(), infly);
+ }
+
+ void AssertResourceManagerStats(const NRm::IKqpResourceManager* rm, ui64 scanQueryMemory, ui32 executionUnits) {
+ auto stats = rm->GetLocalResources();
+ UNIT_ASSERT_VALUES_EQUAL(scanQueryMemory, stats.Memory[NRm::EKqpMemoryPool::ScanQuery]);
+ UNIT_ASSERT_VALUES_EQUAL(executionUnits, stats.ExecutionUnits);
+ }
+
+ UNIT_TEST_SUITE(KqpRm);
+ UNIT_TEST(SingleTask);
+ UNIT_TEST(ManyTasks);
+ UNIT_TEST(NotEnoughMemory);
+ UNIT_TEST(NotEnoughExecutionUnits);
+ UNIT_TEST(ResourceBrokerNotEnoughResources);
+ UNIT_TEST(Snapshot);
+ UNIT_TEST_SUITE_END();
+
+ void SingleTask();
+ void ManyTasks();
+ void NotEnoughMemory();
+ void NotEnoughExecutionUnits();
+ void ResourceBrokerNotEnoughResources();
+ void Snapshot();
+
+private:
+ THolder<TTestBasicRuntime> Runtime;
+ TIntrusivePtr<NMonitoring::TDynamicCounters> Counters;
+ // TIntrusivePtr<TKqpCounters> KqpCounters;
+ TActorId ResourceBrokerActorId;
+ TActorId ResourceManagerActorId;
+};
+UNIT_TEST_SUITE_REGISTRATION(KqpRm);
+
+
+void KqpRm::SingleTask() {
+ CreateKqpResourceManager(MakeKqpResourceManagerConfig());
+ NKikimr::TActorSystemStub stub;
+
+ auto* rm = GetKqpResourceManager(ResourceManagerActorId.NodeId());
+
+ NRm::TKqpResourcesRequest request;
+ request.ExecutionUnits = 10;
+ request.MemoryPool = NRm::EKqpMemoryPool::ScanQuery;
+ request.Memory = 100;
+
+ bool allocated = rm->AllocateResources(1, 2, request);
+ UNIT_ASSERT(allocated);
+
+ AssertResourceManagerStats(rm, 900, 90);
+ AssertResourceBrokerSensors(0, 100, 0, 0, 1);
+
+ rm->FreeResources(1, 2);
+ AssertResourceManagerStats(rm, 1000, 100);
+ AssertResourceBrokerSensors(0, 0, 0, 1, 0);
+}
+
+void KqpRm::ManyTasks() {
+ CreateKqpResourceManager(MakeKqpResourceManagerConfig());
+ NKikimr::TActorSystemStub stub;
+
+ auto* rm = GetKqpResourceManager(ResourceManagerActorId.NodeId());
+
+ NRm::TKqpResourcesRequest request;
+ request.ExecutionUnits = 10;
+ request.MemoryPool = NRm::EKqpMemoryPool::ScanQuery;
+ request.Memory = 100;
+
+ for (ui32 i = 1; i < 10; ++i) {
+ bool allocated = rm->AllocateResources(1, i, request);
+ UNIT_ASSERT(allocated);
+
+ AssertResourceManagerStats(rm, 1000 - 100 * i, 100 - 10 * i);
+ AssertResourceBrokerSensors(0, 100 * i, 0, 0, i);
+ }
+
+ // invalid taskId
+ rm->FreeResources(1, 0);
+ AssertResourceManagerStats(rm, 100, 10);
+ AssertResourceBrokerSensors(0, 900, 0, 0, 9);
+
+ // invalid txId
+ rm->FreeResources(10, 1);
+ AssertResourceManagerStats(rm, 100, 10);
+ AssertResourceBrokerSensors(0, 900, 0, 0, 9);
+
+ rm->FreeResources(1, 1);
+ AssertResourceManagerStats(rm, 200, 20);
+ AssertResourceBrokerSensors(0, 800, 0, 1, 8);
+
+ rm->FreeResources(1);
+ AssertResourceManagerStats(rm, 1000, 100);
+ AssertResourceBrokerSensors(0, 0, 0, 9, 0);
+}
+
+void KqpRm::NotEnoughMemory() {
+ CreateKqpResourceManager(MakeKqpResourceManagerConfig());
+ NKikimr::TActorSystemStub stub;
+
+ auto* rm = GetKqpResourceManager(ResourceManagerActorId.NodeId());
+
+ NRm::TKqpResourcesRequest request;
+ request.ExecutionUnits = 10;
+ request.MemoryPool = NRm::EKqpMemoryPool::ScanQuery;
+ request.Memory = 10'000;
+
+ bool allocated = rm->AllocateResources(1, 2, request);
+ UNIT_ASSERT(!allocated);
+
+ AssertResourceManagerStats(rm, 1000, 100);
+ AssertResourceBrokerSensors(0, 0, 0, 0, 0);
+}
+
+void KqpRm::NotEnoughExecutionUnits() {
+ CreateKqpResourceManager(MakeKqpResourceManagerConfig());
+ NKikimr::TActorSystemStub stub;
+
+ auto* rm = GetKqpResourceManager(ResourceManagerActorId.NodeId());
+
+ NRm::TKqpResourcesRequest request;
+ request.ExecutionUnits = 1000;
+ request.MemoryPool = NRm::EKqpMemoryPool::ScanQuery;
+ request.Memory = 100;
+
+ bool allocated = rm->AllocateResources(1, 2, request);
+ UNIT_ASSERT(!allocated);
+
+ AssertResourceManagerStats(rm, 1000, 100);
+ AssertResourceBrokerSensors(0, 0, 0, 0, 0);
+}
+
+void KqpRm::ResourceBrokerNotEnoughResources() {
+ auto config = MakeKqpResourceManagerConfig();
+ config.SetQueryMemoryLimit(100000000);
+ CreateKqpResourceManager(config);
+ NKikimr::TActorSystemStub stub;
+
+ auto* rm = GetKqpResourceManager(ResourceManagerActorId.NodeId());
+
+ NRm::TKqpResourcesRequest request;
+ request.ExecutionUnits = 10;
+ request.MemoryPool = NRm::EKqpMemoryPool::ScanQuery;
+ request.Memory = 1'000;
+
+ bool allocated = rm->AllocateResources(1, 2, request);
+ UNIT_ASSERT(allocated);
+
+ request.MemoryPool = NRm::EKqpMemoryPool::ScanQuery;
+ request.Memory = 100'000;
+ allocated = rm->AllocateResources(1, 2, request);
+ UNIT_ASSERT(!allocated);
+
+ AssertResourceManagerStats(rm, config.GetQueryMemoryLimit() - 1000, 90);
+ AssertResourceBrokerSensors(0, 1000, 0, 0, 1);
+}
+
+void KqpRm::Snapshot() {
+ CreateKqpResourceManager(MakeKqpResourceManagerConfig());
+ NKikimr::TActorSystemStub stub;
+
+ auto* rm = GetKqpResourceManager(ResourceManagerActorId.NodeId());
+
+ NRm::TKqpResourcesRequest request;
+ request.ExecutionUnits = 10;
+ request.MemoryPool = NRm::EKqpMemoryPool::ScanQuery;
+ request.Memory = 100;
+
+ bool allocated = rm->AllocateResources(1, 2, request);
+ UNIT_ASSERT(allocated);
+
+ allocated = rm->AllocateResources(2, 1, request);
+ UNIT_ASSERT(allocated);
+
+ AssertResourceManagerStats(rm, 800, 80);
+ AssertResourceBrokerSensors(0, 200, 0, 0, 2);
+
+ Runtime->DispatchEvents(TDispatchOptions(), TDuration::MilliSeconds(500));
+
+ {
+ TVector<NKikimrKqp::TKqpNodeResources> snapshot;
+ std::atomic<int> ready = 0;
+ rm->RequestClusterResourcesInfo([&](TVector<NKikimrKqp::TKqpNodeResources>&& resources) {
+ snapshot = std::move(resources);
+ ready = 1;
+ });
+
+ while (ready.load() != 1) {
+ Runtime->DispatchEvents(TDispatchOptions(), TDuration::MilliSeconds(100));
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(1, snapshot.size());
+ UNIT_ASSERT_VALUES_EQUAL(80, snapshot[0].GetExecutionUnits());
+ UNIT_ASSERT_VALUES_EQUAL(1, snapshot[0].GetMemory().size());
+ UNIT_ASSERT_VALUES_EQUAL(1, snapshot[0].GetMemory()[0].GetPool());
+ UNIT_ASSERT_VALUES_EQUAL(800, snapshot[0].GetMemory()[0].GetAvailable());
+ }
+
+ rm->FreeResources(1);
+ rm->FreeResources(2);
+ AssertResourceManagerStats(rm, 1000, 100);
+ AssertResourceBrokerSensors(0, 0, 0, 2, 0);
+
+ Runtime->DispatchEvents(TDispatchOptions(), TDuration::MilliSeconds(500));
+
+ {
+ TVector<NKikimrKqp::TKqpNodeResources> snapshot;
+ std::atomic<int> ready = 0;
+ rm->RequestClusterResourcesInfo([&](TVector<NKikimrKqp::TKqpNodeResources>&& resources) {
+ snapshot = std::move(resources);
+ ready = 1;
+ });
+
+ while (ready.load() != 1) {
+ Runtime->DispatchEvents(TDispatchOptions(), TDuration::MilliSeconds(100));
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(1, snapshot.size());
+ UNIT_ASSERT_VALUES_EQUAL(100, snapshot[0].GetExecutionUnits());
+ UNIT_ASSERT_VALUES_EQUAL(1, snapshot[0].GetMemory().size());
+ UNIT_ASSERT_VALUES_EQUAL(1, snapshot[0].GetMemory()[0].GetPool());
+ UNIT_ASSERT_VALUES_EQUAL(1000, snapshot[0].GetMemory()[0].GetAvailable());
+ }
+}
+
+} // namespace NKqp
+} // namespace NKikimr
diff --git a/ydb/core/kqp/rm/kqp_snapshot_manager.cpp b/ydb/core/kqp/rm/kqp_snapshot_manager.cpp
index dc064ed418..e461d85ab9 100644
--- a/ydb/core/kqp/rm/kqp_snapshot_manager.cpp
+++ b/ydb/core/kqp/rm/kqp_snapshot_manager.cpp
@@ -1,54 +1,54 @@
-#include "kqp_snapshot_manager.h"
-
+#include "kqp_snapshot_manager.h"
+
#include <ydb/core/tx/tx_proxy/proxy.h>
#include <ydb/core/tx/long_tx_service/public/events.h>
#include <ydb/core/actorlib_impl/long_timer.h>
-
-#include <library/cpp/actors/core/actor_bootstrapped.h>
-#include <library/cpp/actors/core/hfunc.h>
-#include <library/cpp/actors/core/log.h>
-
-namespace NKikimr {
-namespace NKqp {
-
-#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream)
-#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream)
-
-namespace {
-
-class TSnapshotManagerActor: public TActorBootstrapped<TSnapshotManagerActor> {
-public:
+
+#include <library/cpp/actors/core/actor_bootstrapped.h>
+#include <library/cpp/actors/core/hfunc.h>
+#include <library/cpp/actors/core/log.h>
+
+namespace NKikimr {
+namespace NKqp {
+
+#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream)
+#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream)
+
+namespace {
+
+class TSnapshotManagerActor: public TActorBootstrapped<TSnapshotManagerActor> {
+public:
TSnapshotManagerActor(const TString& database, TDuration queryTimeout)
: Database(database)
, RequestTimeout(queryTimeout)
- {}
-
- void Bootstrap() {
- auto *ev = new IEventHandle(this->SelfId(), this->SelfId(), new TEvents::TEvPoison());
- RequestTimeoutCookieHolder_.Reset(ISchedulerCookie::Make2Way());
- CreateLongTimer(TlsActivationContext->AsActorContext(), RequestTimeout, ev, 0, RequestTimeoutCookieHolder_.Get());
-
- LOG_D("Start KqpSnapshotManager at " << SelfId());
-
- Become(&TThis::StateAwaitRequest);
- }
-
-private:
- STATEFN(StateAwaitRequest) {
- switch (ev->GetTypeRewrite()) {
- hFunc(TEvKqpSnapshot::TEvCreateSnapshotRequest, Handle);
- default:
- HandleUnexpectedEvent("AwaitRequest", ev->GetTypeRewrite());
- }
- }
-
- void Handle(TEvKqpSnapshot::TEvCreateSnapshotRequest::TPtr& ev) {
- ClientActorId = ev->Sender;
- Tables = ev->Get()->Tables;
+ {}
+
+ void Bootstrap() {
+ auto *ev = new IEventHandle(this->SelfId(), this->SelfId(), new TEvents::TEvPoison());
+ RequestTimeoutCookieHolder_.Reset(ISchedulerCookie::Make2Way());
+ CreateLongTimer(TlsActivationContext->AsActorContext(), RequestTimeout, ev, 0, RequestTimeoutCookieHolder_.Get());
+
+ LOG_D("Start KqpSnapshotManager at " << SelfId());
+
+ Become(&TThis::StateAwaitRequest);
+ }
+
+private:
+ STATEFN(StateAwaitRequest) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvKqpSnapshot::TEvCreateSnapshotRequest, Handle);
+ default:
+ HandleUnexpectedEvent("AwaitRequest", ev->GetTypeRewrite());
+ }
+ }
+
+ void Handle(TEvKqpSnapshot::TEvCreateSnapshotRequest::TPtr& ev) {
+ ClientActorId = ev->Sender;
+ Tables = ev->Get()->Tables;
MvccSnapshot = ev->Get()->MvccSnapshot;
-
- LOG_D("KqpSnapshotManager: got snapshot request from " << ClientActorId);
-
+
+ LOG_D("KqpSnapshotManager: got snapshot request from " << ClientActorId);
+
if (MvccSnapshot) {
auto longTxService = NLongTxService::MakeLongTxServiceID(SelfId().NodeId());
Send(longTxService, new NLongTxService::TEvLongTxService::TEvAcquireReadSnapshot(Database));
@@ -67,9 +67,9 @@ private:
Send(MakeTxProxyID(), req.Release());
Become(&TThis::StateAwaitCreation);
- }
+ }
}
-
+
STATEFN(StateAwaitAcquireResult) {
switch (ev->GetTypeRewrite()) {
hFunc(NLongTxService::TEvLongTxService::TEvAcquireReadSnapshotResult, Handle);
@@ -77,17 +77,17 @@ private:
default:
HandleUnexpectedEvent("AwaitAcquireResult", ev->GetTypeRewrite());
}
- }
-
- STATEFN(StateAwaitCreation) {
- switch (ev->GetTypeRewrite()) {
- hFunc(TEvTxUserProxy::TEvProposeTransactionStatus, Handle);
- hFunc(TEvents::TEvPoison, HandlePoison);
- default:
- HandleUnexpectedEvent("AwaitCreation", ev->GetTypeRewrite());
- }
- }
-
+ }
+
+ STATEFN(StateAwaitCreation) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvTxUserProxy::TEvProposeTransactionStatus, Handle);
+ hFunc(TEvents::TEvPoison, HandlePoison);
+ default:
+ HandleUnexpectedEvent("AwaitCreation", ev->GetTypeRewrite());
+ }
+ }
+
void Handle(NLongTxService::TEvLongTxService::TEvAcquireReadSnapshotResult::TPtr& ev) {
Y_VERIFY(MvccSnapshot);
Y_VERIFY(Tables.empty());
@@ -112,149 +112,149 @@ private:
}
}
- void Handle(TEvTxUserProxy::TEvProposeTransactionStatus::TPtr& ev) {
+ void Handle(TEvTxUserProxy::TEvProposeTransactionStatus::TPtr& ev) {
Y_VERIFY(!MvccSnapshot);
- using EStatus = TEvTxUserProxy::TEvProposeTransactionStatus::EStatus;
-
- const auto* msg = ev->Get();
- const auto status = static_cast<EStatus>(msg->Record.GetStatus());
-
- if (status == EStatus::ExecComplete && msg->Record.GetStatusCode() == NKikimrIssues::TStatusIds::SUCCESS) {
- Snapshot = IKqpGateway::TKqpSnapshot(msg->Record.GetStep(), msg->Record.GetTxId());
-
- LOG_D("KqpSnapshotManager: snapshot " << Snapshot.Step << ":" << Snapshot.TxId << " created");
-
- bool sent = Send(ClientActorId, new TEvKqpSnapshot::TEvCreateSnapshotResponse(
- Snapshot, NKikimrIssues::TStatusIds::SUCCESS, /* issues */ {}));
- Y_VERIFY_DEBUG(sent);
-
- Become(&TThis::StateRefreshing);
- ScheduleRefresh();
- } else {
- NYql::TIssues issues;
- NYql::IssuesFromMessage(msg->Record.GetIssues(), issues);
-
- LOG_E("KqpSnapshotManager: CreateSnapshot got unexpected status " << status << ": " << issues.ToString());
- ReplyErrorAndDie(msg->Record.GetStatusCode(), std::move(issues));
- }
- }
-
- STATEFN(StateRefreshing) {
- switch (ev->GetTypeRewrite()) {
- hFunc(TEvents::TEvWakeup, HandleRefreshTimeout);
- hFunc(TEvTxUserProxy::TEvProposeTransactionStatus, HandleRefreshStatus);
- hFunc(TEvKqpSnapshot::TEvDiscardSnapshot, HandleDiscardRequest);
- hFunc(TEvents::TEvPoison, HandlePoison);
- default:
- HandleUnexpectedEvent("Refreshing", ev->GetTypeRewrite());
- }
- }
-
- void HandleRefreshTimeout(TEvents::TEvWakeup::TPtr&) {
- auto req = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>();
- req->Record.SetExecTimeoutPeriod(RequestTimeout.MilliSeconds());
- auto* refreshSnapshot = req->Record.MutableTransaction()->MutableRefreshVolatileSnapshot();
- for (const TString& tablePath : Tables) {
- refreshSnapshot->AddTables()->SetTablePath(tablePath);
- }
- refreshSnapshot->SetIgnoreSystemViews(true);
- refreshSnapshot->SetSnapshotStep(Snapshot.Step);
- refreshSnapshot->SetSnapshotTxId(Snapshot.TxId);
-
- LOG_D("KqpSnapshotManager: refreshing snapshot");
-
- Send(MakeTxProxyID(), req.Release());
- ScheduleRefresh();
- }
-
- void HandleRefreshStatus(TEvTxUserProxy::TEvProposeTransactionStatus::TPtr& ev) {
- using EStatus = TEvTxUserProxy::TEvProposeTransactionStatus::EStatus;
-
- const auto* msg = ev->Get();
- const auto status = static_cast<EStatus>(msg->Record.GetStatus());
- if (status != EStatus::ExecComplete || msg->Record.GetStatusCode() != NKikimrIssues::TStatusIds::SUCCESS) {
- NYql::TIssues issues;
- NYql::IssuesFromMessage(msg->Record.GetIssues(), issues);
-
- LOG_E("KqpSnapshotManager: RefreshSnapshot got unexpected status=" << status
- << ", issues:" << issues.ToString());
- ReplyErrorAndDie(msg->Record.GetStatusCode(), std::move(issues));
- }
- }
-
- void HandleDiscardRequest(TEvKqpSnapshot::TEvDiscardSnapshot::TPtr& ev) {
- const auto* msg = ev->Get();
- Y_ASSERT(msg->Snapshot == Snapshot);
- LOG_D("KqpSnapshotManager: discarding snapshot; shutting down");
- SendDiscard();
- PassAway();
- }
-
- void HandlePoison(TEvents::TEvPoison::TPtr&) {
- LOG_D("KqpSnapshotManager: shutting down on timeout");
- ReplyErrorAndDie(NKikimrIssues::TStatusIds::TIMEOUT, {});
- }
-
-private:
- void SendDiscard() {
- auto req = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>();
- req->Record.SetExecTimeoutPeriod(RequestTimeout.MilliSeconds());
- auto* discardSnapshot = req->Record.MutableTransaction()->MutableDiscardVolatileSnapshot();
- for (const TString& tablePath : Tables) {
- discardSnapshot->AddTables()->SetTablePath(tablePath);
- }
- discardSnapshot->SetIgnoreSystemViews(true);
- discardSnapshot->SetSnapshotStep(Snapshot.Step);
- discardSnapshot->SetSnapshotTxId(Snapshot.TxId);
-
- Send(MakeTxProxyID(), req.Release());
- }
-
- void ScheduleRefresh() {
- Schedule(RefreshInterval, new TEvents::TEvWakeup());
- }
-
- void HandleUnexpectedEvent(const TString& state, ui32 eventType) {
+ using EStatus = TEvTxUserProxy::TEvProposeTransactionStatus::EStatus;
+
+ const auto* msg = ev->Get();
+ const auto status = static_cast<EStatus>(msg->Record.GetStatus());
+
+ if (status == EStatus::ExecComplete && msg->Record.GetStatusCode() == NKikimrIssues::TStatusIds::SUCCESS) {
+ Snapshot = IKqpGateway::TKqpSnapshot(msg->Record.GetStep(), msg->Record.GetTxId());
+
+ LOG_D("KqpSnapshotManager: snapshot " << Snapshot.Step << ":" << Snapshot.TxId << " created");
+
+ bool sent = Send(ClientActorId, new TEvKqpSnapshot::TEvCreateSnapshotResponse(
+ Snapshot, NKikimrIssues::TStatusIds::SUCCESS, /* issues */ {}));
+ Y_VERIFY_DEBUG(sent);
+
+ Become(&TThis::StateRefreshing);
+ ScheduleRefresh();
+ } else {
+ NYql::TIssues issues;
+ NYql::IssuesFromMessage(msg->Record.GetIssues(), issues);
+
+ LOG_E("KqpSnapshotManager: CreateSnapshot got unexpected status " << status << ": " << issues.ToString());
+ ReplyErrorAndDie(msg->Record.GetStatusCode(), std::move(issues));
+ }
+ }
+
+ STATEFN(StateRefreshing) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvents::TEvWakeup, HandleRefreshTimeout);
+ hFunc(TEvTxUserProxy::TEvProposeTransactionStatus, HandleRefreshStatus);
+ hFunc(TEvKqpSnapshot::TEvDiscardSnapshot, HandleDiscardRequest);
+ hFunc(TEvents::TEvPoison, HandlePoison);
+ default:
+ HandleUnexpectedEvent("Refreshing", ev->GetTypeRewrite());
+ }
+ }
+
+ void HandleRefreshTimeout(TEvents::TEvWakeup::TPtr&) {
+ auto req = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>();
+ req->Record.SetExecTimeoutPeriod(RequestTimeout.MilliSeconds());
+ auto* refreshSnapshot = req->Record.MutableTransaction()->MutableRefreshVolatileSnapshot();
+ for (const TString& tablePath : Tables) {
+ refreshSnapshot->AddTables()->SetTablePath(tablePath);
+ }
+ refreshSnapshot->SetIgnoreSystemViews(true);
+ refreshSnapshot->SetSnapshotStep(Snapshot.Step);
+ refreshSnapshot->SetSnapshotTxId(Snapshot.TxId);
+
+ LOG_D("KqpSnapshotManager: refreshing snapshot");
+
+ Send(MakeTxProxyID(), req.Release());
+ ScheduleRefresh();
+ }
+
+ void HandleRefreshStatus(TEvTxUserProxy::TEvProposeTransactionStatus::TPtr& ev) {
+ using EStatus = TEvTxUserProxy::TEvProposeTransactionStatus::EStatus;
+
+ const auto* msg = ev->Get();
+ const auto status = static_cast<EStatus>(msg->Record.GetStatus());
+ if (status != EStatus::ExecComplete || msg->Record.GetStatusCode() != NKikimrIssues::TStatusIds::SUCCESS) {
+ NYql::TIssues issues;
+ NYql::IssuesFromMessage(msg->Record.GetIssues(), issues);
+
+ LOG_E("KqpSnapshotManager: RefreshSnapshot got unexpected status=" << status
+ << ", issues:" << issues.ToString());
+ ReplyErrorAndDie(msg->Record.GetStatusCode(), std::move(issues));
+ }
+ }
+
+ void HandleDiscardRequest(TEvKqpSnapshot::TEvDiscardSnapshot::TPtr& ev) {
+ const auto* msg = ev->Get();
+ Y_ASSERT(msg->Snapshot == Snapshot);
+ LOG_D("KqpSnapshotManager: discarding snapshot; shutting down");
+ SendDiscard();
+ PassAway();
+ }
+
+ void HandlePoison(TEvents::TEvPoison::TPtr&) {
+ LOG_D("KqpSnapshotManager: shutting down on timeout");
+ ReplyErrorAndDie(NKikimrIssues::TStatusIds::TIMEOUT, {});
+ }
+
+private:
+ void SendDiscard() {
+ auto req = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>();
+ req->Record.SetExecTimeoutPeriod(RequestTimeout.MilliSeconds());
+ auto* discardSnapshot = req->Record.MutableTransaction()->MutableDiscardVolatileSnapshot();
+ for (const TString& tablePath : Tables) {
+ discardSnapshot->AddTables()->SetTablePath(tablePath);
+ }
+ discardSnapshot->SetIgnoreSystemViews(true);
+ discardSnapshot->SetSnapshotStep(Snapshot.Step);
+ discardSnapshot->SetSnapshotTxId(Snapshot.TxId);
+
+ Send(MakeTxProxyID(), req.Release());
+ }
+
+ void ScheduleRefresh() {
+ Schedule(RefreshInterval, new TEvents::TEvWakeup());
+ }
+
+ void HandleUnexpectedEvent(const TString& state, ui32 eventType) {
LOG_E("KqpSnapshotManager: unexpected event, state: " << state
- << ", event type: " << eventType);
- ReplyErrorAndDie(NKikimrIssues::TStatusIds::INTERNAL_ERROR, {});
- }
-
- void ReplyErrorAndDie(NKikimrIssues::TStatusIds::EStatusCode status, NYql::TIssues&& issues) {
+ << ", event type: " << eventType);
+ ReplyErrorAndDie(NKikimrIssues::TStatusIds::INTERNAL_ERROR, {});
+ }
+
+ void ReplyErrorAndDie(NKikimrIssues::TStatusIds::EStatusCode status, NYql::TIssues&& issues) {
if (CurrentStateFunc() == &TThis::StateAwaitCreation || CurrentStateFunc() == &TThis::StateAwaitAcquireResult) {
- Send(ClientActorId, new TEvKqpSnapshot::TEvCreateSnapshotResponse(
- IKqpGateway::TKqpSnapshot::InvalidSnapshot, status, std::move(issues)));
- } else {
- SendDiscard();
- }
- PassAway();
- }
-
-private:
+ Send(ClientActorId, new TEvKqpSnapshot::TEvCreateSnapshotResponse(
+ IKqpGateway::TKqpSnapshot::InvalidSnapshot, status, std::move(issues)));
+ } else {
+ SendDiscard();
+ }
+ PassAway();
+ }
+
+private:
const TString Database;
- TVector<TString> Tables;
- TActorId ClientActorId;
- IKqpGateway::TKqpSnapshot Snapshot;
-
+ TVector<TString> Tables;
+ TActorId ClientActorId;
+ IKqpGateway::TKqpSnapshot Snapshot;
+
bool MvccSnapshot = false;
- TSchedulerCookieHolder RequestTimeoutCookieHolder_;
-
- const double SnapshotToRequestTimeoutRatio = 1.5;
- const double RefreshToRequestTimeoutRatio = 0.5;
- const TDuration MaxRefreshDuration = TDuration::Seconds(10);
-
- TDuration RequestTimeout;
- TDuration SnapshotTimeout = RequestTimeout * SnapshotToRequestTimeoutRatio;
- TDuration RefreshInterval = Min(RequestTimeout * RefreshToRequestTimeoutRatio, MaxRefreshDuration);
-};
-
-} // anonymous namespace
-
+ TSchedulerCookieHolder RequestTimeoutCookieHolder_;
+
+ const double SnapshotToRequestTimeoutRatio = 1.5;
+ const double RefreshToRequestTimeoutRatio = 0.5;
+ const TDuration MaxRefreshDuration = TDuration::Seconds(10);
+
+ TDuration RequestTimeout;
+ TDuration SnapshotTimeout = RequestTimeout * SnapshotToRequestTimeoutRatio;
+ TDuration RefreshInterval = Min(RequestTimeout * RefreshToRequestTimeoutRatio, MaxRefreshDuration);
+};
+
+} // anonymous namespace
+
IActor* CreateKqpSnapshotManager(const TString& database, TDuration queryTimeout) {
return new TSnapshotManagerActor(database, queryTimeout);
-}
-
-} // namespace NKqp
-} // namespace NKikimr
+}
+
+} // namespace NKqp
+} // namespace NKikimr
diff --git a/ydb/core/kqp/rm/kqp_snapshot_manager.h b/ydb/core/kqp/rm/kqp_snapshot_manager.h
index 8ff36fd7f6..ee56469308 100644
--- a/ydb/core/kqp/rm/kqp_snapshot_manager.h
+++ b/ydb/core/kqp/rm/kqp_snapshot_manager.h
@@ -1,53 +1,53 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/kqp/common/kqp_common.h>
#include <ydb/core/kqp/common/kqp_gateway.h>
-
-#include <library/cpp/actors/core/actor.h>
-
-
-namespace NKikimr {
-namespace NKqp {
-
-struct TEvKqpSnapshot {
- struct TEvCreateSnapshotRequest : public TEventLocal<TEvCreateSnapshotRequest,
- TKqpSnapshotEvents::EvCreateSnapshotRequest>
- {
- explicit TEvCreateSnapshotRequest(const TVector<TString>& tables)
+
+#include <library/cpp/actors/core/actor.h>
+
+
+namespace NKikimr {
+namespace NKqp {
+
+struct TEvKqpSnapshot {
+ struct TEvCreateSnapshotRequest : public TEventLocal<TEvCreateSnapshotRequest,
+ TKqpSnapshotEvents::EvCreateSnapshotRequest>
+ {
+ explicit TEvCreateSnapshotRequest(const TVector<TString>& tables)
: Tables(tables)
, MvccSnapshot(false){}
-
+
explicit TEvCreateSnapshotRequest()
: Tables({})
, MvccSnapshot(true){}
- const TVector<TString> Tables;
+ const TVector<TString> Tables;
const bool MvccSnapshot;
- };
-
- struct TEvCreateSnapshotResponse : public TEventLocal<TEvCreateSnapshotResponse,
- TKqpSnapshotEvents::EvCreateSnapshotResponse>
- {
- TEvCreateSnapshotResponse(const IKqpGateway::TKqpSnapshot& snapshot,
- NKikimrIssues::TStatusIds::EStatusCode status, NYql::TIssues&& issues)
- : Snapshot(snapshot)
- , Status(status)
- , Issues(std::move(issues)) {}
-
- const IKqpGateway::TKqpSnapshot Snapshot;
- const NKikimrIssues::TStatusIds::EStatusCode Status;
- const NYql::TIssues Issues;
- };
-
- struct TEvDiscardSnapshot : public TEventLocal<TEvDiscardSnapshot, TKqpSnapshotEvents::EvDiscardSnapshot> {
- explicit TEvDiscardSnapshot(const IKqpGateway::TKqpSnapshot& snapshot)
- : Snapshot(snapshot)
- {}
- const IKqpGateway::TKqpSnapshot Snapshot;
- };
-};
-
+ };
+
+ struct TEvCreateSnapshotResponse : public TEventLocal<TEvCreateSnapshotResponse,
+ TKqpSnapshotEvents::EvCreateSnapshotResponse>
+ {
+ TEvCreateSnapshotResponse(const IKqpGateway::TKqpSnapshot& snapshot,
+ NKikimrIssues::TStatusIds::EStatusCode status, NYql::TIssues&& issues)
+ : Snapshot(snapshot)
+ , Status(status)
+ , Issues(std::move(issues)) {}
+
+ const IKqpGateway::TKqpSnapshot Snapshot;
+ const NKikimrIssues::TStatusIds::EStatusCode Status;
+ const NYql::TIssues Issues;
+ };
+
+ struct TEvDiscardSnapshot : public TEventLocal<TEvDiscardSnapshot, TKqpSnapshotEvents::EvDiscardSnapshot> {
+ explicit TEvDiscardSnapshot(const IKqpGateway::TKqpSnapshot& snapshot)
+ : Snapshot(snapshot)
+ {}
+ const IKqpGateway::TKqpSnapshot Snapshot;
+ };
+};
+
NActors::IActor* CreateKqpSnapshotManager(const TString& database, TDuration queryTimeout);
-
-} // namespace NKqp
-} // namespace NKikimr
+
+} // namespace NKqp
+} // namespace NKikimr
diff --git a/ydb/core/kqp/rm/ut/ya.make b/ydb/core/kqp/rm/ut/ya.make
index e74b603f3d..d7162df823 100644
--- a/ydb/core/kqp/rm/ut/ya.make
+++ b/ydb/core/kqp/rm/ut/ya.make
@@ -1,22 +1,22 @@
UNITTEST_FOR(ydb/core/kqp/rm)
-
-OWNER(g:kikimr)
-
-FORK_SUBTESTS()
-
-IF (SANITIZER_TYPE OR WITH_VALGRIND)
- SIZE(MEDIUM)
-ENDIF()
-
-SRCS(
- kqp_resource_estimation_ut.cpp
- kqp_rm_ut.cpp
-)
-
-PEERDIR(
+
+OWNER(g:kikimr)
+
+FORK_SUBTESTS()
+
+IF (SANITIZER_TYPE OR WITH_VALGRIND)
+ SIZE(MEDIUM)
+ENDIF()
+
+SRCS(
+ kqp_resource_estimation_ut.cpp
+ kqp_rm_ut.cpp
+)
+
+PEERDIR(
ydb/core/kqp/ut/common
-)
-
+)
+
YQL_LAST_ABI_VERSION()
-
-END()
+
+END()
diff --git a/ydb/core/kqp/rm/ya.make b/ydb/core/kqp/rm/ya.make
index 7d321dae36..0d8a759639 100644
--- a/ydb/core/kqp/rm/ya.make
+++ b/ydb/core/kqp/rm/ya.make
@@ -1,14 +1,14 @@
-LIBRARY()
-
+LIBRARY()
+
OWNER(g:kikimr)
-
-SRCS(
- kqp_resource_estimation.cpp
+
+SRCS(
+ kqp_resource_estimation.cpp
kqp_snapshot_manager.cpp
- kqp_rm.cpp
-)
-
-PEERDIR(
+ kqp_rm.cpp
+)
+
+PEERDIR(
library/cpp/actors/core
ydb/core/actorlib_impl
ydb/core/base
@@ -19,12 +19,12 @@ PEERDIR(
ydb/core/mon
ydb/core/protos
ydb/core/tablet
-)
-
+)
+
YQL_LAST_ABI_VERSION()
-END()
-
-RECURSE_FOR_TESTS(
- ut
-)
+END()
+
+RECURSE_FOR_TESTS(
+ ut
+)
diff --git a/ydb/core/kqp/runtime/kqp_channel_storage.cpp b/ydb/core/kqp/runtime/kqp_channel_storage.cpp
index 0fdc2107c5..d39d009090 100644
--- a/ydb/core/kqp/runtime/kqp_channel_storage.cpp
+++ b/ydb/core/kqp/runtime/kqp_channel_storage.cpp
@@ -1,244 +1,244 @@
-#include "kqp_channel_storage.h"
-#include "kqp_spilling.h"
-#include "kqp_spilling_file.h"
-
+#include "kqp_channel_storage.h"
+#include "kqp_spilling.h"
+#include "kqp_spilling_file.h"
+
#include <ydb/library/yql/utils/yql_panic.h>
-
-#include <library/cpp/actors/core/actor_bootstrapped.h>
-#include <library/cpp/actors/core/hfunc.h>
-#include <library/cpp/actors/core/log.h>
-
-#include <util/generic/buffer.h>
-#include <util/generic/map.h>
-#include <util/generic/set.h>
-#include <util/generic/size_literals.h>
-
-
-namespace NKikimr::NKqp {
-
-using namespace NActors;
-using namespace NYql;
-using namespace NYql::NDq;
-
-#define LOG(...) do { if (Y_UNLIKELY(LogFunc)) { LogFunc(__VA_ARGS__); } } while (0)
-
-#define LOG_D(s) \
- LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_COMPUTE, "TxId: " << TxId << ", channelId: " << ChannelId << ". " << s)
-#define LOG_I(s) \
- LOG_INFO_S(*TlsActivationContext, NKikimrServices::KQP_COMPUTE, "TxId: " << TxId << ", channelId: " << ChannelId << ". " << s)
-#define LOG_E(s) \
- LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_COMPUTE, "TxId: " << TxId << ", channelId: " << ChannelId << ". " << s)
-#define LOG_C(s) \
- LOG_CRIT_S(*TlsActivationContext, NKikimrServices::KQP_COMPUTE, "TxId: " << TxId << ", channelId: " << ChannelId << ". " << s)
-#define LOG_W(s) \
- LOG_WARN_S(*TlsActivationContext, NKikimrServices::KQP_COMPUTE, "TxId: " << TxId << ", channelId: " << ChannelId << ". " << s)
-
-namespace {
-
-constexpr ui32 MAX_INFLIGHT_BLOBS_COUNT = 10;
-constexpr ui64 MAX_INFLIGHT_BLOBS_SIZE = 50_MB;
-
-class TKqpChannelStorageActor : public TActorBootstrapped<TKqpChannelStorageActor> {
- using TBase = TActorBootstrapped<TKqpChannelStorageActor>;
-
-public:
- TKqpChannelStorageActor(ui64 txId, ui64 channelId, IDqChannelStorage::TWakeUpCallback&& wakeUp)
- : TxId(txId)
- , ChannelId(channelId)
- , WakeUp(std::move(wakeUp)) {}
-
- void Bootstrap() {
- auto spillingActor = CreateKqpLocalFileSpillingActor(TxId, TStringBuilder() << "ChannelId: " << ChannelId,
- SelfId(), true);
- SpillingActorId = Register(spillingActor);
-
- Become(&TKqpChannelStorageActor::WorkState);
- }
-
+
+#include <library/cpp/actors/core/actor_bootstrapped.h>
+#include <library/cpp/actors/core/hfunc.h>
+#include <library/cpp/actors/core/log.h>
+
+#include <util/generic/buffer.h>
+#include <util/generic/map.h>
+#include <util/generic/set.h>
+#include <util/generic/size_literals.h>
+
+
+namespace NKikimr::NKqp {
+
+using namespace NActors;
+using namespace NYql;
+using namespace NYql::NDq;
+
+#define LOG(...) do { if (Y_UNLIKELY(LogFunc)) { LogFunc(__VA_ARGS__); } } while (0)
+
+#define LOG_D(s) \
+ LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_COMPUTE, "TxId: " << TxId << ", channelId: " << ChannelId << ". " << s)
+#define LOG_I(s) \
+ LOG_INFO_S(*TlsActivationContext, NKikimrServices::KQP_COMPUTE, "TxId: " << TxId << ", channelId: " << ChannelId << ". " << s)
+#define LOG_E(s) \
+ LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_COMPUTE, "TxId: " << TxId << ", channelId: " << ChannelId << ". " << s)
+#define LOG_C(s) \
+ LOG_CRIT_S(*TlsActivationContext, NKikimrServices::KQP_COMPUTE, "TxId: " << TxId << ", channelId: " << ChannelId << ". " << s)
+#define LOG_W(s) \
+ LOG_WARN_S(*TlsActivationContext, NKikimrServices::KQP_COMPUTE, "TxId: " << TxId << ", channelId: " << ChannelId << ". " << s)
+
+namespace {
+
+constexpr ui32 MAX_INFLIGHT_BLOBS_COUNT = 10;
+constexpr ui64 MAX_INFLIGHT_BLOBS_SIZE = 50_MB;
+
+class TKqpChannelStorageActor : public TActorBootstrapped<TKqpChannelStorageActor> {
+ using TBase = TActorBootstrapped<TKqpChannelStorageActor>;
+
+public:
+ TKqpChannelStorageActor(ui64 txId, ui64 channelId, IDqChannelStorage::TWakeUpCallback&& wakeUp)
+ : TxId(txId)
+ , ChannelId(channelId)
+ , WakeUp(std::move(wakeUp)) {}
+
+ void Bootstrap() {
+ auto spillingActor = CreateKqpLocalFileSpillingActor(TxId, TStringBuilder() << "ChannelId: " << ChannelId,
+ SelfId(), true);
+ SpillingActorId = Register(spillingActor);
+
+ Become(&TKqpChannelStorageActor::WorkState);
+ }
+
static constexpr char ActorName[] = "KQP_CHANNEL_STORAGE";
-protected:
- void PassAway() override {
- Send(SpillingActorId, new TEvents::TEvPoison);
- TBase::PassAway();
- }
-
-private:
- STATEFN(WorkState) {
- switch (ev->GetTypeRewrite()) {
- hFunc(TEvKqpSpilling::TEvWriteResult, HandleWork);
- hFunc(TEvKqpSpilling::TEvReadResult, HandleWork);
- hFunc(TEvKqpSpilling::TEvError, HandleWork);
- default:
- Y_FAIL("TKqpChannelStorageActor::WorkState unexpected event type: %" PRIx32 " event: %s",
- ev->GetTypeRewrite(),
- ev->HasEvent() ? ev->GetBase()->ToString().c_str() : "serialized?");
- }
- }
-
- void HandleWork(TEvKqpSpilling::TEvWriteResult::TPtr& ev) {
- auto& msg = *ev->Get();
- LOG_D("[TEvWriteResult] blobId: " << msg.BlobId);
-
- auto it = WritingBlobs.find(msg.BlobId);
- if (it == WritingBlobs.end()) {
- LOG_E("Got unexpected TEvWriteResult, blobId: " << msg.BlobId);
-
- Error = "Internal error";
-
- Send(SpillingActorId, new TEvents::TEvPoison);
- return;
- }
-
- ui64 size = it->second;
- WritingBlobsSize -= size;
- WritingBlobs.erase(it);
-
- StoredBlobsCount++;
- StoredBlobsSize += size;
- }
-
- void HandleWork(TEvKqpSpilling::TEvReadResult::TPtr& ev) {
- auto& msg = *ev->Get();
- LOG_D("[TEvReadResult] blobId: " << msg.BlobId << ", size: " << msg.Blob.size());
-
- if (LoadingBlobs.erase(msg.BlobId) != 1) {
- LOG_E("[TEvReadResult] unexpected, blobId: " << msg.BlobId << ", size: " << msg.Blob.size());
- return;
- }
-
- LoadedBlobs[msg.BlobId].Swap(msg.Blob);
- YQL_ENSURE(LoadedBlobs[msg.BlobId].size() != 0);
-
- if (LoadedBlobs.size() == 1) {
- WakeUp();
- }
- }
-
- void HandleWork(TEvKqpSpilling::TEvError::TPtr& ev) {
- auto& msg = *ev->Get();
- LOG_D("[TEvError] " << msg.Message);
-
- Error.ConstructInPlace(msg.Message);
- }
-
-public:
- [[nodiscard]]
- const TMaybe<TString>& GetError() const {
- return Error;
- }
-
- bool IsEmpty() const {
- return WritingBlobs.empty() && StoredBlobsCount == 0 && LoadedBlobs.empty();
- }
-
- bool IsFull() const {
- return WritingBlobs.size() > MAX_INFLIGHT_BLOBS_COUNT || WritingBlobsSize > MAX_INFLIGHT_BLOBS_SIZE;
- }
-
- void Put(ui64 blobId, TBuffer&& blob) {
- FailOnError();
-
- // TODO: timeout
- // TODO: limit inflight events
-
- ui64 size = blob.size();
-
- Send(SpillingActorId, new TEvKqpSpilling::TEvWrite(blobId, std::move(blob)));
-
- WritingBlobs.emplace(blobId, size);
- WritingBlobsSize += size;
- }
-
- bool Get(ui64 blobId, TBuffer& blob) {
- FailOnError();
-
- auto loadedIt = LoadedBlobs.find(blobId);
- if (loadedIt != LoadedBlobs.end()) {
- YQL_ENSURE(loadedIt->second.size() != 0);
- blob.Swap(loadedIt->second);
- LoadedBlobs.erase(loadedIt);
- return true;
- }
-
- auto result = LoadingBlobs.emplace(blobId);
- if (result.second) {
- Send(SpillingActorId, new TEvKqpSpilling::TEvRead(blobId, true));
- }
-
- return false;
- }
-
- void Terminate() {
- PassAway();
- }
-
-private:
- void FailOnError() {
- if (Error) {
- LOG_E("TxId: " << TxId << ", channelId: " << ChannelId << ", error: " << *Error);
- ythrow TDqChannelStorageException() << "TxId: " << TxId << ", channelId: " << ChannelId
- << ", error: " << *Error;
- }
- }
-
-private:
- const ui64 TxId;
- const ui64 ChannelId;
- IDqChannelStorage::TWakeUpCallback WakeUp;
+protected:
+ void PassAway() override {
+ Send(SpillingActorId, new TEvents::TEvPoison);
+ TBase::PassAway();
+ }
+
+private:
+ STATEFN(WorkState) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvKqpSpilling::TEvWriteResult, HandleWork);
+ hFunc(TEvKqpSpilling::TEvReadResult, HandleWork);
+ hFunc(TEvKqpSpilling::TEvError, HandleWork);
+ default:
+ Y_FAIL("TKqpChannelStorageActor::WorkState unexpected event type: %" PRIx32 " event: %s",
+ ev->GetTypeRewrite(),
+ ev->HasEvent() ? ev->GetBase()->ToString().c_str() : "serialized?");
+ }
+ }
+
+ void HandleWork(TEvKqpSpilling::TEvWriteResult::TPtr& ev) {
+ auto& msg = *ev->Get();
+ LOG_D("[TEvWriteResult] blobId: " << msg.BlobId);
+
+ auto it = WritingBlobs.find(msg.BlobId);
+ if (it == WritingBlobs.end()) {
+ LOG_E("Got unexpected TEvWriteResult, blobId: " << msg.BlobId);
+
+ Error = "Internal error";
+
+ Send(SpillingActorId, new TEvents::TEvPoison);
+ return;
+ }
+
+ ui64 size = it->second;
+ WritingBlobsSize -= size;
+ WritingBlobs.erase(it);
+
+ StoredBlobsCount++;
+ StoredBlobsSize += size;
+ }
+
+ void HandleWork(TEvKqpSpilling::TEvReadResult::TPtr& ev) {
+ auto& msg = *ev->Get();
+ LOG_D("[TEvReadResult] blobId: " << msg.BlobId << ", size: " << msg.Blob.size());
+
+ if (LoadingBlobs.erase(msg.BlobId) != 1) {
+ LOG_E("[TEvReadResult] unexpected, blobId: " << msg.BlobId << ", size: " << msg.Blob.size());
+ return;
+ }
+
+ LoadedBlobs[msg.BlobId].Swap(msg.Blob);
+ YQL_ENSURE(LoadedBlobs[msg.BlobId].size() != 0);
+
+ if (LoadedBlobs.size() == 1) {
+ WakeUp();
+ }
+ }
+
+ void HandleWork(TEvKqpSpilling::TEvError::TPtr& ev) {
+ auto& msg = *ev->Get();
+ LOG_D("[TEvError] " << msg.Message);
+
+ Error.ConstructInPlace(msg.Message);
+ }
+
+public:
+ [[nodiscard]]
+ const TMaybe<TString>& GetError() const {
+ return Error;
+ }
+
+ bool IsEmpty() const {
+ return WritingBlobs.empty() && StoredBlobsCount == 0 && LoadedBlobs.empty();
+ }
+
+ bool IsFull() const {
+ return WritingBlobs.size() > MAX_INFLIGHT_BLOBS_COUNT || WritingBlobsSize > MAX_INFLIGHT_BLOBS_SIZE;
+ }
+
+ void Put(ui64 blobId, TBuffer&& blob) {
+ FailOnError();
+
+ // TODO: timeout
+ // TODO: limit inflight events
+
+ ui64 size = blob.size();
+
+ Send(SpillingActorId, new TEvKqpSpilling::TEvWrite(blobId, std::move(blob)));
+
+ WritingBlobs.emplace(blobId, size);
+ WritingBlobsSize += size;
+ }
+
+ bool Get(ui64 blobId, TBuffer& blob) {
+ FailOnError();
+
+ auto loadedIt = LoadedBlobs.find(blobId);
+ if (loadedIt != LoadedBlobs.end()) {
+ YQL_ENSURE(loadedIt->second.size() != 0);
+ blob.Swap(loadedIt->second);
+ LoadedBlobs.erase(loadedIt);
+ return true;
+ }
+
+ auto result = LoadingBlobs.emplace(blobId);
+ if (result.second) {
+ Send(SpillingActorId, new TEvKqpSpilling::TEvRead(blobId, true));
+ }
+
+ return false;
+ }
+
+ void Terminate() {
+ PassAway();
+ }
+
+private:
+ void FailOnError() {
+ if (Error) {
+ LOG_E("TxId: " << TxId << ", channelId: " << ChannelId << ", error: " << *Error);
+ ythrow TDqChannelStorageException() << "TxId: " << TxId << ", channelId: " << ChannelId
+ << ", error: " << *Error;
+ }
+ }
+
+private:
+ const ui64 TxId;
+ const ui64 ChannelId;
+ IDqChannelStorage::TWakeUpCallback WakeUp;
TActorId SpillingActorId;
-
- TMap<ui64, ui64> WritingBlobs; // blobId -> blobSize
- ui64 WritingBlobsSize = 0;
-
- ui32 StoredBlobsCount = 0;
- ui64 StoredBlobsSize = 0;
-
- TSet<ui64> LoadingBlobs;
- TMap<ui64, TBuffer> LoadedBlobs;
-
- TMaybe<TString> Error;
-};
-
-
-class TKqpChannelStorage : public IDqChannelStorage {
-public:
- TKqpChannelStorage(ui64 txId, ui64 channelId, TWakeUpCallback&& wakeUp, const TActorContext& ctx)
- {
- SelfActor = new TKqpChannelStorageActor(txId, channelId, std::move(wakeUp));
+
+ TMap<ui64, ui64> WritingBlobs; // blobId -> blobSize
+ ui64 WritingBlobsSize = 0;
+
+ ui32 StoredBlobsCount = 0;
+ ui64 StoredBlobsSize = 0;
+
+ TSet<ui64> LoadingBlobs;
+ TMap<ui64, TBuffer> LoadedBlobs;
+
+ TMaybe<TString> Error;
+};
+
+
+class TKqpChannelStorage : public IDqChannelStorage {
+public:
+ TKqpChannelStorage(ui64 txId, ui64 channelId, TWakeUpCallback&& wakeUp, const TActorContext& ctx)
+ {
+ SelfActor = new TKqpChannelStorageActor(txId, channelId, std::move(wakeUp));
ctx.RegisterWithSameMailbox(SelfActor);
- }
-
- ~TKqpChannelStorage() {
- SelfActor->Terminate();
- }
-
- bool IsEmpty() const override {
- return SelfActor->IsEmpty();
- }
-
- bool IsFull() const override {
- return SelfActor->IsFull();
- }
-
- void Put(ui64 blobId, TBuffer&& blob) override {
- SelfActor->Put(blobId, std::move(blob));
- }
-
- bool Get(ui64 blobId, TBuffer& blob) override {
- return SelfActor->Get(blobId, blob);
- }
-
-private:
- TKqpChannelStorageActor* SelfActor;
-};
-
-} // anonymous namespace
-
-IDqChannelStorage::TPtr CreateKqpChannelStorage(ui64 txId, ui64 channelId, IDqChannelStorage::TWakeUpCallback wakeUp,
- const TActorContext& ctx)
-{
- return new TKqpChannelStorage(txId, channelId, std::move(wakeUp), ctx);
-}
-
-} // namespace NKikimr::NKqp
+ }
+
+ ~TKqpChannelStorage() {
+ SelfActor->Terminate();
+ }
+
+ bool IsEmpty() const override {
+ return SelfActor->IsEmpty();
+ }
+
+ bool IsFull() const override {
+ return SelfActor->IsFull();
+ }
+
+ void Put(ui64 blobId, TBuffer&& blob) override {
+ SelfActor->Put(blobId, std::move(blob));
+ }
+
+ bool Get(ui64 blobId, TBuffer& blob) override {
+ return SelfActor->Get(blobId, blob);
+ }
+
+private:
+ TKqpChannelStorageActor* SelfActor;
+};
+
+} // anonymous namespace
+
+IDqChannelStorage::TPtr CreateKqpChannelStorage(ui64 txId, ui64 channelId, IDqChannelStorage::TWakeUpCallback wakeUp,
+ const TActorContext& ctx)
+{
+ return new TKqpChannelStorage(txId, channelId, std::move(wakeUp), ctx);
+}
+
+} // namespace NKikimr::NKqp
diff --git a/ydb/core/kqp/runtime/kqp_channel_storage.h b/ydb/core/kqp/runtime/kqp_channel_storage.h
index 8c693741ae..ec149722b8 100644
--- a/ydb/core/kqp/runtime/kqp_channel_storage.h
+++ b/ydb/core/kqp/runtime/kqp_channel_storage.h
@@ -1,12 +1,12 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/dq/common/dq_common.h>
#include <ydb/library/yql/dq/runtime/dq_channel_storage.h>
-#include <library/cpp/actors/core/actor.h>
-
-namespace NKikimr::NKqp {
-
-NYql::NDq::IDqChannelStorage::TPtr CreateKqpChannelStorage(ui64 txId, ui64 channelId,
- NYql::NDq::IDqChannelStorage::TWakeUpCallback wakeUpCb, const NActors::TActorContext& ctx);
-
-} // namespace NKikimr::NKqp
+#include <library/cpp/actors/core/actor.h>
+
+namespace NKikimr::NKqp {
+
+NYql::NDq::IDqChannelStorage::TPtr CreateKqpChannelStorage(ui64 txId, ui64 channelId,
+ NYql::NDq::IDqChannelStorage::TWakeUpCallback wakeUpCb, const NActors::TActorContext& ctx);
+
+} // namespace NKikimr::NKqp
diff --git a/ydb/core/kqp/runtime/kqp_compute.cpp b/ydb/core/kqp/runtime/kqp_compute.cpp
index 33539a7558..b4d788fc9c 100644
--- a/ydb/core/kqp/runtime/kqp_compute.cpp
+++ b/ydb/core/kqp/runtime/kqp_compute.cpp
@@ -11,92 +11,92 @@ namespace NKikimr {
namespace NMiniKQL {
TComputationNodeFactory GetKqpBaseComputeFactory(const TKqpComputeContextBase* computeCtx) {
- return NYql::NDq::GetDqBaseComputeFactory(computeCtx);
-}
-
-namespace {
-
-class TKqpEnsureWrapper : public TMutableCodegeneratorNode<TKqpEnsureWrapper> {
- using TBaseComputation = TMutableCodegeneratorNode<TKqpEnsureWrapper>;
-public:
- TKqpEnsureWrapper(TComputationMutables& mutables, IComputationNode* value, IComputationNode* predicate,
- IComputationNode* issueCode, IComputationNode* message)
- : TBaseComputation(mutables, value->GetRepresentation())
- , Arg(value)
- , Predicate(predicate)
- , IssueCode(issueCode)
- , Message(message)
- {
- }
-
- NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const {
- const auto& predicate = Predicate->GetValue(ctx);
- if (predicate && predicate.Get<bool>()) {
- return Arg->GetValue(ctx).Release();
- }
-
- Throw(this, &ctx);
- }
-
-#ifndef MKQL_DISABLE_CODEGEN
- Value* DoGenerateGetValue(const TCodegenContext& ctx, BasicBlock*& block) const {
- auto& context = ctx.Codegen->GetContext();
-
- const auto predicate = GetNodeValue(Predicate, ctx, block);
- const auto pass = CastInst::Create(Instruction::Trunc, predicate, Type::getInt1Ty(context), "bool", block);
-
- const auto kill = BasicBlock::Create(context, "kill", ctx.Func);
- const auto good = BasicBlock::Create(context, "good", ctx.Func);
-
- BranchInst::Create(good, kill, pass, block);
-
- block = kill;
- const auto doFunc = ConstantInt::get(Type::getInt64Ty(context), GetMethodPtr(&TKqpEnsureWrapper::Throw));
- const auto doFuncArg = ConstantInt::get(Type::getInt64Ty(context), (ui64)this);
- const auto doFuncPtr = CastInst::Create(Instruction::IntToPtr, doFunc, PointerType::getUnqual(FunctionType::get(Type::getVoidTy(context), { Type::getInt64Ty(context), ctx.Ctx->getType() }, false)), "thrower", block);
- CallInst::Create(doFuncPtr, { doFuncArg, ctx.Ctx }, "", block)->setTailCall();
- new UnreachableInst(context, block);
-
- block = good;
- return GetNodeValue(Arg, ctx, block);;
- }
-#endif
-
-private:
- [[noreturn]]
- static void Throw(TKqpEnsureWrapper const* thisPtr, TComputationContext* ctxPtr) {
- auto issueCode = thisPtr->IssueCode->GetValue(*ctxPtr);
- auto message = thisPtr->Message->GetValue(*ctxPtr);
-
- throw TKqpEnsureFail(issueCode.Get<ui32>(), TString(message.AsStringRef().Data()));
- }
-
- void RegisterDependencies() const final {
- DependsOn(Arg);
- DependsOn(Predicate);
- }
-
- IComputationNode* const Arg;
- IComputationNode* const Predicate;
- IComputationNode* const IssueCode;
- IComputationNode* const Message;
-};
-
-} // namespace
-
-IComputationNode* WrapKqpEnsure(TCallable& callable, const TComputationNodeFactoryContext& ctx) {
- MKQL_ENSURE(callable.GetInputsCount() == 4, "Expected 4 args");
- bool isOptional;
- auto unpackedType = UnpackOptionalData(callable.GetInput(1), isOptional);
- MKQL_ENSURE(unpackedType->GetSchemeType() == NUdf::TDataType<bool>::Id, "Expected bool");
-
- auto value = LocateNode(ctx.NodeLocator, callable, 0);
- auto predicate = LocateNode(ctx.NodeLocator, callable, 1);
- auto issueCode = LocateNode(ctx.NodeLocator, callable, 2);
- auto message = LocateNode(ctx.NodeLocator, callable, 3);
-
- return new TKqpEnsureWrapper(ctx.Mutables, value, predicate, issueCode, message);
+ return NYql::NDq::GetDqBaseComputeFactory(computeCtx);
}
-} // namespace NMiniKQL
+namespace {
+
+class TKqpEnsureWrapper : public TMutableCodegeneratorNode<TKqpEnsureWrapper> {
+ using TBaseComputation = TMutableCodegeneratorNode<TKqpEnsureWrapper>;
+public:
+ TKqpEnsureWrapper(TComputationMutables& mutables, IComputationNode* value, IComputationNode* predicate,
+ IComputationNode* issueCode, IComputationNode* message)
+ : TBaseComputation(mutables, value->GetRepresentation())
+ , Arg(value)
+ , Predicate(predicate)
+ , IssueCode(issueCode)
+ , Message(message)
+ {
+ }
+
+ NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const {
+ const auto& predicate = Predicate->GetValue(ctx);
+ if (predicate && predicate.Get<bool>()) {
+ return Arg->GetValue(ctx).Release();
+ }
+
+ Throw(this, &ctx);
+ }
+
+#ifndef MKQL_DISABLE_CODEGEN
+ Value* DoGenerateGetValue(const TCodegenContext& ctx, BasicBlock*& block) const {
+ auto& context = ctx.Codegen->GetContext();
+
+ const auto predicate = GetNodeValue(Predicate, ctx, block);
+ const auto pass = CastInst::Create(Instruction::Trunc, predicate, Type::getInt1Ty(context), "bool", block);
+
+ const auto kill = BasicBlock::Create(context, "kill", ctx.Func);
+ const auto good = BasicBlock::Create(context, "good", ctx.Func);
+
+ BranchInst::Create(good, kill, pass, block);
+
+ block = kill;
+ const auto doFunc = ConstantInt::get(Type::getInt64Ty(context), GetMethodPtr(&TKqpEnsureWrapper::Throw));
+ const auto doFuncArg = ConstantInt::get(Type::getInt64Ty(context), (ui64)this);
+ const auto doFuncPtr = CastInst::Create(Instruction::IntToPtr, doFunc, PointerType::getUnqual(FunctionType::get(Type::getVoidTy(context), { Type::getInt64Ty(context), ctx.Ctx->getType() }, false)), "thrower", block);
+ CallInst::Create(doFuncPtr, { doFuncArg, ctx.Ctx }, "", block)->setTailCall();
+ new UnreachableInst(context, block);
+
+ block = good;
+ return GetNodeValue(Arg, ctx, block);;
+ }
+#endif
+
+private:
+ [[noreturn]]
+ static void Throw(TKqpEnsureWrapper const* thisPtr, TComputationContext* ctxPtr) {
+ auto issueCode = thisPtr->IssueCode->GetValue(*ctxPtr);
+ auto message = thisPtr->Message->GetValue(*ctxPtr);
+
+ throw TKqpEnsureFail(issueCode.Get<ui32>(), TString(message.AsStringRef().Data()));
+ }
+
+ void RegisterDependencies() const final {
+ DependsOn(Arg);
+ DependsOn(Predicate);
+ }
+
+ IComputationNode* const Arg;
+ IComputationNode* const Predicate;
+ IComputationNode* const IssueCode;
+ IComputationNode* const Message;
+};
+
+} // namespace
+
+IComputationNode* WrapKqpEnsure(TCallable& callable, const TComputationNodeFactoryContext& ctx) {
+ MKQL_ENSURE(callable.GetInputsCount() == 4, "Expected 4 args");
+ bool isOptional;
+ auto unpackedType = UnpackOptionalData(callable.GetInput(1), isOptional);
+ MKQL_ENSURE(unpackedType->GetSchemeType() == NUdf::TDataType<bool>::Id, "Expected bool");
+
+ auto value = LocateNode(ctx.NodeLocator, callable, 0);
+ auto predicate = LocateNode(ctx.NodeLocator, callable, 1);
+ auto issueCode = LocateNode(ctx.NodeLocator, callable, 2);
+ auto message = LocateNode(ctx.NodeLocator, callable, 3);
+
+ return new TKqpEnsureWrapper(ctx.Mutables, value, predicate, issueCode, message);
+}
+
+} // namespace NMiniKQL
} // namespace NKikimr
diff --git a/ydb/core/kqp/runtime/kqp_compute.h b/ydb/core/kqp/runtime/kqp_compute.h
index 8b034613eb..c55a4e2a21 100644
--- a/ydb/core/kqp/runtime/kqp_compute.h
+++ b/ydb/core/kqp/runtime/kqp_compute.h
@@ -16,37 +16,37 @@ public:
NTable::TTag Tag;
NScheme::TTypeId Type;
};
-
- // used only at then building of a computation graph, to inject taskId in runtime nodes
- void SetCurrentTaskId(ui64 taskId) { CurrentTaskId = taskId; }
- ui64 GetCurrentTaskId() const { return CurrentTaskId; }
-
-private:
- ui64 CurrentTaskId = 0;
+
+ // used only at then building of a computation graph, to inject taskId in runtime nodes
+ void SetCurrentTaskId(ui64 taskId) { CurrentTaskId = taskId; }
+ ui64 GetCurrentTaskId() const { return CurrentTaskId; }
+
+private:
+ ui64 CurrentTaskId = 0;
};
TComputationNodeFactory GetKqpBaseComputeFactory(const TKqpComputeContextBase* computeCtx);
-class TKqpEnsureFail : public yexception {
-public:
- TKqpEnsureFail(ui32 code, TString&& message)
- : Code(code)
- , Message(std::move(message)) {}
-
- ui32 GetCode() const {
- return Code;
- }
-
- const TString& GetMessage() const {
- return Message;
- }
-
-private:
- ui32 Code;
- TString Message;
-};
-
-IComputationNode* WrapKqpEnsure(TCallable& callable, const TComputationNodeFactoryContext& ctx);
-
-} // namespace NMiniKQL
+class TKqpEnsureFail : public yexception {
+public:
+ TKqpEnsureFail(ui32 code, TString&& message)
+ : Code(code)
+ , Message(std::move(message)) {}
+
+ ui32 GetCode() const {
+ return Code;
+ }
+
+ const TString& GetMessage() const {
+ return Message;
+ }
+
+private:
+ ui32 Code;
+ TString Message;
+};
+
+IComputationNode* WrapKqpEnsure(TCallable& callable, const TComputationNodeFactoryContext& ctx);
+
+} // namespace NMiniKQL
} // namespace NKikimr
diff --git a/ydb/core/kqp/runtime/kqp_effects.cpp b/ydb/core/kqp/runtime/kqp_effects.cpp
index 3243765c14..b178192f27 100644
--- a/ydb/core/kqp/runtime/kqp_effects.cpp
+++ b/ydb/core/kqp/runtime/kqp_effects.cpp
@@ -35,5 +35,5 @@ IDqOutputConsumer::TPtr CreateKqpApplyEffectsConsumer(NUdf::IApplyContext* apply
return MakeIntrusive<TKqpApplyEffectsConsumer>(applyCtx);
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/runtime/kqp_output_stream.cpp b/ydb/core/kqp/runtime/kqp_output_stream.cpp
index 9248c8c1fb..640706d9b4 100644
--- a/ydb/core/kqp/runtime/kqp_output_stream.cpp
+++ b/ydb/core/kqp/runtime/kqp_output_stream.cpp
@@ -12,63 +12,63 @@ using namespace NMiniKQL;
namespace {
-using namespace NYql;
-using namespace NDq;
-using namespace NUdf;
-
-
-class TKqpOutputRangePartitionConsumer : public IDqOutputConsumer {
+using namespace NYql;
+using namespace NDq;
+using namespace NUdf;
+
+
+class TKqpOutputRangePartitionConsumer : public IDqOutputConsumer {
public:
- TKqpOutputRangePartitionConsumer(const TTypeEnvironment& typeEnv,
+ TKqpOutputRangePartitionConsumer(const TTypeEnvironment& typeEnv,
TVector<NYql::NDq::IDqOutput::TPtr>&& outputs, TVector<TKqpRangePartition>&& partitions,
- TVector<TDataTypeId>&& keyColumnTypes, TVector<ui32>&& keyColumnIndices)
- : TypeEnv(typeEnv)
+ TVector<TDataTypeId>&& keyColumnTypes, TVector<ui32>&& keyColumnIndices)
+ : TypeEnv(typeEnv)
, Outputs(std::move(outputs))
, Partitions(std::move(partitions))
, KeyColumnTypes(std::move(keyColumnTypes))
, KeyColumnIndices(std::move(keyColumnIndices))
- {
+ {
MKQL_ENSURE_S(!Partitions.empty());
MKQL_ENSURE_S(KeyColumnTypes.size() == KeyColumnIndices.size());
SortPartitions(Partitions, KeyColumnTypes, [](const auto& partition) { return partition.Range; });
}
- bool IsFull() const override {
+ bool IsFull() const override {
return AnyOf(Outputs, [](const auto& output) { return output->IsFull(); });
- }
+ }
- void Consume(TUnboxedValue&& value) final {
- ui32 partitionIndex = FindKeyPartitionIndex(TypeEnv, value, Partitions, KeyColumnTypes, KeyColumnIndices,
- [](const auto& partition) { return partition.Range; });
+ void Consume(TUnboxedValue&& value) final {
+ ui32 partitionIndex = FindKeyPartitionIndex(TypeEnv, value, Partitions, KeyColumnTypes, KeyColumnIndices,
+ [](const auto& partition) { return partition.Range; });
Outputs[partitionIndex]->Push(std::move(value));
- }
+ }
- void Finish() final {
+ void Finish() final {
for (auto& output : Outputs) {
output->Finish();
}
}
private:
- const TTypeEnvironment& TypeEnv;
+ const TTypeEnvironment& TypeEnv;
TVector<NYql::NDq::IDqOutput::TPtr> Outputs;
TVector<TKqpRangePartition> Partitions;
- TVector<TDataTypeId> KeyColumnTypes;
+ TVector<TDataTypeId> KeyColumnTypes;
TVector<ui32> KeyColumnIndices;
};
} // namespace
-NYql::NDq::IDqOutputConsumer::TPtr CreateOutputRangePartitionConsumer(
+NYql::NDq::IDqOutputConsumer::TPtr CreateOutputRangePartitionConsumer(
TVector<NYql::NDq::IDqOutput::TPtr>&& outputs, TVector<TKqpRangePartition>&& partitions,
- TVector<NUdf::TDataTypeId>&& keyColumnTypes, TVector<ui32>&& keyColumnIndices,
- const NMiniKQL::TTypeEnvironment& typeEnv)
-{
+ TVector<NUdf::TDataTypeId>&& keyColumnTypes, TVector<ui32>&& keyColumnIndices,
+ const NMiniKQL::TTypeEnvironment& typeEnv)
+{
return MakeIntrusive<TKqpOutputRangePartitionConsumer>(typeEnv, std::move(outputs), std::move(partitions),
- std::move(keyColumnTypes), std::move(keyColumnIndices));
+ std::move(keyColumnTypes), std::move(keyColumnIndices));
}
-} // namespace NKqp
-} // namespace NKikimr
+} // namespace NKqp
+} // namespace NKikimr
diff --git a/ydb/core/kqp/runtime/kqp_program_builder.cpp b/ydb/core/kqp/runtime/kqp_program_builder.cpp
index fdf8341b29..b5169c19d8 100644
--- a/ydb/core/kqp/runtime/kqp_program_builder.cpp
+++ b/ydb/core/kqp/runtime/kqp_program_builder.cpp
@@ -15,11 +15,11 @@ TType* GetRowType(const TProgramBuilder& builder, const TArrayRef<TKqpTableColum
TStructTypeBuilder rowTypeBuilder(builder.GetTypeEnvironment());
for (auto& column : columns) {
TType* type = nullptr;
- if (column.Type == NUdf::TDataType<NUdf::TDecimal>::Id) {
+ if (column.Type == NUdf::TDataType<NUdf::TDecimal>::Id) {
type = TDataDecimalType::Create(NScheme::DECIMAL_PRECISION, NScheme::DECIMAL_SCALE, builder.GetTypeEnvironment());
- } else {
+ } else {
type = TDataType::Create(column.Type, builder.GetTypeEnvironment());
- }
+ }
if (!column.NotNull) {
type = TOptionalType::Create(type, builder.GetTypeEnvironment());
@@ -124,19 +124,19 @@ TType* MakeWideFlowType(TProgramBuilder& builder, TStructType* rowType) {
TKqpProgramBuilder::TKqpProgramBuilder(const TTypeEnvironment& env, const IFunctionRegistry& functionRegistry)
: TProgramBuilder(env, functionRegistry) {}
-TRuntimeNode TKqpProgramBuilder::KqpReadTable(const TTableId& tableId, const TKqpKeyRange& range,
+TRuntimeNode TKqpProgramBuilder::KqpReadTable(const TTableId& tableId, const TKqpKeyRange& range,
const TArrayRef<TKqpTableColumn>& columns)
{
auto rowType = GetRowType(*this, columns);
- auto returnType = NewFlowType(rowType);
+ auto returnType = NewFlowType(rowType);
TCallableBuilder builder(Env, __func__, returnType);
builder.Add(BuildTableIdLiteral(tableId, *this));
builder.Add(BuildKeyRangeNode(*this, range));
builder.Add(BuildColumnTags(*this, columns));
builder.Add(BuildSkipNullKeysNode(*this, range));
- builder.Add(range.ItemsLimit ? range.ItemsLimit : NewNull());
- builder.Add(NewDataLiteral(range.Reverse));
+ builder.Add(range.ItemsLimit ? range.ItemsLimit : NewNull());
+ builder.Add(NewDataLiteral(range.Reverse));
return TRuntimeNode(builder.Build(), false);
}
@@ -158,8 +158,8 @@ TRuntimeNode TKqpProgramBuilder::KqpWideReadTable(const TTableId& tableId, const
builder.Add(BuildKeyRangeNode(*this, range));
builder.Add(BuildColumnTags(*this, columns));
builder.Add(BuildSkipNullKeysNode(*this, range));
- builder.Add(range.ItemsLimit ? range.ItemsLimit : NewNull());
- builder.Add(NewDataLiteral(range.Reverse));
+ builder.Add(range.ItemsLimit ? range.ItemsLimit : NewNull());
+ builder.Add(NewDataLiteral(range.Reverse));
return TRuntimeNode(builder.Build(), false);
}
@@ -242,32 +242,32 @@ TRuntimeNode TKqpProgramBuilder::KqpEffects(const TArrayRef<const TRuntimeNode>&
return TRuntimeNode(builder.Build(), false);
}
-TRuntimeNode TKqpProgramBuilder::KqpEnsure(TRuntimeNode value, TRuntimeNode predicate, TRuntimeNode issueCode,
- TRuntimeNode message)
-{
- bool isOptional;
- const auto unpackedType = UnpackOptionalData(predicate, isOptional);
- MKQL_ENSURE(unpackedType->GetSchemeType() == NUdf::TDataType<bool>::Id, "Expected bool.");
-
- const auto& issueCodeType = issueCode.GetStaticType();
- MKQL_ENSURE(issueCodeType->IsData(), "Expected data.");
-
- const auto& issueCodeTypeData = static_cast<const TDataType&>(*issueCodeType);
- MKQL_ENSURE(issueCodeTypeData.GetSchemeType() == NUdf::TDataType<ui32>::Id, "Expected uint32.");
-
- const auto& messageType = message.GetStaticType();
- MKQL_ENSURE(messageType->IsData(), "Expected data.");
-
- const auto& messageTypeData = static_cast<const TDataType&>(*messageType);
- MKQL_ENSURE(messageTypeData.GetSchemeType() == NUdf::TDataType<NUdf::TUtf8>::Id, "Expected string or utf8.");
-
- TCallableBuilder callableBuilder(Env, __func__, value.GetStaticType());
- callableBuilder.Add(value);
- callableBuilder.Add(predicate);
- callableBuilder.Add(issueCode);
- callableBuilder.Add(message);
- return TRuntimeNode(callableBuilder.Build(), false);
-}
-
-} // namespace NMiniKQL
+TRuntimeNode TKqpProgramBuilder::KqpEnsure(TRuntimeNode value, TRuntimeNode predicate, TRuntimeNode issueCode,
+ TRuntimeNode message)
+{
+ bool isOptional;
+ const auto unpackedType = UnpackOptionalData(predicate, isOptional);
+ MKQL_ENSURE(unpackedType->GetSchemeType() == NUdf::TDataType<bool>::Id, "Expected bool.");
+
+ const auto& issueCodeType = issueCode.GetStaticType();
+ MKQL_ENSURE(issueCodeType->IsData(), "Expected data.");
+
+ const auto& issueCodeTypeData = static_cast<const TDataType&>(*issueCodeType);
+ MKQL_ENSURE(issueCodeTypeData.GetSchemeType() == NUdf::TDataType<ui32>::Id, "Expected uint32.");
+
+ const auto& messageType = message.GetStaticType();
+ MKQL_ENSURE(messageType->IsData(), "Expected data.");
+
+ const auto& messageTypeData = static_cast<const TDataType&>(*messageType);
+ MKQL_ENSURE(messageTypeData.GetSchemeType() == NUdf::TDataType<NUdf::TUtf8>::Id, "Expected string or utf8.");
+
+ TCallableBuilder callableBuilder(Env, __func__, value.GetStaticType());
+ callableBuilder.Add(value);
+ callableBuilder.Add(predicate);
+ callableBuilder.Add(issueCode);
+ callableBuilder.Add(message);
+ return TRuntimeNode(callableBuilder.Build(), false);
+}
+
+} // namespace NMiniKQL
} // namespace NKikimr
diff --git a/ydb/core/kqp/runtime/kqp_program_builder.h b/ydb/core/kqp/runtime/kqp_program_builder.h
index 61466f3952..348d25eeea 100644
--- a/ydb/core/kqp/runtime/kqp_program_builder.h
+++ b/ydb/core/kqp/runtime/kqp_program_builder.h
@@ -28,9 +28,9 @@ struct TKqpKeyRange {
TKqpKeyTuple ToTuple;
bool FromInclusive = false;
bool ToInclusive = false;
- TSmallVec<bool> SkipNullKeys;
- TRuntimeNode ItemsLimit;
- bool Reverse = false;
+ TSmallVec<bool> SkipNullKeys;
+ TRuntimeNode ItemsLimit;
+ bool Reverse = false;
};
struct TKqpKeyRanges {
@@ -44,7 +44,7 @@ class TKqpProgramBuilder: public TProgramBuilder {
public:
TKqpProgramBuilder(const TTypeEnvironment& env, const IFunctionRegistry& functionRegistry);
- TRuntimeNode KqpReadTable(const TTableId& tableId, const TKqpKeyRange& range,
+ TRuntimeNode KqpReadTable(const TTableId& tableId, const TKqpKeyRange& range,
const TArrayRef<TKqpTableColumn>& columns);
TRuntimeNode KqpWideReadTable(const TTableId& tableId, const TKqpKeyRange& range,
@@ -62,9 +62,9 @@ public:
TRuntimeNode KqpDeleteRows(const TTableId& tableId, const TRuntimeNode& rows);
TRuntimeNode KqpEffects(const TArrayRef<const TRuntimeNode>& effects);
-
- TRuntimeNode KqpEnsure(TRuntimeNode value, TRuntimeNode predicate, TRuntimeNode issueCode, TRuntimeNode message);
+
+ TRuntimeNode KqpEnsure(TRuntimeNode value, TRuntimeNode predicate, TRuntimeNode issueCode, TRuntimeNode message);
};
-} // namespace NMiniKQL
+} // namespace NMiniKQL
} // namespace NKikimr
diff --git a/ydb/core/kqp/runtime/kqp_read_table.cpp b/ydb/core/kqp/runtime/kqp_read_table.cpp
index b88dbf2289..358ebd0a3e 100644
--- a/ydb/core/kqp/runtime/kqp_read_table.cpp
+++ b/ydb/core/kqp/runtime/kqp_read_table.cpp
@@ -131,18 +131,18 @@ void ParseWideReadColumns(const TCallable& callable, const TRuntimeNode& tagsNod
if (IsSystemColumn(columnId)) {
systemColumns.push_back({columnId, AS_TYPE(TDataType, memberType)->GetSchemeType()});
- } else {
+ } else {
columns.push_back({columnId, AS_TYPE(TDataType, memberType)->GetSchemeType()});
- }
- }
+ }
+ }
}
TParseReadTableResult ParseWideReadTable(TCallable& callable) {
- MKQL_ENSURE_S(callable.GetInputsCount() >= 4);
+ MKQL_ENSURE_S(callable.GetInputsCount() >= 4);
TParseReadTableResult result;
- result.CallableId = 0; // callable.GetUniqueId();
+ result.CallableId = 0; // callable.GetUniqueId();
auto tableNode = callable.GetInput(0);
auto rangeNode = callable.GetInput(1);
@@ -163,27 +163,27 @@ TParseReadTableResult ParseWideReadTable(TCallable& callable) {
ParseWideReadColumns(callable, tagsNode, result.Columns, result.SystemColumns);
- auto skipNullKeys = AS_VALUE(TListLiteral, callable.GetInput(3));
- result.SkipNullKeys.reserve(skipNullKeys->GetItemsCount());
- for (ui32 i = 0; i < skipNullKeys->GetItemsCount(); ++i) {
- result.SkipNullKeys.push_back(AS_VALUE(TDataLiteral, skipNullKeys->GetItems()[i])->AsValue().Get<bool>());
- }
-
- if (callable.GetInputsCount() >= 5) {
- auto node = callable.GetInput(4).GetNode();
- if (node->GetType()->GetKind() == TType::EKind::Callable) {
- MKQL_ENSURE_S(AS_TYPE(TDataType, AS_TYPE(TCallableType, node->GetType())->GetReturnType())->GetSchemeType()
- == NUdf::TDataType<ui64>::Id, "ItemsLimit must be () -> ui64");
- result.ItemsLimit = node;
- } else {
- MKQL_ENSURE_S(node->GetType()->GetKind() == TType::EKind::Null, "ItemsLimit expected to be Callable or Null");
- }
- }
-
- if (callable.GetInputsCount() >= 6) {
- result.Reverse = AS_VALUE(TDataLiteral, callable.GetInput(5))->AsValue().Get<bool>();
- }
-
+ auto skipNullKeys = AS_VALUE(TListLiteral, callable.GetInput(3));
+ result.SkipNullKeys.reserve(skipNullKeys->GetItemsCount());
+ for (ui32 i = 0; i < skipNullKeys->GetItemsCount(); ++i) {
+ result.SkipNullKeys.push_back(AS_VALUE(TDataLiteral, skipNullKeys->GetItems()[i])->AsValue().Get<bool>());
+ }
+
+ if (callable.GetInputsCount() >= 5) {
+ auto node = callable.GetInput(4).GetNode();
+ if (node->GetType()->GetKind() == TType::EKind::Callable) {
+ MKQL_ENSURE_S(AS_TYPE(TDataType, AS_TYPE(TCallableType, node->GetType())->GetReturnType())->GetSchemeType()
+ == NUdf::TDataType<ui64>::Id, "ItemsLimit must be () -> ui64");
+ result.ItemsLimit = node;
+ } else {
+ MKQL_ENSURE_S(node->GetType()->GetKind() == TType::EKind::Null, "ItemsLimit expected to be Callable or Null");
+ }
+ }
+
+ if (callable.GetInputsCount() >= 6) {
+ result.Reverse = AS_VALUE(TDataLiteral, callable.GetInput(5))->AsValue().Get<bool>();
+ }
+
return result;
}
@@ -229,7 +229,7 @@ TParseReadTableRangesResult ParseWideReadTableRanges(TCallable& callable) {
return result;
}
-namespace {
+namespace {
class TKqpScanWideReadTableWrapperBase : public TStatelessWideFlowCodegeneratorNode<TKqpScanWideReadTableWrapperBase> {
using TBase = TStatelessWideFlowCodegeneratorNode<TKqpScanWideReadTableWrapperBase>;
@@ -377,8 +377,8 @@ private:
TParseReadTableRangesResult ParseResult;
};
-} // namespace
-
+} // namespace
+
IComputationNode* WrapKqpScanWideReadTableRanges(TCallable& callable, const TComputationNodeFactoryContext& ctx,
TKqpScanComputeContext& computeCtx)
{
@@ -424,5 +424,5 @@ IComputationNode* WrapKqpScanWideReadTable(TCallable& callable, const TComputati
return new TKqpScanWideReadTableWrapper(computeCtx, parseResult, fromNode, toNode, std::move(representations));
}
-} // namespace NMiniKQL
-} // namespace NKikimr
+} // namespace NMiniKQL
+} // namespace NKikimr
diff --git a/ydb/core/kqp/runtime/kqp_read_table.h b/ydb/core/kqp/runtime/kqp_read_table.h
index 44c8bfb5c9..99c9cc3f05 100644
--- a/ydb/core/kqp/runtime/kqp_read_table.h
+++ b/ydb/core/kqp/runtime/kqp_read_table.h
@@ -20,8 +20,8 @@ struct TParseReadTableResultBase {
TSmallVec<TKqpComputeContextBase::TColumn> Columns;
TSmallVec<TKqpComputeContextBase::TColumn> SystemColumns;
TSmallVec<bool> SkipNullKeys;
- TNode* ItemsLimit = nullptr;
- bool Reverse = false;
+ TNode* ItemsLimit = nullptr;
+ bool Reverse = false;
};
struct TParseReadTableResult : TParseReadTableResultBase {
diff --git a/ydb/core/kqp/runtime/kqp_runtime_impl.h b/ydb/core/kqp/runtime/kqp_runtime_impl.h
index 8520d10fe1..004af2a5d6 100644
--- a/ydb/core/kqp/runtime/kqp_runtime_impl.h
+++ b/ydb/core/kqp/runtime/kqp_runtime_impl.h
@@ -10,19 +10,19 @@ namespace NKqp {
struct TKqpRangePartition {
TKeyDesc::TPartitionRangeInfo Range;
- ui64 ChannelId = std::numeric_limits<ui64>::max();
+ ui64 ChannelId = std::numeric_limits<ui64>::max();
};
TTableId ParseTableId(const NMiniKQL::TRuntimeNode& node);
NUdf::TDataTypeId UnwrapDataTypeFromStruct(const NMiniKQL::TStructType& structType, ui32 index);
-NYql::NDq::IDqOutputConsumer::TPtr CreateOutputRangePartitionConsumer(
+NYql::NDq::IDqOutputConsumer::TPtr CreateOutputRangePartitionConsumer(
TVector<NYql::NDq::IDqOutput::TPtr>&& outputs, TVector<TKqpRangePartition>&& partitions,
- TVector<NUdf::TDataTypeId>&& keyColumnTypes, TVector<ui32>&& keyColumnIndices,
- const NMiniKQL::TTypeEnvironment& typeEnv);
+ TVector<NUdf::TDataTypeId>&& keyColumnTypes, TVector<ui32>&& keyColumnIndices,
+ const NMiniKQL::TTypeEnvironment& typeEnv);
NYql::NDq::IDqOutputConsumer::TPtr CreateKqpApplyEffectsConsumer(NUdf::IApplyContext* applyCtx);
-} // namespace NKqp
-} // namespace NKikimr
+} // namespace NKqp
+} // namespace NKikimr
diff --git a/ydb/core/kqp/runtime/kqp_scan_data.cpp b/ydb/core/kqp/runtime/kqp_scan_data.cpp
index 47f6bcc8ec..c4d8e24196 100644
--- a/ydb/core/kqp/runtime/kqp_scan_data.cpp
+++ b/ydb/core/kqp/runtime/kqp_scan_data.cpp
@@ -249,7 +249,7 @@ std::pair<ui64, ui64> GetUnboxedValueSizeForTests(const NUdf::TUnboxedValue& val
}
TKqpScanComputeContext::TScanData::TScanData(const TTableId& tableId, const TTableRange& range,
- const TSmallVec<TColumn>& columns, const TSmallVec<TColumn>& systemColumns, const TSmallVec<bool>& skipNullKeys)
+ const TSmallVec<TColumn>& columns, const TSmallVec<TColumn>& systemColumns, const TSmallVec<bool>& skipNullKeys)
: TableId(tableId)
, Range(range)
, SkipNullKeys(skipNullKeys)
@@ -257,37 +257,37 @@ TKqpScanComputeContext::TScanData::TScanData(const TTableId& tableId, const TTab
, SystemColumns(systemColumns)
{}
-TKqpScanComputeContext::TScanData::TScanData(const NKikimrTxDataShard::TKqpTransaction_TScanTaskMeta& meta,
- NYql::NDqProto::EDqStatsMode statsMode)
-{
- const auto& tableMeta = meta.GetTable();
- TableId = TTableId(tableMeta.GetTableId().GetOwnerId(), tableMeta.GetTableId().GetTableId(),
- tableMeta.GetSysViewInfo(), tableMeta.GetSchemaVersion());
- TablePath = meta.GetTable().GetTablePath();
-
- std::copy(meta.GetSkipNullKeys().begin(), meta.GetSkipNullKeys().end(), std::back_inserter(SkipNullKeys));
-
- Columns.reserve(meta.GetColumns().size());
- for (const auto& column : meta.GetColumns()) {
- NMiniKQL::TKqpScanComputeContext::TColumn c;
- c.Tag = column.GetId();
- c.Type = column.GetType();
-
- if (!IsSystemColumn(c.Tag)) {
- Columns.emplace_back(std::move(c));
- } else {
- SystemColumns.emplace_back(std::move(c));
- }
- }
-
- if (statsMode >= NYql::NDqProto::DQ_STATS_MODE_BASIC) {
- BasicStats = std::make_unique<TBasicStats>();
- }
- if (Y_UNLIKELY(statsMode >= NYql::NDqProto::DQ_STATS_MODE_PROFILE)) {
- ProfileStats = std::make_unique<TProfileStats>();
- }
-}
-
+TKqpScanComputeContext::TScanData::TScanData(const NKikimrTxDataShard::TKqpTransaction_TScanTaskMeta& meta,
+ NYql::NDqProto::EDqStatsMode statsMode)
+{
+ const auto& tableMeta = meta.GetTable();
+ TableId = TTableId(tableMeta.GetTableId().GetOwnerId(), tableMeta.GetTableId().GetTableId(),
+ tableMeta.GetSysViewInfo(), tableMeta.GetSchemaVersion());
+ TablePath = meta.GetTable().GetTablePath();
+
+ std::copy(meta.GetSkipNullKeys().begin(), meta.GetSkipNullKeys().end(), std::back_inserter(SkipNullKeys));
+
+ Columns.reserve(meta.GetColumns().size());
+ for (const auto& column : meta.GetColumns()) {
+ NMiniKQL::TKqpScanComputeContext::TColumn c;
+ c.Tag = column.GetId();
+ c.Type = column.GetType();
+
+ if (!IsSystemColumn(c.Tag)) {
+ Columns.emplace_back(std::move(c));
+ } else {
+ SystemColumns.emplace_back(std::move(c));
+ }
+ }
+
+ if (statsMode >= NYql::NDqProto::DQ_STATS_MODE_BASIC) {
+ BasicStats = std::make_unique<TBasicStats>();
+ }
+ if (Y_UNLIKELY(statsMode >= NYql::NDqProto::DQ_STATS_MODE_PROFILE)) {
+ ProfileStats = std::make_unique<TProfileStats>();
+ }
+}
+
ui64 TKqpScanComputeContext::TScanData::AddRows(const TVector<TOwnedCellVec>& batch, TMaybe<ui64> shardId, const THolderFactory& holderFactory) {
if (Finished || batch.empty()) {
@@ -394,34 +394,34 @@ NUdf::TUnboxedValue TKqpScanComputeContext::TScanData::TakeRow() {
return row;
}
-void TKqpScanComputeContext::AddTableScan(ui32, const TTableId& tableId, const TTableRange& range,
- const TSmallVec<TColumn>& columns, const TSmallVec<TColumn>& systemColumns, const TSmallVec<bool>& skipNullKeys)
+void TKqpScanComputeContext::AddTableScan(ui32, const TTableId& tableId, const TTableRange& range,
+ const TSmallVec<TColumn>& columns, const TSmallVec<TColumn>& systemColumns, const TSmallVec<bool>& skipNullKeys)
+{
+ auto scanData = TKqpScanComputeContext::TScanData(tableId, range, columns, systemColumns, skipNullKeys);
+
+ if (Y_UNLIKELY(StatsMode >= NYql::NDqProto::DQ_STATS_MODE_BASIC)) {
+ scanData.BasicStats = std::make_unique<TScanData::TBasicStats>();
+ }
+
+ if (Y_UNLIKELY(StatsMode >= NYql::NDqProto::DQ_STATS_MODE_PROFILE)) {
+ scanData.ProfileStats = std::make_unique<TScanData::TProfileStats>();
+ }
+
+ auto result = Scans.emplace(0, std::move(scanData));
+ Y_ENSURE(result.second);
+}
+
+void TKqpScanComputeContext::AddTableScan(ui32, const NKikimrTxDataShard::TKqpTransaction_TScanTaskMeta& meta,
+ NYql::NDqProto::EDqStatsMode statsMode)
{
- auto scanData = TKqpScanComputeContext::TScanData(tableId, range, columns, systemColumns, skipNullKeys);
-
- if (Y_UNLIKELY(StatsMode >= NYql::NDqProto::DQ_STATS_MODE_BASIC)) {
- scanData.BasicStats = std::make_unique<TScanData::TBasicStats>();
- }
-
- if (Y_UNLIKELY(StatsMode >= NYql::NDqProto::DQ_STATS_MODE_PROFILE)) {
- scanData.ProfileStats = std::make_unique<TScanData::TProfileStats>();
- }
-
- auto result = Scans.emplace(0, std::move(scanData));
+ auto scanData = TKqpScanComputeContext::TScanData(meta, statsMode);
+
+ auto result = Scans.emplace(0, std::move(scanData));
Y_ENSURE(result.second);
}
-void TKqpScanComputeContext::AddTableScan(ui32, const NKikimrTxDataShard::TKqpTransaction_TScanTaskMeta& meta,
- NYql::NDqProto::EDqStatsMode statsMode)
-{
- auto scanData = TKqpScanComputeContext::TScanData(meta, statsMode);
-
- auto result = Scans.emplace(0, std::move(scanData));
- Y_ENSURE(result.second);
-}
-
-TKqpScanComputeContext::TScanData& TKqpScanComputeContext::GetTableScan(ui32) {
- auto scanData = Scans.FindPtr(0);
+TKqpScanComputeContext::TScanData& TKqpScanComputeContext::GetTableScan(ui32) {
+ auto scanData = Scans.FindPtr(0);
Y_ENSURE(scanData);
return *scanData;
@@ -431,13 +431,13 @@ TMap<ui32, TKqpScanComputeContext::TScanData>& TKqpScanComputeContext::GetTableS
return Scans;
}
-const TMap<ui32, TKqpScanComputeContext::TScanData>& TKqpScanComputeContext::GetTableScans() const {
- return Scans;
-}
-
-TIntrusivePtr<IKqpTableReader> TKqpScanComputeContext::ReadTable(ui32) const {
- auto scanData = Scans.FindPtr(0);
- Y_ENSURE(scanData);
+const TMap<ui32, TKqpScanComputeContext::TScanData>& TKqpScanComputeContext::GetTableScans() const {
+ return Scans;
+}
+
+TIntrusivePtr<IKqpTableReader> TKqpScanComputeContext::ReadTable(ui32) const {
+ auto scanData = Scans.FindPtr(0);
+ Y_ENSURE(scanData);
Y_ENSURE(scanData->TableReader);
return scanData->TableReader;
@@ -487,5 +487,5 @@ TIntrusivePtr<IKqpTableReader> CreateKqpTableReader(TKqpScanComputeContext::TSca
return MakeIntrusive<TKqpTableReader>(scanData);
}
-} // namespace NMiniKQL
+} // namespace NMiniKQL
} // namespace NKikimr
diff --git a/ydb/core/kqp/runtime/kqp_scan_data.h b/ydb/core/kqp/runtime/kqp_scan_data.h
index e0ae6364b7..c8e7d82b32 100644
--- a/ydb/core/kqp/runtime/kqp_scan_data.h
+++ b/ydb/core/kqp/runtime/kqp_scan_data.h
@@ -15,19 +15,19 @@
#include <contrib/libs/apache/arrow/cpp/src/arrow/api.h>
-namespace NKikimrTxDataShard {
- class TKqpTransaction_TScanTaskMeta;
-}
-
+namespace NKikimrTxDataShard {
+ class TKqpTransaction_TScanTaskMeta;
+}
+
namespace NKikimr {
namespace NMiniKQL {
std::pair<ui64, ui64> GetUnboxedValueSizeForTests(const NUdf::TUnboxedValue& value, NScheme::TTypeId type);
class IKqpTableReader : public TSimpleRefCount<IKqpTableReader> {
-public:
+public:
virtual ~IKqpTableReader() = default;
-
+
virtual NUdf::EFetchStatus Next(NUdf::TUnboxedValue& result) = 0;
virtual EFetchResult Next(NUdf::TUnboxedValue* const* output) = 0;
};
@@ -39,9 +39,9 @@ public:
TScanData(TScanData&&) = default; // needed to create TMap<ui32, TScanData> Scans
TScanData(const TTableId& tableId, const TTableRange& range, const TSmallVec<TColumn>& columns,
const TSmallVec<TColumn>& systemColumns, const TSmallVec<bool>& skipNullKeys);
-
- TScanData(const NKikimrTxDataShard::TKqpTransaction_TScanTaskMeta& meta, NYql::NDqProto::EDqStatsMode statsMode);
-
+
+ TScanData(const NKikimrTxDataShard::TKqpTransaction_TScanTaskMeta& meta, NYql::NDqProto::EDqStatsMode statsMode);
+
~TScanData() {
TString msg = TStringBuilder() << "Buffer in TScanData was not cleared, data is leaking: "
<< "Queue of UnboxedValues must be emptied under allocator using Clear method, but has " << RowBatches.size() << " elements!";
@@ -58,9 +58,9 @@ public:
const TSmallVec<TColumn>& GetSystemColumns() const {
return SystemColumns;
}
-
+
ui64 AddRows(const TVector<TOwnedCellVec>& batch, TMaybe<ui64> shardId, const THolderFactory& holderFactory);
-
+
ui64 AddRows(const arrow::RecordBatch& batch, TMaybe<ui64> shardId, const THolderFactory& holderFactory);
NUdf::TUnboxedValue TakeRow();
@@ -68,59 +68,59 @@ public:
bool IsEmpty() const {
return RowBatches.empty();
}
-
+
ui64 GetStoredBytes() const {
return StoredBytes;
}
-
+
void Finish() {
Finished = true;
}
-
+
bool IsFinished() const {
return Finished;
}
-
+
void Clear() {
RowBatches.clear();
}
public:
- ui64 TaskId = 0;
+ ui64 TaskId = 0;
TTableId TableId;
- TString TablePath;
+ TString TablePath;
TSerializedTableRange Range;
- TSmallVec<bool> SkipNullKeys;
-
- // shared with actor via TableReader
+ TSmallVec<bool> SkipNullKeys;
+
+ // shared with actor via TableReader
TIntrusivePtr<IKqpTableReader> TableReader;
-
- struct TBasicStats {
- size_t Rows = 0;
- size_t Bytes = 0;
+
+ struct TBasicStats {
+ size_t Rows = 0;
+ size_t Bytes = 0;
ui32 AffectedShards = 0;
- };
-
- struct TProfileStats {
- size_t PageFaults = 0;
- size_t Messages = 0;
- size_t MessagesByPageFault = 0;
-
- // Produce statistics
- TDuration ScanCpuTime;
- TDuration ScanWaitTime; // IScan waiting data time
- };
-
- std::unique_ptr<TBasicStats> BasicStats;
- std::unique_ptr<TProfileStats> ProfileStats;
-
+ };
+
+ struct TProfileStats {
+ size_t PageFaults = 0;
+ size_t Messages = 0;
+ size_t MessagesByPageFault = 0;
+
+ // Produce statistics
+ TDuration ScanCpuTime;
+ TDuration ScanWaitTime; // IScan waiting data time
+ };
+
+ std::unique_ptr<TBasicStats> BasicStats;
+ std::unique_ptr<TProfileStats> ProfileStats;
+
private:
struct RowBatch {
TUnboxedValueVector Batch;
TMaybe<ui64> ShardId;
ui64 CurrentRow = 0;
};
-
+
TSmallVec<TColumn> Columns;
TSmallVec<TColumn> SystemColumns;
TQueue<RowBatch> RowBatches;
@@ -129,20 +129,20 @@ public:
};
public:
- explicit TKqpScanComputeContext(NYql::NDqProto::EDqStatsMode statsMode)
- : StatsMode(statsMode) {}
+ explicit TKqpScanComputeContext(NYql::NDqProto::EDqStatsMode statsMode)
+ : StatsMode(statsMode) {}
TIntrusivePtr<IKqpTableReader> ReadTable(ui32 callableId) const;
void AddTableScan(ui32 callableId, const TTableId& tableId, const TTableRange& range,
- const TSmallVec<TColumn>& columns, const TSmallVec<TColumn>& systemColumns, const TSmallVec<bool>& skipNullKeys);
+ const TSmallVec<TColumn>& columns, const TSmallVec<TColumn>& systemColumns, const TSmallVec<bool>& skipNullKeys);
+
+ void AddTableScan(ui32 callableId, const NKikimrTxDataShard::TKqpTransaction_TScanTaskMeta& meta,
+ NYql::NDqProto::EDqStatsMode statsMode);
- void AddTableScan(ui32 callableId, const NKikimrTxDataShard::TKqpTransaction_TScanTaskMeta& meta,
- NYql::NDqProto::EDqStatsMode statsMode);
-
TScanData& GetTableScan(ui32 callableId);
TMap<ui32, TScanData>& GetTableScans();
- const TMap<ui32, TScanData>& GetTableScans() const;
+ const TMap<ui32, TScanData>& GetTableScans() const;
void Clear() {
for (auto& scan: Scans) {
@@ -152,11 +152,11 @@ public:
}
private:
- const NYql::NDqProto::EDqStatsMode StatsMode;
+ const NYql::NDqProto::EDqStatsMode StatsMode;
TMap<ui32, TScanData> Scans;
};
TIntrusivePtr<IKqpTableReader> CreateKqpTableReader(TKqpScanComputeContext::TScanData& scanData);
-} // namespace NMiniKQL
+} // namespace NMiniKQL
} // namespace NKikimr
diff --git a/ydb/core/kqp/runtime/kqp_spilling.h b/ydb/core/kqp/runtime/kqp_spilling.h
index b17e531365..d0290dfd15 100644
--- a/ydb/core/kqp/runtime/kqp_spilling.h
+++ b/ydb/core/kqp/runtime/kqp_spilling.h
@@ -1,52 +1,52 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/kqp/common/kqp_common.h>
#include <ydb/core/protos/config.pb.h>
-
-#include <util/generic/buffer.h>
-
-namespace NKikimr::NKqp {
-
-struct TEvKqpSpilling {
- struct TEvWrite : public TEventLocal<TEvWrite, TKqpSpillingEvents::EvWrite> {
- ui64 BlobId;
- TBuffer Blob;
- TMaybe<TDuration> Timeout;
-
- TEvWrite(ui64 blobId, TBuffer&& blob, TMaybe<TDuration> timeout = {})
- : BlobId(blobId), Blob(std::move(blob)), Timeout(timeout) {}
- };
-
- struct TEvWriteResult : public TEventLocal<TEvWriteResult, TKqpSpillingEvents::EvWriteResult> {
- ui64 BlobId;
-
- TEvWriteResult(ui64 blobId)
- : BlobId(blobId) {}
- };
-
- struct TEvRead : public TEventLocal<TEvRead, TKqpSpillingEvents::EvRead> {
- ui64 BlobId;
- bool RemoveBlob;
- TMaybe<TDuration> Timeout;
-
- TEvRead(ui64 blobId, bool removeBlob = false, TMaybe<TDuration> timeout = {})
- : BlobId(blobId), RemoveBlob(removeBlob), Timeout(timeout) {}
- };
-
- struct TEvReadResult : public TEventLocal<TEvReadResult, TKqpSpillingEvents::EvReadResult> {
- ui64 BlobId;
- TBuffer Blob;
-
- TEvReadResult(ui64 blobId, TBuffer&& blob)
- : BlobId(blobId), Blob(std::move(blob)) {}
- };
-
- struct TEvError : public TEventLocal<TEvError, TKqpSpillingEvents::EvError> {
- TString Message;
-
- TEvError(const TString& message)
- : Message(message) {}
- };
-};
-
-} // namespace NKikimr::NKqp
+
+#include <util/generic/buffer.h>
+
+namespace NKikimr::NKqp {
+
+struct TEvKqpSpilling {
+ struct TEvWrite : public TEventLocal<TEvWrite, TKqpSpillingEvents::EvWrite> {
+ ui64 BlobId;
+ TBuffer Blob;
+ TMaybe<TDuration> Timeout;
+
+ TEvWrite(ui64 blobId, TBuffer&& blob, TMaybe<TDuration> timeout = {})
+ : BlobId(blobId), Blob(std::move(blob)), Timeout(timeout) {}
+ };
+
+ struct TEvWriteResult : public TEventLocal<TEvWriteResult, TKqpSpillingEvents::EvWriteResult> {
+ ui64 BlobId;
+
+ TEvWriteResult(ui64 blobId)
+ : BlobId(blobId) {}
+ };
+
+ struct TEvRead : public TEventLocal<TEvRead, TKqpSpillingEvents::EvRead> {
+ ui64 BlobId;
+ bool RemoveBlob;
+ TMaybe<TDuration> Timeout;
+
+ TEvRead(ui64 blobId, bool removeBlob = false, TMaybe<TDuration> timeout = {})
+ : BlobId(blobId), RemoveBlob(removeBlob), Timeout(timeout) {}
+ };
+
+ struct TEvReadResult : public TEventLocal<TEvReadResult, TKqpSpillingEvents::EvReadResult> {
+ ui64 BlobId;
+ TBuffer Blob;
+
+ TEvReadResult(ui64 blobId, TBuffer&& blob)
+ : BlobId(blobId), Blob(std::move(blob)) {}
+ };
+
+ struct TEvError : public TEventLocal<TEvError, TKqpSpillingEvents::EvError> {
+ TString Message;
+
+ TEvError(const TString& message)
+ : Message(message) {}
+ };
+};
+
+} // namespace NKikimr::NKqp
diff --git a/ydb/core/kqp/runtime/kqp_spilling_file.cpp b/ydb/core/kqp/runtime/kqp_spilling_file.cpp
index 0b37a20f26..0dd0a4f7a3 100644
--- a/ydb/core/kqp/runtime/kqp_spilling_file.cpp
+++ b/ydb/core/kqp/runtime/kqp_spilling_file.cpp
@@ -1,965 +1,965 @@
-#include "kqp_spilling.h"
-#include "kqp_spilling_file.h"
-
+#include "kqp_spilling.h"
+#include "kqp_spilling_file.h"
+
#include <ydb/core/actorlib_impl/long_timer.h>
#include <ydb/core/base/appdata.h>
#include <ydb/core/kqp/kqp.h>
#include <ydb/core/mon/mon.h>
-
+
#include <ydb/library/yql/utils/yql_panic.h>
-
-#include <library/cpp/actors/core/actor_bootstrapped.h>
-#include <library/cpp/actors/core/hfunc.h>
-#include <library/cpp/actors/core/log.h>
-#include <library/cpp/monlib/service/pages/templates.h>
-
-#include <util/folder/path.h>
-#include <util/stream/file.h>
-#include <util/thread/pool.h>
-
-namespace NKikimr::NKqp {
-
-using namespace NActors;
-
-namespace {
-
-// Read, write, and execute by owner only
-constexpr int DIR_MODE = S_IRWXU;
-
-#define LOG_D(s) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_BLOBS_STORAGE, s)
-#define LOG_I(s) LOG_INFO_S(*TlsActivationContext, NKikimrServices::KQP_BLOBS_STORAGE, s)
-#define LOG_E(s) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_BLOBS_STORAGE, s)
-#define LOG_C(s) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::KQP_BLOBS_STORAGE, s)
-
-#define A_LOG_D(s) LOG_DEBUG_S(*ActorSystem, NKikimrServices::KQP_BLOBS_STORAGE, s)
-#define A_LOG_E(s) LOG_ERROR_S(*ActorSystem, NKikimrServices::KQP_BLOBS_STORAGE, s)
-
-#define REMOVE_FILES 1
-
-// Local File Storage Events
-struct TEvKqpSpillingLocalFile {
- enum EEv {
- EvOpenFile = EventSpaceBegin(TKikimrEvents::ES_PRIVATE),
- EvCloseFile,
-
- LastEvent = EvCloseFile
- };
-
- struct TEvOpenFile : public TEventLocal<TEvOpenFile, EvOpenFile> {
- ui64 TxId;
- TString Description; // for viewer & logs only
- bool RemoveBlobsAfterRead;
-
- TEvOpenFile(ui64 txId, const TString& description, bool removeBlobsAfterRead)
- : TxId(txId), Description(description), RemoveBlobsAfterRead(removeBlobsAfterRead) {}
- };
-
- struct TEvCloseFile : public TEventLocal<TEvCloseFile, EvCloseFile> {
- TMaybe<TString> Error;
-
- TEvCloseFile() = default;
- TEvCloseFile(const TString& error)
- : Error(error) {}
- };
-};
-
-
-// It is a simple proxy between client and spilling service with one feature --
-// it provides human-readable description for logs and viewer.
-class TKqpLocalFileSpillingActor : public TActorBootstrapped<TKqpLocalFileSpillingActor> {
-public:
+
+#include <library/cpp/actors/core/actor_bootstrapped.h>
+#include <library/cpp/actors/core/hfunc.h>
+#include <library/cpp/actors/core/log.h>
+#include <library/cpp/monlib/service/pages/templates.h>
+
+#include <util/folder/path.h>
+#include <util/stream/file.h>
+#include <util/thread/pool.h>
+
+namespace NKikimr::NKqp {
+
+using namespace NActors;
+
+namespace {
+
+// Read, write, and execute by owner only
+constexpr int DIR_MODE = S_IRWXU;
+
+#define LOG_D(s) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_BLOBS_STORAGE, s)
+#define LOG_I(s) LOG_INFO_S(*TlsActivationContext, NKikimrServices::KQP_BLOBS_STORAGE, s)
+#define LOG_E(s) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_BLOBS_STORAGE, s)
+#define LOG_C(s) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::KQP_BLOBS_STORAGE, s)
+
+#define A_LOG_D(s) LOG_DEBUG_S(*ActorSystem, NKikimrServices::KQP_BLOBS_STORAGE, s)
+#define A_LOG_E(s) LOG_ERROR_S(*ActorSystem, NKikimrServices::KQP_BLOBS_STORAGE, s)
+
+#define REMOVE_FILES 1
+
+// Local File Storage Events
+struct TEvKqpSpillingLocalFile {
+ enum EEv {
+ EvOpenFile = EventSpaceBegin(TKikimrEvents::ES_PRIVATE),
+ EvCloseFile,
+
+ LastEvent = EvCloseFile
+ };
+
+ struct TEvOpenFile : public TEventLocal<TEvOpenFile, EvOpenFile> {
+ ui64 TxId;
+ TString Description; // for viewer & logs only
+ bool RemoveBlobsAfterRead;
+
+ TEvOpenFile(ui64 txId, const TString& description, bool removeBlobsAfterRead)
+ : TxId(txId), Description(description), RemoveBlobsAfterRead(removeBlobsAfterRead) {}
+ };
+
+ struct TEvCloseFile : public TEventLocal<TEvCloseFile, EvCloseFile> {
+ TMaybe<TString> Error;
+
+ TEvCloseFile() = default;
+ TEvCloseFile(const TString& error)
+ : Error(error) {}
+ };
+};
+
+
+// It is a simple proxy between client and spilling service with one feature --
+// it provides human-readable description for logs and viewer.
+class TKqpLocalFileSpillingActor : public TActorBootstrapped<TKqpLocalFileSpillingActor> {
+public:
TKqpLocalFileSpillingActor(ui64 txId, const TString& details, const TActorId& client, bool removeBlobsAfterRead)
- : TxId(txId)
- , Details(details)
- , ClientActorId(client)
- , RemoveBlobsAfterRead(removeBlobsAfterRead) {}
-
- void Bootstrap() {
- ServiceActorId = MakeKqpLocalFileSpillingServiceID(SelfId().NodeId());
- YQL_ENSURE(ServiceActorId);
-
- LOG_D("Register LocalFileSpillingActor " << SelfId() << " at service " << ServiceActorId);
- Send(ServiceActorId, new TEvKqpSpillingLocalFile::TEvOpenFile(TxId, Details, RemoveBlobsAfterRead));
-
- Become(&TKqpLocalFileSpillingActor::WorkState);
- }
-
+ : TxId(txId)
+ , Details(details)
+ , ClientActorId(client)
+ , RemoveBlobsAfterRead(removeBlobsAfterRead) {}
+
+ void Bootstrap() {
+ ServiceActorId = MakeKqpLocalFileSpillingServiceID(SelfId().NodeId());
+ YQL_ENSURE(ServiceActorId);
+
+ LOG_D("Register LocalFileSpillingActor " << SelfId() << " at service " << ServiceActorId);
+ Send(ServiceActorId, new TEvKqpSpillingLocalFile::TEvOpenFile(TxId, Details, RemoveBlobsAfterRead));
+
+ Become(&TKqpLocalFileSpillingActor::WorkState);
+ }
+
static constexpr char ActorName[] = "KQP_LOCAL_FILE_SPILLING";
-private:
- STRICT_STFUNC(WorkState,
- hFunc(TEvKqpSpilling::TEvWrite, HandleWork)
- hFunc(TEvKqpSpilling::TEvWriteResult, HandleWork)
- hFunc(TEvKqpSpilling::TEvRead, HandleWork)
- hFunc(TEvKqpSpilling::TEvReadResult, HandleWork)
- hFunc(TEvKqpSpilling::TEvError, HandleWork)
- hFunc(TEvents::TEvPoison, HandleWork)
- );
-
- void HandleWork(TEvKqpSpilling::TEvWrite::TPtr& ev) {
- ValidateSender(ev->Sender);
-
- Send(ServiceActorId, ev->Release().Release());
- }
-
- void HandleWork(TEvKqpSpilling::TEvWriteResult::TPtr& ev) {
- if (!Send(ClientActorId, ev->Release().Release())) {
- ClientLost();
- }
- }
-
- void HandleWork(TEvKqpSpilling::TEvRead::TPtr& ev) {
- ValidateSender(ev->Sender);
-
- Send(ServiceActorId, ev->Release().Release());
- }
-
- void HandleWork(TEvKqpSpilling::TEvReadResult::TPtr& ev) {
- if (!Send(ClientActorId, ev->Release().Release())) {
- ClientLost();
- }
- }
-
- void HandleWork(TEvKqpSpilling::TEvError::TPtr& ev) {
- Send(ClientActorId, ev->Release().Release());
- }
-
- void HandleWork(TEvents::TEvPoison::TPtr& ev) {
- ValidateSender(ev->Sender);
-
- Send(ServiceActorId, new TEvKqpSpillingLocalFile::TEvCloseFile);
- PassAway();
- }
-
-private:
+private:
+ STRICT_STFUNC(WorkState,
+ hFunc(TEvKqpSpilling::TEvWrite, HandleWork)
+ hFunc(TEvKqpSpilling::TEvWriteResult, HandleWork)
+ hFunc(TEvKqpSpilling::TEvRead, HandleWork)
+ hFunc(TEvKqpSpilling::TEvReadResult, HandleWork)
+ hFunc(TEvKqpSpilling::TEvError, HandleWork)
+ hFunc(TEvents::TEvPoison, HandleWork)
+ );
+
+ void HandleWork(TEvKqpSpilling::TEvWrite::TPtr& ev) {
+ ValidateSender(ev->Sender);
+
+ Send(ServiceActorId, ev->Release().Release());
+ }
+
+ void HandleWork(TEvKqpSpilling::TEvWriteResult::TPtr& ev) {
+ if (!Send(ClientActorId, ev->Release().Release())) {
+ ClientLost();
+ }
+ }
+
+ void HandleWork(TEvKqpSpilling::TEvRead::TPtr& ev) {
+ ValidateSender(ev->Sender);
+
+ Send(ServiceActorId, ev->Release().Release());
+ }
+
+ void HandleWork(TEvKqpSpilling::TEvReadResult::TPtr& ev) {
+ if (!Send(ClientActorId, ev->Release().Release())) {
+ ClientLost();
+ }
+ }
+
+ void HandleWork(TEvKqpSpilling::TEvError::TPtr& ev) {
+ Send(ClientActorId, ev->Release().Release());
+ }
+
+ void HandleWork(TEvents::TEvPoison::TPtr& ev) {
+ ValidateSender(ev->Sender);
+
+ Send(ServiceActorId, new TEvKqpSpillingLocalFile::TEvCloseFile);
+ PassAway();
+ }
+
+private:
void ValidateSender(const TActorId& sender) {
- YQL_ENSURE(ClientActorId == sender, "" << ClientActorId << " != " << sender);
- }
-
- void ClientLost() {
- Send(ServiceActorId, new TEvKqpSpillingLocalFile::TEvCloseFile("Client lost"));
- PassAway();
- }
-
-private:
- const ui64 TxId;
- const TString Details;
+ YQL_ENSURE(ClientActorId == sender, "" << ClientActorId << " != " << sender);
+ }
+
+ void ClientLost() {
+ Send(ServiceActorId, new TEvKqpSpillingLocalFile::TEvCloseFile("Client lost"));
+ PassAway();
+ }
+
+private:
+ const ui64 TxId;
+ const TString Details;
const TActorId ClientActorId;
- const bool RemoveBlobsAfterRead;
+ const bool RemoveBlobsAfterRead;
TActorId ServiceActorId;
-};
-
-class TKqpLocalFileSpillingService : public TActorBootstrapped<TKqpLocalFileSpillingService> {
-private:
- struct TEvPrivate {
- enum EEv {
- EvCloseFileResponse = TEvKqpSpillingLocalFile::EEv::LastEvent + 1,
- EvWriteFileResponse,
- EvReadFileResponse,
-
- LastEvent
- };
-
- static_assert(EEv::LastEvent - EventSpaceBegin(TKikimrEvents::ES_PRIVATE) < 16);
-
- struct TEvCloseFileResponse : public TEventLocal<TEvCloseFileResponse, EvCloseFileResponse> {
+};
+
+class TKqpLocalFileSpillingService : public TActorBootstrapped<TKqpLocalFileSpillingService> {
+private:
+ struct TEvPrivate {
+ enum EEv {
+ EvCloseFileResponse = TEvKqpSpillingLocalFile::EEv::LastEvent + 1,
+ EvWriteFileResponse,
+ EvReadFileResponse,
+
+ LastEvent
+ };
+
+ static_assert(EEv::LastEvent - EventSpaceBegin(TKikimrEvents::ES_PRIVATE) < 16);
+
+ struct TEvCloseFileResponse : public TEventLocal<TEvCloseFileResponse, EvCloseFileResponse> {
TActorId Client;
- TDuration WaitTime;
- TDuration WorkTime;
- };
-
- struct TEvWriteFileResponse : public TEventLocal<TEvWriteFileResponse, EvWriteFileResponse> {
+ TDuration WaitTime;
+ TDuration WorkTime;
+ };
+
+ struct TEvWriteFileResponse : public TEventLocal<TEvWriteFileResponse, EvWriteFileResponse> {
TActorId Client;
- TDuration WaitTime;
- TDuration WorkTime;
- ui64 BlobId = 0;
- THolder<TFileHandle> NewFileHandle;
- TMaybe<TString> Error;
- };
-
- struct TEvReadFileResponse : public TEventLocal<TEvReadFileResponse, EvReadFileResponse> {
+ TDuration WaitTime;
+ TDuration WorkTime;
+ ui64 BlobId = 0;
+ THolder<TFileHandle> NewFileHandle;
+ TMaybe<TString> Error;
+ };
+
+ struct TEvReadFileResponse : public TEventLocal<TEvReadFileResponse, EvReadFileResponse> {
TActorId Client;
- TDuration WaitTime;
- TDuration WorkTime;
- ui64 BlobId = 0;
- TBuffer Blob;
- bool Removed = false;
- TMaybe<TString> Error;
- };
- };
-
- struct TFileDesc;
+ TDuration WaitTime;
+ TDuration WorkTime;
+ ui64 BlobId = 0;
+ TBuffer Blob;
+ bool Removed = false;
+ TMaybe<TString> Error;
+ };
+ };
+
+ struct TFileDesc;
using TFilesIt = __yhashtable_iterator<std::pair<const TActorId, TFileDesc>>;
-
-public:
- TKqpLocalFileSpillingService(const NKikimrConfig::TTableServiceConfig::TSpillingServiceConfig::TLocalFileConfig& config,
- TIntrusivePtr<TKqpCounters> counters)
- : Config(config)
- , Counters(counters)
- {
- IoThreadPool = CreateThreadPool(Config.GetIoThreadPool().GetWorkersCount(),
- Config.GetIoThreadPool().GetQueueSize(), IThreadPool::TParams().SetThreadNamePrefix("KqpSpilling"));
- }
-
- void Bootstrap() {
- Root = Config.GetRoot();
- Root /= (TStringBuilder() << "node_" << SelfId().NodeId());
-
- LOG_I("Init KQP local file spilling service at " << Root << ", actor: " << SelfId());
-
- try {
- if (Root.IsSymlink()) {
- throw TIoException() << Root << " is a symlink, can not start Spilling Service";
- }
- Root.ForceDelete();
- Root.MkDirs(DIR_MODE);
- } catch (...) {
- LOG_E(CurrentExceptionMessage());
- Become(&TKqpLocalFileSpillingService::BrokenState);
- return;
- }
-
- NActors::TMon* mon = AppData()->Mon;
- if (mon) {
- NMonitoring::TIndexMonPage* actorsMonPage = mon->RegisterIndexPage("actors", "Actors");
- mon->RegisterActorPage(actorsMonPage, "kqp_spilling_file", "KQP Local File Spilling Service", false,
- TlsActivationContext->ExecutorThread.ActorSystem, SelfId());
- }
-
- Become(&TKqpLocalFileSpillingService::WorkState);
- }
-
+
+public:
+ TKqpLocalFileSpillingService(const NKikimrConfig::TTableServiceConfig::TSpillingServiceConfig::TLocalFileConfig& config,
+ TIntrusivePtr<TKqpCounters> counters)
+ : Config(config)
+ , Counters(counters)
+ {
+ IoThreadPool = CreateThreadPool(Config.GetIoThreadPool().GetWorkersCount(),
+ Config.GetIoThreadPool().GetQueueSize(), IThreadPool::TParams().SetThreadNamePrefix("KqpSpilling"));
+ }
+
+ void Bootstrap() {
+ Root = Config.GetRoot();
+ Root /= (TStringBuilder() << "node_" << SelfId().NodeId());
+
+ LOG_I("Init KQP local file spilling service at " << Root << ", actor: " << SelfId());
+
+ try {
+ if (Root.IsSymlink()) {
+ throw TIoException() << Root << " is a symlink, can not start Spilling Service";
+ }
+ Root.ForceDelete();
+ Root.MkDirs(DIR_MODE);
+ } catch (...) {
+ LOG_E(CurrentExceptionMessage());
+ Become(&TKqpLocalFileSpillingService::BrokenState);
+ return;
+ }
+
+ NActors::TMon* mon = AppData()->Mon;
+ if (mon) {
+ NMonitoring::TIndexMonPage* actorsMonPage = mon->RegisterIndexPage("actors", "Actors");
+ mon->RegisterActorPage(actorsMonPage, "kqp_spilling_file", "KQP Local File Spilling Service", false,
+ TlsActivationContext->ExecutorThread.ActorSystem, SelfId());
+ }
+
+ Become(&TKqpLocalFileSpillingService::WorkState);
+ }
+
static constexpr char ActorName[] = "KQP_LOCAL_FILE_SPILLING_SERVICE";
-protected:
- void PassAway() override {
- IoThreadPool->Stop();
- IActor::PassAway();
- }
-
-private:
- STATEFN(BrokenState) {
- switch (ev->GetTypeRewrite()) {
- case TEvKqpSpillingLocalFile::TEvOpenFile::EventType:
- case TEvKqpSpillingLocalFile::TEvCloseFile::EventType:
- case TEvKqpSpilling::TEvWrite::EventType:
- case TEvKqpSpilling::TEvRead::EventType: {
- HandleBroken(ev->Sender);
- break;
- }
- hFunc(NMon::TEvHttpInfo, HandleBroken);
- cFunc(TEvents::TEvPoison::EventType, PassAway);
- default:
- Y_VERIFY_DEBUG(false, "%s: unexpected message type 0x%08" PRIx32, __func__, ev->GetTypeRewrite());
- }
- }
-
- void HandleBroken(const TActorId& from) {
- LOG_E("Service is broken, send error to client " << from);
- Send(from, new TEvKqpSpilling::TEvError("Service not started"));
- }
-
- void HandleBroken(NMon::TEvHttpInfo::TPtr& ev) {
- Send(ev->Sender, new NMon::TEvHttpInfoRes("<html><h2>Service is not started due to IO error</h2></html>"));
- }
-
-private:
- STRICT_STFUNC(WorkState,
- hFunc(TEvKqpSpillingLocalFile::TEvOpenFile, HandleWork)
- hFunc(TEvKqpSpillingLocalFile::TEvCloseFile, HandleWork)
- hFunc(TEvPrivate::TEvCloseFileResponse, HandleWork)
- hFunc(TEvKqpSpilling::TEvWrite, HandleWork)
- hFunc(TEvPrivate::TEvWriteFileResponse, HandleWork)
- hFunc(TEvKqpSpilling::TEvRead, HandleWork)
- hFunc(TEvPrivate::TEvReadFileResponse, HandleWork)
- hFunc(NMon::TEvHttpInfo, HandleWork)
- cFunc(TEvents::TEvPoison::EventType, PassAway)
- );
-
- void HandleWork(TEvKqpSpillingLocalFile::TEvOpenFile::TPtr& ev) {
- auto& msg = *ev->Get();
- LOG_D("[OpenFile] TxId: " << msg.TxId << ", desc: " << msg.Description << ", from: " << ev->Sender
- << ", removeBlobsAfterRead: " << msg.RemoveBlobsAfterRead);
-
- auto it = Files.find(ev->Sender);
- if (it != Files.end()) {
- LOG_E("[OpenFile] Can not open file: already exists. TxId: " << msg.TxId << ", desc: " << msg.Description);
-
- Send(ev->Sender, new TEvKqpSpilling::TEvError("File already exists"));
- return;
- }
-
- auto& fd = Files[ev->Sender];
- fd.TxId = msg.TxId;
- fd.Description = std::move(msg.Description);
- fd.RemoveBlobsAfterRead = msg.RemoveBlobsAfterRead;
- fd.OpenAt = TInstant::Now();
- }
-
- void HandleWork(TEvKqpSpillingLocalFile::TEvCloseFile::TPtr& ev) {
- auto& msg = *ev->Get();
- LOG_D("[CloseFile] from: " << ev->Sender << ", error: " << msg.Error);
-
- auto it = Files.find(ev->Sender);
- if (it == Files.end()) {
- LOG_E("[CloseFile] Can not close file: not found. From: " << ev->Sender << ", error: " << msg.Error);
- return;
- }
-
- if (it->second.CloseAt) {
- LOG_E("[CloseFile] Can not close file: closing. From: " << ev->Sender << ", error: " << msg.Error);
- return;
- }
-
- CloseFile(it, msg.Error);
- }
-
- void CloseFile(TFilesIt it, TMaybe<TString>& error) {
- auto& fd = it->second;
-
- fd.CloseAt = TInstant::Now();
- fd.Error.Swap(error);
-
- if (!fd.PartsList.empty()) {
- auto closeOp = MakeHolder<TCloseFileOp>();
- closeOp->Client = it->first;
- closeOp->Service = SelfId();
- closeOp->ActorSystem = TlsActivationContext->ActorSystem();
- closeOp->FileHandles.reserve(fd.PartsList.size());
- closeOp->FileNames.reserve(fd.PartsList.size());
- for (auto& fp : fd.PartsList) {
- closeOp->FileHandles.emplace_back(std::move(fp.FileHandle));
- closeOp->FileNames.emplace_back(std::move(fp.FileName));
- }
-
- RunOp("CloseFile", std::move(closeOp), fd);
- } else {
- MoveFileToClosed(it);
- }
- }
-
- void HandleWork(TEvPrivate::TEvCloseFileResponse::TPtr& ev) {
- auto& msg = *ev->Get();
- LOG_D("[CloseFileResponse] from: " << msg.Client);
-
- auto it = Files.find(msg.Client);
- if (it == Files.end()) {
- LOG_E("[CloseFileResponse] Can not find file from: " << msg.Client);
- return;
- }
-
- ui64 blobs = 0;
- for (auto& fp : it->second.PartsList) {
- blobs += fp.Blobs.size();
- }
-
- Counters->SpillingStoredBlobs->Sub(blobs);
- Counters->SpillingTotalSpaceUsed->Sub(it->second.TotalSize);
-
- MoveFileToClosed(it);
- }
-
- void HandleWork(TEvKqpSpilling::TEvWrite::TPtr& ev) {
- auto& msg = *ev->Get();
- LOG_D("[Write] from: " << ev->Sender << ", blobId: " << msg.BlobId << ", bytes: " << msg.Blob.size());
-
- auto it = Files.find(ev->Sender);
- if (it == Files.end()) {
- LOG_E("[Write] File not found. "
- << "From: " << ev->Sender << ", blobId: " << msg.BlobId << ", bytes: " << msg.Blob.size());
-
- Send(ev->Sender, new TEvKqpSpilling::TEvError("File not found"));
- return;
- }
-
- auto& fd = it->second;
-
- if (fd.CloseAt) {
- LOG_E("[Write] File already closed. "
- << "From: " << ev->Sender << ", blobId: " << msg.BlobId << ", bytes: " << msg.Blob.size());
-
- Send(ev->Sender, new TEvKqpSpilling::TEvError("File already closed"));
- return;
- }
-
- if (fd.TotalSize + msg.Blob.size() > Config.GetMaxFileSize()) {
- LOG_E("[Write] File size limit exceeded. "
- << "From: " << ev->Sender << ", blobId: " << msg.BlobId << ", bytes: " << msg.Blob.size());
-
- Send(ev->Sender, new TEvKqpSpilling::TEvError("File size limit exceeded"));
-
- Counters->SpillingTooBigFileErrors->Inc();
- return;
- }
-
- if (TotalSize + msg.Blob.size() > Config.GetMaxTotalSize()) {
- LOG_E("[Write] Total size limit exceeded. "
- << "From: " << ev->Sender << ", blobId: " << msg.BlobId << ", bytes: " << msg.Blob.size());
-
- Send(ev->Sender, new TEvKqpSpilling::TEvError("Total size limit exceeded"));
-
- Counters->SpillingNoSpaceErrors->Inc();
- return;
- }
-
- fd.TotalSize += msg.Blob.size();
- TotalSize += msg.Blob.size();
-
- TFileDesc::TFilePart* fp = fd.PartsList.empty() ? nullptr : &fd.PartsList.back();
-
- bool newFile = false;
- if (!fp || (fd.RemoveBlobsAfterRead && (fp->Size + msg.Blob.size() > Config.GetMaxFilePartSize()))) {
- if (!fd.PartsList.empty()) {
- fd.PartsList.back().Last = false;
- }
-
- fd.PartsList.push_back({});
- fp = &fd.PartsList.back();
- fd.Parts.emplace(msg.BlobId, fp);
-
- auto fname = TStringBuilder() << fd.TxId << "_" << fd.Description << "_" << fd.NextPartListIndex++;
- fp->FileName = (Root / fname).GetPath();
-
- LOG_D("[Write] create new FilePart " << fp->FileName);
- newFile = true;
- } else {
- fd.Parts.emplace(msg.BlobId, fp);
- }
-
- auto& blobDesc = fp->Blobs[msg.BlobId];
- blobDesc.Size = msg.Blob.size();
- blobDesc.Offset = fp->Size;
-
- fp->Size += blobDesc.Size;
-
- auto writeOp = MakeHolder<TWriteFileOp>();
- writeOp->Client = ev->Sender;
- writeOp->Service = SelfId();
- writeOp->ActorSystem = TlsActivationContext->ActorSystem();
- writeOp->FileName = fp->FileName;
- writeOp->CreateFile = newFile;
- writeOp->BlobId = msg.BlobId;
- writeOp->Blob = std::move(msg.Blob);
-
- RunOp("Write", std::move(writeOp), fd);
- }
-
- void HandleWork(TEvPrivate::TEvWriteFileResponse::TPtr& ev) {
- auto& msg = *ev->Get();
- LOG_D("[WriteFileResponse] from: " << msg.Client << ", blobId: " << msg.BlobId << ", error: " << msg.Error);
-
- auto it = Files.find(msg.Client);
- if (it == Files.end()) {
- LOG_E("[WriteFileResponse] Can not write file: not found. "
- << "From: " << msg.Client << ", blobId: " << msg.BlobId << ", error: " << msg.Error);
-
- Send(ev->Sender, new TEvKqpSpilling::TEvError("Internal error"));
- return;
- }
-
- auto& fd = it->second;
-
- fd.Error = std::move(msg.Error);
- fd.TotalWaitTime += msg.WaitTime;
- fd.TotalWorkTime += msg.WorkTime;
-
- if (auto* fp = fd.Parts[msg.BlobId]) {
- auto& blobDesc = fp->Blobs[msg.BlobId];
-
- fp->WaitTime += msg.WaitTime;
- fp->WorkTime += msg.WorkTime;
- fp->WriteBytes += blobDesc.Size;
-
- fd.TotalWriteBytes += blobDesc.Size;
-
- Counters->SpillingStoredBlobs->Inc();
- Counters->SpillingTotalSpaceUsed->Add(blobDesc.Size);
-
- if (msg.NewFileHandle) {
- fp->FileHandle.Swap(msg.NewFileHandle);
- }
- } else {
- LOG_E("[WriteFileResponse] File part not found. From: " << msg.Client << ", blobId: " << msg.BlobId);
- if (!fd.Error) {
- fd.Error = "File part not found";
- }
-
- Counters->SpillingIoErrors->Inc();
- }
-
- if (fd.Error) {
- Send(msg.Client, new TEvKqpSpilling::TEvError(*fd.Error));
-
- fd.Ops.clear();
- CloseFile(it, fd.Error);
- return;
- }
-
- Counters->SpillingWriteBlobs->Inc();
-
- Send(msg.Client, new TEvKqpSpilling::TEvWriteResult(msg.BlobId));
- RunNextOp(fd);
- }
-
- void HandleWork(TEvKqpSpilling::TEvRead::TPtr& ev) {
- auto& msg = *ev->Get();
- LOG_D("[Read] from: " << ev->Sender << ", blobId: " << msg.BlobId);
-
- auto it = Files.find(ev->Sender);
- if (it == Files.end()) {
- LOG_E("[Read] Can not read file: not found. From: " << ev->Sender << ", blobId: " << msg.BlobId);
-
- Send(ev->Sender, new TEvKqpSpilling::TEvError("File not found"));
- return;
- }
-
- auto& fd = it->second;
-
- if (fd.CloseAt) {
- LOG_E("[Read] Can not read file: closed. From: " << ev->Sender << ", blobId: " << msg.BlobId);
-
- Send(ev->Sender, new TEvKqpSpilling::TEvError("Closed"));
- return;
- }
-
- auto partIt = fd.Parts.find(msg.BlobId);
- if (partIt == fd.Parts.end()) {
- LOG_E("[Read] Can not read file: part not found. From: " << ev->Sender << ", blobId: " << msg.BlobId);
-
- Send(ev->Sender, new TEvKqpSpilling::TEvError("File part not found"));
-
- fd.Ops.clear();
- TMaybe<TString> err = "Part not found";
- CloseFile(it, err);
- return;
- }
-
- auto& fp = partIt->second;
-
- auto blobIt = fp->Blobs.find(msg.BlobId);
- if (blobIt == fp->Blobs.end()) {
- LOG_E("[Read] Can not read file: blob not found in the part. From: " << ev->Sender << ", blobId: " << msg.BlobId);
-
- Send(ev->Sender, new TEvKqpSpilling::TEvError("Blob not found in the file part"));
-
- fd.Ops.clear();
- TMaybe<TString> err = "Blob not found in the file part";
- CloseFile(it, err);
- return;
- }
-
- blobIt->second.Read = true;
-
- bool remove = false;
- if (fd.RemoveBlobsAfterRead && !fp->Last) {
- remove = true;
- for (auto& b: partIt->second->Blobs) {
- if (!b.second.Read) {
- remove = false;
- break;
- }
- }
- }
-
- auto readOp = MakeHolder<TReadFileOp>();
- readOp->Client = ev->Sender;
- readOp->Service = SelfId();
- readOp->ActorSystem = TlsActivationContext->ActorSystem();
- readOp->FileName = partIt->second->FileName;
- readOp->BlobId = msg.BlobId;
- readOp->Offset = blobIt->second.Offset;
- readOp->Size = blobIt->second.Size;
- if (remove) {
- readOp->RemoveFile = std::move(fp->FileHandle);
- }
-
- RunOp("Read", std::move(readOp), fd);
- }
-
- void HandleWork(TEvPrivate::TEvReadFileResponse::TPtr& ev) {
- auto& msg = *ev->Get();
- LOG_D("[ReadFileResponse] from: " << msg.Client << ", blobId: " << msg.BlobId << ", removed: " << msg.Removed
- << ", error: " << msg.Error);
-
- auto it = Files.find(msg.Client);
- if (it == Files.end()) {
- LOG_E("[ReadFileResponse] Can not read file: not found. "
- << "From: " << msg.Client << ", blobId: " << msg.BlobId << ", error: " << msg.Error);
-
- Send(ev->Sender, new TEvKqpSpilling::TEvError("Internal error"));
- return;
- }
-
- auto& fd = it->second;
-
- fd.Error = std::move(msg.Error);
- fd.TotalWaitTime += msg.WaitTime;
- fd.TotalWorkTime += msg.WorkTime;
- fd.TotalReadBytes += msg.Blob.size();
-
- auto* fp = fd.Parts[msg.BlobId];
- if (fp) {
- fp->Blobs[msg.BlobId].Read = true;
- fp->WaitTime += msg.WaitTime;
- fp->WorkTime += msg.WorkTime;
- fp->ReadBytes += msg.Blob.size();
-
- if (msg.Removed) {
- fd.TotalSize -= fp->Size;
- TotalSize -= fp->Size;
-
- Counters->SpillingTotalSpaceUsed->Sub(fp->Size);
- Counters->SpillingStoredBlobs->Sub(fp->Blobs.size());
-
- fd.Parts.erase(msg.BlobId);
- fd.PartsList.remove_if([fp](const auto& x) { return &x == fp; });
- }
- } else {
- if (!fd.Error) {
- fd.Error = "Internal error";
- }
- }
-
- if (fd.Error) {
- Send(msg.Client, new TEvKqpSpilling::TEvError(*fd.Error));
-
- fd.Ops.clear();
- CloseFile(it, fd.Error);
- return;
- }
-
- Counters->SpillingReadBlobs->Inc();
-
- Send(msg.Client, new TEvKqpSpilling::TEvReadResult(msg.BlobId, std::move(msg.Blob)));
- RunNextOp(fd);
- }
-
- void HandleWork(NMon::TEvHttpInfo::TPtr& ev) {
- TStringStream s;
-
- TMap<ui64, TVector<const TFileDesc*>> byTx;
- for (const auto& fd : Files) {
- byTx[fd.second.TxId].push_back(&fd.second);
- }
-
- HTML(s) {
- H2() { s << "Configuration"; }
- PRE() {
- s << " - Root: " << Config.GetRoot() << Endl;
- s << " - MaxTotalSize: " << Config.GetMaxTotalSize() << Endl;
- s << " - MaxFileSize: " << Config.GetMaxFileSize() << Endl;
- s << " - MaxFilePartSize: " << Config.GetMaxFilePartSize() << Endl;
- s << " - IO thread pool, workers: " << Config.GetIoThreadPool().GetWorkersCount()
- << ", queue: " << Config.GetIoThreadPool().GetQueueSize() << Endl;
- }
-
- H2() { s << "Active files"; }
- PRE() { s << "Used space: " << TotalSize << Endl; }
-
- for (const auto& tx : byTx) {
- H4() { s << "Transaction " << tx.first; }
- s << "Open files:" << Endl;
- UL() {
- for (const auto* fd : tx.second) {
- LI() { s << fd->Description; }
- PRE() {
- s << " Remove blobs after read: " << fd->RemoveBlobsAfterRead << Endl;
- s << " Open at: " << fd->OpenAt << Endl;
- s << " Error: " << fd->Error << Endl;
- s << " Total size: " << fd->TotalSize << Endl;
- s << " Total work time: " << fd->TotalWorkTime << Endl;
- s << " Total wait time: " << fd->TotalWaitTime << Endl;
- s << " Total write bytes: " << fd->TotalWriteBytes << Endl;
- s << " Total read bytes: " << fd->TotalReadBytes << Endl;
- ui64 blobs = 0;
- for (const auto& fp : fd->PartsList) {
- blobs += fp.Blobs.size();
- }
- s << " Total blobs: " << blobs << Endl;
- s << " Total files: " << fd->PartsList.size() << Endl;
- s << " Files:" << Endl;
- for (const auto& fp : fd->PartsList) {
- s << " - " << fp.FileName << Endl;
- s << " Size: " << fp.Size << Endl;
- s << " Blobs: " << fp.Blobs.size() << Endl;
- s << " Write bytes: " << fp.WriteBytes << Endl;
- s << " Read bytes: " << fp.WriteBytes << Endl;
- }
- }
- }
- }
- }
-
- H2() { s << "Last closed files"; }
- UL() {
- for (auto it = ClosedFiles.rbegin(); it != ClosedFiles.rend(); ++it) {
- auto& fd = *it;
- LI() { s << "Transaction: " << fd.TxId << ", " << fd.Description; }
- PRE() {
- s << " Remove blobs after read: " << fd.RemoveBlobsAfterRead << Endl;
- s << " Open at: " << fd.OpenAt << Endl;
- s << " Close at: " << fd.CloseAt << Endl;
- s << " Error: " << fd.Error << Endl;
- s << " Total work time: " << fd.WorkTime << Endl;
- s << " Total wait time: " << fd.WaitTime << Endl;
- s << " Total write bytes: " << fd.WriteBytes << Endl;
- s << " Total read bytes: " << fd.ReadBytes << Endl;
- }
- }
- }
-
- }
-
- Send(ev->Sender, new NMon::TEvHttpInfoRes(s.Str()));
- }
-
-private:
- void RunOp(TStringBuf opName, THolder<IObjectInQueue> op, TFileDesc& fd) {
- if (fd.HasActiveOp) {
- fd.Ops.emplace_back(opName, std::move(op));
- } else {
- fd.HasActiveOp = true;
- // TODO: retry if fails
- IoThreadPool->SafeAddAndOwn(std::move(op));
- }
- }
-
- void RunNextOp(TFileDesc& fd) {
- fd.HasActiveOp = false;
- if (!fd.Ops.empty()) {
- auto op = std::move(fd.Ops.front().second);
- auto opName = fd.Ops.front().first;
- fd.Ops.pop_front();
-
- RunOp(opName, std::move(op), fd);
- }
- }
-
- void MoveFileToClosed(TFilesIt it) {
- TotalSize -= it->second.TotalSize;
- ClosedFiles.emplace_back(TClosedFileDesc(std::move(it->second)));
- while (ClosedFiles.size() > 100) {
- ClosedFiles.pop_front();
- }
- Files.erase(it);
- }
-
-private:
- struct TCloseFileOp : public IObjectInQueue {
+protected:
+ void PassAway() override {
+ IoThreadPool->Stop();
+ IActor::PassAway();
+ }
+
+private:
+ STATEFN(BrokenState) {
+ switch (ev->GetTypeRewrite()) {
+ case TEvKqpSpillingLocalFile::TEvOpenFile::EventType:
+ case TEvKqpSpillingLocalFile::TEvCloseFile::EventType:
+ case TEvKqpSpilling::TEvWrite::EventType:
+ case TEvKqpSpilling::TEvRead::EventType: {
+ HandleBroken(ev->Sender);
+ break;
+ }
+ hFunc(NMon::TEvHttpInfo, HandleBroken);
+ cFunc(TEvents::TEvPoison::EventType, PassAway);
+ default:
+ Y_VERIFY_DEBUG(false, "%s: unexpected message type 0x%08" PRIx32, __func__, ev->GetTypeRewrite());
+ }
+ }
+
+ void HandleBroken(const TActorId& from) {
+ LOG_E("Service is broken, send error to client " << from);
+ Send(from, new TEvKqpSpilling::TEvError("Service not started"));
+ }
+
+ void HandleBroken(NMon::TEvHttpInfo::TPtr& ev) {
+ Send(ev->Sender, new NMon::TEvHttpInfoRes("<html><h2>Service is not started due to IO error</h2></html>"));
+ }
+
+private:
+ STRICT_STFUNC(WorkState,
+ hFunc(TEvKqpSpillingLocalFile::TEvOpenFile, HandleWork)
+ hFunc(TEvKqpSpillingLocalFile::TEvCloseFile, HandleWork)
+ hFunc(TEvPrivate::TEvCloseFileResponse, HandleWork)
+ hFunc(TEvKqpSpilling::TEvWrite, HandleWork)
+ hFunc(TEvPrivate::TEvWriteFileResponse, HandleWork)
+ hFunc(TEvKqpSpilling::TEvRead, HandleWork)
+ hFunc(TEvPrivate::TEvReadFileResponse, HandleWork)
+ hFunc(NMon::TEvHttpInfo, HandleWork)
+ cFunc(TEvents::TEvPoison::EventType, PassAway)
+ );
+
+ void HandleWork(TEvKqpSpillingLocalFile::TEvOpenFile::TPtr& ev) {
+ auto& msg = *ev->Get();
+ LOG_D("[OpenFile] TxId: " << msg.TxId << ", desc: " << msg.Description << ", from: " << ev->Sender
+ << ", removeBlobsAfterRead: " << msg.RemoveBlobsAfterRead);
+
+ auto it = Files.find(ev->Sender);
+ if (it != Files.end()) {
+ LOG_E("[OpenFile] Can not open file: already exists. TxId: " << msg.TxId << ", desc: " << msg.Description);
+
+ Send(ev->Sender, new TEvKqpSpilling::TEvError("File already exists"));
+ return;
+ }
+
+ auto& fd = Files[ev->Sender];
+ fd.TxId = msg.TxId;
+ fd.Description = std::move(msg.Description);
+ fd.RemoveBlobsAfterRead = msg.RemoveBlobsAfterRead;
+ fd.OpenAt = TInstant::Now();
+ }
+
+ void HandleWork(TEvKqpSpillingLocalFile::TEvCloseFile::TPtr& ev) {
+ auto& msg = *ev->Get();
+ LOG_D("[CloseFile] from: " << ev->Sender << ", error: " << msg.Error);
+
+ auto it = Files.find(ev->Sender);
+ if (it == Files.end()) {
+ LOG_E("[CloseFile] Can not close file: not found. From: " << ev->Sender << ", error: " << msg.Error);
+ return;
+ }
+
+ if (it->second.CloseAt) {
+ LOG_E("[CloseFile] Can not close file: closing. From: " << ev->Sender << ", error: " << msg.Error);
+ return;
+ }
+
+ CloseFile(it, msg.Error);
+ }
+
+ void CloseFile(TFilesIt it, TMaybe<TString>& error) {
+ auto& fd = it->second;
+
+ fd.CloseAt = TInstant::Now();
+ fd.Error.Swap(error);
+
+ if (!fd.PartsList.empty()) {
+ auto closeOp = MakeHolder<TCloseFileOp>();
+ closeOp->Client = it->first;
+ closeOp->Service = SelfId();
+ closeOp->ActorSystem = TlsActivationContext->ActorSystem();
+ closeOp->FileHandles.reserve(fd.PartsList.size());
+ closeOp->FileNames.reserve(fd.PartsList.size());
+ for (auto& fp : fd.PartsList) {
+ closeOp->FileHandles.emplace_back(std::move(fp.FileHandle));
+ closeOp->FileNames.emplace_back(std::move(fp.FileName));
+ }
+
+ RunOp("CloseFile", std::move(closeOp), fd);
+ } else {
+ MoveFileToClosed(it);
+ }
+ }
+
+ void HandleWork(TEvPrivate::TEvCloseFileResponse::TPtr& ev) {
+ auto& msg = *ev->Get();
+ LOG_D("[CloseFileResponse] from: " << msg.Client);
+
+ auto it = Files.find(msg.Client);
+ if (it == Files.end()) {
+ LOG_E("[CloseFileResponse] Can not find file from: " << msg.Client);
+ return;
+ }
+
+ ui64 blobs = 0;
+ for (auto& fp : it->second.PartsList) {
+ blobs += fp.Blobs.size();
+ }
+
+ Counters->SpillingStoredBlobs->Sub(blobs);
+ Counters->SpillingTotalSpaceUsed->Sub(it->second.TotalSize);
+
+ MoveFileToClosed(it);
+ }
+
+ void HandleWork(TEvKqpSpilling::TEvWrite::TPtr& ev) {
+ auto& msg = *ev->Get();
+ LOG_D("[Write] from: " << ev->Sender << ", blobId: " << msg.BlobId << ", bytes: " << msg.Blob.size());
+
+ auto it = Files.find(ev->Sender);
+ if (it == Files.end()) {
+ LOG_E("[Write] File not found. "
+ << "From: " << ev->Sender << ", blobId: " << msg.BlobId << ", bytes: " << msg.Blob.size());
+
+ Send(ev->Sender, new TEvKqpSpilling::TEvError("File not found"));
+ return;
+ }
+
+ auto& fd = it->second;
+
+ if (fd.CloseAt) {
+ LOG_E("[Write] File already closed. "
+ << "From: " << ev->Sender << ", blobId: " << msg.BlobId << ", bytes: " << msg.Blob.size());
+
+ Send(ev->Sender, new TEvKqpSpilling::TEvError("File already closed"));
+ return;
+ }
+
+ if (fd.TotalSize + msg.Blob.size() > Config.GetMaxFileSize()) {
+ LOG_E("[Write] File size limit exceeded. "
+ << "From: " << ev->Sender << ", blobId: " << msg.BlobId << ", bytes: " << msg.Blob.size());
+
+ Send(ev->Sender, new TEvKqpSpilling::TEvError("File size limit exceeded"));
+
+ Counters->SpillingTooBigFileErrors->Inc();
+ return;
+ }
+
+ if (TotalSize + msg.Blob.size() > Config.GetMaxTotalSize()) {
+ LOG_E("[Write] Total size limit exceeded. "
+ << "From: " << ev->Sender << ", blobId: " << msg.BlobId << ", bytes: " << msg.Blob.size());
+
+ Send(ev->Sender, new TEvKqpSpilling::TEvError("Total size limit exceeded"));
+
+ Counters->SpillingNoSpaceErrors->Inc();
+ return;
+ }
+
+ fd.TotalSize += msg.Blob.size();
+ TotalSize += msg.Blob.size();
+
+ TFileDesc::TFilePart* fp = fd.PartsList.empty() ? nullptr : &fd.PartsList.back();
+
+ bool newFile = false;
+ if (!fp || (fd.RemoveBlobsAfterRead && (fp->Size + msg.Blob.size() > Config.GetMaxFilePartSize()))) {
+ if (!fd.PartsList.empty()) {
+ fd.PartsList.back().Last = false;
+ }
+
+ fd.PartsList.push_back({});
+ fp = &fd.PartsList.back();
+ fd.Parts.emplace(msg.BlobId, fp);
+
+ auto fname = TStringBuilder() << fd.TxId << "_" << fd.Description << "_" << fd.NextPartListIndex++;
+ fp->FileName = (Root / fname).GetPath();
+
+ LOG_D("[Write] create new FilePart " << fp->FileName);
+ newFile = true;
+ } else {
+ fd.Parts.emplace(msg.BlobId, fp);
+ }
+
+ auto& blobDesc = fp->Blobs[msg.BlobId];
+ blobDesc.Size = msg.Blob.size();
+ blobDesc.Offset = fp->Size;
+
+ fp->Size += blobDesc.Size;
+
+ auto writeOp = MakeHolder<TWriteFileOp>();
+ writeOp->Client = ev->Sender;
+ writeOp->Service = SelfId();
+ writeOp->ActorSystem = TlsActivationContext->ActorSystem();
+ writeOp->FileName = fp->FileName;
+ writeOp->CreateFile = newFile;
+ writeOp->BlobId = msg.BlobId;
+ writeOp->Blob = std::move(msg.Blob);
+
+ RunOp("Write", std::move(writeOp), fd);
+ }
+
+ void HandleWork(TEvPrivate::TEvWriteFileResponse::TPtr& ev) {
+ auto& msg = *ev->Get();
+ LOG_D("[WriteFileResponse] from: " << msg.Client << ", blobId: " << msg.BlobId << ", error: " << msg.Error);
+
+ auto it = Files.find(msg.Client);
+ if (it == Files.end()) {
+ LOG_E("[WriteFileResponse] Can not write file: not found. "
+ << "From: " << msg.Client << ", blobId: " << msg.BlobId << ", error: " << msg.Error);
+
+ Send(ev->Sender, new TEvKqpSpilling::TEvError("Internal error"));
+ return;
+ }
+
+ auto& fd = it->second;
+
+ fd.Error = std::move(msg.Error);
+ fd.TotalWaitTime += msg.WaitTime;
+ fd.TotalWorkTime += msg.WorkTime;
+
+ if (auto* fp = fd.Parts[msg.BlobId]) {
+ auto& blobDesc = fp->Blobs[msg.BlobId];
+
+ fp->WaitTime += msg.WaitTime;
+ fp->WorkTime += msg.WorkTime;
+ fp->WriteBytes += blobDesc.Size;
+
+ fd.TotalWriteBytes += blobDesc.Size;
+
+ Counters->SpillingStoredBlobs->Inc();
+ Counters->SpillingTotalSpaceUsed->Add(blobDesc.Size);
+
+ if (msg.NewFileHandle) {
+ fp->FileHandle.Swap(msg.NewFileHandle);
+ }
+ } else {
+ LOG_E("[WriteFileResponse] File part not found. From: " << msg.Client << ", blobId: " << msg.BlobId);
+ if (!fd.Error) {
+ fd.Error = "File part not found";
+ }
+
+ Counters->SpillingIoErrors->Inc();
+ }
+
+ if (fd.Error) {
+ Send(msg.Client, new TEvKqpSpilling::TEvError(*fd.Error));
+
+ fd.Ops.clear();
+ CloseFile(it, fd.Error);
+ return;
+ }
+
+ Counters->SpillingWriteBlobs->Inc();
+
+ Send(msg.Client, new TEvKqpSpilling::TEvWriteResult(msg.BlobId));
+ RunNextOp(fd);
+ }
+
+ void HandleWork(TEvKqpSpilling::TEvRead::TPtr& ev) {
+ auto& msg = *ev->Get();
+ LOG_D("[Read] from: " << ev->Sender << ", blobId: " << msg.BlobId);
+
+ auto it = Files.find(ev->Sender);
+ if (it == Files.end()) {
+ LOG_E("[Read] Can not read file: not found. From: " << ev->Sender << ", blobId: " << msg.BlobId);
+
+ Send(ev->Sender, new TEvKqpSpilling::TEvError("File not found"));
+ return;
+ }
+
+ auto& fd = it->second;
+
+ if (fd.CloseAt) {
+ LOG_E("[Read] Can not read file: closed. From: " << ev->Sender << ", blobId: " << msg.BlobId);
+
+ Send(ev->Sender, new TEvKqpSpilling::TEvError("Closed"));
+ return;
+ }
+
+ auto partIt = fd.Parts.find(msg.BlobId);
+ if (partIt == fd.Parts.end()) {
+ LOG_E("[Read] Can not read file: part not found. From: " << ev->Sender << ", blobId: " << msg.BlobId);
+
+ Send(ev->Sender, new TEvKqpSpilling::TEvError("File part not found"));
+
+ fd.Ops.clear();
+ TMaybe<TString> err = "Part not found";
+ CloseFile(it, err);
+ return;
+ }
+
+ auto& fp = partIt->second;
+
+ auto blobIt = fp->Blobs.find(msg.BlobId);
+ if (blobIt == fp->Blobs.end()) {
+ LOG_E("[Read] Can not read file: blob not found in the part. From: " << ev->Sender << ", blobId: " << msg.BlobId);
+
+ Send(ev->Sender, new TEvKqpSpilling::TEvError("Blob not found in the file part"));
+
+ fd.Ops.clear();
+ TMaybe<TString> err = "Blob not found in the file part";
+ CloseFile(it, err);
+ return;
+ }
+
+ blobIt->second.Read = true;
+
+ bool remove = false;
+ if (fd.RemoveBlobsAfterRead && !fp->Last) {
+ remove = true;
+ for (auto& b: partIt->second->Blobs) {
+ if (!b.second.Read) {
+ remove = false;
+ break;
+ }
+ }
+ }
+
+ auto readOp = MakeHolder<TReadFileOp>();
+ readOp->Client = ev->Sender;
+ readOp->Service = SelfId();
+ readOp->ActorSystem = TlsActivationContext->ActorSystem();
+ readOp->FileName = partIt->second->FileName;
+ readOp->BlobId = msg.BlobId;
+ readOp->Offset = blobIt->second.Offset;
+ readOp->Size = blobIt->second.Size;
+ if (remove) {
+ readOp->RemoveFile = std::move(fp->FileHandle);
+ }
+
+ RunOp("Read", std::move(readOp), fd);
+ }
+
+ void HandleWork(TEvPrivate::TEvReadFileResponse::TPtr& ev) {
+ auto& msg = *ev->Get();
+ LOG_D("[ReadFileResponse] from: " << msg.Client << ", blobId: " << msg.BlobId << ", removed: " << msg.Removed
+ << ", error: " << msg.Error);
+
+ auto it = Files.find(msg.Client);
+ if (it == Files.end()) {
+ LOG_E("[ReadFileResponse] Can not read file: not found. "
+ << "From: " << msg.Client << ", blobId: " << msg.BlobId << ", error: " << msg.Error);
+
+ Send(ev->Sender, new TEvKqpSpilling::TEvError("Internal error"));
+ return;
+ }
+
+ auto& fd = it->second;
+
+ fd.Error = std::move(msg.Error);
+ fd.TotalWaitTime += msg.WaitTime;
+ fd.TotalWorkTime += msg.WorkTime;
+ fd.TotalReadBytes += msg.Blob.size();
+
+ auto* fp = fd.Parts[msg.BlobId];
+ if (fp) {
+ fp->Blobs[msg.BlobId].Read = true;
+ fp->WaitTime += msg.WaitTime;
+ fp->WorkTime += msg.WorkTime;
+ fp->ReadBytes += msg.Blob.size();
+
+ if (msg.Removed) {
+ fd.TotalSize -= fp->Size;
+ TotalSize -= fp->Size;
+
+ Counters->SpillingTotalSpaceUsed->Sub(fp->Size);
+ Counters->SpillingStoredBlobs->Sub(fp->Blobs.size());
+
+ fd.Parts.erase(msg.BlobId);
+ fd.PartsList.remove_if([fp](const auto& x) { return &x == fp; });
+ }
+ } else {
+ if (!fd.Error) {
+ fd.Error = "Internal error";
+ }
+ }
+
+ if (fd.Error) {
+ Send(msg.Client, new TEvKqpSpilling::TEvError(*fd.Error));
+
+ fd.Ops.clear();
+ CloseFile(it, fd.Error);
+ return;
+ }
+
+ Counters->SpillingReadBlobs->Inc();
+
+ Send(msg.Client, new TEvKqpSpilling::TEvReadResult(msg.BlobId, std::move(msg.Blob)));
+ RunNextOp(fd);
+ }
+
+ void HandleWork(NMon::TEvHttpInfo::TPtr& ev) {
+ TStringStream s;
+
+ TMap<ui64, TVector<const TFileDesc*>> byTx;
+ for (const auto& fd : Files) {
+ byTx[fd.second.TxId].push_back(&fd.second);
+ }
+
+ HTML(s) {
+ H2() { s << "Configuration"; }
+ PRE() {
+ s << " - Root: " << Config.GetRoot() << Endl;
+ s << " - MaxTotalSize: " << Config.GetMaxTotalSize() << Endl;
+ s << " - MaxFileSize: " << Config.GetMaxFileSize() << Endl;
+ s << " - MaxFilePartSize: " << Config.GetMaxFilePartSize() << Endl;
+ s << " - IO thread pool, workers: " << Config.GetIoThreadPool().GetWorkersCount()
+ << ", queue: " << Config.GetIoThreadPool().GetQueueSize() << Endl;
+ }
+
+ H2() { s << "Active files"; }
+ PRE() { s << "Used space: " << TotalSize << Endl; }
+
+ for (const auto& tx : byTx) {
+ H4() { s << "Transaction " << tx.first; }
+ s << "Open files:" << Endl;
+ UL() {
+ for (const auto* fd : tx.second) {
+ LI() { s << fd->Description; }
+ PRE() {
+ s << " Remove blobs after read: " << fd->RemoveBlobsAfterRead << Endl;
+ s << " Open at: " << fd->OpenAt << Endl;
+ s << " Error: " << fd->Error << Endl;
+ s << " Total size: " << fd->TotalSize << Endl;
+ s << " Total work time: " << fd->TotalWorkTime << Endl;
+ s << " Total wait time: " << fd->TotalWaitTime << Endl;
+ s << " Total write bytes: " << fd->TotalWriteBytes << Endl;
+ s << " Total read bytes: " << fd->TotalReadBytes << Endl;
+ ui64 blobs = 0;
+ for (const auto& fp : fd->PartsList) {
+ blobs += fp.Blobs.size();
+ }
+ s << " Total blobs: " << blobs << Endl;
+ s << " Total files: " << fd->PartsList.size() << Endl;
+ s << " Files:" << Endl;
+ for (const auto& fp : fd->PartsList) {
+ s << " - " << fp.FileName << Endl;
+ s << " Size: " << fp.Size << Endl;
+ s << " Blobs: " << fp.Blobs.size() << Endl;
+ s << " Write bytes: " << fp.WriteBytes << Endl;
+ s << " Read bytes: " << fp.WriteBytes << Endl;
+ }
+ }
+ }
+ }
+ }
+
+ H2() { s << "Last closed files"; }
+ UL() {
+ for (auto it = ClosedFiles.rbegin(); it != ClosedFiles.rend(); ++it) {
+ auto& fd = *it;
+ LI() { s << "Transaction: " << fd.TxId << ", " << fd.Description; }
+ PRE() {
+ s << " Remove blobs after read: " << fd.RemoveBlobsAfterRead << Endl;
+ s << " Open at: " << fd.OpenAt << Endl;
+ s << " Close at: " << fd.CloseAt << Endl;
+ s << " Error: " << fd.Error << Endl;
+ s << " Total work time: " << fd.WorkTime << Endl;
+ s << " Total wait time: " << fd.WaitTime << Endl;
+ s << " Total write bytes: " << fd.WriteBytes << Endl;
+ s << " Total read bytes: " << fd.ReadBytes << Endl;
+ }
+ }
+ }
+
+ }
+
+ Send(ev->Sender, new NMon::TEvHttpInfoRes(s.Str()));
+ }
+
+private:
+ void RunOp(TStringBuf opName, THolder<IObjectInQueue> op, TFileDesc& fd) {
+ if (fd.HasActiveOp) {
+ fd.Ops.emplace_back(opName, std::move(op));
+ } else {
+ fd.HasActiveOp = true;
+ // TODO: retry if fails
+ IoThreadPool->SafeAddAndOwn(std::move(op));
+ }
+ }
+
+ void RunNextOp(TFileDesc& fd) {
+ fd.HasActiveOp = false;
+ if (!fd.Ops.empty()) {
+ auto op = std::move(fd.Ops.front().second);
+ auto opName = fd.Ops.front().first;
+ fd.Ops.pop_front();
+
+ RunOp(opName, std::move(op), fd);
+ }
+ }
+
+ void MoveFileToClosed(TFilesIt it) {
+ TotalSize -= it->second.TotalSize;
+ ClosedFiles.emplace_back(TClosedFileDesc(std::move(it->second)));
+ while (ClosedFiles.size() > 100) {
+ ClosedFiles.pop_front();
+ }
+ Files.erase(it);
+ }
+
+private:
+ struct TCloseFileOp : public IObjectInQueue {
TActorId Client;
TActorId Service;
- TActorSystem* ActorSystem;
- TVector<THolder<TFileHandle>> FileHandles;
- TVector<TString> FileNames;
- TInstant Ts = TInstant::Now();
-
- void Process(void*) override {
- auto now = TInstant::Now();
-
- auto resp = MakeHolder<TEvPrivate::TEvCloseFileResponse>();
- resp->Client = Client;
- resp->WaitTime = now - Ts;
-
- A_LOG_D("[CloseFile async] from: " << Client << ", waitTime: " << resp->WaitTime);
-
- for (ui64 i = 0; i < FileHandles.size(); ++i) {
- try {
- FileHandles[i].Reset();
-#ifdef REMOVE_FILES
- ::unlink(FileNames[i].c_str());
-#endif
- } catch (const yexception& e) {
- A_LOG_E("[CloseFile async] error while closing file " << FileNames[i] << ": " << e.what());
- }
- }
- resp->WorkTime = TInstant::Now() - now;
-
- ActorSystem->Send(Service, resp.Release());
- }
- };
-
- struct TWriteFileOp : public IObjectInQueue {
+ TActorSystem* ActorSystem;
+ TVector<THolder<TFileHandle>> FileHandles;
+ TVector<TString> FileNames;
+ TInstant Ts = TInstant::Now();
+
+ void Process(void*) override {
+ auto now = TInstant::Now();
+
+ auto resp = MakeHolder<TEvPrivate::TEvCloseFileResponse>();
+ resp->Client = Client;
+ resp->WaitTime = now - Ts;
+
+ A_LOG_D("[CloseFile async] from: " << Client << ", waitTime: " << resp->WaitTime);
+
+ for (ui64 i = 0; i < FileHandles.size(); ++i) {
+ try {
+ FileHandles[i].Reset();
+#ifdef REMOVE_FILES
+ ::unlink(FileNames[i].c_str());
+#endif
+ } catch (const yexception& e) {
+ A_LOG_E("[CloseFile async] error while closing file " << FileNames[i] << ": " << e.what());
+ }
+ }
+ resp->WorkTime = TInstant::Now() - now;
+
+ ActorSystem->Send(Service, resp.Release());
+ }
+ };
+
+ struct TWriteFileOp : public IObjectInQueue {
TActorId Client;
TActorId Service;
- TActorSystem* ActorSystem;
- TString FileName;
- bool CreateFile = false;
- ui64 BlobId = 0;
- TBuffer Blob;
- TInstant Ts = TInstant::Now();
-
- void Process(void*) override {
- auto now = TInstant::Now();
- A_LOG_D("[Write async] file: " << FileName << ", blobId: " << BlobId << ", bytes: " << Blob.size()
- << ", offset: " << (CreateFile ? 0 : GetFileLength(FileName)));
-
- auto resp = MakeHolder<TEvPrivate::TEvWriteFileResponse>();
- resp->Client = Client;
- resp->WaitTime = now - Ts;
- resp->BlobId = BlobId;
-
- try {
- TFile file;
- if (CreateFile) {
- file = TFile(FileName, CreateAlways | WrOnly);
+ TActorSystem* ActorSystem;
+ TString FileName;
+ bool CreateFile = false;
+ ui64 BlobId = 0;
+ TBuffer Blob;
+ TInstant Ts = TInstant::Now();
+
+ void Process(void*) override {
+ auto now = TInstant::Now();
+ A_LOG_D("[Write async] file: " << FileName << ", blobId: " << BlobId << ", bytes: " << Blob.size()
+ << ", offset: " << (CreateFile ? 0 : GetFileLength(FileName)));
+
+ auto resp = MakeHolder<TEvPrivate::TEvWriteFileResponse>();
+ resp->Client = Client;
+ resp->WaitTime = now - Ts;
+ resp->BlobId = BlobId;
+
+ try {
+ TFile file;
+ if (CreateFile) {
+ file = TFile(FileName, CreateAlways | WrOnly);
resp->NewFileHandle = MakeHolder<TFileHandle>(FileName, OpenExisting | RdWr);
- } else {
- file = TFile::ForAppend(FileName);
- }
- file.Write(Blob.Data(), Blob.size());
- } catch (const yexception& e) {
- A_LOG_E("[Write async] file: " << FileName << ", io error: " << e.what());
- resp->Error = e.what();
- }
- resp->WorkTime = TInstant::Now() - now;
-
- ActorSystem->Send(Service, resp.Release());
- }
- };
-
- struct TReadFileOp : public IObjectInQueue {
+ } else {
+ file = TFile::ForAppend(FileName);
+ }
+ file.Write(Blob.Data(), Blob.size());
+ } catch (const yexception& e) {
+ A_LOG_E("[Write async] file: " << FileName << ", io error: " << e.what());
+ resp->Error = e.what();
+ }
+ resp->WorkTime = TInstant::Now() - now;
+
+ ActorSystem->Send(Service, resp.Release());
+ }
+ };
+
+ struct TReadFileOp : public IObjectInQueue {
TActorId Client;
TActorId Service;
- TActorSystem* ActorSystem;
- TString FileName;
- ui64 BlobId;
- ui64 Offset;
- ui64 Size;
- THolder<TFileHandle> RemoveFile;
- TInstant Ts = TInstant::Now();
-
- void Process(void*) override {
- auto now = TInstant::Now();
- A_LOG_D("[Read async] file: " << FileName << ", blobId: " << BlobId << ", offset: " << Offset
- << ", size: " << Size << ", remove: " << (bool) RemoveFile);
-
- auto resp = MakeHolder<TEvPrivate::TEvReadFileResponse>();
- resp->Client = Client;
- resp->WaitTime = TInstant::Now() - now;
- resp->BlobId = BlobId;
-
- try {
- resp->Blob.Resize(Size);
-
- TFile f(FileName, OpenExisting | RdOnly);
- f.Seek(Offset, SeekDir::sSet);
-
- auto read = f.Read(resp->Blob.Data(), Size);
- YQL_ENSURE(read == Size, "" << read << " != " << Size);
- YQL_ENSURE(resp->Blob.size() == Size);
-
- if (RemoveFile) {
- f.Close();
- RemoveFile.Reset();
-#ifdef REMOVE_FILES
- ::unlink(FileName.c_str());
-#endif
- resp->Removed = true;
- }
- } catch (const yexception& e) {
- A_LOG_E("[Read async] file: " << FileName << ", blobId: " << BlobId << ", offset: " << Offset
- << ", error: " << e.what());
- resp->Error = e.what();
- }
- resp->WorkTime = TInstant::Now() - now;
-
- ActorSystem->Send(Service, resp.Release());
- }
- };
-
-private:
- const NKikimrConfig::TTableServiceConfig::TSpillingServiceConfig::TLocalFileConfig Config;
- TFsPath Root;
- TIntrusivePtr<TKqpCounters> Counters;
-
- struct TFileDesc {
- ui64 TxId = 0;
- TString Description;
- bool RemoveBlobsAfterRead = false;
- TInstant OpenAt;
-
- TInstant CloseAt;
- TMaybe<TString> Error;
-
- struct TBlobDesc {
- ui64 Offset = 0;
- ui64 Size = 0;
- bool Read = false;
- };
-
- struct TFilePart {
- TString FileName;
- THolder<TFileHandle> FileHandle;
-
- THashMap<ui64, TBlobDesc> Blobs;
-
- ui64 WriteBytes = 0;
- ui64 ReadBytes = 0;
- ui64 Size = 0; // also it is an offset for the next blob
- TDuration WorkTime;
- TDuration WaitTime;
- bool Last = true;
- };
-
- ui64 TotalSize = 0;
-
- TDuration TotalWorkTime;
- TDuration TotalWaitTime;
- ui64 TotalWriteBytes = 0;
- ui64 TotalReadBytes = 0;
-
- THashMap<ui64, TFilePart*> Parts;
- TList<TFilePart> PartsList;
- ui32 NextPartListIndex = 0;
-
- TList<std::pair<TString, THolder<IObjectInQueue>>> Ops;
- bool HasActiveOp = false;
- };
-
- struct TClosedFileDesc {
- ui64 TxId;
- TString Description;
- bool RemoveBlobsAfterRead;
- TInstant OpenAt;
- TInstant CloseAt;
- TDuration WorkTime;
- TDuration WaitTime;
- ui64 WriteBytes;
- ui64 ReadBytes;
- TMaybe<TString> Error;
-
- TClosedFileDesc(TFileDesc&& fd)
- : TxId(fd.TxId)
- , Description(std::move(fd.Description))
- , RemoveBlobsAfterRead(fd.RemoveBlobsAfterRead)
- , OpenAt(fd.OpenAt)
- , CloseAt(fd.CloseAt)
- , WorkTime(fd.TotalWorkTime)
- , WaitTime(fd.TotalWaitTime)
- , WriteBytes(fd.TotalWriteBytes)
- , ReadBytes(fd.TotalReadBytes)
- , Error(std::move(fd.Error)) {}
- };
-
- THolder<IThreadPool> IoThreadPool;
+ TActorSystem* ActorSystem;
+ TString FileName;
+ ui64 BlobId;
+ ui64 Offset;
+ ui64 Size;
+ THolder<TFileHandle> RemoveFile;
+ TInstant Ts = TInstant::Now();
+
+ void Process(void*) override {
+ auto now = TInstant::Now();
+ A_LOG_D("[Read async] file: " << FileName << ", blobId: " << BlobId << ", offset: " << Offset
+ << ", size: " << Size << ", remove: " << (bool) RemoveFile);
+
+ auto resp = MakeHolder<TEvPrivate::TEvReadFileResponse>();
+ resp->Client = Client;
+ resp->WaitTime = TInstant::Now() - now;
+ resp->BlobId = BlobId;
+
+ try {
+ resp->Blob.Resize(Size);
+
+ TFile f(FileName, OpenExisting | RdOnly);
+ f.Seek(Offset, SeekDir::sSet);
+
+ auto read = f.Read(resp->Blob.Data(), Size);
+ YQL_ENSURE(read == Size, "" << read << " != " << Size);
+ YQL_ENSURE(resp->Blob.size() == Size);
+
+ if (RemoveFile) {
+ f.Close();
+ RemoveFile.Reset();
+#ifdef REMOVE_FILES
+ ::unlink(FileName.c_str());
+#endif
+ resp->Removed = true;
+ }
+ } catch (const yexception& e) {
+ A_LOG_E("[Read async] file: " << FileName << ", blobId: " << BlobId << ", offset: " << Offset
+ << ", error: " << e.what());
+ resp->Error = e.what();
+ }
+ resp->WorkTime = TInstant::Now() - now;
+
+ ActorSystem->Send(Service, resp.Release());
+ }
+ };
+
+private:
+ const NKikimrConfig::TTableServiceConfig::TSpillingServiceConfig::TLocalFileConfig Config;
+ TFsPath Root;
+ TIntrusivePtr<TKqpCounters> Counters;
+
+ struct TFileDesc {
+ ui64 TxId = 0;
+ TString Description;
+ bool RemoveBlobsAfterRead = false;
+ TInstant OpenAt;
+
+ TInstant CloseAt;
+ TMaybe<TString> Error;
+
+ struct TBlobDesc {
+ ui64 Offset = 0;
+ ui64 Size = 0;
+ bool Read = false;
+ };
+
+ struct TFilePart {
+ TString FileName;
+ THolder<TFileHandle> FileHandle;
+
+ THashMap<ui64, TBlobDesc> Blobs;
+
+ ui64 WriteBytes = 0;
+ ui64 ReadBytes = 0;
+ ui64 Size = 0; // also it is an offset for the next blob
+ TDuration WorkTime;
+ TDuration WaitTime;
+ bool Last = true;
+ };
+
+ ui64 TotalSize = 0;
+
+ TDuration TotalWorkTime;
+ TDuration TotalWaitTime;
+ ui64 TotalWriteBytes = 0;
+ ui64 TotalReadBytes = 0;
+
+ THashMap<ui64, TFilePart*> Parts;
+ TList<TFilePart> PartsList;
+ ui32 NextPartListIndex = 0;
+
+ TList<std::pair<TString, THolder<IObjectInQueue>>> Ops;
+ bool HasActiveOp = false;
+ };
+
+ struct TClosedFileDesc {
+ ui64 TxId;
+ TString Description;
+ bool RemoveBlobsAfterRead;
+ TInstant OpenAt;
+ TInstant CloseAt;
+ TDuration WorkTime;
+ TDuration WaitTime;
+ ui64 WriteBytes;
+ ui64 ReadBytes;
+ TMaybe<TString> Error;
+
+ TClosedFileDesc(TFileDesc&& fd)
+ : TxId(fd.TxId)
+ , Description(std::move(fd.Description))
+ , RemoveBlobsAfterRead(fd.RemoveBlobsAfterRead)
+ , OpenAt(fd.OpenAt)
+ , CloseAt(fd.CloseAt)
+ , WorkTime(fd.TotalWorkTime)
+ , WaitTime(fd.TotalWaitTime)
+ , WriteBytes(fd.TotalWriteBytes)
+ , ReadBytes(fd.TotalReadBytes)
+ , Error(std::move(fd.Error)) {}
+ };
+
+ THolder<IThreadPool> IoThreadPool;
THashMap<TActorId, TFileDesc> Files;
- TList<const TClosedFileDesc> ClosedFiles;
- ui64 TotalSize = 0;
-};
-
-} // anonymous namespace
-
+ TList<const TClosedFileDesc> ClosedFiles;
+ ui64 TotalSize = 0;
+};
+
+} // anonymous namespace
+
IActor* CreateKqpLocalFileSpillingActor(ui64 txId, const TString& details, const TActorId& client,
- bool removeBlobsAfterRead)
-{
- return new TKqpLocalFileSpillingActor(txId, details, client, removeBlobsAfterRead);
-}
-
-IActor* CreateKqpLocalFileSpillingService(
- const NKikimrConfig::TTableServiceConfig::TSpillingServiceConfig::TLocalFileConfig& config,
- TIntrusivePtr<TKqpCounters> counters)
-{
- return new TKqpLocalFileSpillingService(config, counters);
-}
-
-} // namespace NKikimr::NKqp
+ bool removeBlobsAfterRead)
+{
+ return new TKqpLocalFileSpillingActor(txId, details, client, removeBlobsAfterRead);
+}
+
+IActor* CreateKqpLocalFileSpillingService(
+ const NKikimrConfig::TTableServiceConfig::TSpillingServiceConfig::TLocalFileConfig& config,
+ TIntrusivePtr<TKqpCounters> counters)
+{
+ return new TKqpLocalFileSpillingService(config, counters);
+}
+
+} // namespace NKikimr::NKqp
diff --git a/ydb/core/kqp/runtime/kqp_spilling_file.h b/ydb/core/kqp/runtime/kqp_spilling_file.h
index 3fa14c4a39..14196a8498 100644
--- a/ydb/core/kqp/runtime/kqp_spilling_file.h
+++ b/ydb/core/kqp/runtime/kqp_spilling_file.h
@@ -1,18 +1,18 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/kqp/counters/kqp_counters.h>
#include <ydb/core/protos/config.pb.h>
-
-#include <library/cpp/actors/core/actor.h>
-
-
-namespace NKikimr::NKqp {
-
+
+#include <library/cpp/actors/core/actor.h>
+
+
+namespace NKikimr::NKqp {
+
NActors::IActor* CreateKqpLocalFileSpillingActor(ui64 txId, const TString& details, const NActors::TActorId& client,
- bool removeBlobsAfterRead);
-
-NActors::IActor* CreateKqpLocalFileSpillingService(
- const NKikimrConfig::TTableServiceConfig::TSpillingServiceConfig::TLocalFileConfig& config,
- TIntrusivePtr<TKqpCounters> counters);
-
-} // namespace NKikimr::NKqp
+ bool removeBlobsAfterRead);
+
+NActors::IActor* CreateKqpLocalFileSpillingService(
+ const NKikimrConfig::TTableServiceConfig::TSpillingServiceConfig::TLocalFileConfig& config,
+ TIntrusivePtr<TKqpCounters> counters);
+
+} // namespace NKikimr::NKqp
diff --git a/ydb/core/kqp/runtime/kqp_spilling_file_ut.cpp b/ydb/core/kqp/runtime/kqp_spilling_file_ut.cpp
index cadcaaa533..c3efc85cbc 100644
--- a/ydb/core/kqp/runtime/kqp_spilling_file_ut.cpp
+++ b/ydb/core/kqp/runtime/kqp_spilling_file_ut.cpp
@@ -1,418 +1,418 @@
-#include "kqp_spilling_file.h"
-#include "kqp_spilling.h"
-
+#include "kqp_spilling_file.h"
+#include "kqp_spilling.h"
+
#include <ydb/core/kqp/kqp.h>
#include <ydb/core/testlib/basics/appdata.h>
#include <ydb/core/testlib/basics/runtime.h>
-
+
#include <library/cpp/testing/unittest/registar.h>
-
-#include <util/system/fs.h>
-
-namespace NKikimr::NKqp {
-
-using namespace NActors;
-using namespace NKikimr;
-using namespace NKikimr::NMiniKQL;
-using namespace NKikimr::NKqp;
-using namespace NYql;
-
-namespace {
-
-TBuffer CreateBlob(ui32 size, char symbol) {
- TBuffer blob(size);
- blob.Fill(symbol, size);
- return blob;
-}
-
-void AssertEquals(const TBuffer& lhs, const TBuffer& rhs) {
- TStringBuf l{lhs.data(), lhs.size()};
- TStringBuf r{rhs.data(), rhs.size()};
- UNIT_ASSERT_STRINGS_EQUAL(l, r);
-}
-
-TIntrusivePtr<NMonitoring::TDynamicCounters> Counters() {
- static auto counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
- return counters;
-}
-
+
+#include <util/system/fs.h>
+
+namespace NKikimr::NKqp {
+
+using namespace NActors;
+using namespace NKikimr;
+using namespace NKikimr::NMiniKQL;
+using namespace NKikimr::NKqp;
+using namespace NYql;
+
+namespace {
+
+TBuffer CreateBlob(ui32 size, char symbol) {
+ TBuffer blob(size);
+ blob.Fill(symbol, size);
+ return blob;
+}
+
+void AssertEquals(const TBuffer& lhs, const TBuffer& rhs) {
+ TStringBuf l{lhs.data(), lhs.size()};
+ TStringBuf r{rhs.data(), rhs.size()};
+ UNIT_ASSERT_STRINGS_EQUAL(l, r);
+}
+
+TIntrusivePtr<NMonitoring::TDynamicCounters> Counters() {
+ static auto counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
+ return counters;
+}
+
TActorId StartSpillingService(TTestBasicRuntime& runtime, ui64 maxTotalSize = 1000, ui64 maxFileSize = 500,
- ui64 maxFilePartSize = 100, const TString& root = "./kqp_spilling/")
-{
- Cerr << "cwd: " << NFs::CurrentWorkingDirectory() << Endl;
-
- NKikimrConfig::TTableServiceConfig::TSpillingServiceConfig::TLocalFileConfig config;
- config.SetEnable(true);
- config.SetRoot(root);
- config.SetMaxTotalSize(maxTotalSize);
- config.SetMaxFileSize(maxFileSize);
- config.SetMaxFilePartSize(maxFilePartSize);
-
- auto counters = Counters();
- counters->ResetCounters();
- auto kqpCounters = MakeIntrusive<TKqpCounters>(counters);
-
- auto* spillingService = CreateKqpLocalFileSpillingService(config, kqpCounters);
- auto spillingServiceActorId = runtime.Register(spillingService);
- runtime.EnableScheduleForActor(spillingServiceActorId);
- runtime.RegisterService(MakeKqpLocalFileSpillingServiceID(runtime.GetNodeId()), spillingServiceActorId);
-
- return spillingServiceActorId;
-}
-
+ ui64 maxFilePartSize = 100, const TString& root = "./kqp_spilling/")
+{
+ Cerr << "cwd: " << NFs::CurrentWorkingDirectory() << Endl;
+
+ NKikimrConfig::TTableServiceConfig::TSpillingServiceConfig::TLocalFileConfig config;
+ config.SetEnable(true);
+ config.SetRoot(root);
+ config.SetMaxTotalSize(maxTotalSize);
+ config.SetMaxFileSize(maxFileSize);
+ config.SetMaxFilePartSize(maxFilePartSize);
+
+ auto counters = Counters();
+ counters->ResetCounters();
+ auto kqpCounters = MakeIntrusive<TKqpCounters>(counters);
+
+ auto* spillingService = CreateKqpLocalFileSpillingService(config, kqpCounters);
+ auto spillingServiceActorId = runtime.Register(spillingService);
+ runtime.EnableScheduleForActor(spillingServiceActorId);
+ runtime.RegisterService(MakeKqpLocalFileSpillingServiceID(runtime.GetNodeId()), spillingServiceActorId);
+
+ return spillingServiceActorId;
+}
+
TActorId StartSpillingActor(TTestBasicRuntime& runtime, const TActorId& client, bool removeBlobsAfterRead = true) {
- auto *spillingActor = CreateKqpLocalFileSpillingActor(1, "test", client, removeBlobsAfterRead);
- auto spillingActorId = runtime.Register(spillingActor);
- runtime.EnableScheduleForActor(spillingActorId);
-
- return spillingActorId;
-}
-
-void WaitBootstrap(TTestBasicRuntime& runtime) {
- TDispatchOptions options;
- options.FinalEvents.emplace_back(TEvents::TSystem::Bootstrap, 1);
- UNIT_ASSERT(runtime.DispatchEvents(options));
-}
-
-void SetupLogs(TTestBasicRuntime& runtime) {
- runtime.SetLogPriority(NKikimrServices::KQP_BLOBS_STORAGE, NActors::NLog::PRI_ERROR);
-}
-
-} // anonymous namespace
-
-Y_UNIT_TEST_SUITE(KqpSpillingFileTests) {
-
-Y_UNIT_TEST(Simple) {
- TTestBasicRuntime runtime{1, false};
- runtime.Initialize(TAppPrepare().Unwrap());
- SetupLogs(runtime);
-
- auto spillingService = StartSpillingService(runtime);
- auto tester = runtime.AllocateEdgeActor();
- auto spillingActor = StartSpillingActor(runtime, tester);
-
- WaitBootstrap(runtime);
-
- // put blob 1
- {
- auto ev = new TEvKqpSpilling::TEvWrite(1, CreateBlob(10, 'a'));
- runtime.Send(new IEventHandle(spillingActor, tester, ev));
-
- auto resp = runtime.GrabEdgeEvent<TEvKqpSpilling::TEvWriteResult>(tester, TDuration::Seconds(1));
- UNIT_ASSERT_VALUES_EQUAL(1, resp->Get()->BlobId);
- }
-
- // put blob 2
- {
- auto ev = new TEvKqpSpilling::TEvWrite(2, CreateBlob(11, 'z'));
- runtime.Send(new IEventHandle(spillingActor, tester, ev));
-
- auto resp = runtime.GrabEdgeEvent<TEvKqpSpilling::TEvWriteResult>(tester, TDuration::Seconds(1));
- UNIT_ASSERT_VALUES_EQUAL(2, resp->Get()->BlobId);
- }
-
- // get blob 1
- {
- auto ev = new TEvKqpSpilling::TEvRead(1);
- runtime.Send(new IEventHandle(spillingActor, tester, ev));
-
- auto resp = runtime.GrabEdgeEvent<TEvKqpSpilling::TEvReadResult>(tester, TDuration::Seconds(1));
- UNIT_ASSERT_VALUES_EQUAL(1, resp->Get()->BlobId);
-
- TBuffer expected = CreateBlob(10, 'a');
- AssertEquals(expected, resp->Get()->Blob);
- }
-
- // get blob 2
- {
- auto ev = new TEvKqpSpilling::TEvRead(2);
- runtime.Send(new IEventHandle(spillingActor, tester, ev));
-
- auto resp = runtime.GrabEdgeEvent<TEvKqpSpilling::TEvReadResult>(tester, TDuration::Seconds(1));
- UNIT_ASSERT_VALUES_EQUAL(2, resp->Get()->BlobId);
-
- TBuffer expected = CreateBlob(11, 'z');
- AssertEquals(expected, resp->Get()->Blob);
- }
-
- // terminate
- {
- runtime.Send(new IEventHandle(spillingActor, tester, new TEvents::TEvPoison));
-
- std::atomic<bool> done = false;
- runtime.SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& event) {
- if (event->GetRecipientRewrite() == spillingService) {
- if (event->GetTypeRewrite() == 2146435074 /* EvCloseFileResponse */ ) {
- done = true;
- }
- }
- return TTestActorRuntimeBase::EEventAction::PROCESS;
- });
-
- TDispatchOptions options;
- options.CustomFinalCondition = [&]() {
- return (bool) done;
- };
-
- runtime.DispatchEvents(options, TDuration::Seconds(1));
- }
-}
-
-Y_UNIT_TEST(Write_TotalSizeLimitExceeded) {
- TTestBasicRuntime runtime{1, false};
- runtime.Initialize(TAppPrepare().Unwrap());
- SetupLogs(runtime);
-
- StartSpillingService(runtime, 100, 1000, 1000);
- auto tester = runtime.AllocateEdgeActor();
- auto spillingActor = StartSpillingActor(runtime, tester);
-
- WaitBootstrap(runtime);
-
- {
- auto ev = new TEvKqpSpilling::TEvWrite(1, CreateBlob(51, 'a'));
- runtime.Send(new IEventHandle(spillingActor, tester, ev));
-
- auto resp = runtime.GrabEdgeEvent<TEvKqpSpilling::TEvWriteResult>(tester);
- UNIT_ASSERT_VALUES_EQUAL(1, resp->Get()->BlobId);
- }
-
- {
- auto ev = new TEvKqpSpilling::TEvWrite(2, CreateBlob(50, 'b'));
- runtime.Send(new IEventHandle(spillingActor, tester, ev));
-
- auto resp = runtime.GrabEdgeEvent<TEvKqpSpilling::TEvError>(tester);
- UNIT_ASSERT_STRINGS_EQUAL("Total size limit exceeded", resp->Get()->Message);
- }
-}
-
-Y_UNIT_TEST(Write_FileSizeLimitExceeded) {
- TTestBasicRuntime runtime{1, false};
- runtime.Initialize(TAppPrepare().Unwrap());
- SetupLogs(runtime);
-
- StartSpillingService(runtime, 1000, 100, 1000);
- auto tester = runtime.AllocateEdgeActor();
- auto spillingActor = StartSpillingActor(runtime, tester);
-
- WaitBootstrap(runtime);
-
- {
- auto ev = new TEvKqpSpilling::TEvWrite(1, CreateBlob(51, 'a'));
- runtime.Send(new IEventHandle(spillingActor, tester, ev));
-
- auto resp = runtime.GrabEdgeEvent<TEvKqpSpilling::TEvWriteResult>(tester);
- UNIT_ASSERT_VALUES_EQUAL(1, resp->Get()->BlobId);
- }
-
- {
- auto ev = new TEvKqpSpilling::TEvWrite(2, CreateBlob(50, 'b'));
- runtime.Send(new IEventHandle(spillingActor, tester, ev));
-
- auto resp = runtime.GrabEdgeEvent<TEvKqpSpilling::TEvError>(tester);
- UNIT_ASSERT_STRINGS_EQUAL("File size limit exceeded", resp->Get()->Message);
- }
-}
-
-Y_UNIT_TEST(MultipleFileParts) {
- TTestBasicRuntime runtime{1, false};
- runtime.Initialize(TAppPrepare().Unwrap());
- SetupLogs(runtime);
-
- StartSpillingService(runtime, 1000, 100, 25);
- auto tester = runtime.AllocateEdgeActor();
- auto spillingActor = StartSpillingActor(runtime, tester);
-
- WaitBootstrap(runtime);
-
- const TString filePrefix = TStringBuilder() << NFs::CurrentWorkingDirectory() << "/kqp_spilling/node_1/1_test_";
-
- for (ui32 i = 0; i < 5; ++i) {
- // Cerr << "---- store blob #" << i << Endl;
- auto ev = new TEvKqpSpilling::TEvWrite(i, CreateBlob(20, 'a' + i));
- runtime.Send(new IEventHandle(spillingActor, tester, ev));
-
- auto resp = runtime.GrabEdgeEvent<TEvKqpSpilling::TEvWriteResult>(tester);
- UNIT_ASSERT_VALUES_EQUAL(i, resp->Get()->BlobId);
-
- UNIT_ASSERT(NFs::Exists(TStringBuilder() << filePrefix << i));
- }
-
- for (i32 i = 4; i >= 0; --i) {
- // Cerr << "---- load blob #" << i << Endl;
- auto ev = new TEvKqpSpilling::TEvRead(i, true);
- runtime.Send(new IEventHandle(spillingActor, tester, ev));
-
- auto resp = runtime.GrabEdgeEvent<TEvKqpSpilling::TEvReadResult>(tester);
- UNIT_ASSERT_VALUES_EQUAL(i, resp->Get()->BlobId);
- TBuffer expected = CreateBlob(20, 'a' + i);
- AssertEquals(expected, resp->Get()->Blob);
-
- if (i == 4) {
- // do not remove last file
- UNIT_ASSERT(NFs::Exists(TStringBuilder() << filePrefix << i));
- } else {
- UNIT_ASSERT(!NFs::Exists(TStringBuilder() << filePrefix << i));
- }
- }
-}
-
-Y_UNIT_TEST(SingleFilePart) {
- TTestBasicRuntime runtime{1, false};
- runtime.Initialize(TAppPrepare().Unwrap());
- SetupLogs(runtime);
-
- StartSpillingService(runtime, 1000, 100, 25);
- auto tester = runtime.AllocateEdgeActor();
- auto spillingActor = StartSpillingActor(runtime, tester, false);
-
- WaitBootstrap(runtime);
-
- const TString filePrefix = TStringBuilder() << NFs::CurrentWorkingDirectory() << "/kqp_spilling/node_1/1_test_";
-
- for (ui32 i = 0; i < 5; ++i) {
- // Cerr << "---- store blob #" << i << Endl;
- auto ev = new TEvKqpSpilling::TEvWrite(i, CreateBlob(20, 'a' + i));
- runtime.Send(new IEventHandle(spillingActor, tester, ev));
-
- auto resp = runtime.GrabEdgeEvent<TEvKqpSpilling::TEvWriteResult>(tester);
- UNIT_ASSERT_VALUES_EQUAL(i, resp->Get()->BlobId);
-
- UNIT_ASSERT(NFs::Exists(TStringBuilder() << filePrefix << 0));
- if (i > 0) {
- UNIT_ASSERT(!NFs::Exists(TStringBuilder() << filePrefix << i));
- }
- }
-
- for (i32 i = 4; i >= 0; --i) {
- // Cerr << "---- load blob #" << i << Endl;
- auto ev = new TEvKqpSpilling::TEvRead(i, true);
- runtime.Send(new IEventHandle(spillingActor, tester, ev));
-
- auto resp = runtime.GrabEdgeEvent<TEvKqpSpilling::TEvReadResult>(tester);
- UNIT_ASSERT_VALUES_EQUAL(i, resp->Get()->BlobId);
- TBuffer expected = CreateBlob(20, 'a' + i);
- AssertEquals(expected, resp->Get()->Blob);
-
- UNIT_ASSERT(NFs::Exists(TStringBuilder() << filePrefix << 0));
- }
-}
-
-Y_UNIT_TEST(ReadError) {
- TTestBasicRuntime runtime{1, false};
- runtime.Initialize(TAppPrepare().Unwrap());
- SetupLogs(runtime);
-
- StartSpillingService(runtime);
- auto tester = runtime.AllocateEdgeActor();
- auto spillingActor = StartSpillingActor(runtime, tester);
-
- WaitBootstrap(runtime);
-
- {
- auto ev = new TEvKqpSpilling::TEvWrite(0, CreateBlob(20, 'a'));
- runtime.Send(new IEventHandle(spillingActor, tester, ev));
-
- auto resp = runtime.GrabEdgeEvent<TEvKqpSpilling::TEvWriteResult>(tester);
- UNIT_ASSERT_VALUES_EQUAL(0, resp->Get()->BlobId);
- }
-
- ::unlink((NFs::CurrentWorkingDirectory() + "/kqp_spilling/node_1/1_test_0").c_str());
-
- {
- auto ev = new TEvKqpSpilling::TEvRead(0, true);
- runtime.Send(new IEventHandle(spillingActor, tester, ev));
-
- auto resp = runtime.GrabEdgeEvent<TEvKqpSpilling::TEvError>(tester);
- auto& err = resp->Get()->Message;
- UNIT_ASSERT_C(err.Contains("No such file or directory"), err);
- UNIT_ASSERT_C(err.Contains("can't open \"kqp_spilling/node_1/1_test_0\" with mode RdOnly"), err);
- }
-}
-
-
-struct THttpRequest : NMonitoring::IHttpRequest {
- HTTP_METHOD Method;
- TCgiParameters CgiParameters;
- THttpHeaders HttpHeaders;
-
- THttpRequest(HTTP_METHOD method)
- : Method(method)
- {}
-
- ~THttpRequest() {}
-
- const char* GetURI() const override {
- return "";
- }
-
- const char* GetPath() const override {
- return "";
- }
-
- const TCgiParameters& GetParams() const override {
- return CgiParameters;
- }
-
- const TCgiParameters& GetPostParams() const override {
- return CgiParameters;
- }
-
- TStringBuf GetPostContent() const override {
- return TString();
- }
-
- HTTP_METHOD GetMethod() const override {
- return Method;
- }
-
- const THttpHeaders& GetHeaders() const override {
- return HttpHeaders;
- }
-};
-
-Y_UNIT_TEST(StartError) {
- TTestBasicRuntime runtime{1, false};
- runtime.Initialize(TAppPrepare().Unwrap());
- SetupLogs(runtime);
-
- auto spillingService = StartSpillingService(runtime, 100, 500, 100, "/root/kqp_spilling/");
- auto tester = runtime.AllocateEdgeActor();
- auto spillingActor = StartSpillingActor(runtime, tester);
-
- WaitBootstrap(runtime);
-
- // put blob 1
- {
- auto ev = new TEvKqpSpilling::TEvWrite(1, CreateBlob(10, 'a'));
- runtime.Send(new IEventHandle(spillingActor, tester, ev));
-
- auto resp = runtime.GrabEdgeEvent<TEvKqpSpilling::TEvError>(tester, TDuration::Seconds(1));
- UNIT_ASSERT_EQUAL("Service not started", resp->Get()->Message);
- }
-
- // get blob 1
- {
- auto ev = new TEvKqpSpilling::TEvRead(1);
- runtime.Send(new IEventHandle(spillingActor, tester, ev));
-
- auto resp = runtime.GrabEdgeEvent<TEvKqpSpilling::TEvError>(tester, TDuration::Seconds(1));
- UNIT_ASSERT_EQUAL("Service not started", resp->Get()->Message);
- }
-
- // mon
- {
- THttpRequest httpReq(HTTP_METHOD_GET);
- NMonitoring::TMonService2HttpRequest monReq(nullptr, &httpReq, nullptr, nullptr, "", nullptr);
-
- runtime.Send(new IEventHandle(spillingService, tester, new NMon::TEvHttpInfo(monReq)));
-
- auto resp = runtime.GrabEdgeEvent<NMon::TEvHttpInfoRes>(tester, TDuration::Seconds(1));
- UNIT_ASSERT_EQUAL("<html><h2>Service is not started due to IO error</h2></html>",
- ((NMon::TEvHttpInfoRes*) resp->Get())->Answer);
- }
-}
-
-} // suite
-
-} // namespace NKikimr::NKqp
+ auto *spillingActor = CreateKqpLocalFileSpillingActor(1, "test", client, removeBlobsAfterRead);
+ auto spillingActorId = runtime.Register(spillingActor);
+ runtime.EnableScheduleForActor(spillingActorId);
+
+ return spillingActorId;
+}
+
+void WaitBootstrap(TTestBasicRuntime& runtime) {
+ TDispatchOptions options;
+ options.FinalEvents.emplace_back(TEvents::TSystem::Bootstrap, 1);
+ UNIT_ASSERT(runtime.DispatchEvents(options));
+}
+
+void SetupLogs(TTestBasicRuntime& runtime) {
+ runtime.SetLogPriority(NKikimrServices::KQP_BLOBS_STORAGE, NActors::NLog::PRI_ERROR);
+}
+
+} // anonymous namespace
+
+Y_UNIT_TEST_SUITE(KqpSpillingFileTests) {
+
+Y_UNIT_TEST(Simple) {
+ TTestBasicRuntime runtime{1, false};
+ runtime.Initialize(TAppPrepare().Unwrap());
+ SetupLogs(runtime);
+
+ auto spillingService = StartSpillingService(runtime);
+ auto tester = runtime.AllocateEdgeActor();
+ auto spillingActor = StartSpillingActor(runtime, tester);
+
+ WaitBootstrap(runtime);
+
+ // put blob 1
+ {
+ auto ev = new TEvKqpSpilling::TEvWrite(1, CreateBlob(10, 'a'));
+ runtime.Send(new IEventHandle(spillingActor, tester, ev));
+
+ auto resp = runtime.GrabEdgeEvent<TEvKqpSpilling::TEvWriteResult>(tester, TDuration::Seconds(1));
+ UNIT_ASSERT_VALUES_EQUAL(1, resp->Get()->BlobId);
+ }
+
+ // put blob 2
+ {
+ auto ev = new TEvKqpSpilling::TEvWrite(2, CreateBlob(11, 'z'));
+ runtime.Send(new IEventHandle(spillingActor, tester, ev));
+
+ auto resp = runtime.GrabEdgeEvent<TEvKqpSpilling::TEvWriteResult>(tester, TDuration::Seconds(1));
+ UNIT_ASSERT_VALUES_EQUAL(2, resp->Get()->BlobId);
+ }
+
+ // get blob 1
+ {
+ auto ev = new TEvKqpSpilling::TEvRead(1);
+ runtime.Send(new IEventHandle(spillingActor, tester, ev));
+
+ auto resp = runtime.GrabEdgeEvent<TEvKqpSpilling::TEvReadResult>(tester, TDuration::Seconds(1));
+ UNIT_ASSERT_VALUES_EQUAL(1, resp->Get()->BlobId);
+
+ TBuffer expected = CreateBlob(10, 'a');
+ AssertEquals(expected, resp->Get()->Blob);
+ }
+
+ // get blob 2
+ {
+ auto ev = new TEvKqpSpilling::TEvRead(2);
+ runtime.Send(new IEventHandle(spillingActor, tester, ev));
+
+ auto resp = runtime.GrabEdgeEvent<TEvKqpSpilling::TEvReadResult>(tester, TDuration::Seconds(1));
+ UNIT_ASSERT_VALUES_EQUAL(2, resp->Get()->BlobId);
+
+ TBuffer expected = CreateBlob(11, 'z');
+ AssertEquals(expected, resp->Get()->Blob);
+ }
+
+ // terminate
+ {
+ runtime.Send(new IEventHandle(spillingActor, tester, new TEvents::TEvPoison));
+
+ std::atomic<bool> done = false;
+ runtime.SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& event) {
+ if (event->GetRecipientRewrite() == spillingService) {
+ if (event->GetTypeRewrite() == 2146435074 /* EvCloseFileResponse */ ) {
+ done = true;
+ }
+ }
+ return TTestActorRuntimeBase::EEventAction::PROCESS;
+ });
+
+ TDispatchOptions options;
+ options.CustomFinalCondition = [&]() {
+ return (bool) done;
+ };
+
+ runtime.DispatchEvents(options, TDuration::Seconds(1));
+ }
+}
+
+Y_UNIT_TEST(Write_TotalSizeLimitExceeded) {
+ TTestBasicRuntime runtime{1, false};
+ runtime.Initialize(TAppPrepare().Unwrap());
+ SetupLogs(runtime);
+
+ StartSpillingService(runtime, 100, 1000, 1000);
+ auto tester = runtime.AllocateEdgeActor();
+ auto spillingActor = StartSpillingActor(runtime, tester);
+
+ WaitBootstrap(runtime);
+
+ {
+ auto ev = new TEvKqpSpilling::TEvWrite(1, CreateBlob(51, 'a'));
+ runtime.Send(new IEventHandle(spillingActor, tester, ev));
+
+ auto resp = runtime.GrabEdgeEvent<TEvKqpSpilling::TEvWriteResult>(tester);
+ UNIT_ASSERT_VALUES_EQUAL(1, resp->Get()->BlobId);
+ }
+
+ {
+ auto ev = new TEvKqpSpilling::TEvWrite(2, CreateBlob(50, 'b'));
+ runtime.Send(new IEventHandle(spillingActor, tester, ev));
+
+ auto resp = runtime.GrabEdgeEvent<TEvKqpSpilling::TEvError>(tester);
+ UNIT_ASSERT_STRINGS_EQUAL("Total size limit exceeded", resp->Get()->Message);
+ }
+}
+
+Y_UNIT_TEST(Write_FileSizeLimitExceeded) {
+ TTestBasicRuntime runtime{1, false};
+ runtime.Initialize(TAppPrepare().Unwrap());
+ SetupLogs(runtime);
+
+ StartSpillingService(runtime, 1000, 100, 1000);
+ auto tester = runtime.AllocateEdgeActor();
+ auto spillingActor = StartSpillingActor(runtime, tester);
+
+ WaitBootstrap(runtime);
+
+ {
+ auto ev = new TEvKqpSpilling::TEvWrite(1, CreateBlob(51, 'a'));
+ runtime.Send(new IEventHandle(spillingActor, tester, ev));
+
+ auto resp = runtime.GrabEdgeEvent<TEvKqpSpilling::TEvWriteResult>(tester);
+ UNIT_ASSERT_VALUES_EQUAL(1, resp->Get()->BlobId);
+ }
+
+ {
+ auto ev = new TEvKqpSpilling::TEvWrite(2, CreateBlob(50, 'b'));
+ runtime.Send(new IEventHandle(spillingActor, tester, ev));
+
+ auto resp = runtime.GrabEdgeEvent<TEvKqpSpilling::TEvError>(tester);
+ UNIT_ASSERT_STRINGS_EQUAL("File size limit exceeded", resp->Get()->Message);
+ }
+}
+
+Y_UNIT_TEST(MultipleFileParts) {
+ TTestBasicRuntime runtime{1, false};
+ runtime.Initialize(TAppPrepare().Unwrap());
+ SetupLogs(runtime);
+
+ StartSpillingService(runtime, 1000, 100, 25);
+ auto tester = runtime.AllocateEdgeActor();
+ auto spillingActor = StartSpillingActor(runtime, tester);
+
+ WaitBootstrap(runtime);
+
+ const TString filePrefix = TStringBuilder() << NFs::CurrentWorkingDirectory() << "/kqp_spilling/node_1/1_test_";
+
+ for (ui32 i = 0; i < 5; ++i) {
+ // Cerr << "---- store blob #" << i << Endl;
+ auto ev = new TEvKqpSpilling::TEvWrite(i, CreateBlob(20, 'a' + i));
+ runtime.Send(new IEventHandle(spillingActor, tester, ev));
+
+ auto resp = runtime.GrabEdgeEvent<TEvKqpSpilling::TEvWriteResult>(tester);
+ UNIT_ASSERT_VALUES_EQUAL(i, resp->Get()->BlobId);
+
+ UNIT_ASSERT(NFs::Exists(TStringBuilder() << filePrefix << i));
+ }
+
+ for (i32 i = 4; i >= 0; --i) {
+ // Cerr << "---- load blob #" << i << Endl;
+ auto ev = new TEvKqpSpilling::TEvRead(i, true);
+ runtime.Send(new IEventHandle(spillingActor, tester, ev));
+
+ auto resp = runtime.GrabEdgeEvent<TEvKqpSpilling::TEvReadResult>(tester);
+ UNIT_ASSERT_VALUES_EQUAL(i, resp->Get()->BlobId);
+ TBuffer expected = CreateBlob(20, 'a' + i);
+ AssertEquals(expected, resp->Get()->Blob);
+
+ if (i == 4) {
+ // do not remove last file
+ UNIT_ASSERT(NFs::Exists(TStringBuilder() << filePrefix << i));
+ } else {
+ UNIT_ASSERT(!NFs::Exists(TStringBuilder() << filePrefix << i));
+ }
+ }
+}
+
+Y_UNIT_TEST(SingleFilePart) {
+ TTestBasicRuntime runtime{1, false};
+ runtime.Initialize(TAppPrepare().Unwrap());
+ SetupLogs(runtime);
+
+ StartSpillingService(runtime, 1000, 100, 25);
+ auto tester = runtime.AllocateEdgeActor();
+ auto spillingActor = StartSpillingActor(runtime, tester, false);
+
+ WaitBootstrap(runtime);
+
+ const TString filePrefix = TStringBuilder() << NFs::CurrentWorkingDirectory() << "/kqp_spilling/node_1/1_test_";
+
+ for (ui32 i = 0; i < 5; ++i) {
+ // Cerr << "---- store blob #" << i << Endl;
+ auto ev = new TEvKqpSpilling::TEvWrite(i, CreateBlob(20, 'a' + i));
+ runtime.Send(new IEventHandle(spillingActor, tester, ev));
+
+ auto resp = runtime.GrabEdgeEvent<TEvKqpSpilling::TEvWriteResult>(tester);
+ UNIT_ASSERT_VALUES_EQUAL(i, resp->Get()->BlobId);
+
+ UNIT_ASSERT(NFs::Exists(TStringBuilder() << filePrefix << 0));
+ if (i > 0) {
+ UNIT_ASSERT(!NFs::Exists(TStringBuilder() << filePrefix << i));
+ }
+ }
+
+ for (i32 i = 4; i >= 0; --i) {
+ // Cerr << "---- load blob #" << i << Endl;
+ auto ev = new TEvKqpSpilling::TEvRead(i, true);
+ runtime.Send(new IEventHandle(spillingActor, tester, ev));
+
+ auto resp = runtime.GrabEdgeEvent<TEvKqpSpilling::TEvReadResult>(tester);
+ UNIT_ASSERT_VALUES_EQUAL(i, resp->Get()->BlobId);
+ TBuffer expected = CreateBlob(20, 'a' + i);
+ AssertEquals(expected, resp->Get()->Blob);
+
+ UNIT_ASSERT(NFs::Exists(TStringBuilder() << filePrefix << 0));
+ }
+}
+
+Y_UNIT_TEST(ReadError) {
+ TTestBasicRuntime runtime{1, false};
+ runtime.Initialize(TAppPrepare().Unwrap());
+ SetupLogs(runtime);
+
+ StartSpillingService(runtime);
+ auto tester = runtime.AllocateEdgeActor();
+ auto spillingActor = StartSpillingActor(runtime, tester);
+
+ WaitBootstrap(runtime);
+
+ {
+ auto ev = new TEvKqpSpilling::TEvWrite(0, CreateBlob(20, 'a'));
+ runtime.Send(new IEventHandle(spillingActor, tester, ev));
+
+ auto resp = runtime.GrabEdgeEvent<TEvKqpSpilling::TEvWriteResult>(tester);
+ UNIT_ASSERT_VALUES_EQUAL(0, resp->Get()->BlobId);
+ }
+
+ ::unlink((NFs::CurrentWorkingDirectory() + "/kqp_spilling/node_1/1_test_0").c_str());
+
+ {
+ auto ev = new TEvKqpSpilling::TEvRead(0, true);
+ runtime.Send(new IEventHandle(spillingActor, tester, ev));
+
+ auto resp = runtime.GrabEdgeEvent<TEvKqpSpilling::TEvError>(tester);
+ auto& err = resp->Get()->Message;
+ UNIT_ASSERT_C(err.Contains("No such file or directory"), err);
+ UNIT_ASSERT_C(err.Contains("can't open \"kqp_spilling/node_1/1_test_0\" with mode RdOnly"), err);
+ }
+}
+
+
+struct THttpRequest : NMonitoring::IHttpRequest {
+ HTTP_METHOD Method;
+ TCgiParameters CgiParameters;
+ THttpHeaders HttpHeaders;
+
+ THttpRequest(HTTP_METHOD method)
+ : Method(method)
+ {}
+
+ ~THttpRequest() {}
+
+ const char* GetURI() const override {
+ return "";
+ }
+
+ const char* GetPath() const override {
+ return "";
+ }
+
+ const TCgiParameters& GetParams() const override {
+ return CgiParameters;
+ }
+
+ const TCgiParameters& GetPostParams() const override {
+ return CgiParameters;
+ }
+
+ TStringBuf GetPostContent() const override {
+ return TString();
+ }
+
+ HTTP_METHOD GetMethod() const override {
+ return Method;
+ }
+
+ const THttpHeaders& GetHeaders() const override {
+ return HttpHeaders;
+ }
+};
+
+Y_UNIT_TEST(StartError) {
+ TTestBasicRuntime runtime{1, false};
+ runtime.Initialize(TAppPrepare().Unwrap());
+ SetupLogs(runtime);
+
+ auto spillingService = StartSpillingService(runtime, 100, 500, 100, "/root/kqp_spilling/");
+ auto tester = runtime.AllocateEdgeActor();
+ auto spillingActor = StartSpillingActor(runtime, tester);
+
+ WaitBootstrap(runtime);
+
+ // put blob 1
+ {
+ auto ev = new TEvKqpSpilling::TEvWrite(1, CreateBlob(10, 'a'));
+ runtime.Send(new IEventHandle(spillingActor, tester, ev));
+
+ auto resp = runtime.GrabEdgeEvent<TEvKqpSpilling::TEvError>(tester, TDuration::Seconds(1));
+ UNIT_ASSERT_EQUAL("Service not started", resp->Get()->Message);
+ }
+
+ // get blob 1
+ {
+ auto ev = new TEvKqpSpilling::TEvRead(1);
+ runtime.Send(new IEventHandle(spillingActor, tester, ev));
+
+ auto resp = runtime.GrabEdgeEvent<TEvKqpSpilling::TEvError>(tester, TDuration::Seconds(1));
+ UNIT_ASSERT_EQUAL("Service not started", resp->Get()->Message);
+ }
+
+ // mon
+ {
+ THttpRequest httpReq(HTTP_METHOD_GET);
+ NMonitoring::TMonService2HttpRequest monReq(nullptr, &httpReq, nullptr, nullptr, "", nullptr);
+
+ runtime.Send(new IEventHandle(spillingService, tester, new NMon::TEvHttpInfo(monReq)));
+
+ auto resp = runtime.GrabEdgeEvent<NMon::TEvHttpInfoRes>(tester, TDuration::Seconds(1));
+ UNIT_ASSERT_EQUAL("<html><h2>Service is not started due to IO error</h2></html>",
+ ((NMon::TEvHttpInfoRes*) resp->Get())->Answer);
+ }
+}
+
+} // suite
+
+} // namespace NKikimr::NKqp
diff --git a/ydb/core/kqp/runtime/kqp_tasks_runner.cpp b/ydb/core/kqp/runtime/kqp_tasks_runner.cpp
index a65a96e13f..70b1038e6c 100644
--- a/ydb/core/kqp/runtime/kqp_tasks_runner.cpp
+++ b/ydb/core/kqp/runtime/kqp_tasks_runner.cpp
@@ -10,219 +10,219 @@
#include <util/generic/vector.h>
namespace NKikimr {
-
-namespace NMiniKQL {
-class TKqpScanComputeContext;
-} // namespace NMiniKQL
-
+
+namespace NMiniKQL {
+class TKqpScanComputeContext;
+} // namespace NMiniKQL
+
namespace NKqp {
using namespace NMiniKQL;
using namespace NYql;
using namespace NDq;
-IDqOutputConsumer::TPtr KqpBuildOutputConsumer(const NDqProto::TTaskOutput& outputDesc, const TType* type,
+IDqOutputConsumer::TPtr KqpBuildOutputConsumer(const NDqProto::TTaskOutput& outputDesc, const TType* type,
NUdf::IApplyContext* applyCtx, const TTypeEnvironment& typeEnv, TVector<IDqOutput::TPtr>&& outputs)
-{
- switch (outputDesc.GetTypeCase()) {
- case NDqProto::TTaskOutput::kRangePartition: {
- TVector<NUdf::TDataTypeId> keyColumnTypes;
- TVector<ui32> keyColumnIndices;
- GetColumnsInfo(type, outputDesc.GetRangePartition().GetKeyColumns(), keyColumnTypes, keyColumnIndices);
- YQL_ENSURE(!keyColumnTypes.empty());
-
- TVector<TKqpRangePartition> partitions;
- partitions.reserve(outputDesc.GetRangePartition().PartitionsSize());
-
- for (auto& partitionDesc : outputDesc.GetRangePartition().GetPartitions()) {
- TKqpRangePartition partition;
- partition.ChannelId = partitionDesc.GetChannelId();
- partition.Range.EndKeyPrefix = TSerializedCellVec(partitionDesc.GetEndKeyPrefix());
- partition.Range.IsInclusive = partitionDesc.GetIsInclusive();
- partition.Range.IsPoint = partitionDesc.GetIsPoint();
-
- partitions.emplace_back(std::move(partition));
+{
+ switch (outputDesc.GetTypeCase()) {
+ case NDqProto::TTaskOutput::kRangePartition: {
+ TVector<NUdf::TDataTypeId> keyColumnTypes;
+ TVector<ui32> keyColumnIndices;
+ GetColumnsInfo(type, outputDesc.GetRangePartition().GetKeyColumns(), keyColumnTypes, keyColumnIndices);
+ YQL_ENSURE(!keyColumnTypes.empty());
+
+ TVector<TKqpRangePartition> partitions;
+ partitions.reserve(outputDesc.GetRangePartition().PartitionsSize());
+
+ for (auto& partitionDesc : outputDesc.GetRangePartition().GetPartitions()) {
+ TKqpRangePartition partition;
+ partition.ChannelId = partitionDesc.GetChannelId();
+ partition.Range.EndKeyPrefix = TSerializedCellVec(partitionDesc.GetEndKeyPrefix());
+ partition.Range.IsInclusive = partitionDesc.GetIsInclusive();
+ partition.Range.IsPoint = partitionDesc.GetIsPoint();
+
+ partitions.emplace_back(std::move(partition));
}
return CreateOutputRangePartitionConsumer(std::move(outputs), std::move(partitions),
- std::move(keyColumnTypes), std::move(keyColumnIndices), typeEnv);
- }
+ std::move(keyColumnTypes), std::move(keyColumnIndices), typeEnv);
+ }
- case NDqProto::TTaskOutput::kEffects: {
- return CreateKqpApplyEffectsConsumer(applyCtx);
+ case NDqProto::TTaskOutput::kEffects: {
+ return CreateKqpApplyEffectsConsumer(applyCtx);
}
-
- default: {
+
+ default: {
return DqBuildOutputConsumer(outputDesc, type, typeEnv, std::move(outputs));
- }
+ }
+ }
+}
+
+TIntrusivePtr<IDqTaskRunner> CreateKqpTaskRunner(const TDqTaskRunnerContext& execCtx,
+ const TDqTaskRunnerSettings& settings, const TLogFunc& logFunc)
+{
+ return MakeDqTaskRunner(execCtx, settings, logFunc);
+}
+
+
+TKqpTasksRunner::TKqpTasksRunner(const google::protobuf::RepeatedPtrField<NDqProto::TDqTask>& tasks,
+ const TDqTaskRunnerContext& execCtx, const TDqTaskRunnerSettings& settings, const TLogFunc& logFunc)
+ : LogFunc(logFunc)
+ , Alloc(execCtx.Alloc)
+{
+ YQL_ENSURE(execCtx.Alloc);
+ YQL_ENSURE(execCtx.TypeEnv);
+
+ ComputeCtx = dynamic_cast<NMiniKQL::TKqpComputeContextBase*>(execCtx.ComputeCtx);
+ YQL_ENSURE(ComputeCtx);
+
+ auto guard = execCtx.TypeEnv->BindAllocator();
+ try {
+ for (auto& task : tasks) {
+ auto runner = CreateKqpTaskRunner(execCtx, settings, logFunc);
+ if (auto* stats = runner->GetStats()) {
+ Stats.emplace(task.GetId(), stats);
+ }
+ TaskRunners.emplace(task.GetId(), std::move(runner));
+ Tasks.emplace(task.GetId(), &task);
+ }
+ } catch (const TMemoryLimitExceededException&) {
+ TaskRunners.clear();
+ Tasks.clear();
+ throw;
+ }
+}
+
+TKqpTasksRunner::~TKqpTasksRunner() {
+ for (auto& [_, runner] : TaskRunners) {
+ auto guard = runner->BindAllocator();
+ runner.Reset();
+ }
+}
+
+void TKqpTasksRunner::Prepare(const TDqTaskRunnerMemoryLimits& memoryLimits, const IDqTaskRunnerExecutionContext& execCtx)
+{
+ if (State >= EState::Prepared) {
+ return;
+ }
+ YQL_ENSURE(State == EState::Initial, "" << (int) State);
+
+ for (auto& [taskId, taskRunner] : TaskRunners) {
+ ComputeCtx->SetCurrentTaskId(taskId);
+ taskRunner->Prepare(*Tasks[taskId], memoryLimits, execCtx);
+ }
+
+ ComputeCtx->SetCurrentTaskId(std::numeric_limits<ui64>::max());
+
+ State = EState::Prepared;
+}
+
+ERunStatus TKqpTasksRunner::Run(bool applyEffects) {
+ YQL_ENSURE(State >= EState::Prepared, "" << (int) State);
+ State = EState::Running;
+
+ bool hasPendingInputTasks = false;
+ bool hasPendingOutputTasks = false;
+
+ for (auto& [taskId, task] : TaskRunners) {
+ if (Y_UNLIKELY(LogFunc)) {
+ LogFunc(TStringBuilder() << "running task: " << taskId);
+ }
+
+ if (!applyEffects && task->HasEffects()) {
+ // TODO: effects-only task?
+ continue;
+ }
+
+ auto status = task->Run();
+
+ switch (status) {
+ case ERunStatus::PendingInput:
+ case ERunStatus::PendingOutput: {
+ if (applyEffects && task->HasEffects()) {
+ // Keep effects order between tasks, can't process next task before all
+ // effects from the current one are applied.
+ if (Y_UNLIKELY(LogFunc)) {
+ LogFunc(TStringBuilder() << "task: " << taskId << " execution status: " << status
+ << ", stop tasks execution to preserve effects order between tasks");
+ }
+ return status;
+ }
+ hasPendingInputTasks = status == ERunStatus::PendingInput;
+ hasPendingOutputTasks = status == ERunStatus::PendingOutput;
+ break;
+ }
+ case ERunStatus::Finished:
+ break;
+ }
+
+ if (Y_UNLIKELY(LogFunc)) {
+ LogFunc(TStringBuilder() << "task: " << taskId << " execution status: " << status);
+ }
}
-}
-TIntrusivePtr<IDqTaskRunner> CreateKqpTaskRunner(const TDqTaskRunnerContext& execCtx,
- const TDqTaskRunnerSettings& settings, const TLogFunc& logFunc)
-{
- return MakeDqTaskRunner(execCtx, settings, logFunc);
+ if (hasPendingOutputTasks) {
+ return ERunStatus::PendingOutput;
+ }
+ if (hasPendingInputTasks) {
+ return ERunStatus::PendingInput;
+ }
+
+ return ERunStatus::Finished;
+}
+
+std::pair<bool, bool> TKqpTasksRunner::TransferData(ui64 fromTask, ui64 fromChannelId, ui64 toTask, ui64 toChannelId) {
+ auto src = GetOutputChannel(fromTask, fromChannelId);
+ auto dst = GetInputChannel(toTask, toChannelId);
+
+ bool transferred = false;
+ bool finished = false;
+
+ // todo: transfer data as-is from input- to output- channel (KIKIMR-10658)
+ NDqProto::TData data;
+ if (src->Pop(data, std::numeric_limits<ui64>::max())) {
+ transferred = true;
+ dst->Push(std::move(data));
+ }
+
+ if (src->IsFinished()) {
+ finished = true;
+ dst->Finish();
+ }
+
+ return std::make_pair(transferred, finished);
+}
+
+IDqTaskRunner& TKqpTasksRunner::GetTaskRunner(ui64 taskId) {
+ auto task = TaskRunners.FindPtr(taskId);
+ YQL_ENSURE(task);
+
+ return **task;
+}
+
+const IDqTaskRunner& TKqpTasksRunner::GetTaskRunner(ui64 taskId) const {
+ auto task = TaskRunners.FindPtr(taskId);
+ YQL_ENSURE(task);
+
+ return **task;
+}
+
+const NYql::NDqProto::TDqTask& TKqpTasksRunner::GetTask(ui64 taskId) const {
+ return *Tasks.at(taskId);
+}
+
+TGuard<NMiniKQL::TScopedAlloc> TKqpTasksRunner::BindAllocator(TMaybe<ui64> memoryLimit) {
+ if (memoryLimit) {
+ Alloc->SetLimit(*memoryLimit);
+ }
+ return TGuard(*Alloc);
+}
+
+TIntrusivePtr<TKqpTasksRunner> CreateKqpTasksRunner(const google::protobuf::RepeatedPtrField<NDqProto::TDqTask>& tasks,
+ const TDqTaskRunnerContext& execCtx, const TDqTaskRunnerSettings& settings, const TLogFunc& logFunc)
+{
+ return new TKqpTasksRunner(tasks, execCtx, settings, logFunc);
}
-
-TKqpTasksRunner::TKqpTasksRunner(const google::protobuf::RepeatedPtrField<NDqProto::TDqTask>& tasks,
- const TDqTaskRunnerContext& execCtx, const TDqTaskRunnerSettings& settings, const TLogFunc& logFunc)
- : LogFunc(logFunc)
- , Alloc(execCtx.Alloc)
-{
- YQL_ENSURE(execCtx.Alloc);
- YQL_ENSURE(execCtx.TypeEnv);
-
- ComputeCtx = dynamic_cast<NMiniKQL::TKqpComputeContextBase*>(execCtx.ComputeCtx);
- YQL_ENSURE(ComputeCtx);
-
- auto guard = execCtx.TypeEnv->BindAllocator();
- try {
- for (auto& task : tasks) {
- auto runner = CreateKqpTaskRunner(execCtx, settings, logFunc);
- if (auto* stats = runner->GetStats()) {
- Stats.emplace(task.GetId(), stats);
- }
- TaskRunners.emplace(task.GetId(), std::move(runner));
- Tasks.emplace(task.GetId(), &task);
- }
- } catch (const TMemoryLimitExceededException&) {
- TaskRunners.clear();
- Tasks.clear();
- throw;
- }
-}
-
-TKqpTasksRunner::~TKqpTasksRunner() {
- for (auto& [_, runner] : TaskRunners) {
- auto guard = runner->BindAllocator();
- runner.Reset();
- }
-}
-
-void TKqpTasksRunner::Prepare(const TDqTaskRunnerMemoryLimits& memoryLimits, const IDqTaskRunnerExecutionContext& execCtx)
-{
- if (State >= EState::Prepared) {
- return;
- }
- YQL_ENSURE(State == EState::Initial, "" << (int) State);
-
- for (auto& [taskId, taskRunner] : TaskRunners) {
- ComputeCtx->SetCurrentTaskId(taskId);
- taskRunner->Prepare(*Tasks[taskId], memoryLimits, execCtx);
- }
-
- ComputeCtx->SetCurrentTaskId(std::numeric_limits<ui64>::max());
-
- State = EState::Prepared;
-}
-
-ERunStatus TKqpTasksRunner::Run(bool applyEffects) {
- YQL_ENSURE(State >= EState::Prepared, "" << (int) State);
- State = EState::Running;
-
- bool hasPendingInputTasks = false;
- bool hasPendingOutputTasks = false;
-
- for (auto& [taskId, task] : TaskRunners) {
- if (Y_UNLIKELY(LogFunc)) {
- LogFunc(TStringBuilder() << "running task: " << taskId);
- }
-
- if (!applyEffects && task->HasEffects()) {
- // TODO: effects-only task?
- continue;
- }
-
- auto status = task->Run();
-
- switch (status) {
- case ERunStatus::PendingInput:
- case ERunStatus::PendingOutput: {
- if (applyEffects && task->HasEffects()) {
- // Keep effects order between tasks, can't process next task before all
- // effects from the current one are applied.
- if (Y_UNLIKELY(LogFunc)) {
- LogFunc(TStringBuilder() << "task: " << taskId << " execution status: " << status
- << ", stop tasks execution to preserve effects order between tasks");
- }
- return status;
- }
- hasPendingInputTasks = status == ERunStatus::PendingInput;
- hasPendingOutputTasks = status == ERunStatus::PendingOutput;
- break;
- }
- case ERunStatus::Finished:
- break;
- }
-
- if (Y_UNLIKELY(LogFunc)) {
- LogFunc(TStringBuilder() << "task: " << taskId << " execution status: " << status);
- }
- }
-
- if (hasPendingOutputTasks) {
- return ERunStatus::PendingOutput;
- }
- if (hasPendingInputTasks) {
- return ERunStatus::PendingInput;
- }
-
- return ERunStatus::Finished;
-}
-
-std::pair<bool, bool> TKqpTasksRunner::TransferData(ui64 fromTask, ui64 fromChannelId, ui64 toTask, ui64 toChannelId) {
- auto src = GetOutputChannel(fromTask, fromChannelId);
- auto dst = GetInputChannel(toTask, toChannelId);
-
- bool transferred = false;
- bool finished = false;
-
- // todo: transfer data as-is from input- to output- channel (KIKIMR-10658)
- NDqProto::TData data;
- if (src->Pop(data, std::numeric_limits<ui64>::max())) {
- transferred = true;
- dst->Push(std::move(data));
- }
-
- if (src->IsFinished()) {
- finished = true;
- dst->Finish();
- }
-
- return std::make_pair(transferred, finished);
-}
-
-IDqTaskRunner& TKqpTasksRunner::GetTaskRunner(ui64 taskId) {
- auto task = TaskRunners.FindPtr(taskId);
- YQL_ENSURE(task);
-
- return **task;
-}
-
-const IDqTaskRunner& TKqpTasksRunner::GetTaskRunner(ui64 taskId) const {
- auto task = TaskRunners.FindPtr(taskId);
- YQL_ENSURE(task);
-
- return **task;
-}
-
-const NYql::NDqProto::TDqTask& TKqpTasksRunner::GetTask(ui64 taskId) const {
- return *Tasks.at(taskId);
-}
-
-TGuard<NMiniKQL::TScopedAlloc> TKqpTasksRunner::BindAllocator(TMaybe<ui64> memoryLimit) {
- if (memoryLimit) {
- Alloc->SetLimit(*memoryLimit);
- }
- return TGuard(*Alloc);
-}
-
-TIntrusivePtr<TKqpTasksRunner> CreateKqpTasksRunner(const google::protobuf::RepeatedPtrField<NDqProto::TDqTask>& tasks,
- const TDqTaskRunnerContext& execCtx, const TDqTaskRunnerSettings& settings, const TLogFunc& logFunc)
-{
- return new TKqpTasksRunner(tasks, execCtx, settings, logFunc);
-}
-
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/runtime/kqp_tasks_runner.h b/ydb/core/kqp/runtime/kqp_tasks_runner.h
index 3fd9b6c3e1..6ae8bf1db7 100644
--- a/ydb/core/kqp/runtime/kqp_tasks_runner.h
+++ b/ydb/core/kqp/runtime/kqp_tasks_runner.h
@@ -1,74 +1,74 @@
#pragma once
-#include "kqp_compute.h"
+#include "kqp_compute.h"
#include <ydb/library/yql/dq/runtime/dq_tasks_runner.h>
namespace NKikimr {
-namespace NKqp {
-
-NYql::NDq::IDqOutputConsumer::TPtr KqpBuildOutputConsumer(const NYql::NDqProto::TTaskOutput& outputDesc,
- const NMiniKQL::TType* type, NUdf::IApplyContext* applyCtx, const NMiniKQL::TTypeEnvironment& typeEnv,
+namespace NKqp {
+
+NYql::NDq::IDqOutputConsumer::TPtr KqpBuildOutputConsumer(const NYql::NDqProto::TTaskOutput& outputDesc,
+ const NMiniKQL::TType* type, NUdf::IApplyContext* applyCtx, const NMiniKQL::TTypeEnvironment& typeEnv,
TVector<NYql::NDq::IDqOutput::TPtr>&& outputs);
-
-TIntrusivePtr<NYql::NDq::IDqTaskRunner> CreateKqpTaskRunner(const NYql::NDq::TDqTaskRunnerContext& execCtx,
- const NYql::NDq::TDqTaskRunnerSettings& settings, const NYql::NDq::TLogFunc& logFunc);
-
-
-class TKqpTasksRunner : public TSimpleRefCount<TKqpTasksRunner>, private TNonCopyable {
-public:
- TKqpTasksRunner(const google::protobuf::RepeatedPtrField<NYql::NDqProto::TDqTask>& tasks,
- const NYql::NDq::TDqTaskRunnerContext& execCtx, const NYql::NDq::TDqTaskRunnerSettings& settings,
- const NYql::NDq::TLogFunc& logFunc);
-
- ~TKqpTasksRunner();
-
- void Prepare(const NYql::NDq::TDqTaskRunnerMemoryLimits& memoryLimits,
- const NYql::NDq::IDqTaskRunnerExecutionContext& execCtx);
-
- NYql::NDq::ERunStatus Run(bool applyEffects);
-
- /// @returns (data_was_transferred, finished)
- std::pair<bool, bool> TransferData(ui64 fromTask, ui64 fromChannelId, ui64 toTask, ui64 toChannelId);
-
- NYql::NDq::IDqTaskRunner& GetTaskRunner(ui64 taskId);
- const NYql::NDq::IDqTaskRunner& GetTaskRunner(ui64 taskId) const;
-
- const NYql::NDqProto::TDqTask& GetTask(ui64 taskId) const;
-
- NYql::NDq::IDqInputChannel::TPtr GetInputChannel(ui64 taskId, ui64 channelId) {
- return GetTaskRunner(taskId).GetInputChannel(channelId);
- }
- NYql::NDq::IDqOutputChannel::TPtr GetOutputChannel(ui64 taskId, ui64 channelId) {
- return GetTaskRunner(taskId).GetOutputChannel(channelId);
- }
-
- // if memoryLimit = Nothing() then don't set memory limit, use existing one (if any)
- // if memoryLimit = 0 then set unlimited
- // otherwise use particular memory limit
- TGuard<NMiniKQL::TScopedAlloc> BindAllocator(TMaybe<ui64> memoryLimit = Nothing());
-
- const TMap<ui64, const NYql::NDq::TDqTaskRunnerStats*> GetTasksStats() const { return Stats; };
-
-private:
- TMap<ui64, TIntrusivePtr<NYql::NDq::IDqTaskRunner>> TaskRunners;
- TMap<ui64, const NYql::NDqProto::TDqTask*> Tasks;
- TMap<ui64, const NYql::NDq::TDqTaskRunnerStats*> Stats;
- NYql::NDq::TLogFunc LogFunc;
- NMiniKQL::TScopedAlloc* Alloc;
- NMiniKQL::TKqpComputeContextBase* ComputeCtx;
-
- enum EState {
- Initial = 0,
- Prepared = 1,
- Running = 2
- };
- EState State = EState::Initial;
-};
-
-
-TIntrusivePtr<TKqpTasksRunner> CreateKqpTasksRunner(const google::protobuf::RepeatedPtrField<NYql::NDqProto::TDqTask>& tasks,
- const NYql::NDq::TDqTaskRunnerContext& execCtx, const NYql::NDq::TDqTaskRunnerSettings& settings,
- const NYql::NDq::TLogFunc& logFunc);
-
+
+TIntrusivePtr<NYql::NDq::IDqTaskRunner> CreateKqpTaskRunner(const NYql::NDq::TDqTaskRunnerContext& execCtx,
+ const NYql::NDq::TDqTaskRunnerSettings& settings, const NYql::NDq::TLogFunc& logFunc);
+
+
+class TKqpTasksRunner : public TSimpleRefCount<TKqpTasksRunner>, private TNonCopyable {
+public:
+ TKqpTasksRunner(const google::protobuf::RepeatedPtrField<NYql::NDqProto::TDqTask>& tasks,
+ const NYql::NDq::TDqTaskRunnerContext& execCtx, const NYql::NDq::TDqTaskRunnerSettings& settings,
+ const NYql::NDq::TLogFunc& logFunc);
+
+ ~TKqpTasksRunner();
+
+ void Prepare(const NYql::NDq::TDqTaskRunnerMemoryLimits& memoryLimits,
+ const NYql::NDq::IDqTaskRunnerExecutionContext& execCtx);
+
+ NYql::NDq::ERunStatus Run(bool applyEffects);
+
+ /// @returns (data_was_transferred, finished)
+ std::pair<bool, bool> TransferData(ui64 fromTask, ui64 fromChannelId, ui64 toTask, ui64 toChannelId);
+
+ NYql::NDq::IDqTaskRunner& GetTaskRunner(ui64 taskId);
+ const NYql::NDq::IDqTaskRunner& GetTaskRunner(ui64 taskId) const;
+
+ const NYql::NDqProto::TDqTask& GetTask(ui64 taskId) const;
+
+ NYql::NDq::IDqInputChannel::TPtr GetInputChannel(ui64 taskId, ui64 channelId) {
+ return GetTaskRunner(taskId).GetInputChannel(channelId);
+ }
+ NYql::NDq::IDqOutputChannel::TPtr GetOutputChannel(ui64 taskId, ui64 channelId) {
+ return GetTaskRunner(taskId).GetOutputChannel(channelId);
+ }
+
+ // if memoryLimit = Nothing() then don't set memory limit, use existing one (if any)
+ // if memoryLimit = 0 then set unlimited
+ // otherwise use particular memory limit
+ TGuard<NMiniKQL::TScopedAlloc> BindAllocator(TMaybe<ui64> memoryLimit = Nothing());
+
+ const TMap<ui64, const NYql::NDq::TDqTaskRunnerStats*> GetTasksStats() const { return Stats; };
+
+private:
+ TMap<ui64, TIntrusivePtr<NYql::NDq::IDqTaskRunner>> TaskRunners;
+ TMap<ui64, const NYql::NDqProto::TDqTask*> Tasks;
+ TMap<ui64, const NYql::NDq::TDqTaskRunnerStats*> Stats;
+ NYql::NDq::TLogFunc LogFunc;
+ NMiniKQL::TScopedAlloc* Alloc;
+ NMiniKQL::TKqpComputeContextBase* ComputeCtx;
+
+ enum EState {
+ Initial = 0,
+ Prepared = 1,
+ Running = 2
+ };
+ EState State = EState::Initial;
+};
+
+
+TIntrusivePtr<TKqpTasksRunner> CreateKqpTasksRunner(const google::protobuf::RepeatedPtrField<NYql::NDqProto::TDqTask>& tasks,
+ const NYql::NDq::TDqTaskRunnerContext& execCtx, const NYql::NDq::TDqTaskRunnerSettings& settings,
+ const NYql::NDq::TLogFunc& logFunc);
+
} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/runtime/kqp_transport.cpp b/ydb/core/kqp/runtime/kqp_transport.cpp
index e9f9864153..8c1b3262d0 100644
--- a/ydb/core/kqp/runtime/kqp_transport.cpp
+++ b/ydb/core/kqp/runtime/kqp_transport.cpp
@@ -15,75 +15,75 @@ namespace NKqp {
using namespace NMiniKQL;
using namespace NYql;
-TKqpProtoBuilder::TSelfHosted::TSelfHosted(const IFunctionRegistry& funcRegistry)
+TKqpProtoBuilder::TSelfHosted::TSelfHosted(const IFunctionRegistry& funcRegistry)
: Alloc(TAlignedPagePoolCounters(), funcRegistry.SupportsSizedAllocators())
, TypeEnv(Alloc)
- , MemInfo("KqpProtoBuilder")
- , HolderFactory(Alloc.Ref(), MemInfo)
+ , MemInfo("KqpProtoBuilder")
+ , HolderFactory(Alloc.Ref(), MemInfo)
+{
+}
+
+TKqpProtoBuilder::TKqpProtoBuilder(const IFunctionRegistry& funcRegistry)
+ : SelfHosted(MakeHolder<TSelfHosted>(funcRegistry))
+{
+ Alloc = &SelfHosted->Alloc;
+ TypeEnv = &SelfHosted->TypeEnv;
+ HolderFactory = &SelfHosted->HolderFactory;
+
+ Alloc->Release();
+}
+
+TKqpProtoBuilder::TKqpProtoBuilder(TScopedAlloc* alloc, TTypeEnvironment* typeEnv, THolderFactory* holderFactory)
+ : Alloc(alloc)
+ , TypeEnv(typeEnv)
+ , HolderFactory(holderFactory)
{
}
-TKqpProtoBuilder::TKqpProtoBuilder(const IFunctionRegistry& funcRegistry)
- : SelfHosted(MakeHolder<TSelfHosted>(funcRegistry))
-{
- Alloc = &SelfHosted->Alloc;
- TypeEnv = &SelfHosted->TypeEnv;
- HolderFactory = &SelfHosted->HolderFactory;
-
- Alloc->Release();
-}
-
-TKqpProtoBuilder::TKqpProtoBuilder(TScopedAlloc* alloc, TTypeEnvironment* typeEnv, THolderFactory* holderFactory)
- : Alloc(alloc)
- , TypeEnv(typeEnv)
- , HolderFactory(holderFactory)
-{
-}
-
TKqpProtoBuilder::~TKqpProtoBuilder() {
- if (SelfHosted) {
- SelfHosted->Alloc.Acquire();
- }
+ if (SelfHosted) {
+ SelfHosted->Alloc.Acquire();
+ }
}
-NKikimrMiniKQL::TType TKqpProtoBuilder::ApplyColumnHints(const NKikimrMiniKQL::TType& srcRowType,
- const TVector<TString>& columnHints)
-{
- YQL_ENSURE(srcRowType.GetKind() == NKikimrMiniKQL::Struct);
- YQL_ENSURE(srcRowType.GetStruct().MemberSize() == columnHints.size(),
- "" << srcRowType.GetStruct().MemberSize() << " != " << columnHints.size());
-
- TMap<TString, size_t> memberIndices;
- for (size_t i = 0; i < srcRowType.GetStruct().MemberSize(); ++i) {
- memberIndices[srcRowType.GetStruct().GetMember(i).GetName()] = i;
- }
-
- NKikimrMiniKQL::TType dstRowType;
- dstRowType.SetKind(NKikimrMiniKQL::Struct);
-
- for (auto& columnName : columnHints) {
- auto* memberIndex = memberIndices.FindPtr(columnName);
- YQL_ENSURE(memberIndex);
-
- auto* newMember = dstRowType.MutableStruct()->AddMember();
- newMember->SetName(columnName);
- newMember->MutableType()->CopyFrom(srcRowType.GetStruct().GetMember(*memberIndex).GetType());
- }
-
- return dstRowType;
-}
-
-void TKqpProtoBuilder::BuildValue(const TVector<NDqProto::TData>& data, const NKikimrMiniKQL::TType& valueType,
- NKikimrMiniKQL::TResult* result)
+NKikimrMiniKQL::TType TKqpProtoBuilder::ApplyColumnHints(const NKikimrMiniKQL::TType& srcRowType,
+ const TVector<TString>& columnHints)
{
- THolder<TGuard<TScopedAlloc>> guard;
- if (SelfHosted) {
- guard = MakeHolder<TGuard<TScopedAlloc>>(*Alloc);
- }
+ YQL_ENSURE(srcRowType.GetKind() == NKikimrMiniKQL::Struct);
+ YQL_ENSURE(srcRowType.GetStruct().MemberSize() == columnHints.size(),
+ "" << srcRowType.GetStruct().MemberSize() << " != " << columnHints.size());
+
+ TMap<TString, size_t> memberIndices;
+ for (size_t i = 0; i < srcRowType.GetStruct().MemberSize(); ++i) {
+ memberIndices[srcRowType.GetStruct().GetMember(i).GetName()] = i;
+ }
+
+ NKikimrMiniKQL::TType dstRowType;
+ dstRowType.SetKind(NKikimrMiniKQL::Struct);
+
+ for (auto& columnName : columnHints) {
+ auto* memberIndex = memberIndices.FindPtr(columnName);
+ YQL_ENSURE(memberIndex);
+
+ auto* newMember = dstRowType.MutableStruct()->AddMember();
+ newMember->SetName(columnName);
+ newMember->MutableType()->CopyFrom(srcRowType.GetStruct().GetMember(*memberIndex).GetType());
+ }
- result->MutableType()->CopyFrom(valueType);
+ return dstRowType;
+}
+
+void TKqpProtoBuilder::BuildValue(const TVector<NDqProto::TData>& data, const NKikimrMiniKQL::TType& valueType,
+ NKikimrMiniKQL::TResult* result)
+{
+ THolder<TGuard<TScopedAlloc>> guard;
+ if (SelfHosted) {
+ guard = MakeHolder<TGuard<TScopedAlloc>>(*Alloc);
+ }
- auto mkqlType = ImportTypeFromProto(result->GetType(), *TypeEnv);
+ result->MutableType()->CopyFrom(valueType);
+
+ auto mkqlType = ImportTypeFromProto(result->GetType(), *TypeEnv);
TUnboxedValueVector buffer;
auto transportVersion = NDqProto::EDataTransportVersion::DATA_TRANSPORT_VERSION_UNSPECIFIED;
@@ -111,38 +111,38 @@ void TKqpProtoBuilder::BuildValue(const TVector<NDqProto::TData>& data, const NK
}
YQL_ENSURE(buffer.size() == 1, "Actual buffer size: " << data.size());
- ExportValueToProto(mkqlType, buffer[0], *result->MutableValue());
-}
-
-void TKqpProtoBuilder::BuildValue(TUnboxedValueVector& rows, const NKikimrMiniKQL::TType& valueType,
- NKikimrMiniKQL::TResult* result)
-{
- YQL_ENSURE(rows.size() == 1, "Actual buffer size: " << rows.size());
-
- result->MutableType()->CopyFrom(valueType);
- auto mkqlType = ImportTypeFromProto(result->GetType(), *TypeEnv);
- ExportValueToProto(mkqlType, rows[0], *result->MutableValue());
+ ExportValueToProto(mkqlType, buffer[0], *result->MutableValue());
+}
+
+void TKqpProtoBuilder::BuildValue(TUnboxedValueVector& rows, const NKikimrMiniKQL::TType& valueType,
+ NKikimrMiniKQL::TResult* result)
+{
+ YQL_ENSURE(rows.size() == 1, "Actual buffer size: " << rows.size());
+
+ result->MutableType()->CopyFrom(valueType);
+ auto mkqlType = ImportTypeFromProto(result->GetType(), *TypeEnv);
+ ExportValueToProto(mkqlType, rows[0], *result->MutableValue());
}
-void TKqpProtoBuilder::BuildStream(const TVector<NDqProto::TData>& data, const NKikimrMiniKQL::TType& srcRowType,
- const NKikimrMiniKQL::TType* dstRowType, NKikimrMiniKQL::TResult* result)
+void TKqpProtoBuilder::BuildStream(const TVector<NDqProto::TData>& data, const NKikimrMiniKQL::TType& srcRowType,
+ const NKikimrMiniKQL::TType* dstRowType, NKikimrMiniKQL::TResult* result)
{
- YQL_ENSURE(srcRowType.GetKind() == NKikimrMiniKQL::Struct);
-
- auto* mkqlSrcRowType = NMiniKQL::ImportTypeFromProto(srcRowType, *TypeEnv);
- auto* mkqlSrcRowStructType = static_cast<TStructType*>(mkqlSrcRowType);
-
- result->MutableType()->SetKind(NKikimrMiniKQL::List);
- auto* newRowType = result->MutableType()->MutableList()->MutableItem();
-
- TListType* mkqlSrcRowsType = nullptr;
- TMap<TStringBuf, ui32> memberIndices;
-
- THolder<TGuard<TScopedAlloc>> guard;
- if (SelfHosted) {
- guard = MakeHolder<TGuard<TScopedAlloc>>(*Alloc);
- }
-
+ YQL_ENSURE(srcRowType.GetKind() == NKikimrMiniKQL::Struct);
+
+ auto* mkqlSrcRowType = NMiniKQL::ImportTypeFromProto(srcRowType, *TypeEnv);
+ auto* mkqlSrcRowStructType = static_cast<TStructType*>(mkqlSrcRowType);
+
+ result->MutableType()->SetKind(NKikimrMiniKQL::List);
+ auto* newRowType = result->MutableType()->MutableList()->MutableItem();
+
+ TListType* mkqlSrcRowsType = nullptr;
+ TMap<TStringBuf, ui32> memberIndices;
+
+ THolder<TGuard<TScopedAlloc>> guard;
+ if (SelfHosted) {
+ guard = MakeHolder<TGuard<TScopedAlloc>>(*Alloc);
+ }
+
auto transportVersion = NDqProto::EDataTransportVersion::DATA_TRANSPORT_VERSION_UNSPECIFIED;
if (!data.empty()) {
switch (data.front().GetTransportVersion()) {
@@ -164,123 +164,123 @@ void TKqpProtoBuilder::BuildStream(const TVector<NDqProto::TData>& data, const N
}
NDq::TDqDataSerializer dataSerializer(*TypeEnv, *HolderFactory, transportVersion);
- if (dstRowType) {
- YQL_ENSURE(dstRowType->GetKind() == NKikimrMiniKQL::Struct);
- newRowType->CopyFrom(*dstRowType);
-
- for (ui32 index = 0; index < srcRowType.GetStruct().MemberSize(); ++index) {
- memberIndices[srcRowType.GetStruct().GetMember(index).GetName()] = index;
- }
- } else {
- newRowType->CopyFrom(srcRowType);
- mkqlSrcRowsType = TListType::Create(mkqlSrcRowType, *TypeEnv);
- }
-
+ if (dstRowType) {
+ YQL_ENSURE(dstRowType->GetKind() == NKikimrMiniKQL::Struct);
+ newRowType->CopyFrom(*dstRowType);
+
+ for (ui32 index = 0; index < srcRowType.GetStruct().MemberSize(); ++index) {
+ memberIndices[srcRowType.GetStruct().GetMember(index).GetName()] = index;
+ }
+ } else {
+ newRowType->CopyFrom(srcRowType);
+ mkqlSrcRowsType = TListType::Create(mkqlSrcRowType, *TypeEnv);
+ }
+
for (auto& part : data) {
- if (part.GetRows() == 0) {
- continue;
- }
-
- TUnboxedValueVector rows;
- dataSerializer.Deserialize(part, mkqlSrcRowType, rows);
-
- if (dstRowType) {
- for (auto& srcRow : rows) {
- auto* dstRow = result->MutableValue()->MutableList()->Add()->MutableStruct();
- for (auto& dstMember : dstRowType->GetStruct().GetMember()) {
- ui32* srcMemberIndex = memberIndices.FindPtr(dstMember.GetName());
- YQL_ENSURE(srcMemberIndex);
-
- auto* memberType = mkqlSrcRowStructType->GetMemberType(*srcMemberIndex);
- ExportValueToProto(memberType, srcRow.GetElement(*srcMemberIndex), *dstRow->Add());
- }
- }
- } else {
- ExportValueToProto(mkqlSrcRowsType, HolderFactory->VectorAsArray(rows), *result->MutableValue());
- }
+ if (part.GetRows() == 0) {
+ continue;
+ }
+
+ TUnboxedValueVector rows;
+ dataSerializer.Deserialize(part, mkqlSrcRowType, rows);
+
+ if (dstRowType) {
+ for (auto& srcRow : rows) {
+ auto* dstRow = result->MutableValue()->MutableList()->Add()->MutableStruct();
+ for (auto& dstMember : dstRowType->GetStruct().GetMember()) {
+ ui32* srcMemberIndex = memberIndices.FindPtr(dstMember.GetName());
+ YQL_ENSURE(srcMemberIndex);
+
+ auto* memberType = mkqlSrcRowStructType->GetMemberType(*srcMemberIndex);
+ ExportValueToProto(memberType, srcRow.GetElement(*srcMemberIndex), *dstRow->Add());
+ }
+ }
+ } else {
+ ExportValueToProto(mkqlSrcRowsType, HolderFactory->VectorAsArray(rows), *result->MutableValue());
+ }
+ }
+}
+
+void TKqpProtoBuilder::BuildStream(TUnboxedValueVector& rows, const NKikimrMiniKQL::TType& srcRowType,
+ const NKikimrMiniKQL::TType* dstRowType, NKikimrMiniKQL::TResult* result)
+{
+ YQL_ENSURE(srcRowType.GetKind() == NKikimrMiniKQL::Struct);
+
+ auto* mkqlSrcRowType = NMiniKQL::ImportTypeFromProto(srcRowType, *TypeEnv);
+ auto* mkqlSrcRowStructType = static_cast<TStructType*>(mkqlSrcRowType);
+
+ result->MutableType()->SetKind(NKikimrMiniKQL::List);
+ auto* newRowType = result->MutableType()->MutableList()->MutableItem();
+
+ if (dstRowType) {
+ YQL_ENSURE(dstRowType->GetKind() == NKikimrMiniKQL::Struct);
+ newRowType->CopyFrom(*dstRowType);
+
+ TMap<TStringBuf, ui32> memberIndices;
+ for (ui32 index = 0; index < srcRowType.GetStruct().MemberSize(); ++index) {
+ memberIndices[srcRowType.GetStruct().GetMember(index).GetName()] = index;
+ }
+
+ for (auto& srcRow : rows) {
+ auto* dstRow = result->MutableValue()->MutableList()->Add()->MutableStruct();
+ for (auto& dstMember : dstRowType->GetStruct().GetMember()) {
+ ui32* srcMemberIndex = memberIndices.FindPtr(dstMember.GetName());
+ YQL_ENSURE(srcMemberIndex);
+
+ auto* memberType = mkqlSrcRowStructType->GetMemberType(*srcMemberIndex);
+ ExportValueToProto(memberType, srcRow.GetElement(*srcMemberIndex), *dstRow->Add());
+ }
+ }
+ } else {
+ newRowType->CopyFrom(srcRowType);
+ auto mkqlSrcRowsType = TListType::Create(mkqlSrcRowType, *TypeEnv);
+
+ THolder<TGuard<TScopedAlloc>> guard;
+ if (SelfHosted) {
+ guard = MakeHolder<TGuard<TScopedAlloc>>(*Alloc);
+ }
+
+ ExportValueToProto(mkqlSrcRowsType, HolderFactory->VectorAsArray(rows), *result->MutableValue());
}
-}
-
-void TKqpProtoBuilder::BuildStream(TUnboxedValueVector& rows, const NKikimrMiniKQL::TType& srcRowType,
- const NKikimrMiniKQL::TType* dstRowType, NKikimrMiniKQL::TResult* result)
-{
- YQL_ENSURE(srcRowType.GetKind() == NKikimrMiniKQL::Struct);
-
- auto* mkqlSrcRowType = NMiniKQL::ImportTypeFromProto(srcRowType, *TypeEnv);
- auto* mkqlSrcRowStructType = static_cast<TStructType*>(mkqlSrcRowType);
-
- result->MutableType()->SetKind(NKikimrMiniKQL::List);
- auto* newRowType = result->MutableType()->MutableList()->MutableItem();
-
- if (dstRowType) {
- YQL_ENSURE(dstRowType->GetKind() == NKikimrMiniKQL::Struct);
- newRowType->CopyFrom(*dstRowType);
-
- TMap<TStringBuf, ui32> memberIndices;
- for (ui32 index = 0; index < srcRowType.GetStruct().MemberSize(); ++index) {
- memberIndices[srcRowType.GetStruct().GetMember(index).GetName()] = index;
- }
-
- for (auto& srcRow : rows) {
- auto* dstRow = result->MutableValue()->MutableList()->Add()->MutableStruct();
- for (auto& dstMember : dstRowType->GetStruct().GetMember()) {
- ui32* srcMemberIndex = memberIndices.FindPtr(dstMember.GetName());
- YQL_ENSURE(srcMemberIndex);
-
- auto* memberType = mkqlSrcRowStructType->GetMemberType(*srcMemberIndex);
- ExportValueToProto(memberType, srcRow.GetElement(*srcMemberIndex), *dstRow->Add());
- }
- }
- } else {
- newRowType->CopyFrom(srcRowType);
- auto mkqlSrcRowsType = TListType::Create(mkqlSrcRowType, *TypeEnv);
-
- THolder<TGuard<TScopedAlloc>> guard;
- if (SelfHosted) {
- guard = MakeHolder<TGuard<TScopedAlloc>>(*Alloc);
- }
-
- ExportValueToProto(mkqlSrcRowsType, HolderFactory->VectorAsArray(rows), *result->MutableValue());
- }
}
Ydb::ResultSet TKqpProtoBuilder::BuildYdbResultSet(const TVector<NDqProto::TData>& data,
- const NKikimrMiniKQL::TType& srcRowType, const NKikimrMiniKQL::TType* dstRowType)
+ const NKikimrMiniKQL::TType& srcRowType, const NKikimrMiniKQL::TType* dstRowType)
{
- YQL_ENSURE(srcRowType.GetKind() == NKikimrMiniKQL::Struct);
+ YQL_ENSURE(srcRowType.GetKind() == NKikimrMiniKQL::Struct);
+
+ auto* mkqlSrcRowType = NMiniKQL::ImportTypeFromProto(srcRowType, *TypeEnv);
+ auto* mkqlSrcRowStructType = static_cast<TStructType*>(mkqlSrcRowType);
- auto* mkqlSrcRowType = NMiniKQL::ImportTypeFromProto(srcRowType, *TypeEnv);
- auto* mkqlSrcRowStructType = static_cast<TStructType*>(mkqlSrcRowType);
-
Ydb::ResultSet resultSet;
- TMap<TStringBuf, ui32> memberIndices;
-
- if (dstRowType) {
- YQL_ENSURE(dstRowType->GetKind() == NKikimrMiniKQL::Struct);
-
- for (ui32 index = 0; index < mkqlSrcRowStructType->GetMembersCount(); ++index) {
- memberIndices[mkqlSrcRowStructType->GetMemberName(index)] = index;
- }
-
- for (auto& member : dstRowType->GetStruct().GetMember()) {
- auto* column = resultSet.add_columns();
- column->set_name(member.GetName());
+ TMap<TStringBuf, ui32> memberIndices;
+
+ if (dstRowType) {
+ YQL_ENSURE(dstRowType->GetKind() == NKikimrMiniKQL::Struct);
+
+ for (ui32 index = 0; index < mkqlSrcRowStructType->GetMembersCount(); ++index) {
+ memberIndices[mkqlSrcRowStructType->GetMemberName(index)] = index;
+ }
+
+ for (auto& member : dstRowType->GetStruct().GetMember()) {
+ auto* column = resultSet.add_columns();
+ column->set_name(member.GetName());
ConvertMiniKQLTypeToYdbType(member.GetType(), *column->mutable_type());
- }
- } else {
- for (auto& member : srcRowType.GetStruct().GetMember()) {
- auto* column = resultSet.add_columns();
- column->set_name(member.GetName());
+ }
+ } else {
+ for (auto& member : srcRowType.GetStruct().GetMember()) {
+ auto* column = resultSet.add_columns();
+ column->set_name(member.GetName());
ConvertMiniKQLTypeToYdbType(member.GetType(), *column->mutable_type());
- }
+ }
+ }
+
+ THolder<TGuard<TScopedAlloc>> guard;
+ if (SelfHosted) {
+ guard = MakeHolder<TGuard<TScopedAlloc>>(*Alloc);
}
- THolder<TGuard<TScopedAlloc>> guard;
- if (SelfHosted) {
- guard = MakeHolder<TGuard<TScopedAlloc>>(*Alloc);
- }
-
auto transportVersion = NDqProto::EDataTransportVersion::DATA_TRANSPORT_VERSION_UNSPECIFIED;
if (!data.empty()) {
switch (data.front().GetTransportVersion()) {
@@ -301,35 +301,35 @@ Ydb::ResultSet TKqpProtoBuilder::BuildYdbResultSet(const TVector<NDqProto::TData
}
}
NDq::TDqDataSerializer dataSerializer(*TypeEnv, *HolderFactory, transportVersion);
-
+
for (auto& part : data) {
- if (part.GetRows()) {
- TUnboxedValueVector rows;
- dataSerializer.Deserialize(part, mkqlSrcRowType, rows);
-
- if (dstRowType) {
- for (auto& srcRow : rows) {
- auto* dstRow = resultSet.add_rows();
- dstRow->mutable_items()->Reserve(dstRowType->GetStruct().MemberSize());
-
- for (auto& dstMember : dstRowType->GetStruct().GetMember()) {
- ui32* srcMemberIndex = memberIndices.FindPtr(dstMember.GetName());
- YQL_ENSURE(srcMemberIndex);
-
- auto* memberType = mkqlSrcRowStructType->GetMemberType(*srcMemberIndex);
- ExportValueToProto(memberType, srcRow.GetElement(*srcMemberIndex), *dstRow->mutable_items()->Add());
- }
- }
- } else {
- for (auto& row : rows) {
- ExportValueToProto(mkqlSrcRowType, row, *resultSet.add_rows());
- }
- }
+ if (part.GetRows()) {
+ TUnboxedValueVector rows;
+ dataSerializer.Deserialize(part, mkqlSrcRowType, rows);
+
+ if (dstRowType) {
+ for (auto& srcRow : rows) {
+ auto* dstRow = resultSet.add_rows();
+ dstRow->mutable_items()->Reserve(dstRowType->GetStruct().MemberSize());
+
+ for (auto& dstMember : dstRowType->GetStruct().GetMember()) {
+ ui32* srcMemberIndex = memberIndices.FindPtr(dstMember.GetName());
+ YQL_ENSURE(srcMemberIndex);
+
+ auto* memberType = mkqlSrcRowStructType->GetMemberType(*srcMemberIndex);
+ ExportValueToProto(memberType, srcRow.GetElement(*srcMemberIndex), *dstRow->mutable_items()->Add());
+ }
+ }
+ } else {
+ for (auto& row : rows) {
+ ExportValueToProto(mkqlSrcRowType, row, *resultSet.add_rows());
+ }
+ }
}
}
return resultSet;
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/runtime/kqp_transport.h b/ydb/core/kqp/runtime/kqp_transport.h
index 2be9143e15..9fa6adec13 100644
--- a/ydb/core/kqp/runtime/kqp_transport.h
+++ b/ydb/core/kqp/runtime/kqp_transport.h
@@ -13,45 +13,45 @@ namespace NKqp {
class TKqpProtoBuilder : private TNonCopyable {
public:
TKqpProtoBuilder(const NMiniKQL::IFunctionRegistry& funcRegistry);
-
- TKqpProtoBuilder(NMiniKQL::TScopedAlloc* alloc, NMiniKQL::TTypeEnvironment* typeEnv,
- NMiniKQL::THolderFactory* holderFactory);
-
+
+ TKqpProtoBuilder(NMiniKQL::TScopedAlloc* alloc, NMiniKQL::TTypeEnvironment* typeEnv,
+ NMiniKQL::THolderFactory* holderFactory);
+
~TKqpProtoBuilder();
- static NKikimrMiniKQL::TType ApplyColumnHints(const NKikimrMiniKQL::TType& srcRowType,
- const TVector<TString>& columnHints);
-
- void BuildValue(const TVector<NYql::NDqProto::TData>& data, const NKikimrMiniKQL::TType& type,
- NKikimrMiniKQL::TResult* result);
-
- void BuildValue(NMiniKQL::TUnboxedValueVector& rows, const NKikimrMiniKQL::TType& type,
- NKikimrMiniKQL::TResult* result);
-
- void BuildStream(const TVector<NYql::NDqProto::TData>& data, const NKikimrMiniKQL::TType& srcRowType,
- const NKikimrMiniKQL::TType* dstRowType, NKikimrMiniKQL::TResult* result);
-
- void BuildStream(NMiniKQL::TUnboxedValueVector& rows, const NKikimrMiniKQL::TType& srcRowType,
- const NKikimrMiniKQL::TType* dstRowType, NKikimrMiniKQL::TResult* result);
-
+ static NKikimrMiniKQL::TType ApplyColumnHints(const NKikimrMiniKQL::TType& srcRowType,
+ const TVector<TString>& columnHints);
+
+ void BuildValue(const TVector<NYql::NDqProto::TData>& data, const NKikimrMiniKQL::TType& type,
+ NKikimrMiniKQL::TResult* result);
+
+ void BuildValue(NMiniKQL::TUnboxedValueVector& rows, const NKikimrMiniKQL::TType& type,
+ NKikimrMiniKQL::TResult* result);
+
+ void BuildStream(const TVector<NYql::NDqProto::TData>& data, const NKikimrMiniKQL::TType& srcRowType,
+ const NKikimrMiniKQL::TType* dstRowType, NKikimrMiniKQL::TResult* result);
+
+ void BuildStream(NMiniKQL::TUnboxedValueVector& rows, const NKikimrMiniKQL::TType& srcRowType,
+ const NKikimrMiniKQL::TType* dstRowType, NKikimrMiniKQL::TResult* result);
+
Ydb::ResultSet BuildYdbResultSet(const TVector<NYql::NDqProto::TData>& data,
- const NKikimrMiniKQL::TType& srcRowType, const NKikimrMiniKQL::TType* dstRowType = nullptr);
+ const NKikimrMiniKQL::TType& srcRowType, const NKikimrMiniKQL::TType* dstRowType = nullptr);
private:
- NMiniKQL::TScopedAlloc* Alloc = nullptr;
- NMiniKQL::TTypeEnvironment* TypeEnv = nullptr;
- NMiniKQL::THolderFactory* HolderFactory = nullptr;
-
- struct TSelfHosted {
- NMiniKQL::TScopedAlloc Alloc;
- NMiniKQL::TTypeEnvironment TypeEnv;
- NMiniKQL::TMemoryUsageInfo MemInfo;
- NMiniKQL::THolderFactory HolderFactory;
-
- explicit TSelfHosted(const NMiniKQL::IFunctionRegistry& funcRegistry);
- };
- THolder<TSelfHosted> SelfHosted;
+ NMiniKQL::TScopedAlloc* Alloc = nullptr;
+ NMiniKQL::TTypeEnvironment* TypeEnv = nullptr;
+ NMiniKQL::THolderFactory* HolderFactory = nullptr;
+
+ struct TSelfHosted {
+ NMiniKQL::TScopedAlloc Alloc;
+ NMiniKQL::TTypeEnvironment TypeEnv;
+ NMiniKQL::TMemoryUsageInfo MemInfo;
+ NMiniKQL::THolderFactory HolderFactory;
+
+ explicit TSelfHosted(const NMiniKQL::IFunctionRegistry& funcRegistry);
+ };
+ THolder<TSelfHosted> SelfHosted;
};
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/runtime/ut/ya.make b/ydb/core/kqp/runtime/ut/ya.make
index 87b3bd2709..ea3f10523c 100644
--- a/ydb/core/kqp/runtime/ut/ya.make
+++ b/ydb/core/kqp/runtime/ut/ya.make
@@ -1,24 +1,24 @@
UNITTEST_FOR(ydb/core/kqp/runtime)
-
-OWNER(g:kikimr)
-
-FORK_SUBTESTS()
-
-IF (SANITIZER_TYPE OR WITH_VALGRIND)
- SIZE(MEDIUM)
-ENDIF()
-
-SRCS(
+
+OWNER(g:kikimr)
+
+FORK_SUBTESTS()
+
+IF (SANITIZER_TYPE OR WITH_VALGRIND)
+ SIZE(MEDIUM)
+ENDIF()
+
+SRCS(
# kqp_spilling_file_ut.cpp
kqp_scan_data_ut.cpp
-)
-
+)
+
YQL_LAST_ABI_VERSION()
-PEERDIR(
+PEERDIR(
library/cpp/testing/unittest
ydb/core/testlib/basics
ydb/library/yql/public/udf/service/exception_policy
-)
-
-END()
+)
+
+END()
diff --git a/ydb/core/kqp/runtime/ya.make b/ydb/core/kqp/runtime/ya.make
index 5101f111d0..1143b6bc89 100644
--- a/ydb/core/kqp/runtime/ya.make
+++ b/ydb/core/kqp/runtime/ya.make
@@ -6,7 +6,7 @@ OWNER(
)
SRCS(
- kqp_channel_storage.cpp
+ kqp_channel_storage.cpp
kqp_compute.cpp
kqp_compute.h
kqp_effects.cpp
@@ -18,7 +18,7 @@ SRCS(
kqp_runtime_impl.h
kqp_scan_data.cpp
kqp_scan_data.h
- kqp_spilling_file.cpp
+ kqp_spilling_file.cpp
kqp_tasks_runner.cpp
kqp_tasks_runner.h
kqp_transport.h
@@ -47,9 +47,9 @@ PEERDIR(
YQL_LAST_ABI_VERSION()
GENERATE_ENUM_SERIALIZATION(kqp_spilling.h)
-
+
END()
-
-RECURSE_FOR_TESTS(
- ut
-)
+
+RECURSE_FOR_TESTS(
+ ut
+)
diff --git a/ydb/core/kqp/ut/common/kqp_ut_common.cpp b/ydb/core/kqp/ut/common/kqp_ut_common.cpp
index 1f55b41649..9bc7889f92 100644
--- a/ydb/core/kqp/ut/common/kqp_ut_common.cpp
+++ b/ydb/core/kqp/ut/common/kqp_ut_common.cpp
@@ -2,7 +2,7 @@
#include <ydb/core/kqp/provider/yql_kikimr_results.h>
#include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h>
-
+
#include <ydb/library/yql/core/yql_data_provider.h>
#include <ydb/library/yql/utils/backtrace/backtrace.h>
#include <ydb/library/yql/public/udf/udf_helpers.h>
@@ -15,32 +15,32 @@ namespace NKqp {
using namespace NYdb::NTable;
-SIMPLE_UDF(TTestFilter, bool(i64)) {
- Y_UNUSED(valueBuilder);
- const i64 arg = args[0].Get<i64>();
-
- return NUdf::TUnboxedValuePod(arg >= 0);
-}
-
-SIMPLE_UDF(TTestFilterTerminate, bool(i64, i64)) {
- Y_UNUSED(valueBuilder);
- const i64 arg1 = args[0].Get<i64>();
- const i64 arg2 = args[1].Get<i64>();
-
- if (arg1 < arg2) {
- UdfTerminate("Bad filter value.");
- }
-
- return NUdf::TUnboxedValuePod(true);
-}
-
+SIMPLE_UDF(TTestFilter, bool(i64)) {
+ Y_UNUSED(valueBuilder);
+ const i64 arg = args[0].Get<i64>();
+
+ return NUdf::TUnboxedValuePod(arg >= 0);
+}
+
+SIMPLE_UDF(TTestFilterTerminate, bool(i64, i64)) {
+ Y_UNUSED(valueBuilder);
+ const i64 arg1 = args[0].Get<i64>();
+ const i64 arg2 = args[1].Get<i64>();
+
+ if (arg1 < arg2) {
+ UdfTerminate("Bad filter value.");
+ }
+
+ return NUdf::TUnboxedValuePod(true);
+}
+
SIMPLE_UDF(TRandString, char*(ui32)) {
Y_UNUSED(valueBuilder);
const ui32 size = args[0].Get<ui32>();
-
+
auto str = valueBuilder->NewStringNotFilled(size);
auto strRef = str.AsStringRef();
-
+
for (ui32 i = 0; i < size; ++i) {
*(strRef.Data() + i) = '0' + RandomNumber<ui32>() % 10;
}
@@ -50,13 +50,13 @@ SIMPLE_UDF(TRandString, char*(ui32)) {
SIMPLE_MODULE(TTestUdfsModule, TTestFilter, TTestFilterTerminate, TRandString);
-NMiniKQL::IFunctionRegistry* UdfFrFactory(const NScheme::TTypeRegistry& typeRegistry) {
- Y_UNUSED(typeRegistry);
- auto funcRegistry = NMiniKQL::CreateFunctionRegistry(NMiniKQL::CreateBuiltinRegistry())->Clone();
- funcRegistry->AddModule("", "TestUdfs", new TTestUdfsModule());
- return funcRegistry.Release();
-}
-
+NMiniKQL::IFunctionRegistry* UdfFrFactory(const NScheme::TTypeRegistry& typeRegistry) {
+ Y_UNUSED(typeRegistry);
+ auto funcRegistry = NMiniKQL::CreateFunctionRegistry(NMiniKQL::CreateBuiltinRegistry())->Clone();
+ funcRegistry->AddModule("", "TestUdfs", new TTestUdfsModule());
+ return funcRegistry.Release();
+}
+
TVector<NKikimrKqp::TKqpSetting> SyntaxV1Settings() {
auto setting = NKikimrKqp::TKqpSetting();
setting.SetName("_KqpYqlSyntaxVersion");
@@ -66,35 +66,35 @@ TVector<NKikimrKqp::TKqpSetting> SyntaxV1Settings() {
TKikimrRunner::TKikimrRunner(const TKikimrSettings& settings) {
// EnableKikimrBacktraceFormat(); // Very slow, enable only when required locally
-
+
auto mbusPort = PortManager.GetPort();
auto grpcPort = PortManager.GetPort();
- Cerr << "Trying to start KiKiMR, gRPC: " << grpcPort << ", MsgBus: " << mbusPort << Endl;
-
+ Cerr << "Trying to start KiKiMR, gRPC: " << grpcPort << ", MsgBus: " << mbusPort << Endl;
+
TVector<NKikimrKqp::TKqpSetting> effectiveKqpSettings;
- {
- // Allow NewEngine in tests
- NKikimrKqp::TKqpSetting setting;
- setting.SetName("_KqpAllowNewEngine");
- setting.SetValue("true");
- effectiveKqpSettings.push_back(setting);
-
- // Force NewEngine in tests
+ {
+ // Allow NewEngine in tests
+ NKikimrKqp::TKqpSetting setting;
+ setting.SetName("_KqpAllowNewEngine");
+ setting.SetValue("true");
+ effectiveKqpSettings.push_back(setting);
+
+ // Force NewEngine in tests
// setting.SetName("_KqpForceNewEngine");
// setting.SetValue("true");
// effectiveKqpSettings.push_back(setting);
- }
+ }
- bool enableSpilling = false;
+ bool enableSpilling = false;
if (settings.AppConfig.GetTableServiceConfig().GetSpillingServiceConfig().GetLocalFileConfig().GetEnable()) {
- NKikimrKqp::TKqpSetting setting;
- setting.SetName("_KqpEnableSpilling");
- setting.SetValue("true");
- effectiveKqpSettings.push_back(setting);
- enableSpilling = true;
- }
-
+ NKikimrKqp::TKqpSetting setting;
+ setting.SetName("_KqpEnableSpilling");
+ setting.SetValue("true");
+ effectiveKqpSettings.push_back(setting);
+ enableSpilling = true;
+ }
+
effectiveKqpSettings.insert(effectiveKqpSettings.end(), settings.KqpSettings.begin(), settings.KqpSettings.end());
ServerSettings.Reset(MakeHolder<Tests::TServerSettings>(mbusPort));
@@ -393,11 +393,11 @@ void TKikimrRunner::Initialize(const TKikimrSettings& settings) {
// Server->GetRuntime()->SetLogPriority(NKikimrServices::KQP_PROXY, NActors::NLog::PRI_DEBUG);
// Server->GetRuntime()->SetLogPriority(NKikimrServices::KQP_COMPILE_SERVICE, NActors::NLog::PRI_DEBUG);
// Server->GetRuntime()->SetLogPriority(NKikimrServices::KQP_COMPILE_ACTOR, NActors::NLog::PRI_DEBUG);
- // Server->GetRuntime()->SetLogPriority(NKikimrServices::KQP_COMPILE_REQUEST, NActors::NLog::PRI_DEBUG);
+ // Server->GetRuntime()->SetLogPriority(NKikimrServices::KQP_COMPILE_REQUEST, NActors::NLog::PRI_DEBUG);
// Server->GetRuntime()->SetLogPriority(NKikimrServices::KQP_GATEWAY, NActors::NLog::PRI_DEBUG);
// Server->GetRuntime()->SetLogPriority(NKikimrServices::RPC_REQUEST, NActors::NLog::PRI_DEBUG);
// Server->GetRuntime()->SetLogPriority(NKikimrServices::KQP_RESOURCE_MANAGER, NActors::NLog::PRI_DEBUG);
- // Server->GetRuntime()->SetLogPriority(NKikimrServices::KQP_NODE, NActors::NLog::PRI_DEBUG);
+ // Server->GetRuntime()->SetLogPriority(NKikimrServices::KQP_NODE, NActors::NLog::PRI_DEBUG);
// Server->GetRuntime()->SetLogPriority(NKikimrServices::KQP_BLOBS_STORAGE, NActors::NLog::PRI_DEBUG);
Client->InitRootScheme(settings.DomainRoot);
@@ -452,78 +452,78 @@ bool HasIssue(const NYql::TIssues& issues, ui32 code,
return hasIssue;
}
-void PrintQueryStats(const TDataQueryResult& result) {
- if (!result.GetStats().Defined()) {
- return;
- }
-
- auto stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
-
- Cerr << "------- query stats -----------" << Endl;
- for (const auto& qp : stats.query_phases()) {
- Cerr << "-- phase" << Endl
- << " duration: " << qp.duration_us() << Endl
- << " access: " << Endl;
- for (const auto& ta : qp.table_access()) {
- Cerr << " name: " << ta.name() << Endl
- << " reads: " << ta.reads().rows() << Endl
- << " updates: " << ta.updates().rows() << Endl
- << " deletes: " << ta.deletes().rows() << Endl;
- }
- }
-}
-
+void PrintQueryStats(const TDataQueryResult& result) {
+ if (!result.GetStats().Defined()) {
+ return;
+ }
+
+ auto stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
+
+ Cerr << "------- query stats -----------" << Endl;
+ for (const auto& qp : stats.query_phases()) {
+ Cerr << "-- phase" << Endl
+ << " duration: " << qp.duration_us() << Endl
+ << " access: " << Endl;
+ for (const auto& ta : qp.table_access()) {
+ Cerr << " name: " << ta.name() << Endl
+ << " reads: " << ta.reads().rows() << Endl
+ << " updates: " << ta.updates().rows() << Endl
+ << " deletes: " << ta.deletes().rows() << Endl;
+ }
+ }
+}
+
void AssertTableStats(const TDataQueryResult& result, TStringBuf table, const TExpectedTableStats& expectedStats) {
- auto stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
-
+ auto stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
+
ui64 actualReads = 0;
ui64 actualUpdates = 0;
ui64 actualDeletes = 0;
-
- for (const auto& phase : stats.query_phases()) {
- for (const auto& access : phase.table_access()) {
- if (access.name() == table) {
+
+ for (const auto& phase : stats.query_phases()) {
+ for (const auto& access : phase.table_access()) {
+ if (access.name() == table) {
actualReads += access.reads().rows();
actualUpdates += access.updates().rows();
actualDeletes += access.deletes().rows();
- }
- }
- }
-
+ }
+ }
+ }
+
if (expectedStats.ExpectedReads) {
UNIT_ASSERT_EQUAL_C(*expectedStats.ExpectedReads, actualReads, "table: " << table
<< ", reads expected " << *expectedStats.ExpectedReads << ", actual " << actualReads);
- }
-
+ }
+
if (expectedStats.ExpectedUpdates) {
UNIT_ASSERT_EQUAL_C(*expectedStats.ExpectedUpdates, actualUpdates, "table: " << table
<< ", updates expected " << *expectedStats.ExpectedUpdates << ", actual " << actualUpdates);
- }
-
+ }
+
if (expectedStats.ExpectedDeletes) {
UNIT_ASSERT_EQUAL_C(*expectedStats.ExpectedDeletes, actualDeletes, "table: " << table
<< ", deletes expected " << *expectedStats.ExpectedDeletes << ", actual " << actualDeletes);
- }
-}
-
-TDataQueryResult ExecQueryAndTestResult(TSession& session, const TString& query, const NYdb::TParams& params,
- const TString& expectedYson)
-{
- NYdb::NTable::TExecDataQuerySettings settings;
- settings.CollectQueryStats(ECollectQueryStatsMode::Basic);
-
- TDataQueryResult result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), params, settings)
- .ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
- if (!result.GetIssues().Empty()) {
- Cerr << result.GetIssues().ToString() << Endl;
- }
-
- CompareYson(expectedYson, FormatResultSetYson(result.GetResultSet(0)));
-
- return result;
-}
-
+ }
+}
+
+TDataQueryResult ExecQueryAndTestResult(TSession& session, const TString& query, const NYdb::TParams& params,
+ const TString& expectedYson)
+{
+ NYdb::NTable::TExecDataQuerySettings settings;
+ settings.CollectQueryStats(ECollectQueryStatsMode::Basic);
+
+ TDataQueryResult result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), params, settings)
+ .ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ if (!result.GetIssues().Empty()) {
+ Cerr << result.GetIssues().ToString() << Endl;
+ }
+
+ CompareYson(expectedYson, FormatResultSetYson(result.GetResultSet(0)));
+
+ return result;
+}
+
void FillProfile(NYdb::NTable::TScanQueryPart& streamPart, NYson::TYsonWriter& writer, TVector<TString>* profiles,
ui32 profileIndex)
{
@@ -566,33 +566,33 @@ void PrintResultSet(const NYdb::TResultSet& resultSet, NYson::TYsonWriter& write
template<typename TIterator>
TString StreamResultToYsonImpl(TIterator& it, TVector<TString>* profiles) {
- TStringStream out;
+ TStringStream out;
NYson::TYsonWriter writer(&out, NYson::EYsonFormat::Text, ::NYson::EYsonType::Node, true);
- writer.OnBeginList();
-
- ui32 profileIndex = 0;
-
- for (;;) {
- auto streamPart = it.ReadNext().GetValueSync();
- if (!streamPart.IsSuccess()) {
- UNIT_ASSERT_C(streamPart.EOS(), streamPart.GetIssues().ToString());
- break;
- }
-
- if (streamPart.HasResultSet()) {
+ writer.OnBeginList();
+
+ ui32 profileIndex = 0;
+
+ for (;;) {
+ auto streamPart = it.ReadNext().GetValueSync();
+ if (!streamPart.IsSuccess()) {
+ UNIT_ASSERT_C(streamPart.EOS(), streamPart.GetIssues().ToString());
+ break;
+ }
+
+ if (streamPart.HasResultSet()) {
auto resultSet = streamPart.ExtractResultSet();
PrintResultSet(resultSet, writer);
- }
-
+ }
+
FillProfile(streamPart, writer, profiles, profileIndex);
profileIndex++;
- }
-
- writer.OnEndList();
-
- return out.Str();
-}
-
+ }
+
+ writer.OnEndList();
+
+ return out.Str();
+}
+
TString StreamResultToYson(NYdb::NExperimental::TStreamPartIterator& it, TVector<TString>* profiles) {
return StreamResultToYsonImpl(it, profiles);
}
@@ -708,45 +708,45 @@ TString ReadTablePartToYson(NYdb::NTable::TSession session, const TString& table
return NYdb::FormatResultSetYson(streamPart.ExtractPart());
}
-ui32 CountPlanNodesByKv(const NJson::TJsonValue& plan, const TString& key, const TString& value) {
- ui32 result = 0;
+ui32 CountPlanNodesByKv(const NJson::TJsonValue& plan, const TString& key, const TString& value) {
+ ui32 result = 0;
+
+ if (plan.IsArray()) {
+ for (const auto &node: plan.GetArray()) {
+ result += CountPlanNodesByKv(node, key, value);
+ }
+ return result;
+ }
+
+ UNIT_ASSERT(plan.IsMap());
+
+ auto map = plan.GetMap();
+ if (map.contains(key) && map.at(key).GetStringRobust() == value) {
+ return 1;
+ }
+
+ if (map.contains("Plans")) {
+ for (const auto &node: map["Plans"].GetArraySafe()) {
+ result += CountPlanNodesByKv(node, key, value);
+ }
+ }
+
+ if (map.contains("Plan")) {
+ result += CountPlanNodesByKv(map.at("Plan"), key, value);
+ }
+
+ if (map.contains("Operators")) {
+ for (const auto &node : map["Operators"].GetArraySafe()) {
+ result += CountPlanNodesByKv(node, key, value);
+ }
+ }
+
+ return result;
+}
+NJson::TJsonValue FindPlanNodeByKv(const NJson::TJsonValue& plan, const TString& key, const TString& value) {
if (plan.IsArray()) {
for (const auto &node: plan.GetArray()) {
- result += CountPlanNodesByKv(node, key, value);
- }
- return result;
- }
-
- UNIT_ASSERT(plan.IsMap());
-
- auto map = plan.GetMap();
- if (map.contains(key) && map.at(key).GetStringRobust() == value) {
- return 1;
- }
-
- if (map.contains("Plans")) {
- for (const auto &node: map["Plans"].GetArraySafe()) {
- result += CountPlanNodesByKv(node, key, value);
- }
- }
-
- if (map.contains("Plan")) {
- result += CountPlanNodesByKv(map.at("Plan"), key, value);
- }
-
- if (map.contains("Operators")) {
- for (const auto &node : map["Operators"].GetArraySafe()) {
- result += CountPlanNodesByKv(node, key, value);
- }
- }
-
- return result;
-}
-
-NJson::TJsonValue FindPlanNodeByKv(const NJson::TJsonValue& plan, const TString& key, const TString& value) {
- if (plan.IsArray()) {
- for (const auto &node: plan.GetArray()) {
auto stage = FindPlanNodeByKv(node, key, value);
if (stage.IsDefined()) {
return stage;
diff --git a/ydb/core/kqp/ut/common/kqp_ut_common.h b/ydb/core/kqp/ut/common/kqp_ut_common.h
index 9654cb6a11..c1d64b65c3 100644
--- a/ydb/core/kqp/ut/common/kqp_ut_common.h
+++ b/ydb/core/kqp/ut/common/kqp_ut_common.h
@@ -17,28 +17,28 @@
#include <library/cpp/testing/unittest/registar.h>
#include <library/cpp/yson/writer.h>
-#define Y_UNIT_TEST_TWIN(N, OPT) \
- template <bool OPT> \
- struct TTestCase##N : public TCurrentTestCase { \
- TTestCase##N() : TCurrentTestCase() { \
- if constexpr (OPT) { Name_ = #N "+" #OPT; } else { Name_ = #N "-" #OPT; } \
- } \
- static THolder<NUnitTest::TBaseTestCase> CreateOn() { return ::MakeHolder<TTestCase##N<true>>(); } \
- static THolder<NUnitTest::TBaseTestCase> CreateOff() { return ::MakeHolder<TTestCase##N<false>>(); } \
- void Execute_(NUnitTest::TTestContext&) override; \
- }; \
- struct TTestRegistration##N { \
- TTestRegistration##N() { \
- TCurrentTest::AddTest(TTestCase##N<true>::CreateOn); \
- TCurrentTest::AddTest(TTestCase##N<false>::CreateOff); \
- } \
- }; \
- static TTestRegistration##N testRegistration##N; \
- template <bool OPT> \
- void TTestCase##N<OPT>::Execute_(NUnitTest::TTestContext& ut_context Y_DECLARE_UNUSED)
-
-#define Y_UNIT_TEST_NEW_ENGINE(N) Y_UNIT_TEST_TWIN(N, UseNewEngine)
-
+#define Y_UNIT_TEST_TWIN(N, OPT) \
+ template <bool OPT> \
+ struct TTestCase##N : public TCurrentTestCase { \
+ TTestCase##N() : TCurrentTestCase() { \
+ if constexpr (OPT) { Name_ = #N "+" #OPT; } else { Name_ = #N "-" #OPT; } \
+ } \
+ static THolder<NUnitTest::TBaseTestCase> CreateOn() { return ::MakeHolder<TTestCase##N<true>>(); } \
+ static THolder<NUnitTest::TBaseTestCase> CreateOff() { return ::MakeHolder<TTestCase##N<false>>(); } \
+ void Execute_(NUnitTest::TTestContext&) override; \
+ }; \
+ struct TTestRegistration##N { \
+ TTestRegistration##N() { \
+ TCurrentTest::AddTest(TTestCase##N<true>::CreateOn); \
+ TCurrentTest::AddTest(TTestCase##N<false>::CreateOff); \
+ } \
+ }; \
+ static TTestRegistration##N testRegistration##N; \
+ template <bool OPT> \
+ void TTestCase##N<OPT>::Execute_(NUnitTest::TTestContext& ut_context Y_DECLARE_UNUSED)
+
+#define Y_UNIT_TEST_NEW_ENGINE(N) Y_UNIT_TEST_TWIN(N, UseNewEngine)
+
#define Y_UNIT_TEST_QUAD(N, OPT1, OPT2) \
template<bool OPT1, bool OPT2> void N(NUnitTest::TTestContext&); \
struct TTestRegistration##N { \
@@ -53,17 +53,17 @@
template<bool OPT1, bool OPT2> \
void N(NUnitTest::TTestContext&)
-template <bool UseNewEngine, bool ForceVersionV1 = false>
+template <bool UseNewEngine, bool ForceVersionV1 = false>
TString Query(const TString& tmpl) {
- return TStringBuilder()
- << (ForceVersionV1 ? "--!syntax_v1\n" : "")
- << "PRAGMA Kikimr.UseNewEngine = '" << (UseNewEngine ? "true" : "false") << "';" << Endl
- //<< (UseNewEngine ? "PRAGMA Kikimr.UseNewEngine = 'true';" : "")
- << tmpl;
+ return TStringBuilder()
+ << (ForceVersionV1 ? "--!syntax_v1\n" : "")
+ << "PRAGMA Kikimr.UseNewEngine = '" << (UseNewEngine ? "true" : "false") << "';" << Endl
+ //<< (UseNewEngine ? "PRAGMA Kikimr.UseNewEngine = 'true';" : "")
+ << tmpl;
}
-#define Q_(expr) Query<UseNewEngine, false>(expr)
-#define Q1_(expr) Query<UseNewEngine, true>(expr)
+#define Q_(expr) Query<UseNewEngine, false>(expr)
+#define Q1_(expr) Query<UseNewEngine, true>(expr)
namespace NKikimr {
namespace NKqp {
@@ -113,12 +113,12 @@ public:
TKikimrRunner(const TString& authToken = "", const TString& domainRoot = KikimrDefaultUtDomainRoot,
ui32 nodeCount = 1);
- ~TKikimrRunner() {
- Driver->Stop(true);
- Server.Reset();
- Client.Reset();
- }
-
+ ~TKikimrRunner() {
+ Driver->Stop(true);
+ Server.Reset();
+ Client.Reset();
+ }
+
const TString& GetEndpoint() const { return Endpoint; }
const NYdb::TDriver& GetDriver() const { return *Driver; }
NYdb::NScheme::TSchemeClient GetSchemeClient() const { return NYdb::NScheme::TSchemeClient(*Driver); }
@@ -194,43 +194,43 @@ void CompareYson(const TString& expected, const NKikimrMiniKQL::TResult& actual)
bool HasIssue(const NYql::TIssues& issues, ui32 code,
std::function<bool(const NYql::TIssue& issue)> predicate = {});
-void PrintQueryStats(const NYdb::NTable::TDataQueryResult& result);
-
+void PrintQueryStats(const NYdb::NTable::TDataQueryResult& result);
+
struct TExpectedTableStats {
TMaybe<ui64> ExpectedReads;
TMaybe<ui64> ExpectedUpdates;
TMaybe<ui64> ExpectedDeletes;
};
-
+
void AssertTableStats(const NYdb::NTable::TDataQueryResult& result, TStringBuf table,
const TExpectedTableStats& expectedStats);
inline void AssertTableReads(const NYdb::NTable::TDataQueryResult& result, TStringBuf table, ui64 expectedReads) {
AssertTableStats(result, table, { .ExpectedReads = expectedReads });
-}
-
-NYdb::NTable::TDataQueryResult ExecQueryAndTestResult(NYdb::NTable::TSession& session, const TString& query,
- const NYdb::TParams& params, const TString& expectedYson);
-
-inline NYdb::NTable::TDataQueryResult ExecQueryAndTestResult(NYdb::NTable::TSession& session, const TString& query,
- const TString& expectedYson)
-{
- return ExecQueryAndTestResult(session, query, NYdb::TParamsBuilder().Build(), expectedYson);
-}
-
-TString StreamResultToYson(NYdb::NExperimental::TStreamPartIterator& it, TVector<TString>* profiles = nullptr);
+}
+
+NYdb::NTable::TDataQueryResult ExecQueryAndTestResult(NYdb::NTable::TSession& session, const TString& query,
+ const NYdb::TParams& params, const TString& expectedYson);
+
+inline NYdb::NTable::TDataQueryResult ExecQueryAndTestResult(NYdb::NTable::TSession& session, const TString& query,
+ const TString& expectedYson)
+{
+ return ExecQueryAndTestResult(session, query, NYdb::TParamsBuilder().Build(), expectedYson);
+}
+
+TString StreamResultToYson(NYdb::NExperimental::TStreamPartIterator& it, TVector<TString>* profiles = nullptr);
TString StreamResultToYson(NYdb::NTable::TScanQueryPartIterator& it);
TString StreamResultToYson(NYdb::NScripting::TYqlResultPartIterator& it);
-
-ui32 CountPlanNodesByKv(const NJson::TJsonValue& plan, const TString& key, const TString& value);
+
+ui32 CountPlanNodesByKv(const NJson::TJsonValue& plan, const TString& key, const TString& value);
NJson::TJsonValue FindPlanNodeByKv(const NJson::TJsonValue& plan, const TString& key, const TString& value);
TString ReadTablePartToYson(NYdb::NTable::TSession session, const TString& table);
inline void AssertSuccessResult(const NYdb::TStatus& result) {
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-}
-
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+}
+
void CreateSampleTablesWithIndex(NYdb::NTable::TSession& session);
// KQP proxy needs to asynchronously receive tenants info before it is able to serve requests that have
@@ -238,5 +238,5 @@ void CreateSampleTablesWithIndex(NYdb::NTable::TSession& session);
// This method retries a simple query until it succeeds.
void WaitForKqpProxyInit(const NYdb::TDriver& driver);
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/ut/fat/kqp_force_newengine_ut.cpp b/ydb/core/kqp/ut/fat/kqp_force_newengine_ut.cpp
index b5b3e6a503..09fbe8f295 100644
--- a/ydb/core/kqp/ut/fat/kqp_force_newengine_ut.cpp
+++ b/ydb/core/kqp/ut/fat/kqp_force_newengine_ut.cpp
@@ -1,862 +1,862 @@
-#include <ydb/core/kqp/ut/common/kqp_ut_common.h>
-
-#include <ydb/core/base/counters.h>
-#include <ydb/core/kqp/counters/kqp_counters.h>
-#include <ydb/core/kqp/kqp_impl.h>
-
-#include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h>
-
-#include <library/cpp/grpc/client/grpc_client_low.h>
-
-namespace NKikimr {
-namespace NKqp {
-
-using namespace NYdb;
-using namespace NYdb::NTable;
-
-class KqpForceNewEngine : public TTestBase {
-public:
- void SetUp() override {
- FailForcedNewEngineCompilationForTests(false);
- FailForcedNewEngineExecutionForTests(false);
-
- Kikimr.reset(new TKikimrRunner);
- Counters = Kikimr->GetTestServer().GetRuntime()->GetAppData(0).Counters;
- KqpCounters.reset(new TKqpCounters(Counters));
- }
-
- void TearDown() override {
- FailForcedNewEngineCompilationForTests(false);
- FailForcedNewEngineExecutionForTests(false);
- }
-
- TSession Session() {
- return Kikimr->GetTableClient().CreateSession().GetValueSync().GetSession();
- }
-
- void ForceNewEngine(ui32 percent, ui32 level) {
- NGrpc::TGRpcClientLow grpcClient;
- auto grpcContext = grpcClient.CreateContext();
-
- NGrpc::TGRpcClientConfig grpcConfig(Kikimr->GetEndpoint());
- auto grpc = grpcClient.CreateGRpcServiceConnection<NKikimrClient::TGRpcServer>(grpcConfig);
-
- NKikimrClient::TConsoleRequest request;
- auto* action = request.MutableConfigureRequest()->MutableActions()->Add();
- auto* configItem = action->MutableAddConfigItem()->MutableConfigItem();
- configItem->SetKind(NKikimrConsole::TConfigItem::TableServiceConfigItem);
-
- configItem->MutableConfig()->MutableTableServiceConfig()->SetForceNewEnginePercent(percent);
- configItem->MutableConfig()->MutableTableServiceConfig()->SetForceNewEngineLevel(level);
-
- std::atomic<int> done = 0;
- grpc->DoRequest<NKikimrClient::TConsoleRequest, NKikimrClient::TConsoleResponse>(
- request,
- [&done](NGrpc::TGrpcStatus&& status, NKikimrClient::TConsoleResponse&& response) {
- if (status.Ok()) {
- if (response.GetStatus().code() != Ydb::StatusIds::SUCCESS) {
- done = 3;
- } else if (response.GetConfigureResponse().GetStatus().code() != Ydb::StatusIds::SUCCESS) {
- done = 4;
- } else {
- done = 1;
- }
- } else {
- Cerr << "status: " << status.Msg << ", " << status.InternalError << ", " << status.GRpcStatusCode << Endl;
- Cerr << response.DebugString() << Endl;
- done = 2;
- }
- },
- &NKikimrClient::TGRpcServer::Stub::AsyncConsoleRequest,
- {},
- grpcContext.get());
-
- while (done.load() == 0) {
- ::Sleep(TDuration::Seconds(1));
- }
- grpcContext.reset();
- grpcClient.Stop(true);
-
- UNIT_ASSERT_VALUES_EQUAL(done.load(), 1);
- }
-
- void TestNotInteractiveReadOnlyTx(ui32 level, bool withSqlIn = false) {
- auto session = Session();
-
- auto test = [&](ui32 count) {
- KqpCounters->NewEngineForcedQueryCount->Set(0);
- KqpCounters->NewEngineCompatibleQueryCount->Set(0);
-
- for (ui32 i = 0; i < count; ++i) {
- auto query = withSqlIn
- ? R"(
- --!syntax_v1
- DECLARE $values AS List<String>;
- SELECT * FROM `/Root/TwoShard` WHERE Value1 IN $values
- )"
- : R"(
- SELECT * FROM `/Root/TwoShard` WHERE Key = 1
- )";
- auto params = TParamsBuilder()
- .AddParam("$values").BeginList().AddListItem().String("One").EndList().Build().Build();
- auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), std::move(params)).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- CompareYson(R"([[[1u];["One"];[-1]]])", FormatResultSetYson(result.GetResultSet(0)));
- }
-
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->ForceNewEngineCompileErrors->Val());
- };
-
- {
- test(2);
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
-
- {
- ForceNewEngine(50, level);
- test(20);
- bool diff = KqpCounters->NewEngineForcedQueryCount->Val() * KqpCounters->NewEngineCompatibleQueryCount->Val() != 0;
- if (level == 0 && withSqlIn) {
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
- } else {
- UNIT_ASSERT_C(diff, "forced: " << KqpCounters->NewEngineForcedQueryCount->Val()
- << ", compatible: " << KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
- }
-
- {
- ForceNewEngine(100, level);
- test(2);
- if (level == 0 && withSqlIn) {
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
- } else {
- UNIT_ASSERT_VALUES_EQUAL(2, KqpCounters->NewEngineForcedQueryCount->Val());
- }
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
-
- {
- ForceNewEngine(0, level);
- test(2);
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
- }
-
- void TestNotInteractiveWriteOnlyTx(ui32 level) {
- auto session = Session();
-
- auto test = [&](ui32 count) {
- KqpCounters->NewEngineForcedQueryCount->Set(0);
- KqpCounters->NewEngineCompatibleQueryCount->Set(0);
-
- for (ui32 i = 0; i < count; ++i) {
- auto result = session.ExecuteDataQuery(R"(
- REPLACE INTO `/Root/TwoShard` (Key, Value1) VALUES (1, "OneOne")
- )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
- }
-
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->ForceNewEngineCompileErrors->Val());
- };
-
- {
- test(2);
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
-
- {
- ForceNewEngine(100, level);
- test(2);
- UNIT_ASSERT_VALUES_EQUAL(level == 3 ? 2 : 0, KqpCounters->NewEngineForcedQueryCount->Val());
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
- }
-
- void TestNotInteractiveReadOnlyTxFailedNECompilation(ui32 level) {
- auto session = Session();
-
- auto test = [&]() {
- KqpCounters->ForceNewEngineCompileErrors->Set(0);
- KqpCounters->NewEngineForcedQueryCount->Set(0);
- KqpCounters->NewEngineCompatibleQueryCount->Set(0);
-
- auto result = session.ExecuteDataQuery(R"(
- SELECT * FROM `/Root/KeyValue` LIMIT 1
- )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
- };
-
- {
- ForceNewEngine(100, level);
- FailForcedNewEngineCompilationForTests();
- test();
- UNIT_ASSERT_VALUES_EQUAL(1, KqpCounters->ForceNewEngineCompileErrors->Val());
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
- }
-
- void TestNotInteractiveReadOnlyTxFallback(ui32 level) {
- auto session = Session();
-
- auto test = [&]() {
- KqpCounters->ForceNewEngineCompileErrors->Set(0);
- KqpCounters->NewEngineForcedQueryCount->Set(0);
- KqpCounters->NewEngineCompatibleQueryCount->Set(0);
-
- auto result = session.ExecuteDataQuery(R"(
- SELECT * FROM `/Root/KeyValue` LIMIT 1
- )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- if (level == 0 || level == 1) {
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
- } else {
- UNIT_ASSERT_C(!result.IsSuccess(), result.GetIssues().ToString());
- }
- };
-
- {
- ForceNewEngine(100, level);
- FailForcedNewEngineExecutionForTests();
- test();
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->ForceNewEngineCompileErrors->Val());
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val()); // failed request is not counted
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
- UNIT_ASSERT_VALUES_EQUAL(level == 0 || level == 1 ? 1 : 0, GetServiceCounters(Counters, "kqp")->GetCounter("Requests/OldEngineFallback", true)->Val());
- UNIT_ASSERT_VALUES_EQUAL(level == 0 || level == 1 ? 0 : 1, GetServiceCounters(Counters, "kqp")->GetCounter("Requests/ForceNewEngineExecError", true)->Val());
- }
- }
-
- void TestInteractiveReadWriteEx(ui32 level, bool addBeginTx, /* bool prependWrite, */ bool appendRead, bool addCommitTx) {
- UNIT_ASSERT(level == 2);
-
- auto session = Session();
-
- auto test = [&](ui32 count) {
- KqpCounters->NewEngineForcedQueryCount->Set(0);
- KqpCounters->NewEngineCompatibleQueryCount->Set(0);
-
- for (ui32 i = 0; i < count; ++i) {
- auto txControl = TTxControl::BeginTx();
-
- if (addBeginTx) {
- auto tx = session.BeginTransaction(TTxSettings::SerializableRW()).ExtractValueSync().GetTransaction();
- UNIT_ASSERT(tx.IsActive());
-
- txControl = TTxControl::Tx(tx);
- }
-
- // TODO: prependWrite
-
- auto result = session.ExecuteDataQuery(R"(
- SELECT * FROM `/Root/KeyValue` LIMIT 1
- )", txControl).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- txControl = TTxControl::Tx(*result.GetTransaction());
-
- if (appendRead || addCommitTx) {
- // do nothing
- } else {
- txControl = txControl.CommitTx();
- }
-
- result = session.ExecuteDataQuery(Sprintf(R"(
- UPDATE `/Root/TwoShard` SET Value2 = %d WHERE Key = 1
- )", i), txControl).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- if (appendRead) {
- txControl = TTxControl::Tx(*result.GetTransaction());
- if (!addCommitTx) {
- txControl = txControl.CommitTx();
- }
-
- result = session.ExecuteDataQuery(R"(
- SELECT * FROM `/Root/KeyValue` LIMIT 2
- )", txControl).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- if (addCommitTx) {
- auto txResult = result.GetTransaction()->Commit().ExtractValueSync();
- UNIT_ASSERT_C(txResult.IsSuccess(), txResult.GetIssues().ToString());
- }
- } else {
- if (addCommitTx) {
- auto txResult = result.GetTransaction()->Commit().ExtractValueSync();
- UNIT_ASSERT_C(txResult.IsSuccess(), txResult.GetIssues().ToString());
- }
- }
- }
-
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->ForceNewEngineCompileErrors->Val());
- };
-
- {
- test(2);
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
-
- {
- ForceNewEngine(50, level);
- test(20);
- bool diff = KqpCounters->NewEngineForcedQueryCount->Val() * KqpCounters->NewEngineCompatibleQueryCount->Val() != 0;
- UNIT_ASSERT_C(diff, "forced: " << KqpCounters->NewEngineForcedQueryCount->Val()
- << ", compatible: " << KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
-
- {
- ForceNewEngine(100, level);
- test(2);
- UNIT_ASSERT_VALUES_EQUAL(2 * (appendRead && addCommitTx ? 2 : 1), KqpCounters->NewEngineForcedQueryCount->Val());
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
-
- {
- ForceNewEngine(0, level);
- test(2);
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
- }
-
-
- UNIT_TEST_SUITE(KqpForceNewEngine);
- UNIT_TEST(Level0_NotInteractiveReadOnly);
- UNIT_TEST(Level0_NotInteractiveReadOnlySqlIn);
- UNIT_TEST(Level0_NotInteractiveWriteOnly);
- UNIT_TEST(Level0_NotInteractiveReadWrite);
- UNIT_TEST(Level0_InteractiveReadOnly);
- UNIT_TEST(Level0_CompilationFailure);
- UNIT_TEST(Level0_Fallback);
-
- UNIT_TEST(Level1_NotInteractiveReadOnly);
- UNIT_TEST(Level1_NotInteractiveReadOnlySqlIn);
- UNIT_TEST(Level1_NotInteractiveWriteOnly);
- UNIT_TEST(Level1_NotInteractiveReadWrite);
- UNIT_TEST(Level1_InteractiveReadOnly);
- UNIT_TEST(Level1_CompilationFailure);
- UNIT_TEST(Level1_Fallback);
-
- UNIT_TEST(Level2_NotInteractiveReadOnly);
- UNIT_TEST(Level2_NotInteractiveWriteOnly);
- UNIT_TEST(Level2_InteractiveReadOnly);
- UNIT_TEST(Level2_InteractiveReadWrite);
- UNIT_TEST(Level2_InteractiveBeginReadWrite);
- UNIT_TEST(Level2_InteractiveReadWriteCommit);
- UNIT_TEST(Level2_InteractiveBeginReadWriteCommit);
- UNIT_TEST(Level2_InteractiveReadWriteRead);
- UNIT_TEST(Level2_InteractiveBeginReadWriteRead);
- UNIT_TEST(Level2_InteractiveReadWriteReadCommit);
- UNIT_TEST(Level2_InteractiveBeginReadWriteReadCommit);
- UNIT_TEST(Level2_InteractiveWriteOnly);
- UNIT_TEST(Level2_CompilationFailure);
- UNIT_TEST(Level2_NoFallback);
-
- UNIT_TEST(Level3_NotInteractiveReadOnly);
- UNIT_TEST(Level3_NotInteractiveWriteOnly);
- UNIT_TEST(Level3_InteractiveReadOnly);
- UNIT_TEST(Level3_InteractiveReadWrite);
- UNIT_TEST(Level3_CompilationFailure);
- UNIT_TEST(Level3_NoFallback);
- UNIT_TEST_SUITE_END();
-
- void Level0_NotInteractiveReadOnly();
- void Level0_NotInteractiveReadOnlySqlIn();
- void Level0_NotInteractiveWriteOnly();
- void Level0_NotInteractiveReadWrite();
- void Level0_InteractiveReadOnly();
- void Level0_CompilationFailure();
- void Level0_Fallback();
-
- void Level1_NotInteractiveReadOnly();
- void Level1_NotInteractiveReadOnlySqlIn();
- void Level1_NotInteractiveWriteOnly();
- void Level1_NotInteractiveReadWrite();
- void Level1_InteractiveReadOnly();
- void Level1_CompilationFailure();
- void Level1_Fallback();
-
- void Level2_NotInteractiveReadOnly();
- void Level2_NotInteractiveWriteOnly();
- void Level2_InteractiveReadOnly();
- void Level2_InteractiveReadWrite();
- void Level2_InteractiveBeginReadWrite();
- void Level2_InteractiveReadWriteCommit();
- void Level2_InteractiveBeginReadWriteCommit();
- void Level2_InteractiveReadWriteRead();
- void Level2_InteractiveBeginReadWriteRead();
- void Level2_InteractiveReadWriteReadCommit();
- void Level2_InteractiveBeginReadWriteReadCommit();
- void Level2_InteractiveWriteOnly();
- void Level2_CompilationFailure();
- void Level2_NoFallback();
-
- void Level3_NotInteractiveReadOnly();
- void Level3_NotInteractiveWriteOnly();
- void Level3_InteractiveReadOnly();
- void Level3_InteractiveReadWrite();
- void Level3_CompilationFailure();
- void Level3_NoFallback();
-
-private:
- std::unique_ptr<TKikimrRunner> Kikimr;
- NMonitoring::TDynamicCounterPtr Counters;
- std::unique_ptr<TKqpCounters> KqpCounters;
-};
-UNIT_TEST_SUITE_REGISTRATION(KqpForceNewEngine);
-
-/////////// LEVEL 0 ////////////////////////////////////////////////////////////////////////////////////////////////////
-void KqpForceNewEngine::Level0_NotInteractiveReadOnly() {
- TestNotInteractiveReadOnlyTx(0);
-}
-
-void KqpForceNewEngine::Level0_NotInteractiveReadOnlySqlIn() {
- TestNotInteractiveReadOnlyTx(0, /* withSqlIn */ true);
-}
-
-void KqpForceNewEngine::Level0_NotInteractiveWriteOnly() {
- TestNotInteractiveWriteOnlyTx(0);
-}
-
-void KqpForceNewEngine::Level0_NotInteractiveReadWrite() {
- auto session = Session();
-
- auto test = [&](ui32 count) {
- KqpCounters->NewEngineForcedQueryCount->Set(0);
- KqpCounters->NewEngineCompatibleQueryCount->Set(0);
-
- for (ui32 i = 0; i < count; ++i) {
- auto result = session.ExecuteDataQuery(Sprintf(R"(
- UPDATE `/Root/TwoShard` SET Value2 = %d WHERE Key = 1
- )", i), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
- }
-
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->ForceNewEngineCompileErrors->Val());
- };
-
- {
- test(2);
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
-
- {
- ForceNewEngine(100, 0);
- test(2);
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
-}
-
-void KqpForceNewEngine::Level0_InteractiveReadOnly() {
- auto session = Session();
-
- auto test = [&](ui32 count) {
- KqpCounters->NewEngineForcedQueryCount->Set(0);
- KqpCounters->NewEngineCompatibleQueryCount->Set(0);
-
- for (ui32 i = 0; i < count; ++i) {
- auto result = session.ExecuteDataQuery(R"(
- SELECT * FROM `/Root/TwoShard` WHERE Key = 1
- )", TTxControl::BeginTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
- CompareYson(R"([[[1u];["One"];[-1]]])", FormatResultSetYson(result.GetResultSet(0)));
-
- auto tx = *result.GetTransaction();
-
- result = session.ExecuteDataQuery(R"(
- SELECT * FROM `/Root/TwoShard` WHERE Key = 2
- )", TTxControl::Tx(tx).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
- CompareYson(R"([[[2u];["Two"];[0]]])", FormatResultSetYson(result.GetResultSet(0)));
- }
-
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->ForceNewEngineCompileErrors->Val());
- };
-
- {
- test(2);
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
-
- {
- ForceNewEngine(100, 0);
- test(2);
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
-}
-
-void KqpForceNewEngine::Level0_CompilationFailure() {
- TestNotInteractiveReadOnlyTxFailedNECompilation(0);
-}
-
-void KqpForceNewEngine::Level0_Fallback() {
- TestNotInteractiveReadOnlyTxFallback(0);
-}
-
-/////////// LEVEL 1 ////////////////////////////////////////////////////////////////////////////////////////////////////
-void KqpForceNewEngine::Level1_NotInteractiveReadOnly() {
- TestNotInteractiveReadOnlyTx(1);
-}
-
-void KqpForceNewEngine::Level1_NotInteractiveReadOnlySqlIn() {
- TestNotInteractiveReadOnlyTx(1, /* withSqlIn */ true);
-}
-
-void KqpForceNewEngine::Level1_NotInteractiveWriteOnly() {
- TestNotInteractiveWriteOnlyTx(1);
-}
-
-void KqpForceNewEngine::Level1_NotInteractiveReadWrite() {
- auto session = Session();
-
- auto test = [&](ui32 count) {
- KqpCounters->NewEngineForcedQueryCount->Set(0);
- KqpCounters->NewEngineCompatibleQueryCount->Set(0);
-
- for (ui32 i = 0; i < count; ++i) {
- auto result = session.ExecuteDataQuery(Sprintf(R"(
- UPDATE `/Root/TwoShard` SET Value2 = %d WHERE Key = 1
- )", i), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
- }
-
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->ForceNewEngineCompileErrors->Val());
- };
-
- {
- test(2);
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
-
- {
- ForceNewEngine(100, 1);
- test(2);
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
-}
-
-void KqpForceNewEngine::Level1_InteractiveReadOnly() {
- auto session = Session();
-
- auto test = [&](ui32 count) {
- KqpCounters->NewEngineForcedQueryCount->Set(0);
- KqpCounters->NewEngineCompatibleQueryCount->Set(0);
-
- for (ui32 i = 0; i < count; ++i) {
- auto result = session.ExecuteDataQuery(R"(
- SELECT * FROM `/Root/TwoShard` WHERE Key = 1
- )", TTxControl::BeginTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
- CompareYson(R"([[[1u];["One"];[-1]]])", FormatResultSetYson(result.GetResultSet(0)));
-
- auto tx = *result.GetTransaction();
-
- result = session.ExecuteDataQuery(R"(
- SELECT * FROM `/Root/TwoShard` WHERE Key = 2
- )", TTxControl::Tx(tx).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
- CompareYson(R"([[[2u];["Two"];[0]]])", FormatResultSetYson(result.GetResultSet(0)));
- }
-
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->ForceNewEngineCompileErrors->Val());
- };
-
- {
- test(2);
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
-
- {
- ForceNewEngine(100, 1);
- test(2);
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
-}
-
-void KqpForceNewEngine::Level1_CompilationFailure() {
- TestNotInteractiveReadOnlyTxFailedNECompilation(1);
-}
-
-void KqpForceNewEngine::Level1_Fallback() {
- TestNotInteractiveReadOnlyTxFallback(1);
-}
-
-/////////// LEVEL 2 ////////////////////////////////////////////////////////////////////////////////////////////////////
-void KqpForceNewEngine::Level2_NotInteractiveReadOnly() {
- TestNotInteractiveReadOnlyTx(2);
-}
-
-void KqpForceNewEngine::Level2_NotInteractiveWriteOnly() {
- TestNotInteractiveWriteOnlyTx(2);
-}
-
-void KqpForceNewEngine::Level2_InteractiveReadOnly() {
- auto session = Session();
-
- auto test = [&](ui32 count) {
- KqpCounters->NewEngineForcedQueryCount->Set(0);
- KqpCounters->NewEngineCompatibleQueryCount->Set(0);
-
- for (ui32 i = 0; i < count; ++i) {
- auto result = session.ExecuteDataQuery(R"(
- SELECT * FROM `/Root/TwoShard` WHERE Key = 1
- )", TTxControl::BeginTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
- CompareYson(R"([[[1u];["One"];[-1]]])", FormatResultSetYson(result.GetResultSet(0)));
-
- auto tx = *result.GetTransaction();
-
- result = session.ExecuteDataQuery(R"(
- SELECT * FROM `/Root/TwoShard` WHERE Key = 2
- )", TTxControl::Tx(tx).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
- CompareYson(R"([[[2u];["Two"];[0]]])", FormatResultSetYson(result.GetResultSet(0)));
- }
-
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->ForceNewEngineCompileErrors->Val());
- };
-
- {
- test(2);
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
-
- {
- ForceNewEngine(50, 2);
- test(20);
- bool diff = KqpCounters->NewEngineForcedQueryCount->Val() * KqpCounters->NewEngineCompatibleQueryCount->Val() != 0;
- UNIT_ASSERT_C(diff, "forced: " << KqpCounters->NewEngineForcedQueryCount->Val()
- << ", compatible: " << KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
-
- {
- ForceNewEngine(100, 2);
- test(2);
- UNIT_ASSERT_VALUES_EQUAL(4, KqpCounters->NewEngineForcedQueryCount->Val());
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
-
- {
- ForceNewEngine(0, 2);
- test(2);
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
-}
-
-void KqpForceNewEngine::Level2_InteractiveReadWrite() {
- TestInteractiveReadWriteEx(/* level */ 2, /* addBeginTx */ false, /* appendRead */ false, /* addCommitTx */ false);
-}
-
-void KqpForceNewEngine::Level2_InteractiveBeginReadWrite() {
- TestInteractiveReadWriteEx(/* level */ 2, /* addBeginTx */ true, /* appendRead */ false, /* addCommitTx */ false);
-}
-
-void KqpForceNewEngine::Level2_InteractiveReadWriteCommit() {
- TestInteractiveReadWriteEx(/* level */ 2, /* addBeginTx */ false, /* appendRead */ false, /* addCommitTx */ true);
-}
-
-void KqpForceNewEngine::Level2_InteractiveBeginReadWriteCommit() {
- TestInteractiveReadWriteEx(/* level */ 2, /* addBeginTx */ true, /* appendRead */ false, /* addCommitTx */ true);
-}
-
-void KqpForceNewEngine::Level2_InteractiveReadWriteRead() {
- TestInteractiveReadWriteEx(/* level */ 2, /* addBeginTx */ false, /* appendRead */ true, /* addCommitTx */ false);
-}
-
-void KqpForceNewEngine::Level2_InteractiveBeginReadWriteRead() {
- TestInteractiveReadWriteEx(/* level */ 2, /* addBeginTx */ true, /* appendRead */ true, /* addCommitTx */ false);
-}
-
-void KqpForceNewEngine::Level2_InteractiveReadWriteReadCommit() {
- TestInteractiveReadWriteEx(/* level */ 2, /* addBeginTx */ false, /* appendRead */ true, /* addCommitTx */ true);
-}
-
-void KqpForceNewEngine::Level2_InteractiveBeginReadWriteReadCommit() {
- TestInteractiveReadWriteEx(/* level */ 2, /* addBeginTx */ true, /* appendRead */ true, /* addCommitTx */ true);
-}
-
-void KqpForceNewEngine::Level2_InteractiveWriteOnly() {
- auto session = Session();
-
- auto test = [&](ui32 count) {
- KqpCounters->NewEngineForcedQueryCount->Set(0);
- KqpCounters->NewEngineCompatibleQueryCount->Set(0);
-
- for (ui32 i = 0; i < count; ++i) {
- auto result = session.ExecuteDataQuery(R"(
- REPLACE INTO `/Root/TwoShard` (Key, Value1) VALUES (1, "OneOne")
- )", TTxControl::BeginTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- result = session.ExecuteDataQuery(R"(
- REPLACE INTO `/Root/KeyValue` (Key, Value) VALUES (1, "OneOne")
- )", TTxControl::Tx(*result.GetTransaction()).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
- }
-
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->ForceNewEngineCompileErrors->Val());
- };
-
- {
- test(2);
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
-
- {
- ForceNewEngine(100, 2);
- test(2);
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
-}
-
-void KqpForceNewEngine::Level2_CompilationFailure() {
- TestNotInteractiveReadOnlyTxFailedNECompilation(2);
-}
-
-void KqpForceNewEngine::Level2_NoFallback() {
- TestNotInteractiveReadOnlyTxFallback(2);
-}
-
-/////////// LEVEL 3 ////////////////////////////////////////////////////////////////////////////////////////////////////
-void KqpForceNewEngine::Level3_NotInteractiveReadOnly() {
- TestNotInteractiveReadOnlyTx(3);
-}
-
-void KqpForceNewEngine::Level3_NotInteractiveWriteOnly() {
- TestNotInteractiveWriteOnlyTx(3);
-}
-
-void KqpForceNewEngine::Level3_InteractiveReadOnly() {
- auto session = Session();
-
- auto test = [&](ui32 count) {
- KqpCounters->NewEngineForcedQueryCount->Set(0);
- KqpCounters->NewEngineCompatibleQueryCount->Set(0);
-
- for (ui32 i = 0; i < count; ++i) {
- auto result = session.ExecuteDataQuery(R"(
- SELECT * FROM `/Root/TwoShard` WHERE Key = 1
- )", TTxControl::BeginTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
- CompareYson(R"([[[1u];["One"];[-1]]])", FormatResultSetYson(result.GetResultSet(0)));
-
- auto tx = *result.GetTransaction();
-
- result = session.ExecuteDataQuery(R"(
- SELECT * FROM `/Root/TwoShard` WHERE Key = 2
- )", TTxControl::Tx(tx).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
- CompareYson(R"([[[2u];["Two"];[0]]])", FormatResultSetYson(result.GetResultSet(0)));
- }
-
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->ForceNewEngineCompileErrors->Val());
- };
-
- {
- test(2);
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
-
- {
- ForceNewEngine(50, 3);
- test(20);
- bool diff = KqpCounters->NewEngineForcedQueryCount->Val() * KqpCounters->NewEngineCompatibleQueryCount->Val() != 0;
- UNIT_ASSERT_C(diff, "forced: " << KqpCounters->NewEngineForcedQueryCount->Val()
- << ", compatible: " << KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
-
- {
- ForceNewEngine(100, 3);
- test(2);
- UNIT_ASSERT_VALUES_EQUAL(4, KqpCounters->NewEngineForcedQueryCount->Val());
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
-
- {
- ForceNewEngine(0, 3);
- test(2);
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
-}
-
-void KqpForceNewEngine::Level3_InteractiveReadWrite() {
- auto session = Session();
-
- auto test = [&](ui32 count) {
- KqpCounters->NewEngineForcedQueryCount->Set(0);
- KqpCounters->NewEngineCompatibleQueryCount->Set(0);
-
- for (ui32 i = 0; i < count; ++i) {
- auto result = session.ExecuteDataQuery(R"(
- SELECT * FROM `/Root/KeyValue` LIMIT 1
- )", TTxControl::BeginTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- result = session.ExecuteDataQuery(Sprintf(R"(
- UPDATE `/Root/TwoShard` SET Value2 = %d WHERE Key = 1
- )", i), TTxControl::Tx(*result.GetTransaction()).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
- }
-
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->ForceNewEngineCompileErrors->Val());
- };
-
- {
- test(2);
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
-
- {
- ForceNewEngine(50, 3);
- test(20);
- bool diff = KqpCounters->NewEngineForcedQueryCount->Val() * KqpCounters->NewEngineCompatibleQueryCount->Val() != 0;
- UNIT_ASSERT_C(diff, "forced: " << KqpCounters->NewEngineForcedQueryCount->Val()
- << ", compatible: " << KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
-
- {
- ForceNewEngine(100, 3);
- test(2);
- UNIT_ASSERT_VALUES_EQUAL(4, KqpCounters->NewEngineForcedQueryCount->Val());
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
-
- {
- ForceNewEngine(0, 3);
- test(2);
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
- UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
- }
-}
-
-void KqpForceNewEngine::Level3_CompilationFailure() {
- TestNotInteractiveReadOnlyTxFailedNECompilation(3);
-}
-
-void KqpForceNewEngine::Level3_NoFallback() {
- TestNotInteractiveReadOnlyTxFallback(3);
-}
-
-} // namespace NKqp
-} // namespace NKikimr
+#include <ydb/core/kqp/ut/common/kqp_ut_common.h>
+
+#include <ydb/core/base/counters.h>
+#include <ydb/core/kqp/counters/kqp_counters.h>
+#include <ydb/core/kqp/kqp_impl.h>
+
+#include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h>
+
+#include <library/cpp/grpc/client/grpc_client_low.h>
+
+namespace NKikimr {
+namespace NKqp {
+
+using namespace NYdb;
+using namespace NYdb::NTable;
+
+class KqpForceNewEngine : public TTestBase {
+public:
+ void SetUp() override {
+ FailForcedNewEngineCompilationForTests(false);
+ FailForcedNewEngineExecutionForTests(false);
+
+ Kikimr.reset(new TKikimrRunner);
+ Counters = Kikimr->GetTestServer().GetRuntime()->GetAppData(0).Counters;
+ KqpCounters.reset(new TKqpCounters(Counters));
+ }
+
+ void TearDown() override {
+ FailForcedNewEngineCompilationForTests(false);
+ FailForcedNewEngineExecutionForTests(false);
+ }
+
+ TSession Session() {
+ return Kikimr->GetTableClient().CreateSession().GetValueSync().GetSession();
+ }
+
+ void ForceNewEngine(ui32 percent, ui32 level) {
+ NGrpc::TGRpcClientLow grpcClient;
+ auto grpcContext = grpcClient.CreateContext();
+
+ NGrpc::TGRpcClientConfig grpcConfig(Kikimr->GetEndpoint());
+ auto grpc = grpcClient.CreateGRpcServiceConnection<NKikimrClient::TGRpcServer>(grpcConfig);
+
+ NKikimrClient::TConsoleRequest request;
+ auto* action = request.MutableConfigureRequest()->MutableActions()->Add();
+ auto* configItem = action->MutableAddConfigItem()->MutableConfigItem();
+ configItem->SetKind(NKikimrConsole::TConfigItem::TableServiceConfigItem);
+
+ configItem->MutableConfig()->MutableTableServiceConfig()->SetForceNewEnginePercent(percent);
+ configItem->MutableConfig()->MutableTableServiceConfig()->SetForceNewEngineLevel(level);
+
+ std::atomic<int> done = 0;
+ grpc->DoRequest<NKikimrClient::TConsoleRequest, NKikimrClient::TConsoleResponse>(
+ request,
+ [&done](NGrpc::TGrpcStatus&& status, NKikimrClient::TConsoleResponse&& response) {
+ if (status.Ok()) {
+ if (response.GetStatus().code() != Ydb::StatusIds::SUCCESS) {
+ done = 3;
+ } else if (response.GetConfigureResponse().GetStatus().code() != Ydb::StatusIds::SUCCESS) {
+ done = 4;
+ } else {
+ done = 1;
+ }
+ } else {
+ Cerr << "status: " << status.Msg << ", " << status.InternalError << ", " << status.GRpcStatusCode << Endl;
+ Cerr << response.DebugString() << Endl;
+ done = 2;
+ }
+ },
+ &NKikimrClient::TGRpcServer::Stub::AsyncConsoleRequest,
+ {},
+ grpcContext.get());
+
+ while (done.load() == 0) {
+ ::Sleep(TDuration::Seconds(1));
+ }
+ grpcContext.reset();
+ grpcClient.Stop(true);
+
+ UNIT_ASSERT_VALUES_EQUAL(done.load(), 1);
+ }
+
+ void TestNotInteractiveReadOnlyTx(ui32 level, bool withSqlIn = false) {
+ auto session = Session();
+
+ auto test = [&](ui32 count) {
+ KqpCounters->NewEngineForcedQueryCount->Set(0);
+ KqpCounters->NewEngineCompatibleQueryCount->Set(0);
+
+ for (ui32 i = 0; i < count; ++i) {
+ auto query = withSqlIn
+ ? R"(
+ --!syntax_v1
+ DECLARE $values AS List<String>;
+ SELECT * FROM `/Root/TwoShard` WHERE Value1 IN $values
+ )"
+ : R"(
+ SELECT * FROM `/Root/TwoShard` WHERE Key = 1
+ )";
+ auto params = TParamsBuilder()
+ .AddParam("$values").BeginList().AddListItem().String("One").EndList().Build().Build();
+ auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), std::move(params)).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ CompareYson(R"([[[1u];["One"];[-1]]])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->ForceNewEngineCompileErrors->Val());
+ };
+
+ {
+ test(2);
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+
+ {
+ ForceNewEngine(50, level);
+ test(20);
+ bool diff = KqpCounters->NewEngineForcedQueryCount->Val() * KqpCounters->NewEngineCompatibleQueryCount->Val() != 0;
+ if (level == 0 && withSqlIn) {
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
+ } else {
+ UNIT_ASSERT_C(diff, "forced: " << KqpCounters->NewEngineForcedQueryCount->Val()
+ << ", compatible: " << KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+ }
+
+ {
+ ForceNewEngine(100, level);
+ test(2);
+ if (level == 0 && withSqlIn) {
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
+ } else {
+ UNIT_ASSERT_VALUES_EQUAL(2, KqpCounters->NewEngineForcedQueryCount->Val());
+ }
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+
+ {
+ ForceNewEngine(0, level);
+ test(2);
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+ }
+
+ void TestNotInteractiveWriteOnlyTx(ui32 level) {
+ auto session = Session();
+
+ auto test = [&](ui32 count) {
+ KqpCounters->NewEngineForcedQueryCount->Set(0);
+ KqpCounters->NewEngineCompatibleQueryCount->Set(0);
+
+ for (ui32 i = 0; i < count; ++i) {
+ auto result = session.ExecuteDataQuery(R"(
+ REPLACE INTO `/Root/TwoShard` (Key, Value1) VALUES (1, "OneOne")
+ )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->ForceNewEngineCompileErrors->Val());
+ };
+
+ {
+ test(2);
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+
+ {
+ ForceNewEngine(100, level);
+ test(2);
+ UNIT_ASSERT_VALUES_EQUAL(level == 3 ? 2 : 0, KqpCounters->NewEngineForcedQueryCount->Val());
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+ }
+
+ void TestNotInteractiveReadOnlyTxFailedNECompilation(ui32 level) {
+ auto session = Session();
+
+ auto test = [&]() {
+ KqpCounters->ForceNewEngineCompileErrors->Set(0);
+ KqpCounters->NewEngineForcedQueryCount->Set(0);
+ KqpCounters->NewEngineCompatibleQueryCount->Set(0);
+
+ auto result = session.ExecuteDataQuery(R"(
+ SELECT * FROM `/Root/KeyValue` LIMIT 1
+ )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ };
+
+ {
+ ForceNewEngine(100, level);
+ FailForcedNewEngineCompilationForTests();
+ test();
+ UNIT_ASSERT_VALUES_EQUAL(1, KqpCounters->ForceNewEngineCompileErrors->Val());
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+ }
+
+ void TestNotInteractiveReadOnlyTxFallback(ui32 level) {
+ auto session = Session();
+
+ auto test = [&]() {
+ KqpCounters->ForceNewEngineCompileErrors->Set(0);
+ KqpCounters->NewEngineForcedQueryCount->Set(0);
+ KqpCounters->NewEngineCompatibleQueryCount->Set(0);
+
+ auto result = session.ExecuteDataQuery(R"(
+ SELECT * FROM `/Root/KeyValue` LIMIT 1
+ )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ if (level == 0 || level == 1) {
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ } else {
+ UNIT_ASSERT_C(!result.IsSuccess(), result.GetIssues().ToString());
+ }
+ };
+
+ {
+ ForceNewEngine(100, level);
+ FailForcedNewEngineExecutionForTests();
+ test();
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->ForceNewEngineCompileErrors->Val());
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val()); // failed request is not counted
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
+ UNIT_ASSERT_VALUES_EQUAL(level == 0 || level == 1 ? 1 : 0, GetServiceCounters(Counters, "kqp")->GetCounter("Requests/OldEngineFallback", true)->Val());
+ UNIT_ASSERT_VALUES_EQUAL(level == 0 || level == 1 ? 0 : 1, GetServiceCounters(Counters, "kqp")->GetCounter("Requests/ForceNewEngineExecError", true)->Val());
+ }
+ }
+
+ void TestInteractiveReadWriteEx(ui32 level, bool addBeginTx, /* bool prependWrite, */ bool appendRead, bool addCommitTx) {
+ UNIT_ASSERT(level == 2);
+
+ auto session = Session();
+
+ auto test = [&](ui32 count) {
+ KqpCounters->NewEngineForcedQueryCount->Set(0);
+ KqpCounters->NewEngineCompatibleQueryCount->Set(0);
+
+ for (ui32 i = 0; i < count; ++i) {
+ auto txControl = TTxControl::BeginTx();
+
+ if (addBeginTx) {
+ auto tx = session.BeginTransaction(TTxSettings::SerializableRW()).ExtractValueSync().GetTransaction();
+ UNIT_ASSERT(tx.IsActive());
+
+ txControl = TTxControl::Tx(tx);
+ }
+
+ // TODO: prependWrite
+
+ auto result = session.ExecuteDataQuery(R"(
+ SELECT * FROM `/Root/KeyValue` LIMIT 1
+ )", txControl).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ txControl = TTxControl::Tx(*result.GetTransaction());
+
+ if (appendRead || addCommitTx) {
+ // do nothing
+ } else {
+ txControl = txControl.CommitTx();
+ }
+
+ result = session.ExecuteDataQuery(Sprintf(R"(
+ UPDATE `/Root/TwoShard` SET Value2 = %d WHERE Key = 1
+ )", i), txControl).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ if (appendRead) {
+ txControl = TTxControl::Tx(*result.GetTransaction());
+ if (!addCommitTx) {
+ txControl = txControl.CommitTx();
+ }
+
+ result = session.ExecuteDataQuery(R"(
+ SELECT * FROM `/Root/KeyValue` LIMIT 2
+ )", txControl).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ if (addCommitTx) {
+ auto txResult = result.GetTransaction()->Commit().ExtractValueSync();
+ UNIT_ASSERT_C(txResult.IsSuccess(), txResult.GetIssues().ToString());
+ }
+ } else {
+ if (addCommitTx) {
+ auto txResult = result.GetTransaction()->Commit().ExtractValueSync();
+ UNIT_ASSERT_C(txResult.IsSuccess(), txResult.GetIssues().ToString());
+ }
+ }
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->ForceNewEngineCompileErrors->Val());
+ };
+
+ {
+ test(2);
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+
+ {
+ ForceNewEngine(50, level);
+ test(20);
+ bool diff = KqpCounters->NewEngineForcedQueryCount->Val() * KqpCounters->NewEngineCompatibleQueryCount->Val() != 0;
+ UNIT_ASSERT_C(diff, "forced: " << KqpCounters->NewEngineForcedQueryCount->Val()
+ << ", compatible: " << KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+
+ {
+ ForceNewEngine(100, level);
+ test(2);
+ UNIT_ASSERT_VALUES_EQUAL(2 * (appendRead && addCommitTx ? 2 : 1), KqpCounters->NewEngineForcedQueryCount->Val());
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+
+ {
+ ForceNewEngine(0, level);
+ test(2);
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+ }
+
+
+ UNIT_TEST_SUITE(KqpForceNewEngine);
+ UNIT_TEST(Level0_NotInteractiveReadOnly);
+ UNIT_TEST(Level0_NotInteractiveReadOnlySqlIn);
+ UNIT_TEST(Level0_NotInteractiveWriteOnly);
+ UNIT_TEST(Level0_NotInteractiveReadWrite);
+ UNIT_TEST(Level0_InteractiveReadOnly);
+ UNIT_TEST(Level0_CompilationFailure);
+ UNIT_TEST(Level0_Fallback);
+
+ UNIT_TEST(Level1_NotInteractiveReadOnly);
+ UNIT_TEST(Level1_NotInteractiveReadOnlySqlIn);
+ UNIT_TEST(Level1_NotInteractiveWriteOnly);
+ UNIT_TEST(Level1_NotInteractiveReadWrite);
+ UNIT_TEST(Level1_InteractiveReadOnly);
+ UNIT_TEST(Level1_CompilationFailure);
+ UNIT_TEST(Level1_Fallback);
+
+ UNIT_TEST(Level2_NotInteractiveReadOnly);
+ UNIT_TEST(Level2_NotInteractiveWriteOnly);
+ UNIT_TEST(Level2_InteractiveReadOnly);
+ UNIT_TEST(Level2_InteractiveReadWrite);
+ UNIT_TEST(Level2_InteractiveBeginReadWrite);
+ UNIT_TEST(Level2_InteractiveReadWriteCommit);
+ UNIT_TEST(Level2_InteractiveBeginReadWriteCommit);
+ UNIT_TEST(Level2_InteractiveReadWriteRead);
+ UNIT_TEST(Level2_InteractiveBeginReadWriteRead);
+ UNIT_TEST(Level2_InteractiveReadWriteReadCommit);
+ UNIT_TEST(Level2_InteractiveBeginReadWriteReadCommit);
+ UNIT_TEST(Level2_InteractiveWriteOnly);
+ UNIT_TEST(Level2_CompilationFailure);
+ UNIT_TEST(Level2_NoFallback);
+
+ UNIT_TEST(Level3_NotInteractiveReadOnly);
+ UNIT_TEST(Level3_NotInteractiveWriteOnly);
+ UNIT_TEST(Level3_InteractiveReadOnly);
+ UNIT_TEST(Level3_InteractiveReadWrite);
+ UNIT_TEST(Level3_CompilationFailure);
+ UNIT_TEST(Level3_NoFallback);
+ UNIT_TEST_SUITE_END();
+
+ void Level0_NotInteractiveReadOnly();
+ void Level0_NotInteractiveReadOnlySqlIn();
+ void Level0_NotInteractiveWriteOnly();
+ void Level0_NotInteractiveReadWrite();
+ void Level0_InteractiveReadOnly();
+ void Level0_CompilationFailure();
+ void Level0_Fallback();
+
+ void Level1_NotInteractiveReadOnly();
+ void Level1_NotInteractiveReadOnlySqlIn();
+ void Level1_NotInteractiveWriteOnly();
+ void Level1_NotInteractiveReadWrite();
+ void Level1_InteractiveReadOnly();
+ void Level1_CompilationFailure();
+ void Level1_Fallback();
+
+ void Level2_NotInteractiveReadOnly();
+ void Level2_NotInteractiveWriteOnly();
+ void Level2_InteractiveReadOnly();
+ void Level2_InteractiveReadWrite();
+ void Level2_InteractiveBeginReadWrite();
+ void Level2_InteractiveReadWriteCommit();
+ void Level2_InteractiveBeginReadWriteCommit();
+ void Level2_InteractiveReadWriteRead();
+ void Level2_InteractiveBeginReadWriteRead();
+ void Level2_InteractiveReadWriteReadCommit();
+ void Level2_InteractiveBeginReadWriteReadCommit();
+ void Level2_InteractiveWriteOnly();
+ void Level2_CompilationFailure();
+ void Level2_NoFallback();
+
+ void Level3_NotInteractiveReadOnly();
+ void Level3_NotInteractiveWriteOnly();
+ void Level3_InteractiveReadOnly();
+ void Level3_InteractiveReadWrite();
+ void Level3_CompilationFailure();
+ void Level3_NoFallback();
+
+private:
+ std::unique_ptr<TKikimrRunner> Kikimr;
+ NMonitoring::TDynamicCounterPtr Counters;
+ std::unique_ptr<TKqpCounters> KqpCounters;
+};
+UNIT_TEST_SUITE_REGISTRATION(KqpForceNewEngine);
+
+/////////// LEVEL 0 ////////////////////////////////////////////////////////////////////////////////////////////////////
+void KqpForceNewEngine::Level0_NotInteractiveReadOnly() {
+ TestNotInteractiveReadOnlyTx(0);
+}
+
+void KqpForceNewEngine::Level0_NotInteractiveReadOnlySqlIn() {
+ TestNotInteractiveReadOnlyTx(0, /* withSqlIn */ true);
+}
+
+void KqpForceNewEngine::Level0_NotInteractiveWriteOnly() {
+ TestNotInteractiveWriteOnlyTx(0);
+}
+
+void KqpForceNewEngine::Level0_NotInteractiveReadWrite() {
+ auto session = Session();
+
+ auto test = [&](ui32 count) {
+ KqpCounters->NewEngineForcedQueryCount->Set(0);
+ KqpCounters->NewEngineCompatibleQueryCount->Set(0);
+
+ for (ui32 i = 0; i < count; ++i) {
+ auto result = session.ExecuteDataQuery(Sprintf(R"(
+ UPDATE `/Root/TwoShard` SET Value2 = %d WHERE Key = 1
+ )", i), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->ForceNewEngineCompileErrors->Val());
+ };
+
+ {
+ test(2);
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+
+ {
+ ForceNewEngine(100, 0);
+ test(2);
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+}
+
+void KqpForceNewEngine::Level0_InteractiveReadOnly() {
+ auto session = Session();
+
+ auto test = [&](ui32 count) {
+ KqpCounters->NewEngineForcedQueryCount->Set(0);
+ KqpCounters->NewEngineCompatibleQueryCount->Set(0);
+
+ for (ui32 i = 0; i < count; ++i) {
+ auto result = session.ExecuteDataQuery(R"(
+ SELECT * FROM `/Root/TwoShard` WHERE Key = 1
+ )", TTxControl::BeginTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ CompareYson(R"([[[1u];["One"];[-1]]])", FormatResultSetYson(result.GetResultSet(0)));
+
+ auto tx = *result.GetTransaction();
+
+ result = session.ExecuteDataQuery(R"(
+ SELECT * FROM `/Root/TwoShard` WHERE Key = 2
+ )", TTxControl::Tx(tx).CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ CompareYson(R"([[[2u];["Two"];[0]]])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->ForceNewEngineCompileErrors->Val());
+ };
+
+ {
+ test(2);
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+
+ {
+ ForceNewEngine(100, 0);
+ test(2);
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+}
+
+void KqpForceNewEngine::Level0_CompilationFailure() {
+ TestNotInteractiveReadOnlyTxFailedNECompilation(0);
+}
+
+void KqpForceNewEngine::Level0_Fallback() {
+ TestNotInteractiveReadOnlyTxFallback(0);
+}
+
+/////////// LEVEL 1 ////////////////////////////////////////////////////////////////////////////////////////////////////
+void KqpForceNewEngine::Level1_NotInteractiveReadOnly() {
+ TestNotInteractiveReadOnlyTx(1);
+}
+
+void KqpForceNewEngine::Level1_NotInteractiveReadOnlySqlIn() {
+ TestNotInteractiveReadOnlyTx(1, /* withSqlIn */ true);
+}
+
+void KqpForceNewEngine::Level1_NotInteractiveWriteOnly() {
+ TestNotInteractiveWriteOnlyTx(1);
+}
+
+void KqpForceNewEngine::Level1_NotInteractiveReadWrite() {
+ auto session = Session();
+
+ auto test = [&](ui32 count) {
+ KqpCounters->NewEngineForcedQueryCount->Set(0);
+ KqpCounters->NewEngineCompatibleQueryCount->Set(0);
+
+ for (ui32 i = 0; i < count; ++i) {
+ auto result = session.ExecuteDataQuery(Sprintf(R"(
+ UPDATE `/Root/TwoShard` SET Value2 = %d WHERE Key = 1
+ )", i), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->ForceNewEngineCompileErrors->Val());
+ };
+
+ {
+ test(2);
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+
+ {
+ ForceNewEngine(100, 1);
+ test(2);
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+}
+
+void KqpForceNewEngine::Level1_InteractiveReadOnly() {
+ auto session = Session();
+
+ auto test = [&](ui32 count) {
+ KqpCounters->NewEngineForcedQueryCount->Set(0);
+ KqpCounters->NewEngineCompatibleQueryCount->Set(0);
+
+ for (ui32 i = 0; i < count; ++i) {
+ auto result = session.ExecuteDataQuery(R"(
+ SELECT * FROM `/Root/TwoShard` WHERE Key = 1
+ )", TTxControl::BeginTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ CompareYson(R"([[[1u];["One"];[-1]]])", FormatResultSetYson(result.GetResultSet(0)));
+
+ auto tx = *result.GetTransaction();
+
+ result = session.ExecuteDataQuery(R"(
+ SELECT * FROM `/Root/TwoShard` WHERE Key = 2
+ )", TTxControl::Tx(tx).CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ CompareYson(R"([[[2u];["Two"];[0]]])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->ForceNewEngineCompileErrors->Val());
+ };
+
+ {
+ test(2);
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+
+ {
+ ForceNewEngine(100, 1);
+ test(2);
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+}
+
+void KqpForceNewEngine::Level1_CompilationFailure() {
+ TestNotInteractiveReadOnlyTxFailedNECompilation(1);
+}
+
+void KqpForceNewEngine::Level1_Fallback() {
+ TestNotInteractiveReadOnlyTxFallback(1);
+}
+
+/////////// LEVEL 2 ////////////////////////////////////////////////////////////////////////////////////////////////////
+void KqpForceNewEngine::Level2_NotInteractiveReadOnly() {
+ TestNotInteractiveReadOnlyTx(2);
+}
+
+void KqpForceNewEngine::Level2_NotInteractiveWriteOnly() {
+ TestNotInteractiveWriteOnlyTx(2);
+}
+
+void KqpForceNewEngine::Level2_InteractiveReadOnly() {
+ auto session = Session();
+
+ auto test = [&](ui32 count) {
+ KqpCounters->NewEngineForcedQueryCount->Set(0);
+ KqpCounters->NewEngineCompatibleQueryCount->Set(0);
+
+ for (ui32 i = 0; i < count; ++i) {
+ auto result = session.ExecuteDataQuery(R"(
+ SELECT * FROM `/Root/TwoShard` WHERE Key = 1
+ )", TTxControl::BeginTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ CompareYson(R"([[[1u];["One"];[-1]]])", FormatResultSetYson(result.GetResultSet(0)));
+
+ auto tx = *result.GetTransaction();
+
+ result = session.ExecuteDataQuery(R"(
+ SELECT * FROM `/Root/TwoShard` WHERE Key = 2
+ )", TTxControl::Tx(tx).CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ CompareYson(R"([[[2u];["Two"];[0]]])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->ForceNewEngineCompileErrors->Val());
+ };
+
+ {
+ test(2);
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+
+ {
+ ForceNewEngine(50, 2);
+ test(20);
+ bool diff = KqpCounters->NewEngineForcedQueryCount->Val() * KqpCounters->NewEngineCompatibleQueryCount->Val() != 0;
+ UNIT_ASSERT_C(diff, "forced: " << KqpCounters->NewEngineForcedQueryCount->Val()
+ << ", compatible: " << KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+
+ {
+ ForceNewEngine(100, 2);
+ test(2);
+ UNIT_ASSERT_VALUES_EQUAL(4, KqpCounters->NewEngineForcedQueryCount->Val());
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+
+ {
+ ForceNewEngine(0, 2);
+ test(2);
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+}
+
+void KqpForceNewEngine::Level2_InteractiveReadWrite() {
+ TestInteractiveReadWriteEx(/* level */ 2, /* addBeginTx */ false, /* appendRead */ false, /* addCommitTx */ false);
+}
+
+void KqpForceNewEngine::Level2_InteractiveBeginReadWrite() {
+ TestInteractiveReadWriteEx(/* level */ 2, /* addBeginTx */ true, /* appendRead */ false, /* addCommitTx */ false);
+}
+
+void KqpForceNewEngine::Level2_InteractiveReadWriteCommit() {
+ TestInteractiveReadWriteEx(/* level */ 2, /* addBeginTx */ false, /* appendRead */ false, /* addCommitTx */ true);
+}
+
+void KqpForceNewEngine::Level2_InteractiveBeginReadWriteCommit() {
+ TestInteractiveReadWriteEx(/* level */ 2, /* addBeginTx */ true, /* appendRead */ false, /* addCommitTx */ true);
+}
+
+void KqpForceNewEngine::Level2_InteractiveReadWriteRead() {
+ TestInteractiveReadWriteEx(/* level */ 2, /* addBeginTx */ false, /* appendRead */ true, /* addCommitTx */ false);
+}
+
+void KqpForceNewEngine::Level2_InteractiveBeginReadWriteRead() {
+ TestInteractiveReadWriteEx(/* level */ 2, /* addBeginTx */ true, /* appendRead */ true, /* addCommitTx */ false);
+}
+
+void KqpForceNewEngine::Level2_InteractiveReadWriteReadCommit() {
+ TestInteractiveReadWriteEx(/* level */ 2, /* addBeginTx */ false, /* appendRead */ true, /* addCommitTx */ true);
+}
+
+void KqpForceNewEngine::Level2_InteractiveBeginReadWriteReadCommit() {
+ TestInteractiveReadWriteEx(/* level */ 2, /* addBeginTx */ true, /* appendRead */ true, /* addCommitTx */ true);
+}
+
+void KqpForceNewEngine::Level2_InteractiveWriteOnly() {
+ auto session = Session();
+
+ auto test = [&](ui32 count) {
+ KqpCounters->NewEngineForcedQueryCount->Set(0);
+ KqpCounters->NewEngineCompatibleQueryCount->Set(0);
+
+ for (ui32 i = 0; i < count; ++i) {
+ auto result = session.ExecuteDataQuery(R"(
+ REPLACE INTO `/Root/TwoShard` (Key, Value1) VALUES (1, "OneOne")
+ )", TTxControl::BeginTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ result = session.ExecuteDataQuery(R"(
+ REPLACE INTO `/Root/KeyValue` (Key, Value) VALUES (1, "OneOne")
+ )", TTxControl::Tx(*result.GetTransaction()).CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->ForceNewEngineCompileErrors->Val());
+ };
+
+ {
+ test(2);
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+
+ {
+ ForceNewEngine(100, 2);
+ test(2);
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+}
+
+void KqpForceNewEngine::Level2_CompilationFailure() {
+ TestNotInteractiveReadOnlyTxFailedNECompilation(2);
+}
+
+void KqpForceNewEngine::Level2_NoFallback() {
+ TestNotInteractiveReadOnlyTxFallback(2);
+}
+
+/////////// LEVEL 3 ////////////////////////////////////////////////////////////////////////////////////////////////////
+void KqpForceNewEngine::Level3_NotInteractiveReadOnly() {
+ TestNotInteractiveReadOnlyTx(3);
+}
+
+void KqpForceNewEngine::Level3_NotInteractiveWriteOnly() {
+ TestNotInteractiveWriteOnlyTx(3);
+}
+
+void KqpForceNewEngine::Level3_InteractiveReadOnly() {
+ auto session = Session();
+
+ auto test = [&](ui32 count) {
+ KqpCounters->NewEngineForcedQueryCount->Set(0);
+ KqpCounters->NewEngineCompatibleQueryCount->Set(0);
+
+ for (ui32 i = 0; i < count; ++i) {
+ auto result = session.ExecuteDataQuery(R"(
+ SELECT * FROM `/Root/TwoShard` WHERE Key = 1
+ )", TTxControl::BeginTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ CompareYson(R"([[[1u];["One"];[-1]]])", FormatResultSetYson(result.GetResultSet(0)));
+
+ auto tx = *result.GetTransaction();
+
+ result = session.ExecuteDataQuery(R"(
+ SELECT * FROM `/Root/TwoShard` WHERE Key = 2
+ )", TTxControl::Tx(tx).CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ CompareYson(R"([[[2u];["Two"];[0]]])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->ForceNewEngineCompileErrors->Val());
+ };
+
+ {
+ test(2);
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+
+ {
+ ForceNewEngine(50, 3);
+ test(20);
+ bool diff = KqpCounters->NewEngineForcedQueryCount->Val() * KqpCounters->NewEngineCompatibleQueryCount->Val() != 0;
+ UNIT_ASSERT_C(diff, "forced: " << KqpCounters->NewEngineForcedQueryCount->Val()
+ << ", compatible: " << KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+
+ {
+ ForceNewEngine(100, 3);
+ test(2);
+ UNIT_ASSERT_VALUES_EQUAL(4, KqpCounters->NewEngineForcedQueryCount->Val());
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+
+ {
+ ForceNewEngine(0, 3);
+ test(2);
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+}
+
+void KqpForceNewEngine::Level3_InteractiveReadWrite() {
+ auto session = Session();
+
+ auto test = [&](ui32 count) {
+ KqpCounters->NewEngineForcedQueryCount->Set(0);
+ KqpCounters->NewEngineCompatibleQueryCount->Set(0);
+
+ for (ui32 i = 0; i < count; ++i) {
+ auto result = session.ExecuteDataQuery(R"(
+ SELECT * FROM `/Root/KeyValue` LIMIT 1
+ )", TTxControl::BeginTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ result = session.ExecuteDataQuery(Sprintf(R"(
+ UPDATE `/Root/TwoShard` SET Value2 = %d WHERE Key = 1
+ )", i), TTxControl::Tx(*result.GetTransaction()).CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->ForceNewEngineCompileErrors->Val());
+ };
+
+ {
+ test(2);
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+
+ {
+ ForceNewEngine(50, 3);
+ test(20);
+ bool diff = KqpCounters->NewEngineForcedQueryCount->Val() * KqpCounters->NewEngineCompatibleQueryCount->Val() != 0;
+ UNIT_ASSERT_C(diff, "forced: " << KqpCounters->NewEngineForcedQueryCount->Val()
+ << ", compatible: " << KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+
+ {
+ ForceNewEngine(100, 3);
+ test(2);
+ UNIT_ASSERT_VALUES_EQUAL(4, KqpCounters->NewEngineForcedQueryCount->Val());
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+
+ {
+ ForceNewEngine(0, 3);
+ test(2);
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineForcedQueryCount->Val());
+ UNIT_ASSERT_VALUES_EQUAL(0, KqpCounters->NewEngineCompatibleQueryCount->Val());
+ }
+}
+
+void KqpForceNewEngine::Level3_CompilationFailure() {
+ TestNotInteractiveReadOnlyTxFailedNECompilation(3);
+}
+
+void KqpForceNewEngine::Level3_NoFallback() {
+ TestNotInteractiveReadOnlyTxFallback(3);
+}
+
+} // namespace NKqp
+} // namespace NKikimr
diff --git a/ydb/core/kqp/ut/fat/ya.make b/ydb/core/kqp/ut/fat/ya.make
index 3e1c5df6ce..3c6099c675 100644
--- a/ydb/core/kqp/ut/fat/ya.make
+++ b/ydb/core/kqp/ut/fat/ya.make
@@ -1,29 +1,29 @@
-UNITTEST_FOR(ydb/core/kqp)
-
-OWNER(
- g:kikimr
-)
-
-FORK_SUBTESTS()
-
-TIMEOUT(2400)
-TAG(ya:fat)
-SIZE(LARGE)
-SPLIT_FACTOR(5)
-
-SRCS(
- kqp_force_newengine_ut.cpp
-)
-
-PEERDIR(
- ydb/public/sdk/cpp/client/ydb_proto
- ydb/core/kqp
- ydb/core/kqp/counters
- ydb/core/kqp/host
- ydb/core/kqp/provider
- ydb/core/kqp/ut/common
-)
-
-YQL_LAST_ABI_VERSION()
-
-END()
+UNITTEST_FOR(ydb/core/kqp)
+
+OWNER(
+ g:kikimr
+)
+
+FORK_SUBTESTS()
+
+TIMEOUT(2400)
+TAG(ya:fat)
+SIZE(LARGE)
+SPLIT_FACTOR(5)
+
+SRCS(
+ kqp_force_newengine_ut.cpp
+)
+
+PEERDIR(
+ ydb/public/sdk/cpp/client/ydb_proto
+ ydb/core/kqp
+ ydb/core/kqp/counters
+ ydb/core/kqp/host
+ ydb/core/kqp/provider
+ ydb/core/kqp/ut/common
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/ydb/core/kqp/ut/kqp_acl_ut.cpp b/ydb/core/kqp/ut/kqp_acl_ut.cpp
index e4757b5de5..203c8ba618 100644
--- a/ydb/core/kqp/ut/kqp_acl_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_acl_ut.cpp
@@ -18,7 +18,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngineAcl) {
auto result = session.ExecuteDataQuery(R"(
PRAGMA kikimr.UseNewEngine = "true";
- SELECT * FROM `/Root/TwoShard`;
+ SELECT * FROM `/Root/TwoShard`;
)", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR);
}
@@ -47,17 +47,17 @@ Y_UNIT_TEST_SUITE(KqpNewEngineAcl) {
{
auto result = session.ExecuteDataQuery(R"(
PRAGMA kikimr.UseNewEngine = "true";
- SELECT * FROM `/Root/TwoShard`;
+ SELECT * FROM `/Root/TwoShard`;
)", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR);
}
{
auto result = session.ExecuteDataQuery(R"(
PRAGMA kikimr.UseNewEngine = "true";
- UPSERT INTO `/Root/TwoShard` (Key, Value1, Value2) VALUES
+ UPSERT INTO `/Root/TwoShard` (Key, Value1, Value2) VALUES
(10u, "One", -10);
)", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR);
}
driver.Stop(true);
@@ -86,7 +86,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngineAcl) {
auto result = session.ExecuteDataQuery(R"(
PRAGMA kikimr.UseNewEngine = "true";
- SELECT * FROM `/Root/TwoShard`;
+ SELECT * FROM `/Root/TwoShard`;
)", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
AssertSuccessResult(result);
driver.Stop(true);
@@ -115,7 +115,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngineAcl) {
auto result = session.ExecuteDataQuery(R"(
PRAGMA kikimr.UseNewEngine = "true";
- UPSERT INTO `/Root/TwoShard` (Key, Value1, Value2) VALUES
+ UPSERT INTO `/Root/TwoShard` (Key, Value1, Value2) VALUES
(10u, "One", -10);
)", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
AssertSuccessResult(result);
@@ -145,8 +145,8 @@ Y_UNIT_TEST_SUITE(KqpNewEngineAcl) {
auto session = db.CreateSession().GetValueSync().GetSession();
const char* queryTmpl = R"(
- PRAGMA kikimr.UseNewEngine = "true";
- CREATE TABLE `/Root/PQ/%s` (
+ PRAGMA kikimr.UseNewEngine = "true";
+ CREATE TABLE `/Root/PQ/%s` (
id Int64,
name String,
primary key (id)
@@ -160,5 +160,5 @@ Y_UNIT_TEST_SUITE(KqpNewEngineAcl) {
}
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/ut/kqp_explain_ut.cpp b/ydb/core/kqp/ut/kqp_explain_ut.cpp
index 9617924008..c3a0683636 100644
--- a/ydb/core/kqp/ut/kqp_explain_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_explain_ut.cpp
@@ -461,16 +461,16 @@ Y_UNIT_TEST_SUITE(KqpExplain) {
UPDATE `/Root/EightShard` SET Data=Data-1 WHERE Key In (100,200,300);
DELETE FROM `/Root/EightShard` WHERE Key > 350;
)").ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
NJson::TJsonValue plan;
NJson::ReadJsonTree(result.GetPlan(), &plan, true);
-
- // Cerr << plan << Endl;
-
- auto upsertsCount = CountPlanNodesByKv(plan, "Node Type", "Upsert-ConstantExpr");
- UNIT_ASSERT_VALUES_EQUAL(upsertsCount, 2);
-
+
+ // Cerr << plan << Endl;
+
+ auto upsertsCount = CountPlanNodesByKv(plan, "Node Type", "Upsert-ConstantExpr");
+ UNIT_ASSERT_VALUES_EQUAL(upsertsCount, 2);
+
auto deletesCount = CountPlanNodesByKv(plan, "Node Type", "Delete-ConstantExpr");
UNIT_ASSERT_VALUES_EQUAL(deletesCount, 1);
diff --git a/ydb/core/kqp/ut/kqp_flip_join_ut.cpp b/ydb/core/kqp/ut/kqp_flip_join_ut.cpp
index ab72785898..593e1a1876 100644
--- a/ydb/core/kqp/ut/kqp_flip_join_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_flip_join_ut.cpp
@@ -1,57 +1,57 @@
#include <ydb/core/kqp/ut/common/kqp_ut_common.h>
-
-namespace NKikimr::NKqp {
-
-using namespace NYdb;
-using namespace NYdb::NTable;
-
-static void CreateSampleTables(TSession session) {
- UNIT_ASSERT(session.ExecuteSchemeQuery(R"(
+
+namespace NKikimr::NKqp {
+
+using namespace NYdb;
+using namespace NYdb::NTable;
+
+static void CreateSampleTables(TSession session) {
+ UNIT_ASSERT(session.ExecuteSchemeQuery(R"(
CREATE TABLE [/Root/FJ_Table_1] (
- Key Int32, Fk2 Int32, Fk3 Int32, Value String,
- PRIMARY KEY (Key)
- );
+ Key Int32, Fk2 Int32, Fk3 Int32, Value String,
+ PRIMARY KEY (Key)
+ );
CREATE TABLE [/Root/FJ_Table_2] (
- Key Int32, Fk3 Int32, Fk1 Int32, Value String,
- PRIMARY KEY (Key)
- );
+ Key Int32, Fk3 Int32, Fk1 Int32, Value String,
+ PRIMARY KEY (Key)
+ );
CREATE TABLE [/Root/FJ_Table_3] (
- Key Int32, Fk1 Int32, Fk2 Int32, Value String,
- PRIMARY KEY (Key)
- );
+ Key Int32, Fk1 Int32, Fk2 Int32, Value String,
+ PRIMARY KEY (Key)
+ );
CREATE TABLE [/Root/FJ_Table_4] (
- Key Int32, Value String,
- PRIMARY KEY (Key)
- );
- )").GetValueSync().IsSuccess());
-
- UNIT_ASSERT(session.ExecuteDataQuery(R"(
+ Key Int32, Value String,
+ PRIMARY KEY (Key)
+ );
+ )").GetValueSync().IsSuccess());
+
+ UNIT_ASSERT(session.ExecuteDataQuery(R"(
REPLACE INTO [/Root/FJ_Table_1] (Key, Fk2, Fk3, Value) VALUES
- (1, 101, 1001, "Value11"),
- (2, 102, 1002, "Value12"),
- (3, 103, 1003, "Value13"),
- (4, 104, 1004, "Value14");
+ (1, 101, 1001, "Value11"),
+ (2, 102, 1002, "Value12"),
+ (3, 103, 1003, "Value13"),
+ (4, 104, 1004, "Value14");
REPLACE INTO [/Root/FJ_Table_2] (Key, Fk3, Fk1, Value) VALUES
- (101, 1001, 1, "Value21"),
- (102, 1002, 2, "Value22");
+ (101, 1001, 1, "Value21"),
+ (102, 1002, 2, "Value22");
REPLACE INTO [/Root/FJ_Table_3] (Key, Fk1, Fk2, Value) VALUES
- (1001, 1, 101, "Value31"),
- (1002, 2, 102, "Value32"),
- (1003, 3, 103, "Value33"),
- (1005, 5, 105, "Value35");
+ (1001, 1, 101, "Value31"),
+ (1002, 2, 102, "Value32"),
+ (1003, 3, 103, "Value33"),
+ (1005, 5, 105, "Value35");
REPLACE INTO [/Root/FJ_Table_4] (Key, Value) VALUES
- (1, "Value4_1"),
- (101, "Value4_101"),
- (1001, "Value4_1001");
- )", TTxControl::BeginTx().CommitTx()).GetValueSync().IsSuccess());
-}
-
-static TParams NoParams = TParamsBuilder().Build();
-
-static const bool EnableJoinFlip = false;
-static const bool DisableJoinFlip = true;
-
-static const char* FormatPragma(bool disableFlip) {
+ (1, "Value4_1"),
+ (101, "Value4_101"),
+ (1001, "Value4_1001");
+ )", TTxControl::BeginTx().CommitTx()).GetValueSync().IsSuccess());
+}
+
+static TParams NoParams = TParamsBuilder().Build();
+
+static const bool EnableJoinFlip = false;
+static const bool DisableJoinFlip = true;
+
+static const char* FormatPragma(bool disableFlip) {
if (!disableFlip) {
return "PRAGMA Kikimr.OptDisableJoinReverseTableLookup = 'False';";
}
@@ -59,565 +59,565 @@ static const char* FormatPragma(bool disableFlip) {
}
static const char* FormatLeftSemiPragma(bool disableFlip) {
- if (disableFlip) {
+ if (disableFlip) {
return "PRAGMA Kikimr.OptDisableJoinReverseTableLookupLeftSemi = 'True';";
- }
- return "";
-}
-
-Y_UNIT_TEST_SUITE(KqpFlipJoin) {
-
- // simple inner join, only 2 tables
- Y_UNIT_TEST(Inner_1) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- CreateSampleTables(session);
-
- auto test = [&](bool disableFlip, std::function<void(const TDataQueryResult&)> assertFn) {
- // join on key-column of left table and non-key column of right one
- const TString query = Sprintf(R"(
- PRAGMA Kikimr.UseNewEngine = 'false';
- %s
- SELECT t1.Value, t2.Value
+ }
+ return "";
+}
+
+Y_UNIT_TEST_SUITE(KqpFlipJoin) {
+
+ // simple inner join, only 2 tables
+ Y_UNIT_TEST(Inner_1) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ CreateSampleTables(session);
+
+ auto test = [&](bool disableFlip, std::function<void(const TDataQueryResult&)> assertFn) {
+ // join on key-column of left table and non-key column of right one
+ const TString query = Sprintf(R"(
+ PRAGMA Kikimr.UseNewEngine = 'false';
+ %s
+ SELECT t1.Value, t2.Value
FROM [/Root/FJ_Table_1] AS t1
INNER JOIN [/Root/FJ_Table_2] AS t2 ON t1.Key = t2.Fk1
- ORDER BY t1.Value, t2.Value
- )", FormatPragma(disableFlip));
-
- auto result = ExecQueryAndTestResult(session, query, NoParams,
- R"([[["Value11"];["Value21"]];[["Value12"];["Value22"]]])");
- assertFn(result);
- };
-
- test(DisableJoinFlip, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/FJ_Table_1", 4);
- AssertTableReads(result, "/Root/FJ_Table_2", 2);
- });
-
- test(EnableJoinFlip, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/FJ_Table_1", 2);
- AssertTableReads(result, "/Root/FJ_Table_2", 2);
- });
- }
-
- // hierarchy of joins, flip on the last layer
- Y_UNIT_TEST_NEW_ENGINE(Inner_2) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- CreateSampleTables(session);
-
- auto test = [&](bool disableFlip, std::function<void(const TDataQueryResult&)> assertFn) {
- const TString query = Sprintf(R"(
- %s
- SELECT t1.Value, t2.Value, t3.Value
+ ORDER BY t1.Value, t2.Value
+ )", FormatPragma(disableFlip));
+
+ auto result = ExecQueryAndTestResult(session, query, NoParams,
+ R"([[["Value11"];["Value21"]];[["Value12"];["Value22"]]])");
+ assertFn(result);
+ };
+
+ test(DisableJoinFlip, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/FJ_Table_1", 4);
+ AssertTableReads(result, "/Root/FJ_Table_2", 2);
+ });
+
+ test(EnableJoinFlip, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/FJ_Table_1", 2);
+ AssertTableReads(result, "/Root/FJ_Table_2", 2);
+ });
+ }
+
+ // hierarchy of joins, flip on the last layer
+ Y_UNIT_TEST_NEW_ENGINE(Inner_2) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ CreateSampleTables(session);
+
+ auto test = [&](bool disableFlip, std::function<void(const TDataQueryResult&)> assertFn) {
+ const TString query = Sprintf(R"(
+ %s
+ SELECT t1.Value, t2.Value, t3.Value
FROM `/Root/FJ_Table_3` AS t1
INNER JOIN `/Root/FJ_Table_2` AS t2 ON t1.Key = t2.Fk3
INNER JOIN `/Root/FJ_Table_4` AS t3 ON t2.Key = t3.Key
- ORDER BY t1.Value, t2.Value, t3.Value
- )", FormatPragma(disableFlip));
-
- auto result = ExecQueryAndTestResult(session, Q_(query), NoParams,
- R"([[["Value31"];["Value21"];["Value4_101"]]])");
- assertFn(result);
- };
-
- test(DisableJoinFlip, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/FJ_Table_2", 2);
- AssertTableReads(result, "/Root/FJ_Table_3", 4);
- AssertTableReads(result, "/Root/FJ_Table_4", 1);
- });
-
- test(EnableJoinFlip, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/FJ_Table_2", 2);
- AssertTableReads(result, "/Root/FJ_Table_3", UseNewEngine ? 4 : 2);
- AssertTableReads(result, "/Root/FJ_Table_4", 1);
- });
- }
-
- // hierarchy of joins, flip on the top layer
- Y_UNIT_TEST_NEW_ENGINE(Inner_3) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- CreateSampleTables(session);
-
- auto test = [&](bool disableFlip, std::function<void(const TDataQueryResult&)> assertFn) {
- const TString query = Q_(Sprintf(R"(
- %s
- $join = (
- SELECT t1.Value AS Value1, t2.Value AS Value2, t1.Fk3 AS Fk
+ ORDER BY t1.Value, t2.Value, t3.Value
+ )", FormatPragma(disableFlip));
+
+ auto result = ExecQueryAndTestResult(session, Q_(query), NoParams,
+ R"([[["Value31"];["Value21"];["Value4_101"]]])");
+ assertFn(result);
+ };
+
+ test(DisableJoinFlip, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/FJ_Table_2", 2);
+ AssertTableReads(result, "/Root/FJ_Table_3", 4);
+ AssertTableReads(result, "/Root/FJ_Table_4", 1);
+ });
+
+ test(EnableJoinFlip, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/FJ_Table_2", 2);
+ AssertTableReads(result, "/Root/FJ_Table_3", UseNewEngine ? 4 : 2);
+ AssertTableReads(result, "/Root/FJ_Table_4", 1);
+ });
+ }
+
+ // hierarchy of joins, flip on the top layer
+ Y_UNIT_TEST_NEW_ENGINE(Inner_3) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ CreateSampleTables(session);
+
+ auto test = [&](bool disableFlip, std::function<void(const TDataQueryResult&)> assertFn) {
+ const TString query = Q_(Sprintf(R"(
+ %s
+ $join = (
+ SELECT t1.Value AS Value1, t2.Value AS Value2, t1.Fk3 AS Fk
FROM [/Root/FJ_Table_1] AS t1
INNER JOIN [/Root/FJ_Table_2] AS t2 ON t1.Fk2 = t2.Key
- );
- SELECT t.Value1, t.Value2, t3.Value
+ );
+ SELECT t.Value1, t.Value2, t3.Value
FROM [/Root/FJ_Table_3] AS t3
- INNER JOIN $join AS t ON t3.Key = t.Fk
- ORDER BY t.Value1, t.Value2, t3.Value
- )", FormatPragma(disableFlip)));
-
- auto result = ExecQueryAndTestResult(session, query, NoParams,
- R"([[["Value11"];["Value21"];["Value31"]];[["Value12"];["Value22"];["Value32"]]])");
- assertFn(result);
- };
-
- test(DisableJoinFlip, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/FJ_Table_1", 4);
- AssertTableReads(result, "/Root/FJ_Table_2", 2);
- AssertTableReads(result, "/Root/FJ_Table_3", 4);
- });
-
- test(EnableJoinFlip, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/FJ_Table_1", 4);
- AssertTableReads(result, "/Root/FJ_Table_2", 2);
- AssertTableReads(result, "/Root/FJ_Table_3", UseNewEngine ? 4 : 2);
- });
- }
-
- // simple left semi join, only 2 tables
- Y_UNIT_TEST_NEW_ENGINE(LeftSemi_1) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- CreateSampleTables(session);
-
- auto test = [&](bool disableFlip, std::function<void(const TDataQueryResult&)> assertFn) {
- const TString query = Q_(Sprintf(R"(
- %s
- SELECT t1.Value
+ INNER JOIN $join AS t ON t3.Key = t.Fk
+ ORDER BY t.Value1, t.Value2, t3.Value
+ )", FormatPragma(disableFlip)));
+
+ auto result = ExecQueryAndTestResult(session, query, NoParams,
+ R"([[["Value11"];["Value21"];["Value31"]];[["Value12"];["Value22"];["Value32"]]])");
+ assertFn(result);
+ };
+
+ test(DisableJoinFlip, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/FJ_Table_1", 4);
+ AssertTableReads(result, "/Root/FJ_Table_2", 2);
+ AssertTableReads(result, "/Root/FJ_Table_3", 4);
+ });
+
+ test(EnableJoinFlip, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/FJ_Table_1", 4);
+ AssertTableReads(result, "/Root/FJ_Table_2", 2);
+ AssertTableReads(result, "/Root/FJ_Table_3", UseNewEngine ? 4 : 2);
+ });
+ }
+
+ // simple left semi join, only 2 tables
+ Y_UNIT_TEST_NEW_ENGINE(LeftSemi_1) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ CreateSampleTables(session);
+
+ auto test = [&](bool disableFlip, std::function<void(const TDataQueryResult&)> assertFn) {
+ const TString query = Q_(Sprintf(R"(
+ %s
+ SELECT t1.Value
FROM [/Root/FJ_Table_1] AS t1
LEFT SEMI JOIN [/Root/FJ_Table_2] AS t2 ON t1.Key = t2.Fk1
- ORDER BY t1.Value
- )", FormatLeftSemiPragma(disableFlip)));
-
- auto result = ExecQueryAndTestResult(session, query, NoParams, R"([[["Value11"]];[["Value12"]]])");
- assertFn(result);
- };
-
- test(DisableJoinFlip, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/FJ_Table_1", 4);
- AssertTableReads(result, "/Root/FJ_Table_2", 2);
- });
-
- test(EnableJoinFlip, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/FJ_Table_1", 2);
- AssertTableReads(result, "/Root/FJ_Table_2", 2);
- });
- }
-
- // hierarchy of joins, flip on the last layer
- Y_UNIT_TEST_NEW_ENGINE(LeftSemi_2) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- CreateSampleTables(session);
-
- auto test = [&](bool disableFlip, std::function<void(const TDataQueryResult&)> assertFn) {
- const TString query = Q_(Sprintf(R"(
- %s
- SELECT t1.Key, t1.Value
+ ORDER BY t1.Value
+ )", FormatLeftSemiPragma(disableFlip)));
+
+ auto result = ExecQueryAndTestResult(session, query, NoParams, R"([[["Value11"]];[["Value12"]]])");
+ assertFn(result);
+ };
+
+ test(DisableJoinFlip, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/FJ_Table_1", 4);
+ AssertTableReads(result, "/Root/FJ_Table_2", 2);
+ });
+
+ test(EnableJoinFlip, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/FJ_Table_1", 2);
+ AssertTableReads(result, "/Root/FJ_Table_2", 2);
+ });
+ }
+
+ // hierarchy of joins, flip on the last layer
+ Y_UNIT_TEST_NEW_ENGINE(LeftSemi_2) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ CreateSampleTables(session);
+
+ auto test = [&](bool disableFlip, std::function<void(const TDataQueryResult&)> assertFn) {
+ const TString query = Q_(Sprintf(R"(
+ %s
+ SELECT t1.Key, t1.Value
FROM [/Root/FJ_Table_1] AS t1
LEFT SEMI JOIN [/Root/FJ_Table_2] AS t2 ON t1.Key = t2.Fk1
LEFT SEMI JOIN [/Root/FJ_Table_3] AS t3 ON t1.Key = t3.Fk1
- ORDER BY t1.Key, t1.Value
- )", FormatLeftSemiPragma(disableFlip)));
-
- auto result = ExecQueryAndTestResult(session, query, NoParams, R"([[[1];["Value11"]];[[2];["Value12"]]])");
- assertFn(result);
- };
-
- test(DisableJoinFlip, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/FJ_Table_1", 4);
- AssertTableReads(result, "/Root/FJ_Table_2", 2);
- AssertTableReads(result, "/Root/FJ_Table_3", 4);
- });
-
- test(EnableJoinFlip, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/FJ_Table_1", 2);
- AssertTableReads(result, "/Root/FJ_Table_2", 2);
- AssertTableReads(result, "/Root/FJ_Table_3", 4);
- });
- }
-
- // hierarchy of joins, flip on the top layer
- Y_UNIT_TEST_NEW_ENGINE(LeftSemi_3) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- CreateSampleTables(session);
-
- auto test = [&](bool disableFlip, std::function<void(const TDataQueryResult&)> assertFn) {
- const TString query = Q_(Sprintf(R"(
- %s
- $join = (
- SELECT t1.Value AS Value1, t2.Value AS Value2, t1.Fk3 AS Fk
+ ORDER BY t1.Key, t1.Value
+ )", FormatLeftSemiPragma(disableFlip)));
+
+ auto result = ExecQueryAndTestResult(session, query, NoParams, R"([[[1];["Value11"]];[[2];["Value12"]]])");
+ assertFn(result);
+ };
+
+ test(DisableJoinFlip, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/FJ_Table_1", 4);
+ AssertTableReads(result, "/Root/FJ_Table_2", 2);
+ AssertTableReads(result, "/Root/FJ_Table_3", 4);
+ });
+
+ test(EnableJoinFlip, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/FJ_Table_1", 2);
+ AssertTableReads(result, "/Root/FJ_Table_2", 2);
+ AssertTableReads(result, "/Root/FJ_Table_3", 4);
+ });
+ }
+
+ // hierarchy of joins, flip on the top layer
+ Y_UNIT_TEST_NEW_ENGINE(LeftSemi_3) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ CreateSampleTables(session);
+
+ auto test = [&](bool disableFlip, std::function<void(const TDataQueryResult&)> assertFn) {
+ const TString query = Q_(Sprintf(R"(
+ %s
+ $join = (
+ SELECT t1.Value AS Value1, t2.Value AS Value2, t1.Fk3 AS Fk
FROM [/Root/FJ_Table_1] AS t1
INNER JOIN [/Root/FJ_Table_2] AS t2 ON t1.Fk2 = t2.Key
- );
- SELECT t3.Value
+ );
+ SELECT t3.Value
FROM [/Root/FJ_Table_3] AS t3
- LEFT SEMI JOIN $join AS t ON t3.Key = t.Fk
- ORDER BY t3.Value
- )", FormatLeftSemiPragma(disableFlip)));
-
- auto result = ExecQueryAndTestResult(session, query, NoParams,
- R"([[["Value31"]];[["Value32"]]])");
- assertFn(result);
- };
-
- test(DisableJoinFlip, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/FJ_Table_1", 4);
- AssertTableReads(result, "/Root/FJ_Table_2", 2);
- AssertTableReads(result, "/Root/FJ_Table_3", 4);
- });
-
- test(EnableJoinFlip, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/FJ_Table_1", 4);
- AssertTableReads(result, "/Root/FJ_Table_2", 2);
- AssertTableReads(result, "/Root/FJ_Table_3", 2);
- });
- }
-
- // simple right semi join, only 2 tables
- Y_UNIT_TEST_NEW_ENGINE(RightSemi_1) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- CreateSampleTables(session);
-
- auto test = [&](bool disableFlip, std::function<void(const TDataQueryResult&)> assertFn) {
- const TString query = Q_(Sprintf(R"(
- %s
- SELECT t2.Value
+ LEFT SEMI JOIN $join AS t ON t3.Key = t.Fk
+ ORDER BY t3.Value
+ )", FormatLeftSemiPragma(disableFlip)));
+
+ auto result = ExecQueryAndTestResult(session, query, NoParams,
+ R"([[["Value31"]];[["Value32"]]])");
+ assertFn(result);
+ };
+
+ test(DisableJoinFlip, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/FJ_Table_1", 4);
+ AssertTableReads(result, "/Root/FJ_Table_2", 2);
+ AssertTableReads(result, "/Root/FJ_Table_3", 4);
+ });
+
+ test(EnableJoinFlip, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/FJ_Table_1", 4);
+ AssertTableReads(result, "/Root/FJ_Table_2", 2);
+ AssertTableReads(result, "/Root/FJ_Table_3", 2);
+ });
+ }
+
+ // simple right semi join, only 2 tables
+ Y_UNIT_TEST_NEW_ENGINE(RightSemi_1) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ CreateSampleTables(session);
+
+ auto test = [&](bool disableFlip, std::function<void(const TDataQueryResult&)> assertFn) {
+ const TString query = Q_(Sprintf(R"(
+ %s
+ SELECT t2.Value
FROM [/Root/FJ_Table_1] AS t1
RIGHT SEMI JOIN [/Root/FJ_Table_2] AS t2 ON t1.Key = t2.Fk1
- ORDER BY t2.Value
- )", FormatPragma(disableFlip)));
-
- auto result = ExecQueryAndTestResult(session, query, NoParams, R"([[["Value21"]];[["Value22"]]])");
- assertFn(result);
- };
-
- test(DisableJoinFlip, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/FJ_Table_1", 4);
- AssertTableReads(result, "/Root/FJ_Table_2", 2);
- });
-
- test(EnableJoinFlip, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/FJ_Table_1", UseNewEngine ? 4 : 2);
- AssertTableReads(result, "/Root/FJ_Table_2", 2);
- });
- }
-
- // hierarchy of joins, flip on the last layer
- Y_UNIT_TEST_NEW_ENGINE(RightSemi_2) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- CreateSampleTables(session);
-
- auto test = [&](bool disableFlip, std::function<void(const TDataQueryResult&)> assertFn) {
- const TString query = Q_(Sprintf(R"(
- %s
- SELECT t3.Key, t3.Value
+ ORDER BY t2.Value
+ )", FormatPragma(disableFlip)));
+
+ auto result = ExecQueryAndTestResult(session, query, NoParams, R"([[["Value21"]];[["Value22"]]])");
+ assertFn(result);
+ };
+
+ test(DisableJoinFlip, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/FJ_Table_1", 4);
+ AssertTableReads(result, "/Root/FJ_Table_2", 2);
+ });
+
+ test(EnableJoinFlip, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/FJ_Table_1", UseNewEngine ? 4 : 2);
+ AssertTableReads(result, "/Root/FJ_Table_2", 2);
+ });
+ }
+
+ // hierarchy of joins, flip on the last layer
+ Y_UNIT_TEST_NEW_ENGINE(RightSemi_2) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ CreateSampleTables(session);
+
+ auto test = [&](bool disableFlip, std::function<void(const TDataQueryResult&)> assertFn) {
+ const TString query = Q_(Sprintf(R"(
+ %s
+ SELECT t3.Key, t3.Value
FROM [/Root/FJ_Table_1] AS t1
RIGHT SEMI JOIN [/Root/FJ_Table_2] AS t2 ON t1.Key = t2.Fk1
RIGHT SEMI JOIN [/Root/FJ_Table_3] AS t3 ON t2.Key = t3.Fk2
- ORDER BY t3.Key, t3.Value
- )", FormatPragma(disableFlip)));
-
- auto result = ExecQueryAndTestResult(session, query, NoParams,
- R"([[[1001];["Value31"]];[[1002];["Value32"]]])");
- assertFn(result);
- };
-
- test(DisableJoinFlip, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/FJ_Table_1", 4);
- AssertTableReads(result, "/Root/FJ_Table_2", 2);
- AssertTableReads(result, "/Root/FJ_Table_3", 4);
- });
-
- test(EnableJoinFlip, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/FJ_Table_1", UseNewEngine ? 4 : 2);
- AssertTableReads(result, "/Root/FJ_Table_2", 2);
- AssertTableReads(result, "/Root/FJ_Table_3", 4);
- });
- }
-
- // hierarchy of joins, flip on the top layer
- Y_UNIT_TEST_NEW_ENGINE(RightSemi_3) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- CreateSampleTables(session);
-
- auto test = [&](bool disableFlip, std::function<void(const TDataQueryResult&)> assertFn) {
- const TString query = Q_(Sprintf(R"(
- %s
- $join = (
- SELECT t1.Value AS Value1, t2.Value AS Value2, t1.Fk3 AS Fk3
+ ORDER BY t3.Key, t3.Value
+ )", FormatPragma(disableFlip)));
+
+ auto result = ExecQueryAndTestResult(session, query, NoParams,
+ R"([[[1001];["Value31"]];[[1002];["Value32"]]])");
+ assertFn(result);
+ };
+
+ test(DisableJoinFlip, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/FJ_Table_1", 4);
+ AssertTableReads(result, "/Root/FJ_Table_2", 2);
+ AssertTableReads(result, "/Root/FJ_Table_3", 4);
+ });
+
+ test(EnableJoinFlip, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/FJ_Table_1", UseNewEngine ? 4 : 2);
+ AssertTableReads(result, "/Root/FJ_Table_2", 2);
+ AssertTableReads(result, "/Root/FJ_Table_3", 4);
+ });
+ }
+
+ // hierarchy of joins, flip on the top layer
+ Y_UNIT_TEST_NEW_ENGINE(RightSemi_3) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ CreateSampleTables(session);
+
+ auto test = [&](bool disableFlip, std::function<void(const TDataQueryResult&)> assertFn) {
+ const TString query = Q_(Sprintf(R"(
+ %s
+ $join = (
+ SELECT t1.Value AS Value1, t2.Value AS Value2, t1.Fk3 AS Fk3
FROM [/Root/FJ_Table_1] AS t1
INNER JOIN [/Root/FJ_Table_2] AS t2 ON t1.Fk2 = t2.Key
- );
- SELECT t.Value1, t.Value2
+ );
+ SELECT t.Value1, t.Value2
FROM [/Root/FJ_Table_3] AS t3
- RIGHT SEMI JOIN $join AS t ON t3.Key = t.Fk3
- ORDER BY t.Value1, t.Value2
- )", FormatPragma(disableFlip)));
-
- auto result = ExecQueryAndTestResult(session, query, NoParams,
- R"([[["Value11"];["Value21"]];[["Value12"];["Value22"]]])");
- assertFn(result);
- };
-
- test(DisableJoinFlip, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/FJ_Table_1", 4);
- AssertTableReads(result, "/Root/FJ_Table_2", 2);
- AssertTableReads(result, "/Root/FJ_Table_3", 4);
- });
-
- test(EnableJoinFlip, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/FJ_Table_1", 4);
- AssertTableReads(result, "/Root/FJ_Table_2", 2);
- AssertTableReads(result, "/Root/FJ_Table_3", UseNewEngine ? 4 : 2);
- });
- }
-
- // simple right join, only 2 tables
- Y_UNIT_TEST(Right_1) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- CreateSampleTables(session);
-
- auto test = [&](bool disableFlip, std::function<void(const TDataQueryResult&)> assertFn) {
- const TString query = Sprintf(R"(
- PRAGMA Kikimr.UseNewEngine = 'false';
- %s
- SELECT t2.Value
+ RIGHT SEMI JOIN $join AS t ON t3.Key = t.Fk3
+ ORDER BY t.Value1, t.Value2
+ )", FormatPragma(disableFlip)));
+
+ auto result = ExecQueryAndTestResult(session, query, NoParams,
+ R"([[["Value11"];["Value21"]];[["Value12"];["Value22"]]])");
+ assertFn(result);
+ };
+
+ test(DisableJoinFlip, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/FJ_Table_1", 4);
+ AssertTableReads(result, "/Root/FJ_Table_2", 2);
+ AssertTableReads(result, "/Root/FJ_Table_3", 4);
+ });
+
+ test(EnableJoinFlip, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/FJ_Table_1", 4);
+ AssertTableReads(result, "/Root/FJ_Table_2", 2);
+ AssertTableReads(result, "/Root/FJ_Table_3", UseNewEngine ? 4 : 2);
+ });
+ }
+
+ // simple right join, only 2 tables
+ Y_UNIT_TEST(Right_1) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ CreateSampleTables(session);
+
+ auto test = [&](bool disableFlip, std::function<void(const TDataQueryResult&)> assertFn) {
+ const TString query = Sprintf(R"(
+ PRAGMA Kikimr.UseNewEngine = 'false';
+ %s
+ SELECT t2.Value
FROM [/Root/FJ_Table_1] AS t1
RIGHT JOIN [/Root/FJ_Table_2] AS t2 ON t1.Key = t2.Fk1
- ORDER BY t2.Value
- )", FormatPragma(disableFlip));
-
- auto result = ExecQueryAndTestResult(session, query, NoParams, R"([[["Value21"]];[["Value22"]]])");
- assertFn(result);
- };
-
- test(DisableJoinFlip, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/FJ_Table_1", 4);
- AssertTableReads(result, "/Root/FJ_Table_2", 2);
- });
-
- test(EnableJoinFlip, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/FJ_Table_1", 2);
- AssertTableReads(result, "/Root/FJ_Table_2", 2);
- });
- }
-
- // hierarchy of joins, flip on the last layer
- Y_UNIT_TEST(Right_2) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- CreateSampleTables(session);
-
- auto test = [&](bool disableFlip, std::function<void(const TDataQueryResult&)> assertFn) {
- const TString query = Sprintf(R"(
- PRAGMA Kikimr.UseNewEngine = 'false';
- %s
- SELECT t3.Key, t3.Value
+ ORDER BY t2.Value
+ )", FormatPragma(disableFlip));
+
+ auto result = ExecQueryAndTestResult(session, query, NoParams, R"([[["Value21"]];[["Value22"]]])");
+ assertFn(result);
+ };
+
+ test(DisableJoinFlip, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/FJ_Table_1", 4);
+ AssertTableReads(result, "/Root/FJ_Table_2", 2);
+ });
+
+ test(EnableJoinFlip, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/FJ_Table_1", 2);
+ AssertTableReads(result, "/Root/FJ_Table_2", 2);
+ });
+ }
+
+ // hierarchy of joins, flip on the last layer
+ Y_UNIT_TEST(Right_2) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ CreateSampleTables(session);
+
+ auto test = [&](bool disableFlip, std::function<void(const TDataQueryResult&)> assertFn) {
+ const TString query = Sprintf(R"(
+ PRAGMA Kikimr.UseNewEngine = 'false';
+ %s
+ SELECT t3.Key, t3.Value
FROM [/Root/FJ_Table_1] AS t1
RIGHT JOIN [/Root/FJ_Table_2] AS t2 ON t1.Key = t2.Fk1
RIGHT JOIN [/Root/FJ_Table_3] AS t3 ON t2.Key = t3.Fk2
- ORDER BY t3.Key, t3.Value
- )", FormatPragma(disableFlip));
-
- auto result = ExecQueryAndTestResult(session, query, NoParams,
- R"([[[1001];["Value31"]];[[1002];["Value32"]];[[1003];["Value33"]];[[1005];["Value35"]]])");
- assertFn(result);
- };
-
- test(DisableJoinFlip, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/FJ_Table_1", 4);
- AssertTableReads(result, "/Root/FJ_Table_2", 2);
- AssertTableReads(result, "/Root/FJ_Table_3", 4);
- });
-
- test(EnableJoinFlip, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/FJ_Table_1", 2);
- AssertTableReads(result, "/Root/FJ_Table_2", 2);
- AssertTableReads(result, "/Root/FJ_Table_3", 4);
- });
- }
-
- // hierarchy of joins, flip on the top layer
- Y_UNIT_TEST(Right_3) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- CreateSampleTables(session);
-
- auto test = [&](bool disableFlip, std::function<void(const TDataQueryResult&)> assertFn) {
- const TString query = Sprintf(R"(
- PRAGMA Kikimr.UseNewEngine = 'false';
- %s
- $join = (
- SELECT t1.Value AS Value1, t2.Value AS Value2, t1.Fk3 AS Fk3
+ ORDER BY t3.Key, t3.Value
+ )", FormatPragma(disableFlip));
+
+ auto result = ExecQueryAndTestResult(session, query, NoParams,
+ R"([[[1001];["Value31"]];[[1002];["Value32"]];[[1003];["Value33"]];[[1005];["Value35"]]])");
+ assertFn(result);
+ };
+
+ test(DisableJoinFlip, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/FJ_Table_1", 4);
+ AssertTableReads(result, "/Root/FJ_Table_2", 2);
+ AssertTableReads(result, "/Root/FJ_Table_3", 4);
+ });
+
+ test(EnableJoinFlip, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/FJ_Table_1", 2);
+ AssertTableReads(result, "/Root/FJ_Table_2", 2);
+ AssertTableReads(result, "/Root/FJ_Table_3", 4);
+ });
+ }
+
+ // hierarchy of joins, flip on the top layer
+ Y_UNIT_TEST(Right_3) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ CreateSampleTables(session);
+
+ auto test = [&](bool disableFlip, std::function<void(const TDataQueryResult&)> assertFn) {
+ const TString query = Sprintf(R"(
+ PRAGMA Kikimr.UseNewEngine = 'false';
+ %s
+ $join = (
+ SELECT t1.Value AS Value1, t2.Value AS Value2, t1.Fk3 AS Fk3
FROM [/Root/FJ_Table_1] AS t1
INNER JOIN [/Root/FJ_Table_2] AS t2 ON t1.Fk2 = t2.Key
- );
- SELECT t.Value1, t.Value2
+ );
+ SELECT t.Value1, t.Value2
FROM [/Root/FJ_Table_3] AS t3
- RIGHT JOIN $join AS t ON t3.Key = t.Fk3
- ORDER BY t.Value1, t.Value2
- )", FormatPragma(disableFlip));
-
- auto result = ExecQueryAndTestResult(session, query, NoParams,
- R"([[["Value11"];["Value21"]];[["Value12"];["Value22"]]])");
- assertFn(result);
- };
-
- test(DisableJoinFlip, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/FJ_Table_1", 4);
- AssertTableReads(result, "/Root/FJ_Table_2", 2);
- AssertTableReads(result, "/Root/FJ_Table_3", 4);
- });
-
- test(EnableJoinFlip, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/FJ_Table_1", 4);
- AssertTableReads(result, "/Root/FJ_Table_2", 2);
- AssertTableReads(result, "/Root/FJ_Table_3", 2);
- });
- }
-
- // simple right only join, only 2 tables
- Y_UNIT_TEST(RightOnly_1) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- CreateSampleTables(session);
-
- auto test = [&](bool disableFlip, std::function<void(const TDataQueryResult&)> assertFn) {
- const TString query = Sprintf(R"(
- PRAGMA Kikimr.UseNewEngine = 'false';
- %s
- SELECT t2.Value
+ RIGHT JOIN $join AS t ON t3.Key = t.Fk3
+ ORDER BY t.Value1, t.Value2
+ )", FormatPragma(disableFlip));
+
+ auto result = ExecQueryAndTestResult(session, query, NoParams,
+ R"([[["Value11"];["Value21"]];[["Value12"];["Value22"]]])");
+ assertFn(result);
+ };
+
+ test(DisableJoinFlip, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/FJ_Table_1", 4);
+ AssertTableReads(result, "/Root/FJ_Table_2", 2);
+ AssertTableReads(result, "/Root/FJ_Table_3", 4);
+ });
+
+ test(EnableJoinFlip, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/FJ_Table_1", 4);
+ AssertTableReads(result, "/Root/FJ_Table_2", 2);
+ AssertTableReads(result, "/Root/FJ_Table_3", 2);
+ });
+ }
+
+ // simple right only join, only 2 tables
+ Y_UNIT_TEST(RightOnly_1) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ CreateSampleTables(session);
+
+ auto test = [&](bool disableFlip, std::function<void(const TDataQueryResult&)> assertFn) {
+ const TString query = Sprintf(R"(
+ PRAGMA Kikimr.UseNewEngine = 'false';
+ %s
+ SELECT t2.Value
FROM [/Root/FJ_Table_3] AS t1
RIGHT ONLY JOIN [/Root/FJ_Table_2] AS t2 ON t1.Key = t2.Fk3
- ORDER BY t2.Value
- )", FormatPragma(disableFlip));
-
- auto result = ExecQueryAndTestResult(session, query, NoParams, R"([])");
- assertFn(result);
- };
-
- test(DisableJoinFlip, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/FJ_Table_2", 2);
- AssertTableReads(result, "/Root/FJ_Table_3", 4);
- });
-
- test(EnableJoinFlip, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/FJ_Table_2", 2);
- AssertTableReads(result, "/Root/FJ_Table_3", 2);
- });
- }
-
- // hierarchy of joins, flip on the last layer
- Y_UNIT_TEST(RightOnly_2) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- CreateSampleTables(session);
-
- auto test = [&](bool disableFlip, std::function<void(const TDataQueryResult&)> assertFn) {
- const TString query = Sprintf(R"(
- PRAGMA Kikimr.UseNewEngine = 'false';
- %s
- $join = (
- SELECT t2.Key AS Key, t2.Fk1 AS Fk1, t2.Fk2 AS Fk2, t2.Value AS Value
+ ORDER BY t2.Value
+ )", FormatPragma(disableFlip));
+
+ auto result = ExecQueryAndTestResult(session, query, NoParams, R"([])");
+ assertFn(result);
+ };
+
+ test(DisableJoinFlip, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/FJ_Table_2", 2);
+ AssertTableReads(result, "/Root/FJ_Table_3", 4);
+ });
+
+ test(EnableJoinFlip, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/FJ_Table_2", 2);
+ AssertTableReads(result, "/Root/FJ_Table_3", 2);
+ });
+ }
+
+ // hierarchy of joins, flip on the last layer
+ Y_UNIT_TEST(RightOnly_2) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ CreateSampleTables(session);
+
+ auto test = [&](bool disableFlip, std::function<void(const TDataQueryResult&)> assertFn) {
+ const TString query = Sprintf(R"(
+ PRAGMA Kikimr.UseNewEngine = 'false';
+ %s
+ $join = (
+ SELECT t2.Key AS Key, t2.Fk1 AS Fk1, t2.Fk2 AS Fk2, t2.Value AS Value
FROM [/Root/FJ_Table_1] AS t1
RIGHT ONLY JOIN [/Root/FJ_Table_3] AS t2 ON t1.Key = t2.Fk1
- );
- SELECT t3.Key, t3.Value
- FROM $join AS t
+ );
+ SELECT t3.Key, t3.Value
+ FROM $join AS t
INNER JOIN [/Root/FJ_Table_2] AS t3 ON t.Fk2 = t3.Key
- ORDER BY t3.Key, t3.Value
- )", FormatPragma(disableFlip));
-
+ ORDER BY t3.Key, t3.Value
+ )", FormatPragma(disableFlip));
+
auto result = ExecQueryAndTestResult(session, query, NoParams, R"([])");
- assertFn(result);
- };
-
- test(DisableJoinFlip, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/FJ_Table_1", 4);
+ assertFn(result);
+ };
+
+ test(DisableJoinFlip, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/FJ_Table_1", 4);
AssertTableReads(result, "/Root/FJ_Table_2", 0);
- AssertTableReads(result, "/Root/FJ_Table_3", 4);
- });
-
- test(EnableJoinFlip, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/FJ_Table_3", 4);
+ });
+
+ test(EnableJoinFlip, [](const TDataQueryResult& result) {
AssertTableReads(result, "/Root/FJ_Table_1", 3);
AssertTableReads(result, "/Root/FJ_Table_2", 0);
- AssertTableReads(result, "/Root/FJ_Table_3", 4);
- });
- }
-
- // hierarchy of joins, flip on the top layer
- Y_UNIT_TEST(RightOnly_3) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- CreateSampleTables(session);
-
- auto test = [&](bool disableFlip, std::function<void(const TDataQueryResult&)> assertFn) {
- const TString query = Sprintf(R"(
- PRAGMA Kikimr.UseNewEngine = 'false';
- %s
- $join = (
- SELECT t1.Value AS Value1, t2.Value AS Value2, t1.Fk3 AS Fk3
+ AssertTableReads(result, "/Root/FJ_Table_3", 4);
+ });
+ }
+
+ // hierarchy of joins, flip on the top layer
+ Y_UNIT_TEST(RightOnly_3) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ CreateSampleTables(session);
+
+ auto test = [&](bool disableFlip, std::function<void(const TDataQueryResult&)> assertFn) {
+ const TString query = Sprintf(R"(
+ PRAGMA Kikimr.UseNewEngine = 'false';
+ %s
+ $join = (
+ SELECT t1.Value AS Value1, t2.Value AS Value2, t1.Fk3 AS Fk3
FROM [/Root/FJ_Table_1] AS t1
INNER JOIN [/Root/FJ_Table_2] AS t2 ON t1.Fk2 = t2.Key
- );
- SELECT t.Value1, t.Value2
+ );
+ SELECT t.Value1, t.Value2
FROM [/Root/FJ_Table_4] AS t3
- RIGHT ONLY JOIN $join AS t ON t3.Key = t.Fk3
- ORDER BY t.Value1, t.Value2
- )", FormatPragma(disableFlip));
-
- auto result = ExecQueryAndTestResult(session, query, NoParams,
- R"([[["Value12"];["Value22"]]])");
- assertFn(result);
- };
-
- test(DisableJoinFlip, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/FJ_Table_1", 4);
- AssertTableReads(result, "/Root/FJ_Table_2", 2);
- AssertTableReads(result, "/Root/FJ_Table_4", 3);
- });
-
- test(EnableJoinFlip, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/FJ_Table_1", 4);
- AssertTableReads(result, "/Root/FJ_Table_2", 2);
- AssertTableReads(result, "/Root/FJ_Table_4", 1);
- });
- }
-
-} // Y_UNIT_TEST_SUITE
-
-} // NKikimr::NKqp
+ RIGHT ONLY JOIN $join AS t ON t3.Key = t.Fk3
+ ORDER BY t.Value1, t.Value2
+ )", FormatPragma(disableFlip));
+
+ auto result = ExecQueryAndTestResult(session, query, NoParams,
+ R"([[["Value12"];["Value22"]]])");
+ assertFn(result);
+ };
+
+ test(DisableJoinFlip, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/FJ_Table_1", 4);
+ AssertTableReads(result, "/Root/FJ_Table_2", 2);
+ AssertTableReads(result, "/Root/FJ_Table_4", 3);
+ });
+
+ test(EnableJoinFlip, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/FJ_Table_1", 4);
+ AssertTableReads(result, "/Root/FJ_Table_2", 2);
+ AssertTableReads(result, "/Root/FJ_Table_4", 1);
+ });
+ }
+
+} // Y_UNIT_TEST_SUITE
+
+} // NKikimr::NKqp
diff --git a/ydb/core/kqp/ut/kqp_index_lookup_join_ut.cpp b/ydb/core/kqp/ut/kqp_index_lookup_join_ut.cpp
index 7085b65bd4..e272088932 100644
--- a/ydb/core/kqp/ut/kqp_index_lookup_join_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_index_lookup_join_ut.cpp
@@ -1,167 +1,167 @@
#include <ydb/core/kqp/ut/common/kqp_ut_common.h>
-
+
#include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h>
-
-namespace NKikimr {
-namespace NKqp {
-
-using namespace NYdb;
-using namespace NYdb::NTable;
-
-void PrepareTables(TSession session) {
- UNIT_ASSERT(session.ExecuteSchemeQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- CREATE TABLE `/Root/Left` (
- Key Int32,
- Fk Int32,
- Value String,
- PRIMARY KEY (Key)
- );
- CREATE TABLE `/Root/Right` (
- Key Int32,
- Value String,
- PRIMARY KEY (Key)
- );
- )").GetValueSync().IsSuccess());
-
- UNIT_ASSERT(session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
-
- REPLACE INTO `/Root/Left` (Key, Fk, Value) VALUES
- (1, 101, "Value1"),
- (2, 102, "Value1"),
- (3, 103, "Value2"),
- (4, 104, "Value2"),
- (5, 105, "Value3");
-
- REPLACE INTO `/Root/Right` (Key, Value) VALUES
- (100, "Value20"),
- (101, "Value21"),
- (102, "Value22"),
- (103, "Value23");
- )", TTxControl::BeginTx().CommitTx()).GetValueSync().IsSuccess());
-}
-
-Y_UNIT_TEST_SUITE(KqpIndexLookupJoin) {
-
-template <bool UseNewEngine>
-void Test(const TString& query, const TString& answer, size_t rightTableReads) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- PrepareTables(session);
-
- TExecDataQuerySettings execSettings;
- execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
-
- auto result = session.ExecuteDataQuery(Q_(query), TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- CompareYson(answer, FormatResultSetYson(result.GetResultSet(0)));
-
- auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
+
+namespace NKikimr {
+namespace NKqp {
+
+using namespace NYdb;
+using namespace NYdb::NTable;
+
+void PrepareTables(TSession session) {
+ UNIT_ASSERT(session.ExecuteSchemeQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ CREATE TABLE `/Root/Left` (
+ Key Int32,
+ Fk Int32,
+ Value String,
+ PRIMARY KEY (Key)
+ );
+ CREATE TABLE `/Root/Right` (
+ Key Int32,
+ Value String,
+ PRIMARY KEY (Key)
+ );
+ )").GetValueSync().IsSuccess());
+
+ UNIT_ASSERT(session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+
+ REPLACE INTO `/Root/Left` (Key, Fk, Value) VALUES
+ (1, 101, "Value1"),
+ (2, 102, "Value1"),
+ (3, 103, "Value2"),
+ (4, 104, "Value2"),
+ (5, 105, "Value3");
+
+ REPLACE INTO `/Root/Right` (Key, Value) VALUES
+ (100, "Value20"),
+ (101, "Value21"),
+ (102, "Value22"),
+ (103, "Value23");
+ )", TTxControl::BeginTx().CommitTx()).GetValueSync().IsSuccess());
+}
+
+Y_UNIT_TEST_SUITE(KqpIndexLookupJoin) {
+
+template <bool UseNewEngine>
+void Test(const TString& query, const TString& answer, size_t rightTableReads) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ PrepareTables(session);
+
+ TExecDataQuerySettings execSettings;
+ execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
+
+ auto result = session.ExecuteDataQuery(Q_(query), TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ CompareYson(answer, FormatResultSetYson(result.GetResultSet(0)));
+
+ auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), kikimr.IsUsingSnapshotReads() && !UseNewEngine ? 2 : 3);
-
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/Left");
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 5);
-
- ui32 index = 1;
- if (UseNewEngine) {
- UNIT_ASSERT(stats.query_phases(1).table_access().empty()); // keys extraction for lookups
- index = 2;
- }
-
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(index).table_access().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(index).table_access(0).name(), "/Root/Right");
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(index).table_access(0).reads().rows(), rightTableReads);
-}
-
-Y_UNIT_TEST_NEW_ENGINE(Inner) {
- Test<UseNewEngine>(
- R"(
- SELECT l.Key, l.Fk, l.Value, r.Key, r.Value
- FROM `/Root/Left` AS l
- INNER JOIN `/Root/Right` AS r
- ON l.Fk = r.Key
- WHERE l.Value = 'Value1' -- left table payload filter
- AND r.Value != 'Value22' -- right table payload filter
- )",
- R"([
- [[1];[101];["Value1"];[101];["Value21"]]
- ])",
- 2);
-}
-
-Y_UNIT_TEST_NEW_ENGINE(Left) {
- Test<UseNewEngine>(
- R"(
- SELECT l.Key, l.Fk, l.Value, r.Key, r.Value
- FROM `/Root/Left` AS l
- LEFT JOIN `/Root/Right` AS r
- ON l.Fk = r.Key
- WHERE l.Value != 'Value1' -- left table payload filter
- ORDER BY l.Key
- )",
- R"([
- [[3];[103];["Value2"];[103];["Value23"]];
- [[4];[104];["Value2"];#;#];
- [[5];[105];["Value3"];#;#]
- ])",
- 1);
-}
-
-Y_UNIT_TEST_NEW_ENGINE(LeftOnly) {
- Test<UseNewEngine>(
- R"(
- SELECT l.Key, l.Fk, l.Value
- FROM `/Root/Left` AS l
- LEFT ONLY JOIN `/Root/Right` AS r
- ON l.Fk = r.Key
- WHERE l.Value != 'Value1' -- left table payload filter
- ORDER BY l.Key
- )",
- R"([
- [[4];[104];["Value2"]];
- [[5];[105];["Value3"]]
- ])",
- UseNewEngine ? 1 : 3);
-}
-
-Y_UNIT_TEST_NEW_ENGINE(LeftSemi) {
- Test<UseNewEngine>(
- R"(
- SELECT l.Key, l.Fk, l.Value
- FROM `/Root/Left` AS l
- LEFT SEMI JOIN `/Root/Right` AS r
- ON l.Fk = r.Key
- WHERE l.Value != 'Value1' -- left table payload filter
- )",
- R"([
- [[3];[103];["Value2"]]
- ])",
- UseNewEngine ? 1 : 3);
-}
-
-Y_UNIT_TEST_NEW_ENGINE(RightSemi) {
- Test<UseNewEngine>(
- R"(
- SELECT r.Key, r.Value
- FROM `/Root/Left` AS l
- RIGHT SEMI JOIN `/Root/Right` AS r
- ON l.Fk = r.Key
- WHERE r.Value != 'Value22' -- right table payload filter
- ORDER BY r.Key
- )",
- R"([
- [[101];["Value21"]];
- [[103];["Value23"]]
- ])",
- 3);
-}
-
-} // suite
-
-} // namespace NKqp
-} // namespace NKikimr
+
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/Left");
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 5);
+
+ ui32 index = 1;
+ if (UseNewEngine) {
+ UNIT_ASSERT(stats.query_phases(1).table_access().empty()); // keys extraction for lookups
+ index = 2;
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(index).table_access().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(index).table_access(0).name(), "/Root/Right");
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(index).table_access(0).reads().rows(), rightTableReads);
+}
+
+Y_UNIT_TEST_NEW_ENGINE(Inner) {
+ Test<UseNewEngine>(
+ R"(
+ SELECT l.Key, l.Fk, l.Value, r.Key, r.Value
+ FROM `/Root/Left` AS l
+ INNER JOIN `/Root/Right` AS r
+ ON l.Fk = r.Key
+ WHERE l.Value = 'Value1' -- left table payload filter
+ AND r.Value != 'Value22' -- right table payload filter
+ )",
+ R"([
+ [[1];[101];["Value1"];[101];["Value21"]]
+ ])",
+ 2);
+}
+
+Y_UNIT_TEST_NEW_ENGINE(Left) {
+ Test<UseNewEngine>(
+ R"(
+ SELECT l.Key, l.Fk, l.Value, r.Key, r.Value
+ FROM `/Root/Left` AS l
+ LEFT JOIN `/Root/Right` AS r
+ ON l.Fk = r.Key
+ WHERE l.Value != 'Value1' -- left table payload filter
+ ORDER BY l.Key
+ )",
+ R"([
+ [[3];[103];["Value2"];[103];["Value23"]];
+ [[4];[104];["Value2"];#;#];
+ [[5];[105];["Value3"];#;#]
+ ])",
+ 1);
+}
+
+Y_UNIT_TEST_NEW_ENGINE(LeftOnly) {
+ Test<UseNewEngine>(
+ R"(
+ SELECT l.Key, l.Fk, l.Value
+ FROM `/Root/Left` AS l
+ LEFT ONLY JOIN `/Root/Right` AS r
+ ON l.Fk = r.Key
+ WHERE l.Value != 'Value1' -- left table payload filter
+ ORDER BY l.Key
+ )",
+ R"([
+ [[4];[104];["Value2"]];
+ [[5];[105];["Value3"]]
+ ])",
+ UseNewEngine ? 1 : 3);
+}
+
+Y_UNIT_TEST_NEW_ENGINE(LeftSemi) {
+ Test<UseNewEngine>(
+ R"(
+ SELECT l.Key, l.Fk, l.Value
+ FROM `/Root/Left` AS l
+ LEFT SEMI JOIN `/Root/Right` AS r
+ ON l.Fk = r.Key
+ WHERE l.Value != 'Value1' -- left table payload filter
+ )",
+ R"([
+ [[3];[103];["Value2"]]
+ ])",
+ UseNewEngine ? 1 : 3);
+}
+
+Y_UNIT_TEST_NEW_ENGINE(RightSemi) {
+ Test<UseNewEngine>(
+ R"(
+ SELECT r.Key, r.Value
+ FROM `/Root/Left` AS l
+ RIGHT SEMI JOIN `/Root/Right` AS r
+ ON l.Fk = r.Key
+ WHERE r.Value != 'Value22' -- right table payload filter
+ ORDER BY r.Key
+ )",
+ R"([
+ [[101];["Value21"]];
+ [[103];["Value23"]]
+ ])",
+ 3);
+}
+
+} // suite
+
+} // namespace NKqp
+} // namespace NKikimr
diff --git a/ydb/core/kqp/ut/kqp_indexes_ut.cpp b/ydb/core/kqp/ut/kqp_indexes_ut.cpp
index decf44ccb0..505cf09e6b 100644
--- a/ydb/core/kqp/ut/kqp_indexes_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_indexes_ut.cpp
@@ -30,9 +30,9 @@ namespace {
constexpr const char* TestCluster = "local_ut";
TIntrusivePtr<NKqp::IKqpGateway> GetIcGateway(Tests::TServer& server) {
- auto counters = MakeIntrusive<TKqpRequestCounters>();
- counters->Counters = new TKqpCounters(server.GetRuntime()->GetAppData(0).Counters);
- counters->TxProxyMon = new NTxProxy::TTxProxyMon(server.GetRuntime()->GetAppData(0).Counters);
+ auto counters = MakeIntrusive<TKqpRequestCounters>();
+ counters->Counters = new TKqpCounters(server.GetRuntime()->GetAppData(0).Counters);
+ counters->TxProxyMon = new NTxProxy::TTxProxyMon(server.GetRuntime()->GetAppData(0).Counters);
std::shared_ptr<NYql::IKikimrGateway::IKqpTableMetadataLoader> loader = std::make_shared<TKqpTableMetadataLoader>(server.GetRuntime()->GetAnyNodeActorSystem(), false);
return NKqp::CreateKikimrIcGateway(TestCluster, "/Root", std::move(loader), server.GetRuntime()->GetAnyNodeActorSystem(),
server.GetRuntime()->GetNodeId(0), counters, MakeMiniKQLCompileServiceID());
@@ -127,11 +127,11 @@ Y_UNIT_TEST_SUITE(KqpIndexMetadata) {
server.GetFunctionRegistry());
{
- const TString query = Q_(R"(
- UPSERT INTO `/Root/IndexedTableWithState` (key, value, value2)
- VALUES (1, "qq", "ww")
- )");
-
+ const TString query = Q_(R"(
+ UPSERT INTO `/Root/IndexedTableWithState` (key, value, value2)
+ VALUES (1, "qq", "ww")
+ )");
+
auto explainResult = qp->SyncExplainDataQuery(query, true);
UNIT_ASSERT(explainResult.Success());
@@ -139,24 +139,24 @@ Y_UNIT_TEST_SUITE(KqpIndexMetadata) {
TExprContext exprCtx;
VisitExpr(GetExpr(explainResult.QueryAst, exprCtx, moduleResolver.get()).Ptr(),
[&indexName](const TExprNode::TPtr& exprNode) {
- if (UseNewEngine) {
- if (TMaybeNode<TKqpUpsertRows>(exprNode)) {
- UNIT_ASSERT(!TKqpUpsertRows(exprNode).Table().Path().Value().Contains(indexName));
- }
- if (TMaybeNode<TKqpDeleteRows>(exprNode)) {
- UNIT_ASSERT(!TKqpDeleteRows(exprNode).Table().Path().Value().Contains(indexName));
- }
- } else {
- if (auto maybeupdate = TMaybeNode<TKiUpdateRow>(exprNode)) {
- auto update = maybeupdate.Cast();
- TStringBuf toUpdate = update.Table().Path().Value();
- UNIT_ASSERT(!toUpdate.Contains(indexName));
- }
- if (auto maybeerase = TMaybeNode<TKiEraseRow>(exprNode)) {
- auto erase = maybeerase.Cast();
- TStringBuf toErase = erase.Table().Path().Value();
- UNIT_ASSERT(!toErase.Contains(indexName));
- }
+ if (UseNewEngine) {
+ if (TMaybeNode<TKqpUpsertRows>(exprNode)) {
+ UNIT_ASSERT(!TKqpUpsertRows(exprNode).Table().Path().Value().Contains(indexName));
+ }
+ if (TMaybeNode<TKqpDeleteRows>(exprNode)) {
+ UNIT_ASSERT(!TKqpDeleteRows(exprNode).Table().Path().Value().Contains(indexName));
+ }
+ } else {
+ if (auto maybeupdate = TMaybeNode<TKiUpdateRow>(exprNode)) {
+ auto update = maybeupdate.Cast();
+ TStringBuf toUpdate = update.Table().Path().Value();
+ UNIT_ASSERT(!toUpdate.Contains(indexName));
+ }
+ if (auto maybeerase = TMaybeNode<TKiEraseRow>(exprNode)) {
+ auto erase = maybeerase.Cast();
+ TStringBuf toErase = erase.Table().Path().Value();
+ UNIT_ASSERT(!toErase.Contains(indexName));
+ }
}
return true;
});
@@ -194,10 +194,10 @@ Y_UNIT_TEST_SUITE(KqpIndexMetadata) {
server.GetFunctionRegistry());
{
- const TString query = Q_(R"(
- UPSERT INTO `/Root/IndexedTableWithState` ("key", "value", "value2")
- VALUES (1, "qq", "ww");
- )");
+ const TString query = Q_(R"(
+ UPSERT INTO `/Root/IndexedTableWithState` ("key", "value", "value2")
+ VALUES (1, "qq", "ww");
+ )");
auto explainResult = qp->SyncExplainDataQuery(query, true);
UNIT_ASSERT(explainResult.Success());
@@ -206,32 +206,32 @@ Y_UNIT_TEST_SUITE(KqpIndexMetadata) {
bool indexCleaned = false;
VisitExpr(GetExpr(explainResult.QueryAst, exprCtx, moduleResolver.get()).Ptr(),
[&indexName, &indexUpdated, &indexCleaned](const TExprNode::TPtr& exprNode) mutable {
- if (UseNewEngine) {
- if (TMaybeNode<TKqpUpsertRows>(exprNode)) {
- if (TKqpUpsertRows(exprNode).Table().Path().Value().Contains(indexName)) {
- indexUpdated = true;
- }
+ if (UseNewEngine) {
+ if (TMaybeNode<TKqpUpsertRows>(exprNode)) {
+ if (TKqpUpsertRows(exprNode).Table().Path().Value().Contains(indexName)) {
+ indexUpdated = true;
+ }
}
- if (TMaybeNode<TKqpDeleteRows>(exprNode)) {
- if (TKqpDeleteRows(exprNode).Table().Path().Value().Contains(indexName)) {
- indexCleaned = true;
- }
+ if (TMaybeNode<TKqpDeleteRows>(exprNode)) {
+ if (TKqpDeleteRows(exprNode).Table().Path().Value().Contains(indexName)) {
+ indexCleaned = true;
+ }
+ }
+ } else {
+ if (auto maybeupdate = TMaybeNode<TKiUpdateRow>(exprNode)) {
+ auto update = maybeupdate.Cast();
+ TStringBuf toUpdate = update.Table().Path().Value();
+ if (toUpdate.Contains(indexName)) {
+ indexUpdated = true;
+ }
+ }
+ if (auto maybeerase = TMaybeNode<TKiEraseRow>(exprNode)) {
+ auto erase = maybeerase.Cast();
+ TStringBuf toErase = erase.Table().Path().Value();
+ if (toErase.Contains(indexName)) {
+ indexCleaned = true;
+ }
}
- } else {
- if (auto maybeupdate = TMaybeNode<TKiUpdateRow>(exprNode)) {
- auto update = maybeupdate.Cast();
- TStringBuf toUpdate = update.Table().Path().Value();
- if (toUpdate.Contains(indexName)) {
- indexUpdated = true;
- }
- }
- if (auto maybeerase = TMaybeNode<TKiEraseRow>(exprNode)) {
- auto erase = maybeerase.Cast();
- TStringBuf toErase = erase.Table().Path().Value();
- if (toErase.Contains(indexName)) {
- indexCleaned = true;
- }
- }
}
return true;
});
@@ -259,8 +259,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
NYdb::NTable::TExecDataQuerySettings execSettings;
execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
- const TString query(Q1_(R"(
- SELECT Key FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk IS NULL;
+ const TString query(Q1_(R"(
+ SELECT Key FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk IS NULL;
)"));
auto result = session.ExecuteDataQuery(
@@ -291,8 +291,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
CreateSampleTablesWithIndex(session);
{
- const TString query(Q1_(R"(
- SELECT * FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk IS NULL;
+ const TString query(Q1_(R"(
+ SELECT * FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk IS NULL;
)"));
auto result = session.ExecuteDataQuery(
@@ -322,7 +322,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
const TString query(Q_(R"(
- SELECT * FROM `/Root/SecondaryKeys` : Index WHERE Fk IS NULL;
+ SELECT * FROM `/Root/SecondaryKeys` : Index WHERE Fk IS NULL;
)"));
auto result = session.ExecuteDataQuery(
@@ -344,7 +344,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
auto session = db.CreateSession().GetValueSync().GetSession();
{
- const TString keyColumnName = "key";
+ const TString keyColumnName = "key";
auto builder = TTableBuilder()
.AddNullableColumn(keyColumnName, EPrimitiveType::Uint64);
@@ -358,8 +358,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
}
{
- const TString query1(Q1_(R"(
- DECLARE $items AS List<Struct<'key':Uint64,'index_0':Utf8,'value':Uint32>>;
+ const TString query1(Q1_(R"(
+ DECLARE $items AS List<Struct<'key':Uint64,'index_0':Utf8,'value':Uint32>>;
UPSERT INTO `/Root/Test1`
SELECT * FROM AS_TABLE($items);
)"));
@@ -405,7 +405,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
// Just check table prepared
{
- const auto yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable");
+ const auto yson = ReadTablePartToYson(session, "/Root/SecondaryWithDataColumns/Index/indexImplTable");
const TString expected = R"([[["Secondary1"];["Primary1"];["Value1"]]])";
UNIT_ASSERT_VALUES_EQUAL(yson, expected);
}
@@ -416,8 +416,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
// This test checks the implementation has correct handle not exact semantic for table lookup
// (the lenth of lookup result is not equal to the lengh of the user input)
{
- const TString query1(Q1_(R"(
- UPSERT INTO `/Root/SecondaryWithDataColumns` (Key, Value, ExtPayload) VALUES
+ const TString query1(Q1_(R"(
+ UPSERT INTO `/Root/SecondaryWithDataColumns` (Key, Value, ExtPayload) VALUES
("Primary0", "Value0_", "Something"),
("Primary1", "Value1_", "Something");
)"));
@@ -457,7 +457,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
// Upsert using pk and some other column. Index will be read from table value from user input.
{
const TString query1(Q_(R"(
- UPSERT INTO `/Root/SecondaryWithDataColumns` (Key, Value, ExtPayload) VALUES
+ UPSERT INTO `/Root/SecondaryWithDataColumns` (Key, Value, ExtPayload) VALUES
("Primary1", NULL, "Something");
)"));
@@ -491,13 +491,13 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
.Build()
.Build();
- const TString query1(Q1_(R"(
- DECLARE $rows AS List<Struct<
+ const TString query1(Q1_(R"(
+ DECLARE $rows AS List<Struct<
Key : String?,
Value : String?,
ExtPayload : String?
- >>;
- UPSERT INTO `/Root/SecondaryWithDataColumns`
+ >>;
+ UPSERT INTO `/Root/SecondaryWithDataColumns`
SELECT * FROM AS_TABLE($rows);
)"));
@@ -536,7 +536,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
// Upsert using previous inserved pk and fk, check data column realy updated
{
const TString query1(Q_(R"(
- UPSERT INTO `/Root/SecondaryWithDataColumns` (Key, Index2, Value) VALUES
+ UPSERT INTO `/Root/SecondaryWithDataColumns` (Key, Index2, Value) VALUES
("Primary1", "Secondary1", "Value1_1");
)"));
@@ -556,7 +556,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
// Upsert using previous inserved pk but without fk, check data column still realy updated
{
const TString query1(Q_(R"(
- UPSERT INTO `/Root/SecondaryWithDataColumns` (Key, Value) VALUES
+ UPSERT INTO `/Root/SecondaryWithDataColumns` (Key, Value) VALUES
("Primary1", "Value1_2");
)"));
@@ -576,7 +576,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
// Upsert using new pk without fk
{
const TString query1(Q_(R"(
- UPSERT INTO `/Root/SecondaryWithDataColumns` (Key, Value) VALUES
+ UPSERT INTO `/Root/SecondaryWithDataColumns` (Key, Value) VALUES
("Primary2", "Value2_1");
)"));
@@ -596,7 +596,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
// Upsert using pk, fk, and some other column. Data column in index must have old value
{
const TString query1(Q_(R"(
- UPSERT INTO `/Root/SecondaryWithDataColumns` (Key, Index2, ExtPayload) VALUES
+ UPSERT INTO `/Root/SecondaryWithDataColumns` (Key, Index2, ExtPayload) VALUES
("Primary1", "Secondary1", "Something");
)"));
@@ -616,7 +616,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
// Replace row
{
const TString query1(Q_(R"(
- REPLACE INTO `/Root/SecondaryWithDataColumns` (Key, Index2, Value) VALUES
+ REPLACE INTO `/Root/SecondaryWithDataColumns` (Key, Index2, Value) VALUES
("Primary1", "Secondary1", "Value1_3");
)"));
@@ -636,7 +636,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
// Replace row but no specify data column
{
const TString query1(Q_(R"(
- REPLACE INTO `/Root/SecondaryWithDataColumns` (Key, Index2) VALUES
+ REPLACE INTO `/Root/SecondaryWithDataColumns` (Key, Index2) VALUES
("Primary1", "Secondary1");
)"));
@@ -657,7 +657,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
// Replace row specify data column but no index column
{
const TString query1(Q_(R"(
- REPLACE INTO `/Root/SecondaryWithDataColumns` (Key, Value) VALUES
+ REPLACE INTO `/Root/SecondaryWithDataColumns` (Key, Value) VALUES
("Primary2", "Value2_3");
)"));
@@ -677,7 +677,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
// Replace just by pk
{
const TString query1(Q_(R"(
- REPLACE INTO `/Root/SecondaryWithDataColumns` (Key) VALUES
+ REPLACE INTO `/Root/SecondaryWithDataColumns` (Key) VALUES
("Primary2");
)"));
@@ -697,7 +697,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
// Delete
{
const TString query1(Q_(R"(
- DELETE FROM `/Root/SecondaryWithDataColumns` ON (Key) VALUES
+ DELETE FROM `/Root/SecondaryWithDataColumns` ON (Key) VALUES
("Primary2"), ("Primary1");
)"));
@@ -717,7 +717,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
// Insert new row in empty table
{
const TString query1(Q_(R"(
- INSERT INTO `/Root/SecondaryWithDataColumns` (Key, Index2, Value) VALUES
+ INSERT INTO `/Root/SecondaryWithDataColumns` (Key, Index2, Value) VALUES
("Primary1", "Secondary1", "Value1");
)"));
@@ -737,7 +737,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
// Insert row with same pk
{
const TString query1(R"(
- INSERT INTO `/Root/SecondaryWithDataColumns` (Key, Index2, Value) VALUES
+ INSERT INTO `/Root/SecondaryWithDataColumns` (Key, Index2, Value) VALUES
("Primary1", "Secondary1", "Value1_1");
)");
@@ -760,7 +760,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
// Insert new row but no specify index column
{
const TString query1(Q_(R"(
- INSERT INTO `/Root/SecondaryWithDataColumns` (Key, Value) VALUES
+ INSERT INTO `/Root/SecondaryWithDataColumns` (Key, Value) VALUES
("Primary2", "Value2");
)"));
@@ -780,7 +780,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
// Update on
{
const TString query1(Q_(R"(
- UPDATE `/Root/SecondaryWithDataColumns` ON (Key, Index2, Value) VALUES
+ UPDATE `/Root/SecondaryWithDataColumns` ON (Key, Index2, Value) VALUES
("NonExistPrimary", "Secondary1_1", "Value1_1"),
("Primary1", "Secondary1_1", "Value1_1");
)"));
@@ -802,7 +802,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
const TString query1(Q_(R"(
- UPDATE `/Root/SecondaryWithDataColumns` ON (Key, Value) VALUES
+ UPDATE `/Root/SecondaryWithDataColumns` ON (Key, Value) VALUES
("Primary1", "Value1_2");
)"));
@@ -823,7 +823,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
const TString query1(Q_(R"(
- UPDATE `/Root/SecondaryWithDataColumns` ON (Key, Index2, ExtPayload) VALUES
+ UPDATE `/Root/SecondaryWithDataColumns` ON (Key, Index2, ExtPayload) VALUES
("Primary1", "Secondary1_1", "Something");
)"));
@@ -845,7 +845,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
// Update where - update data column
{
const TString query1(Q_(R"(
- UPDATE `/Root/SecondaryWithDataColumns` SET Value = "Value1_3"
+ UPDATE `/Root/SecondaryWithDataColumns` SET Value = "Value1_3"
WHERE Key = "Primary1";
)"));
@@ -866,7 +866,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
// Update were - do not touch index
{
const TString query1(Q_(R"(
- UPDATE `/Root/SecondaryWithDataColumns` SET ExtPayload = "Something2"
+ UPDATE `/Root/SecondaryWithDataColumns` SET ExtPayload = "Something2"
WHERE Key = "Primary1";
)"));
@@ -887,7 +887,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
// Update wete - update index column
{
const TString query1(Q_(R"(
- UPDATE `/Root/SecondaryWithDataColumns` SET Index2 = "Secondary1_2"
+ UPDATE `/Root/SecondaryWithDataColumns` SET Index2 = "Secondary1_2"
WHERE Key = "Primary1";
)"));
@@ -920,7 +920,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
const TString query1(Q_(R"(
- UPSERT INTO `/Root/SecondaryWithDataColumns` (Key, Index2, Value) VALUES
+ UPSERT INTO `/Root/SecondaryWithDataColumns` (Key, Index2, Value) VALUES
("p2", "s2", "2");
)"));
@@ -933,7 +933,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
const TString query1(Q_(R"(
- UPSERT INTO `/Root/SecondaryKeys` (Key, Fk, Value) VALUES
+ UPSERT INTO `/Root/SecondaryKeys` (Key, Fk, Value) VALUES
(1, 111, "Secondary1");
)"));
@@ -948,8 +948,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
NYdb::NTable::TExecDataQuerySettings execSettings;
execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
- const TString query(Q1_(R"(
- SELECT Value FROM `/Root/SecondaryWithDataColumns` VIEW Index WHERE Index2 = 'Secondary1';
+ const TString query(Q1_(R"(
+ SELECT Value FROM `/Root/SecondaryWithDataColumns` VIEW Index WHERE Index2 = 'Secondary1';
)"));
auto result = session.ExecuteDataQuery(
@@ -980,8 +980,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
NYdb::NTable::TExecDataQuerySettings execSettings;
execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
- const TString query = Q1_(R"(
- SELECT Value FROM `/Root/SecondaryWithDataColumns` VIEW Index WHERE Index2 IN ('Secondary1');
+ const TString query = Q1_(R"(
+ SELECT Value FROM `/Root/SecondaryWithDataColumns` VIEW Index WHERE Index2 IN ('Secondary1');
)");
auto result = session.ExecuteDataQuery(
@@ -994,26 +994,26 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
- UNIT_ASSERT_VALUES_EQUAL_C(stats.query_phases().size(), UseNewEngine ? 2 : 1, stats.DebugString());
+ UNIT_ASSERT_VALUES_EQUAL_C(stats.query_phases().size(), UseNewEngine ? 2 : 1, stats.DebugString());
+
+ ui32 index = 0;
+ if (UseNewEngine) {
+ UNIT_ASSERT(stats.query_phases(index).table_access().empty());
+ index = 1;
+ }
- ui32 index = 0;
- if (UseNewEngine) {
- UNIT_ASSERT(stats.query_phases(index).table_access().empty());
- index = 1;
- }
-
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(index).table_access().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(index).table_access(0).name(), "/Root/SecondaryWithDataColumns/Index/indexImplTable");
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(index).table_access(0).reads().rows(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(index).table_access().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(index).table_access(0).name(), "/Root/SecondaryWithDataColumns/Index/indexImplTable");
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(index).table_access(0).reads().rows(), 1);
}
{
NYdb::NTable::TExecDataQuerySettings execSettings;
execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
- const TString query = Q1_(R"(
- SELECT t2.Value FROM `/Root/SecondaryKeys` as t1
- INNER JOIN `/Root/SecondaryWithDataColumns` VIEW Index as t2 ON t2.Index2 = t1.Value;
+ const TString query = Q1_(R"(
+ SELECT t2.Value FROM `/Root/SecondaryKeys` as t1
+ INNER JOIN `/Root/SecondaryWithDataColumns` VIEW Index as t2 ON t2.Index2 = t1.Value;
)");
auto result = session.ExecuteDataQuery(
@@ -1027,29 +1027,29 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
- int phaseCount = 3; // In mvcc case we don't acquire locks, so that we skip locks check and cleanup
- if (WithMvcc && !UseNewEngine) {
- phaseCount--;
- }
+ int phaseCount = 3; // In mvcc case we don't acquire locks, so that we skip locks check and cleanup
+ if (WithMvcc && !UseNewEngine) {
+ phaseCount--;
+ }
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), phaseCount);
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1);
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/SecondaryKeys");
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 5);
- int idx = UseNewEngine ? 2 : 1;
- UNIT_ASSERT_VALUES_EQUAL_C(stats.query_phases(idx).table_access().size(), 1, stats.DebugString());
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).name(), "/Root/SecondaryWithDataColumns/Index/indexImplTable");
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).reads().rows(), 1);
+ int idx = UseNewEngine ? 2 : 1;
+ UNIT_ASSERT_VALUES_EQUAL_C(stats.query_phases(idx).table_access().size(), 1, stats.DebugString());
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).name(), "/Root/SecondaryWithDataColumns/Index/indexImplTable");
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).reads().rows(), 1);
}
{
NYdb::NTable::TExecDataQuerySettings execSettings;
execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
- const TString query = Q1_(R"(
- SELECT t2.Value FROM `/Root/SecondaryWithDataColumns` VIEW Index as t1
- INNER JOIN `/Root/KeyValue2` as t2 ON t2.Key = t1.Value WHERE t1.Index2 = "s2";
+ const TString query = Q1_(R"(
+ SELECT t2.Value FROM `/Root/SecondaryWithDataColumns` VIEW Index as t1
+ INNER JOIN `/Root/KeyValue2` as t2 ON t2.Key = t1.Value WHERE t1.Index2 = "s2";
)");
auto result = session.ExecuteDataQuery(
@@ -1063,30 +1063,30 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
- int phaseCount = 3; // In mvcc case we don't acquire locks, so that we skip locks check and cleanup
- if (WithMvcc && !UseNewEngine) {
- phaseCount--;
- }
- if (UseNewEngine) {
- phaseCount++;
- }
+ int phaseCount = 3; // In mvcc case we don't acquire locks, so that we skip locks check and cleanup
+ if (WithMvcc && !UseNewEngine) {
+ phaseCount--;
+ }
+ if (UseNewEngine) {
+ phaseCount++;
+ }
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), phaseCount);
- int idx = 0;
- if (UseNewEngine) {
- idx = 1;
- }
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).name(), "/Root/SecondaryWithDataColumns/Index/indexImplTable");
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).reads().rows(), 1);
-
- idx++;
- if (UseNewEngine) {
- idx++;
- }
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).name(), "/Root/KeyValue2");
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).reads().rows(), 1);
+ int idx = 0;
+ if (UseNewEngine) {
+ idx = 1;
+ }
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).name(), "/Root/SecondaryWithDataColumns/Index/indexImplTable");
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).reads().rows(), 1);
+
+ idx++;
+ if (UseNewEngine) {
+ idx++;
+ }
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).name(), "/Root/KeyValue2");
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).reads().rows(), 1);
}
}
@@ -1116,7 +1116,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
const TString query1(Q_(R"(
- UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES
+ UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES
("Primary1", "Secondary1", "Value1"),
("Primary2", "Secondary2", "Value2"),
("Primary3", "Secondary3", "Value3");
@@ -1130,7 +1130,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
}
const TString query1(Q_(R"(
- SELECT Index2 FROM `/Root/TestTable` WHERE Key = "Primary1";
+ SELECT Index2 FROM `/Root/TestTable` WHERE Key = "Primary1";
)"));
// Start tx1, select from table by pk (without touching index table) and without commit
@@ -1147,7 +1147,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
// In other tx, update string
const TString query1(Q_(R"(
- UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES
+ UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES
("Primary1", "Secondary1New", "Value1New")
)"));
@@ -1158,8 +1158,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
UNIT_ASSERT(result.IsSuccess());
}
- const TString query2(Q1_(R"(
- SELECT Index2 FROM `/Root/TestTable` VIEW Index WHERE Index2 = "Secondary1";
+ const TString query2(Q1_(R"(
+ SELECT Index2 FROM `/Root/TestTable` VIEW Index WHERE Index2 = "Secondary1";
)"));
UNIT_ASSERT(result1.GetTransaction());
@@ -1197,7 +1197,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
const TString query1(Q_(R"(
- UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES
+ UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES
("Primary1", "Secondary1", "Value1"),
("Primary2", "Secondary2", "Value2"),
("Primary3", "Secondary3", "Value3");
@@ -1210,8 +1210,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
UNIT_ASSERT(result.IsSuccess());
}
- const TString query1 = Q1_(R"(
- SELECT Index2 FROM `/Root/TestTable` VIEW Index WHERE Index2 = "Secondary1";
+ const TString query1 = Q1_(R"(
+ SELECT Index2 FROM `/Root/TestTable` VIEW Index WHERE Index2 = "Secondary1";
)");
// Start tx1, select from table by index
@@ -1228,7 +1228,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
// In other tx, update string
const TString query(Q_(R"(
- UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES
+ UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES
("Primary1", "Secondary1New", "Value1New")
)"));
@@ -1240,7 +1240,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
}
const TString query2(Q_(R"(
- UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES
+ UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES
("Primary4", "Secondary4", "Value4")
)"));
@@ -1278,7 +1278,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
const TString query1(Q_(R"(
- UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES
+ UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES
("Primary1", "Secondary1", "Value1");
)"));
@@ -1330,8 +1330,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
}
{
- const TString query1 = Q1_(R"(
- UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES
+ const TString query1 = Q1_(R"(
+ UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES
("Primary1", "Secondary1_1", "Value1");
)");
@@ -1344,29 +1344,29 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
.ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), UseNewEngine ? 4 : 3);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), UseNewEngine ? 4 : 3);
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1);
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TestTable");
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 1);
- int idx = UseNewEngine ? 3 : 2;
+ int idx = UseNewEngine ? 3 : 2;
+
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access().size(), 2);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access().size(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).name(), "/Root/TestTable");
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).updates().rows(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).name(), "/Root/TestTable");
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).updates().rows(), 1);
-
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(1).name(), "/Root/TestTable/Index/indexImplTable");
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(1).updates().rows(), 1);
- UNIT_ASSERT(stats.query_phases(idx).table_access(1).has_deletes());
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(1).deletes().rows(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(1).name(), "/Root/TestTable/Index/indexImplTable");
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(1).updates().rows(), 1);
+ UNIT_ASSERT(stats.query_phases(idx).table_access(1).has_deletes());
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(1).deletes().rows(), 1);
}
{
// Upsert without touching index
- const TString query2 = Q1_(R"(
- UPSERT INTO `/Root/TestTable` (Key, Value) VALUES
+ const TString query2 = Q1_(R"(
+ UPSERT INTO `/Root/TestTable` (Key, Value) VALUES
("Primary1", "Value1_1");
)");
@@ -1379,26 +1379,26 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
.ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), UseNewEngine ? 4 : 3);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), UseNewEngine ? 4 : 3);
// One read from main table
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1);
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TestTable");
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 1);
- int idx = UseNewEngine ? 3 : 2;
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access().size(), 2);
+ int idx = UseNewEngine ? 3 : 2;
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access().size(), 2);
// One update of main table
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).name(), "/Root/TestTable");
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).updates().rows(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).name(), "/Root/TestTable");
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).updates().rows(), 1);
// No touching index
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(1).name(), "/Root/TestTable/Index/indexImplTable");
- if (!UseNewEngine) { // BUG
- UNIT_ASSERT(!stats.query_phases(idx).table_access(1).has_updates());
- UNIT_ASSERT(!stats.query_phases(idx).table_access(1).has_deletes());
- }
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(1).name(), "/Root/TestTable/Index/indexImplTable");
+ if (!UseNewEngine) { // BUG
+ UNIT_ASSERT(!stats.query_phases(idx).table_access(1).has_updates());
+ UNIT_ASSERT(!stats.query_phases(idx).table_access(1).has_deletes());
+ }
{
const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/Index/indexImplTable");
@@ -1409,8 +1409,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
// Update without touching index
- const TString query2 = Q1_(R"(
- UPDATE `/Root/TestTable` ON (Key, Value) VALUES
+ const TString query2 = Q1_(R"(
+ UPDATE `/Root/TestTable` ON (Key, Value) VALUES
("Primary1", "Value1_2");
)");
@@ -1423,20 +1423,20 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
.ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), UseNewEngine ? 4 : 2); // BUG
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), UseNewEngine ? 4 : 2); // BUG
- int idx = UseNewEngine ? 1 : 0;
-
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access().size(), 1);
+ int idx = UseNewEngine ? 1 : 0;
+
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access().size(), 1);
// One read of main table
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).name(), "/Root/TestTable");
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).reads().rows(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).name(), "/Root/TestTable");
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).reads().rows(), 1);
// One update of index table
- idx += (UseNewEngine ? 2 : 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).name(), "/Root/TestTable");
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).updates().rows(), 1);
+ idx += (UseNewEngine ? 2 : 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).name(), "/Root/TestTable");
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(idx).table_access(0).updates().rows(), 1);
// Thats it, no phase for index table - we remove it on compile time
@@ -1449,8 +1449,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
// Update without touching index
- const TString query2 = Q1_(R"(
- UPDATE `/Root/TestTable` SET Value = "Value1_3"
+ const TString query2 = Q1_(R"(
+ UPDATE `/Root/TestTable` SET Value = "Value1_3"
WHERE Key = "Primary1";
)");
@@ -1509,8 +1509,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
const TString query1(R"(
- PRAGMA Kikimr.UseNewEngine = 'false';
- UPSERT INTO `/Root/TestTable` (Key, Index) VALUES
+ PRAGMA Kikimr.UseNewEngine = 'false';
+ UPSERT INTO `/Root/TestTable` (Key, Index) VALUES
("Primary1", "Secondary1");
)");
@@ -1549,8 +1549,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
const TString query1(R"(
- PRAGMA Kikimr.UseNewEngine = 'false';
- UPSERT INTO `/Root/TestTable` (Key, Index) VALUES
+ PRAGMA Kikimr.UseNewEngine = 'false';
+ UPSERT INTO `/Root/TestTable` (Key, Index) VALUES
("Primary1", "Secondary1_1");
)");
@@ -1590,8 +1590,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
const TString query1(R"(
- PRAGMA Kikimr.UseNewEngine = 'false';
- UPDATE `/Root/TestTable` ON (Key, Index) VALUES
+ PRAGMA Kikimr.UseNewEngine = 'false';
+ UPDATE `/Root/TestTable` ON (Key, Index) VALUES
("Primary1", "Secondary1_2");
)");
@@ -1630,8 +1630,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
const TString query1(R"(
- PRAGMA Kikimr.UseNewEngine = 'false';
- UPDATE `/Root/TestTable` SET Index = "Secondary1_3"
+ PRAGMA Kikimr.UseNewEngine = 'false';
+ UPDATE `/Root/TestTable` SET Index = "Secondary1_3"
WHERE Key = "Primary1";
)");
@@ -1693,8 +1693,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
const TString query1(R"(
- PRAGMA Kikimr.UseNewEngine = 'false';
- UPSERT INTO `/Root/TestTable` (Key, Index) VALUES
+ PRAGMA Kikimr.UseNewEngine = 'false';
+ UPSERT INTO `/Root/TestTable` (Key, Index) VALUES
("Primary1", "Secondary1");
)");
@@ -1732,8 +1732,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
// Upsert on new pk
const TString query1(R"(
- PRAGMA Kikimr.UseNewEngine = 'false';
- UPSERT INTO `/Root/TestTable` (Key, Index) VALUES
+ PRAGMA Kikimr.UseNewEngine = 'false';
+ UPSERT INTO `/Root/TestTable` (Key, Index) VALUES
("Primary1", "Secondary1_1");
)");
@@ -1772,8 +1772,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
// Update on non existing key - do nothing
const TString query1(R"(
- PRAGMA Kikimr.UseNewEngine = 'false';
- UPDATE `/Root/TestTable` ON (Key, Index) VALUES
+ PRAGMA Kikimr.UseNewEngine = 'false';
+ UPDATE `/Root/TestTable` ON (Key, Index) VALUES
("Primary1", "Secondary1_2");
)");
@@ -1829,8 +1829,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
const TString query1(R"(
- PRAGMA Kikimr.UseNewEngine = 'false';
- REPLACE INTO `/Root/TestTable` (Key, Index2, Value) VALUES
+ PRAGMA Kikimr.UseNewEngine = 'false';
+ REPLACE INTO `/Root/TestTable` (Key, Index2, Value) VALUES
("Primary1", "Secondary1", "Value1");
)");
@@ -1847,7 +1847,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1);
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TestTable");
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 0);
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access().size(), 2);
@@ -1861,8 +1861,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
const TString query1(R"(
- PRAGMA Kikimr.UseNewEngine = 'false';
- REPLACE INTO `/Root/TestTable` (Key, Index2, Value) VALUES
+ PRAGMA Kikimr.UseNewEngine = 'false';
+ REPLACE INTO `/Root/TestTable` (Key, Index2, Value) VALUES
("Primary1", "Secondary1_1", "Value1");
)");
@@ -1924,7 +1924,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
const TString query1(Q_(R"(
- UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES
+ UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES
("Primary1", "SomeOldIndex", "SomeOldValue");
)"));
@@ -1936,7 +1936,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
}
const TString query1(Q_(R"(
- UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES
+ UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES
("Primary1", "Secondary1", "Value1"),
("Primary2", "Secondary2", "Value2"),
("Primary3", "Secondary3", "Value3");
@@ -1961,7 +1961,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
const TString query(Q_(R"(
- SELECT * FROM `/Root/TestTable/Index/indexImplTable`;
+ SELECT * FROM `/Root/TestTable/Index/indexImplTable`;
)"));
auto result = session.ExecuteDataQuery(
@@ -1974,8 +1974,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
}
{
- const TString query(Q1_(R"(
- SELECT Value FROM `/Root/TestTable` VIEW WrongView WHERE Index2 = 'Secondary2';
+ const TString query(Q1_(R"(
+ SELECT Value FROM `/Root/TestTable` VIEW WrongView WHERE Index2 = 'Secondary2';
)"));
auto result = session.ExecuteDataQuery(
@@ -1986,8 +1986,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
}
{
- const TString query(Q1_(R"(
- SELECT Value FROM `/Root/TestTable` VIEW Index WHERE Index2 = 'Secondary2';
+ const TString query(Q1_(R"(
+ SELECT Value FROM `/Root/TestTable` VIEW Index WHERE Index2 = 'Secondary2';
)"));
auto result = session.ExecuteDataQuery(
@@ -2000,7 +2000,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
const TString query(Q_(R"(
- DELETE FROM `/Root/TestTable` ON (Key) VALUES ('Primary1');
+ DELETE FROM `/Root/TestTable` ON (Key) VALUES ('Primary1');
)"));
auto result = session.ExecuteDataQuery(
@@ -2018,7 +2018,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
const TString query(Q_(R"(
- DELETE FROM `/Root/TestTable` WHERE Key = 'Primary2';
+ DELETE FROM `/Root/TestTable` WHERE Key = 'Primary2';
)"));
auto result = session.ExecuteDataQuery(
@@ -2036,7 +2036,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
const TString query(Q_(R"(
- UPDATE `/Root/TestTable` ON (Key, Index2) VALUES ('Primary3', 'Secondary3_1');
+ UPDATE `/Root/TestTable` ON (Key, Index2) VALUES ('Primary3', 'Secondary3_1');
)"));
auto result = session.ExecuteDataQuery(
@@ -2060,7 +2060,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
const TString query(Q_(R"(
- UPDATE `/Root/TestTable` SET Index2 = 'Secondary3_2' WHERE Key = 'Primary3';
+ UPDATE `/Root/TestTable` SET Index2 = 'Secondary3_2' WHERE Key = 'Primary3';
)"));
auto result = session.ExecuteDataQuery(
@@ -2107,7 +2107,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
}
const TString query1(Q_(R"(
- UPSERT INTO `/Root/TestTable` (Key, Index2, Index2A) VALUES
+ UPSERT INTO `/Root/TestTable` (Key, Index2, Index2A) VALUES
("Primary1", "Secondary1", "Secondary1A"),
("Primary2", "Secondary2", "Secondary2A"),
("Primary3", "Secondary3", "Secondary3A");
@@ -2127,7 +2127,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
const TString query(Q_(R"(
- UPDATE `/Root/TestTable` ON (Key, Index2) VALUES ('Primary1', 'Secondary1_1');
+ UPDATE `/Root/TestTable` ON (Key, Index2) VALUES ('Primary1', 'Secondary1_1');
)"));
auto result = session.ExecuteDataQuery(
@@ -2151,7 +2151,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
const TString query(Q_(R"(
- UPDATE `/Root/TestTable` SET Index2 = 'Secondary1_2' WHERE Key = 'Primary1';
+ UPDATE `/Root/TestTable` SET Index2 = 'Secondary1_2' WHERE Key = 'Primary1';
)"));
auto result = session.ExecuteDataQuery(
@@ -2198,7 +2198,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
}
const TString query1(Q_(R"(
- UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES
+ UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES
("Primary1", "Secondary1", "Val1");
)"));
@@ -2216,9 +2216,9 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
- const TString query(Q1_(R"(
- UPDATE `/Root/TestTable` ON (Key, Index2, Value)
- (SELECT Key, Index2, 'Val1_1' as Value FROM `/Root/TestTable` VIEW Index WHERE Index2 = 'Secondary1');
+ const TString query(Q1_(R"(
+ UPDATE `/Root/TestTable` ON (Key, Index2, Value)
+ (SELECT Key, Index2, 'Val1_1' as Value FROM `/Root/TestTable` VIEW Index WHERE Index2 = 'Secondary1');
)"));
auto result = session.ExecuteDataQuery(
@@ -2253,8 +2253,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
CreateSampleTablesWithIndex(session);
{
- const TString query(Q1_(R"(
- SELECT Key FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk = 2;
+ const TString query(Q1_(R"(
+ SELECT Key FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk = 2;
)"));
auto result = session.ExecuteDataQuery(
@@ -2267,8 +2267,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
}
{
- const TString query(Q1_(R"(
- SELECT Key FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk > 1 LIMIT 1;
+ const TString query(Q1_(R"(
+ SELECT Key FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk > 1 LIMIT 1;
)"));
auto result = session.ExecuteDataQuery(
@@ -2281,8 +2281,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
}
{
- const TString query(Q1_(R"(
- SELECT Value FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk = 2;
+ const TString query(Q1_(R"(
+ SELECT Value FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk = 2;
)"));
auto result = session.ExecuteDataQuery(
@@ -2294,8 +2294,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
}
{
- const TString query(Q1_(R"(
- SELECT Value FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk > 1 LIMIT 1;
+ const TString query(Q1_(R"(
+ SELECT Value FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk > 1 LIMIT 1;
)"));
auto result = session.ExecuteDataQuery(
@@ -2307,8 +2307,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
}
{
- const TString query(Q1_(R"(
- SELECT Key FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk = 2 AND Key = 2;
+ const TString query(Q1_(R"(
+ SELECT Key FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk = 2 AND Key = 2;
)"));
auto result = session.ExecuteDataQuery(
@@ -2320,8 +2320,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
}
{
- const TString query(Q1_(R"(
- SELECT Value FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk = 2 AND Key = 2;
+ const TString query(Q1_(R"(
+ SELECT Value FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk = 2 AND Key = 2;
)"));
auto result = session.ExecuteDataQuery(
@@ -2333,8 +2333,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
}
{
- const TString query(Q1_(R"(
- SELECT Value FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk = 2 AND Value = "Payload2";
+ const TString query(Q1_(R"(
+ SELECT Value FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk = 2 AND Value = "Payload2";
)"));
auto result = session.ExecuteDataQuery(
@@ -2346,8 +2346,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
}
{
- const TString query(Q1_(R"(
- SELECT Key FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk = 2 AND Value = "Payload2";
+ const TString query(Q1_(R"(
+ SELECT Key FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk = 2 AND Value = "Payload2";
)"));
auto result = session.ExecuteDataQuery(
@@ -2359,8 +2359,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
}
{
- const TString query(Q1_(R"(
- SELECT Key, Fk, Value FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk = 2 AND Value = "Payload2" ORDER BY Fk, Key;
+ const TString query(Q1_(R"(
+ SELECT Key, Fk, Value FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk = 2 AND Value = "Payload2" ORDER BY Fk, Key;
)"));
auto result = session.ExecuteDataQuery(
@@ -2372,8 +2372,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
}
{
- const TString query(Q1_(R"(
- SELECT Value FROM `/Root/SecondaryKeys` VIEW Index WHERE Key = 2;
+ const TString query(Q1_(R"(
+ SELECT Value FROM `/Root/SecondaryKeys` VIEW Index WHERE Key = 2;
)"));
auto result = session.ExecuteDataQuery(
@@ -2382,15 +2382,15 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
.ExtractValueSync();
if (UseNewEngine) {
- UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_WRONG_INDEX_USAGE,
- [](const NYql::TIssue& issue) {
- return issue.Message.Contains("Given predicate is not suitable for used index: Index");
- }), result.GetIssues().ToString());
+ UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_WRONG_INDEX_USAGE,
+ [](const NYql::TIssue& issue) {
+ return issue.Message.Contains("Given predicate is not suitable for used index: Index");
+ }), result.GetIssues().ToString());
} else {
- UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_WRONG_INDEX_USAGE,
- [](const NYql::TIssue& issue) {
- return issue.Message.Contains("Given predicate is not suitable for used index");
- }), result.GetIssues().ToString());
+ UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_WRONG_INDEX_USAGE,
+ [](const NYql::TIssue& issue) {
+ return issue.Message.Contains("Given predicate is not suitable for used index");
+ }), result.GetIssues().ToString());
}
UNIT_ASSERT(result.IsSuccess());
@@ -2398,8 +2398,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
}
{
- const TString query(Q1_(R"(
- SELECT Fk FROM `/Root/SecondaryKeys` VIEW Index WHERE Key = 2;
+ const TString query(Q1_(R"(
+ SELECT Fk FROM `/Root/SecondaryKeys` VIEW Index WHERE Key = 2;
)"));
auto result = session.ExecuteDataQuery(
@@ -2407,10 +2407,10 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx())
.ExtractValueSync();
if (UseNewEngine) {
- UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_WRONG_INDEX_USAGE,
- [](const NYql::TIssue& issue) {
- return issue.Message.Contains("Given predicate is not suitable for used index: Index");
- }), result.GetIssues().ToString());
+ UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_WRONG_INDEX_USAGE,
+ [](const NYql::TIssue& issue) {
+ return issue.Message.Contains("Given predicate is not suitable for used index: Index");
+ }), result.GetIssues().ToString());
} else {
UNIT_ASSERT_C(result.GetIssues().Empty(), result.GetIssues().ToString());
}
@@ -2419,8 +2419,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
}
{
- const TString query(Q1_(R"(
- SELECT Value, Key FROM `/Root/SecondaryKeys` VIEW Index WHERE Key > 2 ORDER BY Key DESC;
+ const TString query(Q1_(R"(
+ SELECT Value, Key FROM `/Root/SecondaryKeys` VIEW Index WHERE Key > 2 ORDER BY Key DESC;
)"));
auto result = session.ExecuteDataQuery(
@@ -2468,7 +2468,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
const TString query1(Q_(R"(
- UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES
+ UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES
(1, 1001, "Value1"),
(2, 1002, "Value2"),
(3, 1003, "Value3");
@@ -2482,8 +2482,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
}
{
- const TString query(Q1_(R"(
- SELECT * FROM `/Root/TestTable` VIEW Index as t ORDER BY t.Index2 DESC;
+ const TString query(Q1_(R"(
+ SELECT * FROM `/Root/TestTable` VIEW Index as t ORDER BY t.Index2 DESC;
)"));
{
@@ -2514,8 +2514,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
}
{
- const TString query(Q1_(R"(
- SELECT * FROM `/Root/TestTable` VIEW Index as t ORDER BY t.Index2 DESC LIMIT 2;
+ const TString query(Q1_(R"(
+ SELECT * FROM `/Root/TestTable` VIEW Index as t ORDER BY t.Index2 DESC LIMIT 2;
)"));
{
@@ -2547,8 +2547,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
}
{
- const TString query(Q1_(R"(
- SELECT Index2, Key FROM `/Root/TestTable` VIEW Index as t ORDER BY t.Index2 DESC LIMIT 2;
+ const TString query(Q1_(R"(
+ SELECT Index2, Key FROM `/Root/TestTable` VIEW Index as t ORDER BY t.Index2 DESC LIMIT 2;
)"));
{
@@ -2579,8 +2579,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
}
{
- const TString query(Q1_(R"(
- SELECT * FROM `/Root/TestTable` VIEW Index as t WHERE t.Index2 < 1003 ORDER BY t.Index2 DESC;
+ const TString query(Q1_(R"(
+ SELECT * FROM `/Root/TestTable` VIEW Index as t WHERE t.Index2 < 1003 ORDER BY t.Index2 DESC;
)"));
{
@@ -2611,8 +2611,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
}
{
- const TString query(Q1_(R"(
- SELECT * FROM `/Root/TestTable` VIEW Index as t ORDER BY t.Index2;
+ const TString query(Q1_(R"(
+ SELECT * FROM `/Root/TestTable` VIEW Index as t ORDER BY t.Index2;
)"));
{
@@ -2642,8 +2642,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
}
{
- const TString query(Q1_(R"(
- SELECT * FROM `/Root/TestTable` VIEW Index as t ORDER BY t.Index2 LIMIT 2;
+ const TString query(Q1_(R"(
+ SELECT * FROM `/Root/TestTable` VIEW Index as t ORDER BY t.Index2 LIMIT 2;
)"));
{
@@ -2675,8 +2675,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
}
{
- const TString query(Q1_(R"(
- SELECT * FROM `/Root/TestTable` VIEW Index as t
+ const TString query(Q1_(R"(
+ SELECT * FROM `/Root/TestTable` VIEW Index as t
WHERE t.Index2 > 1001
ORDER BY t.Index2 LIMIT 2;
)"));
@@ -2709,8 +2709,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
}
{
- const TString query(Q1_(R"(
- SELECT * FROM `/Root/TestTable` VIEW Index as t
+ const TString query(Q1_(R"(
+ SELECT * FROM `/Root/TestTable` VIEW Index as t
WHERE t.Index2 = 1002
ORDER BY t.Index2 LIMIT 2;
)"));
@@ -2744,8 +2744,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
- const TString query(Q1_(R"(
- SELECT Index2, Key FROM `/Root/TestTable` VIEW Index as t
+ const TString query(Q1_(R"(
+ SELECT Index2, Key FROM `/Root/TestTable` VIEW Index as t
WHERE t.Index2 > 1001
ORDER BY t.Index2 LIMIT 2;
)"));
@@ -2779,8 +2779,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
// Request by Key but using Index table, expect correct result
- const TString query(Q1_(R"(
- SELECT Index2, Key FROM `/Root/TestTable` VIEW Index as t
+ const TString query(Q1_(R"(
+ SELECT Index2, Key FROM `/Root/TestTable` VIEW Index as t
WHERE t.Key > 1
ORDER BY t.Index2 LIMIT 2;
)"));
@@ -2841,7 +2841,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
const TString query1(Q_(R"(
- UPSERT INTO `/Root/TestTable` (id, customer, created, processed) VALUES
+ UPSERT INTO `/Root/TestTable` (id, customer, created, processed) VALUES
(1, "Vasya", CAST('2020-01-01T00:00:01Z' as DATETIME), "Value1"),
(2, "Vova", CAST('2020-01-01T00:00:02Z' as DATETIME), "Value2"),
(3, "Petya", CAST('2020-01-01T00:00:03Z' as DATETIME), "Value3"),
@@ -2860,8 +2860,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
{
- const TString query(Q1_(R"(
- SELECT * FROM `/Root/TestTable` VIEW ix_cust as t WHERE t.customer = "Vasya" ORDER BY t.customer DESC;
+ const TString query(Q1_(R"(
+ SELECT * FROM `/Root/TestTable` VIEW ix_cust as t WHERE t.customer = "Vasya" ORDER BY t.customer DESC;
)"));
{
@@ -2910,8 +2910,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
}
{
- const TString query(Q1_(R"(
- SELECT * FROM `/Root/TestTable` VIEW ix_cust2 as t WHERE t.customer = "Vasya" ORDER BY t.customer DESC, t.created DESC LIMIT 2;
+ const TString query(Q1_(R"(
+ SELECT * FROM `/Root/TestTable` VIEW ix_cust2 as t WHERE t.customer = "Vasya" ORDER BY t.customer DESC, t.created DESC LIMIT 2;
)"));
{
@@ -2961,8 +2961,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
}
{
- const TString query(Q1_(R"(
- SELECT * FROM `/Root/TestTable` VIEW ix_cust3 as t WHERE t.customer = "Vasya" ORDER BY t.customer DESC, t.created DESC LIMIT 2;
+ const TString query(Q1_(R"(
+ SELECT * FROM `/Root/TestTable` VIEW ix_cust3 as t WHERE t.customer = "Vasya" ORDER BY t.customer DESC, t.created DESC LIMIT 2;
)"));
{
@@ -3033,7 +3033,7 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
{
const TString query(Q_(R"(
- REPLACE INTO `/Root/TestTable` (Key, KeyA, Index2, Index2A, Value) VALUES
+ REPLACE INTO `/Root/TestTable` (Key, KeyA, Index2, Index2A, Value) VALUES
("Primary1", 41, "Secondary1", 1, "Value1"),
("Primary2", 42, "Secondary2", 2, "Value2"),
("Primary3", 43, "Secondary3", 3, "Value3");
@@ -3060,7 +3060,7 @@ R"([[["Secondary1"];[1u];["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[
{
const TString query(Q_(R"(
- REPLACE INTO `/Root/TestTable` (Key, KeyA, Value) VALUES
+ REPLACE INTO `/Root/TestTable` (Key, KeyA, Value) VALUES
("Primary1", 41, "Value1_1");
)"));
auto result = session.ExecuteDataQuery(
@@ -3110,7 +3110,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
{
const TString query(Q_(R"(
- INSERT INTO `/Root/TestTable` (Key, Value) VALUES
+ INSERT INTO `/Root/TestTable` (Key, Value) VALUES
("Primary1", "Value1"),
("Primary2", "Value2"),
("Primary3", "Value3");
@@ -3156,7 +3156,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
}
const TString query1(Q_(R"(
- UPSERT INTO `/Root/TestTable` (Key, Value1, Value2) VALUES
+ UPSERT INTO `/Root/TestTable` (Key, Value1, Value2) VALUES
("Primary1", "Val1", "Val2");
)"));
@@ -3223,7 +3223,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
{
const TString query1(Q_(R"(
- UPSERT INTO `/Root/TestTable` (Key, Value1, Value2, Value3) VALUES
+ UPSERT INTO `/Root/TestTable` (Key, Value1, Value2, Value3) VALUES
("Primary1", "Val1", "Val2", 42);
)"));
@@ -3251,8 +3251,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
}
{
- const TString query(Q1_(R"(
- SELECT Value2 FROM `/Root/TestTable` VIEW Index1 WHERE Value1 = 'Val1';
+ const TString query(Q1_(R"(
+ SELECT Value2 FROM `/Root/TestTable` VIEW Index1 WHERE Value1 = 'Val1';
)"));
auto result = session.ExecuteDataQuery(
@@ -3265,7 +3265,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
{
const TString query1(Q_(R"(
- REPLACE INTO `/Root/TestTable` (Key, Value1, Value2, Value3) VALUES
+ REPLACE INTO `/Root/TestTable` (Key, Value1, Value2, Value3) VALUES
("Primary1", "Val1_1", "Val2_1", 43);
)"));
@@ -3293,7 +3293,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
}
{
const TString query1(Q_(R"(
- REPLACE INTO `/Root/TestTable` (Key, Value1, Value2) VALUES
+ REPLACE INTO `/Root/TestTable` (Key, Value1, Value2) VALUES
("Primary1", "Val1_1", "Val2_1");
)"));
@@ -3321,7 +3321,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
}
{
const TString query1(Q_(R"(
- UPDATE `/Root/TestTable` SET 'Value3' = 35;
+ UPDATE `/Root/TestTable` SET 'Value3' = 35;
)"));
auto result = session.ExecuteDataQuery(
@@ -3348,7 +3348,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
}
{
const TString query1(Q_(R"(
- UPDATE `/Root/TestTable` ON (Key, Value3) VALUES ('Primary1', 36);
+ UPDATE `/Root/TestTable` ON (Key, Value3) VALUES ('Primary1', 36);
)"));
auto result = session.ExecuteDataQuery(
@@ -3375,7 +3375,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
}
{
const TString query1(Q_(R"(
- UPDATE `/Root/TestTable` ON (Key, Value1) VALUES ('Primary1', 'Val1_2');
+ UPDATE `/Root/TestTable` ON (Key, Value1) VALUES ('Primary1', 'Val1_2');
)"));
auto result = session.ExecuteDataQuery(
@@ -3402,7 +3402,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
}
{
const TString query1(Q_(R"(
- INSERT INTO `/Root/TestTable` (Key, Value2) VALUES ('Primary2', 'Record2');
+ INSERT INTO `/Root/TestTable` (Key, Value2) VALUES ('Primary2', 'Record2');
)"));
auto result = session.ExecuteDataQuery(
@@ -3429,7 +3429,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
}
{
const TString query1(Q_(R"(
- INSERT INTO `/Root/TestTable` (Key, Value3) VALUES ('Primary3', 37);
+ INSERT INTO `/Root/TestTable` (Key, Value3) VALUES ('Primary3', 37);
)"));
auto result = session.ExecuteDataQuery(
@@ -3456,7 +3456,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
}
{
const TString query1(Q_(R"(
- DELETE FROM `/Root/TestTable` WHERE Key = 'Primary3';
+ DELETE FROM `/Root/TestTable` WHERE Key = 'Primary3';
)"));
auto result = session.ExecuteDataQuery(
@@ -3483,7 +3483,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
}
{
const TString query1(Q_(R"(
- DELETE FROM `/Root/TestTable` ON (Key) VALUES ('Primary2');
+ DELETE FROM `/Root/TestTable` ON (Key) VALUES ('Primary2');
)"));
auto result = session.ExecuteDataQuery(
@@ -3511,7 +3511,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
{
const TString query1(Q_(R"(
- DELETE FROM `/Root/TestTable`;
+ DELETE FROM `/Root/TestTable`;
)"));
auto result = session.ExecuteDataQuery(
@@ -3561,7 +3561,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
{
const TString query1(Q_(R"(
- UPSERT INTO `/Root/TestTable` (Key, KeyA, Value1, Payload) VALUES
+ UPSERT INTO `/Root/TestTable` (Key, KeyA, Value1, Payload) VALUES
("Primary1", 42, "Val1", "SomeData");
)"));
@@ -3590,8 +3590,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
}
{
- const TString query(Q1_(R"(
- SELECT Key FROM `/Root/TestTable` VIEW Index1 WHERE Value1 = 'Val1';
+ const TString query(Q1_(R"(
+ SELECT Key FROM `/Root/TestTable` VIEW Index1 WHERE Value1 = 'Val1';
)"));
auto result = session.ExecuteDataQuery(
@@ -3603,8 +3603,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
}
{
- const TString query(Q1_(R"(
- SELECT Key, Value1, Payload FROM `/Root/TestTable` VIEW Index2 WHERE Key = 'Primary1' ORDER BY Key, Value1;
+ const TString query(Q1_(R"(
+ SELECT Key, Value1, Payload FROM `/Root/TestTable` VIEW Index2 WHERE Key = 'Primary1' ORDER BY Key, Value1;
)"));
auto result = session.ExecuteDataQuery(
@@ -3617,7 +3617,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
{
const TString query1(Q_(R"(
- UPSERT INTO `/Root/TestTable` (Key, KeyA, Value1, Payload) VALUES
+ UPSERT INTO `/Root/TestTable` (Key, KeyA, Value1, Payload) VALUES
("Primary1", 42, "Val1_0", "SomeData2");
)"));
@@ -3648,7 +3648,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
{
const TString query1(Q_(R"(
- REPLACE INTO `/Root/TestTable` (Key, KeyA, Value1) VALUES
+ REPLACE INTO `/Root/TestTable` (Key, KeyA, Value1) VALUES
("Primary1", 42, "Val1_1");
)"));
@@ -3678,7 +3678,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
{
const TString query1(Q_(R"(
- REPLACE INTO `/Root/TestTable` (Key, KeyA) VALUES
+ REPLACE INTO `/Root/TestTable` (Key, KeyA) VALUES
("Primary1", 42);
)"));
@@ -3701,7 +3701,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
{
const TString query1(Q_(R"(
- UPDATE `/Root/TestTable` SET 'Value1' = 'Val1_2';
+ UPDATE `/Root/TestTable` SET 'Value1' = 'Val1_2';
)"));
auto result = session.ExecuteDataQuery(
@@ -3723,7 +3723,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
{
const TString query1(Q_(R"(
- UPDATE `/Root/TestTable` ON (Key, KeyA, Value1) VALUES ('Primary1', 42, 'Val1_3');
+ UPDATE `/Root/TestTable` ON (Key, KeyA, Value1) VALUES ('Primary1', 42, 'Val1_3');
)"));
auto result = session.ExecuteDataQuery(
@@ -3745,7 +3745,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
{
const TString query1(Q_(R"(
- INSERT INTO `/Root/TestTable` (Key, KeyA, Value1) VALUES ('Primary2', 43, 'Val2');
+ INSERT INTO `/Root/TestTable` (Key, KeyA, Value1) VALUES ('Primary2', 43, 'Val2');
)"));
auto result = session.ExecuteDataQuery(
@@ -3767,7 +3767,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
{
const TString query1(Q_(R"(
- DELETE FROM `/Root/TestTable` WHERE Key = 'Primary2';
+ DELETE FROM `/Root/TestTable` WHERE Key = 'Primary2';
)"));
auto result = session.ExecuteDataQuery(
@@ -3789,7 +3789,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
{
const TString query1(Q_(R"(
- DELETE FROM `/Root/TestTable` ON (Key, KeyA) VALUES ('Primary1', 42);
+ DELETE FROM `/Root/TestTable` ON (Key, KeyA) VALUES ('Primary1', 42);
)"));
auto result = session.ExecuteDataQuery(
@@ -3820,11 +3820,11 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
CreateSampleTablesWithIndex(session);
const TString query(R"(
- --!syntax_v1
- PRAGMA Kikimr.UseNewEngine = 'false';
- DECLARE $keys AS List<Tuple<Int32, String>>;
+ --!syntax_v1
+ PRAGMA Kikimr.UseNewEngine = 'false';
+ DECLARE $keys AS List<Tuple<Int32, String>>;
$to_delete = (
- SELECT Key FROM `/Root/SecondaryComplexKeys` VIEW Index WHERE (Fk1, Fk2) in $keys
+ SELECT Key FROM `/Root/SecondaryComplexKeys` VIEW Index WHERE (Fk1, Fk2) in $keys
);
DELETE FROM `/Root/SecondaryComplexKeys` ON
SELECT * FROM $to_delete;
@@ -3898,12 +3898,12 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
}
{
const TString query1(Q_(R"(
- UPSERT INTO `/Root/TestTable1` (Key, Value) VALUES
+ UPSERT INTO `/Root/TestTable1` (Key, Value) VALUES
("Table1Primary3", 3),
("Table1Primary4", 4),
("Table1Primary55", 55);
- UPSERT INTO `/Root/TestTable2` (Key, Value) VALUES
+ UPSERT INTO `/Root/TestTable2` (Key, Value) VALUES
("Table2Primary1", 1),
("Table2Primary2", 2),
("Table2Primary3", 3),
@@ -3922,9 +3922,9 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
{
- const TString query(Q1_(R"(
- SELECT `/Root/TestTable1`.Key FROM `/Root/TestTable1`
- INNER JOIN `/Root/TestTable2` VIEW Index1 as t2 ON t2.Value = `/Root/TestTable1`.Value;
+ const TString query(Q1_(R"(
+ SELECT `/Root/TestTable1`.Key FROM `/Root/TestTable1`
+ INNER JOIN `/Root/TestTable2` VIEW Index1 as t2 ON t2.Value = `/Root/TestTable1`.Value;
)"));
auto result = session.ExecuteDataQuery(
@@ -3955,9 +3955,9 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
}
{
- const TString query(Q1_(R"(
- SELECT `/Root/TestTable1`.Key, `/Root/TestTable1`.Value FROM `/Root/TestTable1`
- INNER JOIN `/Root/TestTable2` VIEW Index1 as t2 ON t2.Value = `/Root/TestTable1`.Value ORDER BY `/Root/TestTable1`.Value DESC;
+ const TString query(Q1_(R"(
+ SELECT `/Root/TestTable1`.Key, `/Root/TestTable1`.Value FROM `/Root/TestTable1`
+ INNER JOIN `/Root/TestTable2` VIEW Index1 as t2 ON t2.Value = `/Root/TestTable1`.Value ORDER BY `/Root/TestTable1`.Value DESC;
)"));
auto result = session.ExecuteDataQuery(
@@ -3988,9 +3988,9 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
}
{
- const TString query(Q1_(R"(
- SELECT `/Root/TestTable1`.Key FROM `/Root/TestTable1`
- LEFT JOIN `/Root/TestTable2` VIEW Index1 as t2 ON t2.Value = `/Root/TestTable1`.Value;
+ const TString query(Q1_(R"(
+ SELECT `/Root/TestTable1`.Key FROM `/Root/TestTable1`
+ LEFT JOIN `/Root/TestTable2` VIEW Index1 as t2 ON t2.Value = `/Root/TestTable1`.Value;
)"));
auto result = session.ExecuteDataQuery(
@@ -4021,9 +4021,9 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
}
{
- const TString query(Q1_(R"(
- SELECT `/Root/TestTable1`.Key, `/Root/TestTable1`.Value FROM `/Root/TestTable1`
- LEFT JOIN `/Root/TestTable2` VIEW Index1 as t2 ON t2.Value = `/Root/TestTable1`.Value ORDER BY `/Root/TestTable1`.Value DESC;
+ const TString query(Q1_(R"(
+ SELECT `/Root/TestTable1`.Key, `/Root/TestTable1`.Value FROM `/Root/TestTable1`
+ LEFT JOIN `/Root/TestTable2` VIEW Index1 as t2 ON t2.Value = `/Root/TestTable1`.Value ORDER BY `/Root/TestTable1`.Value DESC;
)"));
auto result = session.ExecuteDataQuery(
@@ -4093,12 +4093,12 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
}
{
const TString query1(Q_(R"(
- UPSERT INTO `/Root/TestTable1` (Key, Value) VALUES
+ UPSERT INTO `/Root/TestTable1` (Key, Value) VALUES
("Table1Primary3", 3),
("Table1Primary4", 4),
("Table1Primary55", 55);
- UPSERT INTO `/Root/TestTable2` (Key, Value, Value2) VALUES
+ UPSERT INTO `/Root/TestTable2` (Key, Value, Value2) VALUES
("Table2Primary1", 1, "aa"),
("Table2Primary2", 2, "bb"),
("Table2Primary3", 3, "cc"),
@@ -4113,9 +4113,9 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
UNIT_ASSERT(result.IsSuccess());
}
{
- const TString query(Q1_(R"(
- SELECT t1.Key, t2.Value2 FROM `/Root/TestTable1` as t1
- INNER JOIN `/Root/TestTable2` VIEW Index1 as t2 ON t1.Value = t2.Value;
+ const TString query(Q1_(R"(
+ SELECT t1.Key, t2.Value2 FROM `/Root/TestTable1` as t1
+ INNER JOIN `/Root/TestTable2` VIEW Index1 as t2 ON t1.Value = t2.Value;
)"));
auto result = session.ExecuteDataQuery(
@@ -4152,9 +4152,9 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
}
{
- const TString query(Q1_(R"(
- SELECT t1.Key, t2.Value2 FROM `/Root/TestTable1` as t1
- LEFT JOIN `/Root/TestTable2` VIEW Index1 as t2 ON t1.Value = t2.Value;
+ const TString query(Q1_(R"(
+ SELECT t1.Key, t2.Value2 FROM `/Root/TestTable1` as t1
+ LEFT JOIN `/Root/TestTable2` VIEW Index1 as t2 ON t1.Value = t2.Value;
)"));
auto result = session.ExecuteDataQuery(
@@ -4214,7 +4214,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
const TString query1(Q_(R"(
- UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES
+ UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES
("Primary1", "Secondary1", "Value1");
)"));
@@ -4226,8 +4226,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
}
{
- const TString query(Q1_(R"(
- INSERT INTO `/Root/TestTable` VIEW Index (Index2, Key) VALUES('Secondary2', 'Primary2');
+ const TString query(Q1_(R"(
+ INSERT INTO `/Root/TestTable` VIEW Index (Index2, Key) VALUES('Secondary2', 'Primary2');
)"));
auto result = session.ExecuteDataQuery(
@@ -4239,8 +4239,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
}
{
- const TString query(Q1_(R"(
- UPSERT INTO `/Root/TestTable` VIEW Index (Index2, Key) VALUES('Secondary2', 'Primary2');
+ const TString query(Q1_(R"(
+ UPSERT INTO `/Root/TestTable` VIEW Index (Index2, Key) VALUES('Secondary2', 'Primary2');
)"));
auto result = session.ExecuteDataQuery(
@@ -4252,8 +4252,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
}
{
- const TString query(Q1_(R"(
- UPDATE `/Root/TestTable` VIEW Index ON (Index2, Key) VALUES ('Primary1', 'Secondary1');
+ const TString query(Q1_(R"(
+ UPDATE `/Root/TestTable` VIEW Index ON (Index2, Key) VALUES ('Primary1', 'Secondary1');
)"));
auto result = session.ExecuteDataQuery(
@@ -4265,8 +4265,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
}
{
- const TString query(Q1_(R"(
- DELETE FROM `/Root/TestTable` VIEW Index WHERE Index2 = 'Secondary1';
+ const TString query(Q1_(R"(
+ DELETE FROM `/Root/TestTable` VIEW Index WHERE Index2 = 'Secondary1';
)"));
auto result = session.ExecuteDataQuery(
@@ -4353,7 +4353,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
{
const TString query(Q_(R"(
- UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES
+ UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES
("Primary1", "Secondary1", "Value1"),
("Primary1", "Secondary2", "Value2"),
("Primary1", "Secondary3", "Value3");
@@ -4390,7 +4390,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
{
const TString query(Q_(R"(
- UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES
+ UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES
("Primary1", "Secondary1", "Value1"),
("Primary2", "Secondary2", "Value2"),
("Primary1", "Secondary11", "Value3"),
@@ -4427,14 +4427,14 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
{
- const TString query(Q1_(R"(
+ const TString query(Q1_(R"(
DECLARE $rows AS
- List<Struct<
+ List<Struct<
Key: String?,
Index2: String?,
- Value: String?>>;
+ Value: String?>>;
- UPSERT INTO `/Root/TestTable`
+ UPSERT INTO `/Root/TestTable`
SELECT Key, Index2, Value FROM AS_TABLE($rows);
)"));
@@ -4516,7 +4516,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
{
const TString query1(Q_(R"(
- UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES
+ UPSERT INTO `/Root/TestTable` (Key, Index2, Value) VALUES
("Primary1", "Secondary1", "Value1"),
("Primary2", "Secondary2", "Value2");
)"));
@@ -4528,7 +4528,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
const TString query2(Q_(R"(
- DELETE FROM `/Root/TestTable` ON (Key) VALUES
+ DELETE FROM `/Root/TestTable` ON (Key) VALUES
("Primary1"),
("Primary2");
)"));
@@ -4678,8 +4678,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
auto queryId = session.PrepareDataQuery(query).ExtractValueSync().GetQuery();
const auto variants = TVector<std::pair<TTxSettings, EStatus>>{
- {TTxSettings::SerializableRW(), UseNewEngine ? EStatus::PRECONDITION_FAILED : EStatus::GENERIC_ERROR},
- {TTxSettings::OnlineRO(), UseNewEngine ? EStatus::PRECONDITION_FAILED : EStatus::GENERIC_ERROR},
+ {TTxSettings::SerializableRW(), UseNewEngine ? EStatus::PRECONDITION_FAILED : EStatus::GENERIC_ERROR},
+ {TTxSettings::OnlineRO(), UseNewEngine ? EStatus::PRECONDITION_FAILED : EStatus::GENERIC_ERROR},
{TTxSettings::StaleRO(), EStatus::SUCCESS},
};
diff --git a/ydb/core/kqp/ut/kqp_join_ut.cpp b/ydb/core/kqp/ut/kqp_join_ut.cpp
index cff4881ad7..8fa59d31b3 100644
--- a/ydb/core/kqp/ut/kqp_join_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_join_ut.cpp
@@ -61,8 +61,8 @@ static void CreateSampleTables(TSession session) {
)").GetValueSync().IsSuccess());
UNIT_ASSERT(session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
-
+ PRAGMA kikimr.UseNewEngine = "true";
+
REPLACE INTO `/Root/Join1_1` (Key, Fk21, Fk22, Value) VALUES
(1, 101, "One", "Value1"),
(2, 102, "Two", "Value1"),
@@ -93,94 +93,94 @@ static void CreateSampleTables(TSession session) {
)", TTxControl::BeginTx().CommitTx()).GetValueSync().IsSuccess());
}
-static void CreateRightSemiJoinSampleTables(TSession& session) {
- UNIT_ASSERT(session.ExecuteSchemeQuery(R"(
- CREATE TABLE `/Root/RSJ_SimpleKey_1` (
- Key Int32,
- Value String,
- PRIMARY KEY (Key)
- );
- CREATE TABLE `/Root/RSJ_SimpleKey_2` (
- Key Int32,
- Value String,
- PRIMARY KEY (Key)
- );
- CREATE TABLE `/Root/RSJ_CompositeKey_1` (
- Key1 Int32,
- Key2 String,
- Value String,
- PRIMARY KEY (Key1, Key2)
- );
- CREATE TABLE `/Root/RSJ_CompositeKey_2` (
- Key1 Int32,
- Key2 String,
- Value String,
- PRIMARY KEY (Key1, Key2)
- );
- CREATE TABLE `/Root/RSJ_SecondaryKeys_1` (
- Key Int32, SubKey1 Int32, SubKey2 String, Value String,
- PRIMARY KEY (Key),
- INDEX Index GLOBAL ON (SubKey1, SubKey2)
- )
- )").GetValueSync().IsSuccess());
-
- UNIT_ASSERT(session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
-
- REPLACE INTO `/Root/RSJ_SimpleKey_1` (Key, Value) VALUES
- (1, "1.One"),
- (2, "1.Two"),
- (3, "1.Three"),
- (4, "1.Four"),
- (NULL, "1.Null");
-
- REPLACE INTO `/Root/RSJ_SimpleKey_2` (Key, Value) VALUES
- (1, "2.One"),
- (2, "2.Two"),
- (5, "2.Five"),
- (6, "2.Six"),
- (NULL, NULL);
-
- REPLACE INTO `/Root/RSJ_CompositeKey_1` (Key1, Key2, Value) VALUES
- (1, "One", "1.1.One"),
- (2, "Two", "1.2.Two"),
- (3, "Three", "1.3.Three"),
- (6, "Six", "1.6.Six"),
- (7, NULL, "1.7.Null"),
- (NULL, "Eight", "1.Null.Eight"),
- (NULL, NULL, "1.Null.Null");
-
- REPLACE INTO `/Root/RSJ_CompositeKey_2` (Key1, Key2, Value) VALUES
- (1, "One", "2.1.One"),
- (6, "Six", "2.6.Six"),
- (NULL, "Null", "2.Null.Null");
-
- REPLACE INTO `/Root/RSJ_SecondaryKeys_1` (Key, SubKey1, SubKey2, Value) VALUES
- (1, 1, "2.One", "Payload1"), -- SubKey contains in the `/Root/RSJ_SimpleKey_2`.Value
- (5, 5, "2.Five", "Payload2"), -- SubKey contains in the `/Root/RSJ_SimpleKey_2`.Value
- (7, 7, "2.Seven", "Payload3"),
- (8, 8, "2.Eight", "Payload4"),
- (NULL, NULL, NULL, "Payload5")
- )", TTxControl::BeginTx().CommitTx()).GetValueSync().IsSuccess());
-}
-
-static TDataQueryResult ExecQuery(TSession& session, const TString& query, const TParams& params,
- const TString& expected, bool checkRewrite = true)
-{
- auto result = ExecQueryAndTestResult(session, query, params, expected);
-
- if (checkRewrite) {
- auto explain = session.ExplainDataQuery(query).GetValueSync();
- UNIT_ASSERT_C(explain.GetAst().Contains("PartitionByKey"), explain.GetAst());
- }
-
- return result;
-}
-
-static TParams NoParams = TParamsBuilder().Build();
-
+static void CreateRightSemiJoinSampleTables(TSession& session) {
+ UNIT_ASSERT(session.ExecuteSchemeQuery(R"(
+ CREATE TABLE `/Root/RSJ_SimpleKey_1` (
+ Key Int32,
+ Value String,
+ PRIMARY KEY (Key)
+ );
+ CREATE TABLE `/Root/RSJ_SimpleKey_2` (
+ Key Int32,
+ Value String,
+ PRIMARY KEY (Key)
+ );
+ CREATE TABLE `/Root/RSJ_CompositeKey_1` (
+ Key1 Int32,
+ Key2 String,
+ Value String,
+ PRIMARY KEY (Key1, Key2)
+ );
+ CREATE TABLE `/Root/RSJ_CompositeKey_2` (
+ Key1 Int32,
+ Key2 String,
+ Value String,
+ PRIMARY KEY (Key1, Key2)
+ );
+ CREATE TABLE `/Root/RSJ_SecondaryKeys_1` (
+ Key Int32, SubKey1 Int32, SubKey2 String, Value String,
+ PRIMARY KEY (Key),
+ INDEX Index GLOBAL ON (SubKey1, SubKey2)
+ )
+ )").GetValueSync().IsSuccess());
+
+ UNIT_ASSERT(session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+
+ REPLACE INTO `/Root/RSJ_SimpleKey_1` (Key, Value) VALUES
+ (1, "1.One"),
+ (2, "1.Two"),
+ (3, "1.Three"),
+ (4, "1.Four"),
+ (NULL, "1.Null");
+
+ REPLACE INTO `/Root/RSJ_SimpleKey_2` (Key, Value) VALUES
+ (1, "2.One"),
+ (2, "2.Two"),
+ (5, "2.Five"),
+ (6, "2.Six"),
+ (NULL, NULL);
+
+ REPLACE INTO `/Root/RSJ_CompositeKey_1` (Key1, Key2, Value) VALUES
+ (1, "One", "1.1.One"),
+ (2, "Two", "1.2.Two"),
+ (3, "Three", "1.3.Three"),
+ (6, "Six", "1.6.Six"),
+ (7, NULL, "1.7.Null"),
+ (NULL, "Eight", "1.Null.Eight"),
+ (NULL, NULL, "1.Null.Null");
+
+ REPLACE INTO `/Root/RSJ_CompositeKey_2` (Key1, Key2, Value) VALUES
+ (1, "One", "2.1.One"),
+ (6, "Six", "2.6.Six"),
+ (NULL, "Null", "2.Null.Null");
+
+ REPLACE INTO `/Root/RSJ_SecondaryKeys_1` (Key, SubKey1, SubKey2, Value) VALUES
+ (1, 1, "2.One", "Payload1"), -- SubKey contains in the `/Root/RSJ_SimpleKey_2`.Value
+ (5, 5, "2.Five", "Payload2"), -- SubKey contains in the `/Root/RSJ_SimpleKey_2`.Value
+ (7, 7, "2.Seven", "Payload3"),
+ (8, 8, "2.Eight", "Payload4"),
+ (NULL, NULL, NULL, "Payload5")
+ )", TTxControl::BeginTx().CommitTx()).GetValueSync().IsSuccess());
+}
+
+static TDataQueryResult ExecQuery(TSession& session, const TString& query, const TParams& params,
+ const TString& expected, bool checkRewrite = true)
+{
+ auto result = ExecQueryAndTestResult(session, query, params, expected);
+
+ if (checkRewrite) {
+ auto explain = session.ExplainDataQuery(query).GetValueSync();
+ UNIT_ASSERT_C(explain.GetAst().Contains("PartitionByKey"), explain.GetAst());
+ }
+
+ return result;
+}
+
+static TParams NoParams = TParamsBuilder().Build();
+
Y_UNIT_TEST_SUITE(KqpJoin) {
- Y_UNIT_TEST_NEW_ENGINE(IdxLookupLeftPredicate) {
+ Y_UNIT_TEST_NEW_ENGINE(IdxLookupLeftPredicate) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -190,12 +190,12 @@ Y_UNIT_TEST_SUITE(KqpJoin) {
NYdb::NTable::TExecDataQuerySettings execSettings;
execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
- auto result = session.ExecuteDataQuery(Q_(R"(
- SELECT * FROM `/Root/Join1_1` AS t1
- INNER JOIN `/Root/Join1_2` AS t2
- ON t1.Fk21 = t2.Key1 AND t1.Fk22 = t2.Key2
- WHERE t1.Value = 'Value3' AND t2.Value IS NOT NULL
- )"), TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync();
+ auto result = session.ExecuteDataQuery(Q_(R"(
+ SELECT * FROM `/Root/Join1_1` AS t1
+ INNER JOIN `/Root/Join1_2` AS t2
+ ON t1.Fk21 = t2.Key1 AND t1.Fk22 = t2.Key2
+ WHERE t1.Value = 'Value3' AND t2.Value IS NOT NULL
+ )"), TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
CompareYson(R"([[[105];["One"];[5];["Value3"];["Name2"];[105];["One"];["Value27"]]])",
@@ -209,18 +209,18 @@ Y_UNIT_TEST_SUITE(KqpJoin) {
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/Join1_1");
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 8);
- ui32 index = 1;
- if (UseNewEngine) {
- UNIT_ASSERT(stats.query_phases(1).table_access().empty()); // keys extraction for lookups
- index = 2;
- }
-
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(index).table_access().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(index).table_access(0).name(), "/Root/Join1_2");
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(index).table_access(0).reads().rows(), 1);
+ ui32 index = 1;
+ if (UseNewEngine) {
+ UNIT_ASSERT(stats.query_phases(1).table_access().empty()); // keys extraction for lookups
+ index = 2;
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(index).table_access().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(index).table_access(0).name(), "/Root/Join1_2");
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(index).table_access(0).reads().rows(), 1);
}
- Y_UNIT_TEST_NEW_ENGINE(IdxLookupPartialLeftPredicate) {
+ Y_UNIT_TEST_NEW_ENGINE(IdxLookupPartialLeftPredicate) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -230,12 +230,12 @@ Y_UNIT_TEST_SUITE(KqpJoin) {
NYdb::NTable::TExecDataQuerySettings execSettings;
execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
- auto result = session.ExecuteDataQuery(Q_(R"(
- SELECT * FROM `/Root/Join1_1` AS t1
- INNER JOIN `/Root/Join1_2` AS t2
+ auto result = session.ExecuteDataQuery(Q_(R"(
+ SELECT * FROM `/Root/Join1_1` AS t1
+ INNER JOIN `/Root/Join1_2` AS t2
ON t1.Fk21 == t2.Key1
WHERE t1.Value == "Value3";
- )"), TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync();
+ )"), TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
CompareYson(R"([
@@ -245,7 +245,7 @@ Y_UNIT_TEST_SUITE(KqpJoin) {
])", FormatResultSetYson(result.GetResultSet(0)));
auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
- Cerr << stats.DebugString() << Endl;
+ Cerr << stats.DebugString() << Endl;
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), kikimr.IsUsingSnapshotReads() && !UseNewEngine ? 2 : 3);
@@ -253,15 +253,15 @@ Y_UNIT_TEST_SUITE(KqpJoin) {
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/Join1_1");
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 8);
- ui32 index = 1;
- if (UseNewEngine) {
- UNIT_ASSERT(stats.query_phases(1).table_access().empty()); // keys extraction for lookups
- index = 2;
- }
-
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(index).table_access().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(index).table_access(0).name(), "/Root/Join1_2");
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(index).table_access(0).reads().rows(), 3);
+ ui32 index = 1;
+ if (UseNewEngine) {
+ UNIT_ASSERT(stats.query_phases(1).table_access().empty()); // keys extraction for lookups
+ index = 2;
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(index).table_access().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(index).table_access(0).name(), "/Root/Join1_2");
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(index).table_access(0).reads().rows(), 3);
}
Y_UNIT_TEST_NEW_ENGINE(IdxLookupPartialWithTempTable) {
@@ -309,10 +309,10 @@ Y_UNIT_TEST_SUITE(KqpJoin) {
CreateSampleTables(session);
auto result = session.ExecuteDataQuery(Q_(R"(
- SELECT * FROM `/Root/Join1_1` AS t1
- INNER JOIN `/Root/Join1_2` AS t2
+ SELECT * FROM `/Root/Join1_1` AS t1
+ INNER JOIN `/Root/Join1_2` AS t2
ON t1.Fk21 == t2.Key1 AND t1.Fk22 == t2.Key2
- LEFT JOIN `/Root/Join1_3` AS t3
+ LEFT JOIN `/Root/Join1_3` AS t3
ON t2.Fk3 = t3.Key
WHERE t1.Value == "Value5" AND t2.Value == "Value31";
)"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
@@ -321,412 +321,412 @@ Y_UNIT_TEST_SUITE(KqpJoin) {
CompareYson(R"([[[108];["One"];[8];["Value5"];#;[108];["One"];["Value31"];#;#]])",
FormatResultSetYson(result.GetResultSet(0)));
}
-
- // join on not key column => Full Scan
- Y_UNIT_TEST_NEW_ENGINE(RightSemiJoin_FullScan) {
- TKikimrRunner kikimr(SyntaxV1Settings());
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- CreateRightSemiJoinSampleTables(session);
-
- /* join with parameters */
- {
- const TString query = Q_(R"(
- DECLARE $in AS List<Struct<v: String?>>;
- SELECT *
- FROM AS_TABLE($in) AS k RIGHT SEMI JOIN `/Root/RSJ_SimpleKey_1` AS t ON k.v = t.Value
- )");
-
- auto params = TParamsBuilder().AddParam("$in").BeginList()
- .AddListItem().BeginStruct().AddMember("v").OptionalString("1.One").EndStruct()
- .AddListItem().BeginStruct().AddMember("v").OptionalString("1.One").EndStruct() // dup
- .AddListItem().BeginStruct().AddMember("v").OptionalString("1.Two").EndStruct()
- .AddListItem().BeginStruct().AddMember("v").OptionalString("Any").EndStruct() // not exists
- .AddListItem().BeginStruct().AddMember("v").OptionalString(Nothing()).EndStruct() // null
- .EndList().Build().Build();
-
- auto result = ExecQuery(session, query, params, R"([[[1];["1.One"]];[[2];["1.Two"]]])", false);
- AssertTableReads(result, "/Root/RSJ_SimpleKey_1", 5);
- }
-
- /* join with real table */
- {
- const TString query = Q_(R"(
- SELECT *
- FROM `/Root/RSJ_SimpleKey_1` AS l RIGHT SEMI JOIN `/Root/RSJ_SimpleKey_2` AS r on l.Value = r.Value
- )");
-
- auto result = ExecQuery(session, query, NoParams, R"([])", false);
- AssertTableReads(result, "/Root/RSJ_SimpleKey_1", 5);
- AssertTableReads(result, "/Root/RSJ_SimpleKey_2", 5);
- }
- }
-
- // join on key (simple and full) column => index-lookup
- Y_UNIT_TEST_NEW_ENGINE(RightSemiJoin_SimpleKey) {
- TKikimrRunner kikimr(SyntaxV1Settings());
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- CreateRightSemiJoinSampleTables(session);
-
- /* join with parameters */
- {
- const TString query = Q_(R"(
- DECLARE $in AS List<Struct<k: Int32?>>;
- SELECT *
- FROM AS_TABLE($in) AS k RIGHT SEMI JOIN `/Root/RSJ_SimpleKey_1` AS t ON k.k = t.Key
- )");
-
- auto params = TParamsBuilder().AddParam("$in").BeginList()
- .AddListItem().BeginStruct().AddMember("k").OptionalInt32(1).EndStruct()
- .AddListItem().BeginStruct().AddMember("k").OptionalInt32(1).EndStruct() // dup
- .AddListItem().BeginStruct().AddMember("k").OptionalInt32(2).EndStruct()
- .AddListItem().BeginStruct().AddMember("k").OptionalInt32(42).EndStruct() // not exists
- .AddListItem().BeginStruct().AddMember("k").OptionalInt32(Nothing()).EndStruct() // null
- .EndList().Build().Build();
-
- auto result = ExecQuery(session, query, params, R"([[[1];["1.One"]];[[2];["1.Two"]]])");
- AssertTableReads(result, "/Root/RSJ_SimpleKey_1", 2 /* [1, 2] */);
- }
-
- /* join with real table */
- {
- const TString query = Q_(R"(
- SELECT *
- FROM `/Root/RSJ_SimpleKey_1` AS l RIGHT SEMI JOIN `/Root/RSJ_SimpleKey_2` AS r ON l.Key = r.Key
- )");
-
- auto result = ExecQuery(session, query, NoParams, R"([[[1];["2.One"]];[[2];["2.Two"]]])");
- AssertTableReads(result, "/Root/RSJ_SimpleKey_1", 5 /* all keys */);
- AssertTableReads(result, "/Root/RSJ_SimpleKey_2", 2 /* [1, 2] */);
- }
- }
-
- // join on key (complex and full) column => index-lookup
- Y_UNIT_TEST_NEW_ENGINE(RightSemiJoin_ComplexKey) {
- TKikimrRunner kikimr(SyntaxV1Settings());
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- CreateRightSemiJoinSampleTables(session);
-
- /* join with parameters */
- {
- const TString query = Q_(R"(
- DECLARE $in AS List<Struct<k1: Int32?, k2: String?>>;
- SELECT *
- FROM AS_TABLE($in) AS k RIGHT SEMI JOIN `/Root/RSJ_CompositeKey_1` AS t
- ON k.k1 = t.Key1 AND k.k2 = t.Key2
- )");
-
- auto params = TParamsBuilder().AddParam("$in").BeginList()
- .AddListItem().BeginStruct().AddMember("k1").OptionalInt32(1)
- .AddMember("k2").OptionalString("One").EndStruct()
- .AddListItem().BeginStruct().AddMember("k1").OptionalInt32(1)
- .AddMember("k2").OptionalString("One").EndStruct() // dup
- .AddListItem().BeginStruct().AddMember("k1").OptionalInt32(2)
- .AddMember("k2").OptionalString("Two").EndStruct()
- .AddListItem().BeginStruct().AddMember("k1").OptionalInt32(42)
- .AddMember("k2").OptionalString("FortyTwo").EndStruct() // not exists
- .AddListItem().BeginStruct().AddMember("k1").OptionalInt32(Nothing())
- .AddMember("k2").OptionalString("One").EndStruct() // null
- .AddListItem().BeginStruct().AddMember("k1").OptionalInt32(1)
- .AddMember("k2").OptionalString(Nothing()).EndStruct() // null
- .AddListItem().BeginStruct().AddMember("k1").OptionalInt32(Nothing())
- .AddMember("k2").OptionalString(Nothing()).EndStruct() // null
- .EndList().Build().Build();
-
- auto result = ExecQuery(session, query, params, R"([[[1];["One"];["1.1.One"]];[[2];["Two"];["1.2.Two"]]])");
- AssertTableReads(result, "/Root/RSJ_CompositeKey_1", 2 /* [1, 2] */);
- }
-
- /* join with real table */
- {
- const TString query = Q_(R"(
- SELECT *
- FROM `/Root/RSJ_CompositeKey_1` AS l RIGHT SEMI JOIN `/Root/RSJ_CompositeKey_2` AS r
- ON l.Key1 = r.Key1 AND l.Key2 = r.Key2
- )");
-
- auto result = ExecQuery(session, query, NoParams, R"([[[1];["One"];["2.1.One"]];[[6];["Six"];["2.6.Six"]]])");
- AssertTableReads(result, "/Root/RSJ_CompositeKey_1", 7 /* all keys */);
- AssertTableReads(result, "/Root/RSJ_CompositeKey_2", 2 /* [1, 6] */);
- }
- }
-
- // join on key prefix => index-lookup
- Y_UNIT_TEST_NEW_ENGINE(RightSemiJoin_KeyPrefix) {
- TKikimrRunner kikimr(SyntaxV1Settings());
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- CreateRightSemiJoinSampleTables(session);
-
- /* join with parameters */
- {
- const TString query = Q_(R"(
- DECLARE $in AS List<Struct<k: Int32?>>;
- SELECT *
- FROM AS_TABLE($in) AS l RIGHT SEMI JOIN `/Root/RSJ_CompositeKey_1` AS r
- ON l.k = r.Key1
- )");
-
- auto params = TParamsBuilder().AddParam("$in").BeginList()
- .AddListItem().BeginStruct().AddMember("k").OptionalInt32(1).EndStruct()
- .AddListItem().BeginStruct().AddMember("k").OptionalInt32(1).EndStruct() // dup
- .AddListItem().BeginStruct().AddMember("k").OptionalInt32(2).EndStruct()
- .AddListItem().BeginStruct().AddMember("k").OptionalInt32(42).EndStruct() // not exists
- .AddListItem().BeginStruct().AddMember("k").OptionalInt32(Nothing()).EndStruct() // null
- .EndList().Build().Build();
-
- auto result = ExecQuery(session, query, params, R"([[[1];["One"];["1.1.One"]];[[2];["Two"];["1.2.Two"]]])");
- // read of not existing value |42| is not visible in statistics
- AssertTableReads(result, "/Root/RSJ_CompositeKey_1", 2 /* [1, 2, |42|] */);
- }
-
- /* join with real table */
- {
- const TString query = Q_(R"(
- SELECT *
- FROM `/Root/RSJ_SimpleKey_1` AS l RIGHT SEMI JOIN `/Root/RSJ_CompositeKey_1` AS r
- ON l.Key = r.Key1
- )");
-
- auto result = ExecQuery(session, query, NoParams, R"([[[1];["One"];["1.1.One"]];[[2];["Two"];["1.2.Two"]];[[3];["Three"];["1.3.Three"]]])");
- AssertTableReads(result, "/Root/RSJ_SimpleKey_1", 5 /* all rows */);
- AssertTableReads(result, "/Root/RSJ_CompositeKey_1", 3 /* [1, 2, 3] */);
- }
- }
-
- // join on secondary index => index-lookup
+
+ // join on not key column => Full Scan
+ Y_UNIT_TEST_NEW_ENGINE(RightSemiJoin_FullScan) {
+ TKikimrRunner kikimr(SyntaxV1Settings());
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ CreateRightSemiJoinSampleTables(session);
+
+ /* join with parameters */
+ {
+ const TString query = Q_(R"(
+ DECLARE $in AS List<Struct<v: String?>>;
+ SELECT *
+ FROM AS_TABLE($in) AS k RIGHT SEMI JOIN `/Root/RSJ_SimpleKey_1` AS t ON k.v = t.Value
+ )");
+
+ auto params = TParamsBuilder().AddParam("$in").BeginList()
+ .AddListItem().BeginStruct().AddMember("v").OptionalString("1.One").EndStruct()
+ .AddListItem().BeginStruct().AddMember("v").OptionalString("1.One").EndStruct() // dup
+ .AddListItem().BeginStruct().AddMember("v").OptionalString("1.Two").EndStruct()
+ .AddListItem().BeginStruct().AddMember("v").OptionalString("Any").EndStruct() // not exists
+ .AddListItem().BeginStruct().AddMember("v").OptionalString(Nothing()).EndStruct() // null
+ .EndList().Build().Build();
+
+ auto result = ExecQuery(session, query, params, R"([[[1];["1.One"]];[[2];["1.Two"]]])", false);
+ AssertTableReads(result, "/Root/RSJ_SimpleKey_1", 5);
+ }
+
+ /* join with real table */
+ {
+ const TString query = Q_(R"(
+ SELECT *
+ FROM `/Root/RSJ_SimpleKey_1` AS l RIGHT SEMI JOIN `/Root/RSJ_SimpleKey_2` AS r on l.Value = r.Value
+ )");
+
+ auto result = ExecQuery(session, query, NoParams, R"([])", false);
+ AssertTableReads(result, "/Root/RSJ_SimpleKey_1", 5);
+ AssertTableReads(result, "/Root/RSJ_SimpleKey_2", 5);
+ }
+ }
+
+ // join on key (simple and full) column => index-lookup
+ Y_UNIT_TEST_NEW_ENGINE(RightSemiJoin_SimpleKey) {
+ TKikimrRunner kikimr(SyntaxV1Settings());
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ CreateRightSemiJoinSampleTables(session);
+
+ /* join with parameters */
+ {
+ const TString query = Q_(R"(
+ DECLARE $in AS List<Struct<k: Int32?>>;
+ SELECT *
+ FROM AS_TABLE($in) AS k RIGHT SEMI JOIN `/Root/RSJ_SimpleKey_1` AS t ON k.k = t.Key
+ )");
+
+ auto params = TParamsBuilder().AddParam("$in").BeginList()
+ .AddListItem().BeginStruct().AddMember("k").OptionalInt32(1).EndStruct()
+ .AddListItem().BeginStruct().AddMember("k").OptionalInt32(1).EndStruct() // dup
+ .AddListItem().BeginStruct().AddMember("k").OptionalInt32(2).EndStruct()
+ .AddListItem().BeginStruct().AddMember("k").OptionalInt32(42).EndStruct() // not exists
+ .AddListItem().BeginStruct().AddMember("k").OptionalInt32(Nothing()).EndStruct() // null
+ .EndList().Build().Build();
+
+ auto result = ExecQuery(session, query, params, R"([[[1];["1.One"]];[[2];["1.Two"]]])");
+ AssertTableReads(result, "/Root/RSJ_SimpleKey_1", 2 /* [1, 2] */);
+ }
+
+ /* join with real table */
+ {
+ const TString query = Q_(R"(
+ SELECT *
+ FROM `/Root/RSJ_SimpleKey_1` AS l RIGHT SEMI JOIN `/Root/RSJ_SimpleKey_2` AS r ON l.Key = r.Key
+ )");
+
+ auto result = ExecQuery(session, query, NoParams, R"([[[1];["2.One"]];[[2];["2.Two"]]])");
+ AssertTableReads(result, "/Root/RSJ_SimpleKey_1", 5 /* all keys */);
+ AssertTableReads(result, "/Root/RSJ_SimpleKey_2", 2 /* [1, 2] */);
+ }
+ }
+
+ // join on key (complex and full) column => index-lookup
+ Y_UNIT_TEST_NEW_ENGINE(RightSemiJoin_ComplexKey) {
+ TKikimrRunner kikimr(SyntaxV1Settings());
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ CreateRightSemiJoinSampleTables(session);
+
+ /* join with parameters */
+ {
+ const TString query = Q_(R"(
+ DECLARE $in AS List<Struct<k1: Int32?, k2: String?>>;
+ SELECT *
+ FROM AS_TABLE($in) AS k RIGHT SEMI JOIN `/Root/RSJ_CompositeKey_1` AS t
+ ON k.k1 = t.Key1 AND k.k2 = t.Key2
+ )");
+
+ auto params = TParamsBuilder().AddParam("$in").BeginList()
+ .AddListItem().BeginStruct().AddMember("k1").OptionalInt32(1)
+ .AddMember("k2").OptionalString("One").EndStruct()
+ .AddListItem().BeginStruct().AddMember("k1").OptionalInt32(1)
+ .AddMember("k2").OptionalString("One").EndStruct() // dup
+ .AddListItem().BeginStruct().AddMember("k1").OptionalInt32(2)
+ .AddMember("k2").OptionalString("Two").EndStruct()
+ .AddListItem().BeginStruct().AddMember("k1").OptionalInt32(42)
+ .AddMember("k2").OptionalString("FortyTwo").EndStruct() // not exists
+ .AddListItem().BeginStruct().AddMember("k1").OptionalInt32(Nothing())
+ .AddMember("k2").OptionalString("One").EndStruct() // null
+ .AddListItem().BeginStruct().AddMember("k1").OptionalInt32(1)
+ .AddMember("k2").OptionalString(Nothing()).EndStruct() // null
+ .AddListItem().BeginStruct().AddMember("k1").OptionalInt32(Nothing())
+ .AddMember("k2").OptionalString(Nothing()).EndStruct() // null
+ .EndList().Build().Build();
+
+ auto result = ExecQuery(session, query, params, R"([[[1];["One"];["1.1.One"]];[[2];["Two"];["1.2.Two"]]])");
+ AssertTableReads(result, "/Root/RSJ_CompositeKey_1", 2 /* [1, 2] */);
+ }
+
+ /* join with real table */
+ {
+ const TString query = Q_(R"(
+ SELECT *
+ FROM `/Root/RSJ_CompositeKey_1` AS l RIGHT SEMI JOIN `/Root/RSJ_CompositeKey_2` AS r
+ ON l.Key1 = r.Key1 AND l.Key2 = r.Key2
+ )");
+
+ auto result = ExecQuery(session, query, NoParams, R"([[[1];["One"];["2.1.One"]];[[6];["Six"];["2.6.Six"]]])");
+ AssertTableReads(result, "/Root/RSJ_CompositeKey_1", 7 /* all keys */);
+ AssertTableReads(result, "/Root/RSJ_CompositeKey_2", 2 /* [1, 6] */);
+ }
+ }
+
+ // join on key prefix => index-lookup
+ Y_UNIT_TEST_NEW_ENGINE(RightSemiJoin_KeyPrefix) {
+ TKikimrRunner kikimr(SyntaxV1Settings());
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ CreateRightSemiJoinSampleTables(session);
+
+ /* join with parameters */
+ {
+ const TString query = Q_(R"(
+ DECLARE $in AS List<Struct<k: Int32?>>;
+ SELECT *
+ FROM AS_TABLE($in) AS l RIGHT SEMI JOIN `/Root/RSJ_CompositeKey_1` AS r
+ ON l.k = r.Key1
+ )");
+
+ auto params = TParamsBuilder().AddParam("$in").BeginList()
+ .AddListItem().BeginStruct().AddMember("k").OptionalInt32(1).EndStruct()
+ .AddListItem().BeginStruct().AddMember("k").OptionalInt32(1).EndStruct() // dup
+ .AddListItem().BeginStruct().AddMember("k").OptionalInt32(2).EndStruct()
+ .AddListItem().BeginStruct().AddMember("k").OptionalInt32(42).EndStruct() // not exists
+ .AddListItem().BeginStruct().AddMember("k").OptionalInt32(Nothing()).EndStruct() // null
+ .EndList().Build().Build();
+
+ auto result = ExecQuery(session, query, params, R"([[[1];["One"];["1.1.One"]];[[2];["Two"];["1.2.Two"]]])");
+ // read of not existing value |42| is not visible in statistics
+ AssertTableReads(result, "/Root/RSJ_CompositeKey_1", 2 /* [1, 2, |42|] */);
+ }
+
+ /* join with real table */
+ {
+ const TString query = Q_(R"(
+ SELECT *
+ FROM `/Root/RSJ_SimpleKey_1` AS l RIGHT SEMI JOIN `/Root/RSJ_CompositeKey_1` AS r
+ ON l.Key = r.Key1
+ )");
+
+ auto result = ExecQuery(session, query, NoParams, R"([[[1];["One"];["1.1.One"]];[[2];["Two"];["1.2.Two"]];[[3];["Three"];["1.3.Three"]]])");
+ AssertTableReads(result, "/Root/RSJ_SimpleKey_1", 5 /* all rows */);
+ AssertTableReads(result, "/Root/RSJ_CompositeKey_1", 3 /* [1, 2, 3] */);
+ }
+ }
+
+ // join on secondary index => index-lookup
Y_UNIT_TEST_NEW_ENGINE(RightSemiJoin_SecondaryIndex) {
- TKikimrRunner kikimr(SyntaxV1Settings());
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- CreateRightSemiJoinSampleTables(session);
-
- UNIT_ASSERT(session.ExecuteSchemeQuery(R"(
- CREATE TABLE `/Root/RSJ_SimpleKey_3` (
- Key Int32, SubKey String, Value String,
- PRIMARY KEY (Key),
- INDEX SubKeyIndex GLOBAL ON (SubKey)
- )
- )").GetValueSync().IsSuccess());
-
- UNIT_ASSERT(session.ExecuteDataQuery(Q_(R"(
- REPLACE INTO `/Root/RSJ_SimpleKey_3` (Key, SubKey, Value) VALUES
- (1, "2.One", "Payload1"), -- SubKey contains in the `/Root/RSJ_SimpleKey_2`.Value
- (5, "2.Five", "Payload2"), -- SubKey contains in the `/Root/RSJ_SimpleKey_2`.Value
- (7, "2.Seven", "Payload3"),
- (8, "2.Eight", "Payload4"),
- (NULL, NULL, "Payload5")
- )"), TTxControl::BeginTx().CommitTx()).GetValueSync().IsSuccess());
-
- /* join with parameters */
- {
+ TKikimrRunner kikimr(SyntaxV1Settings());
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ CreateRightSemiJoinSampleTables(session);
+
+ UNIT_ASSERT(session.ExecuteSchemeQuery(R"(
+ CREATE TABLE `/Root/RSJ_SimpleKey_3` (
+ Key Int32, SubKey String, Value String,
+ PRIMARY KEY (Key),
+ INDEX SubKeyIndex GLOBAL ON (SubKey)
+ )
+ )").GetValueSync().IsSuccess());
+
+ UNIT_ASSERT(session.ExecuteDataQuery(Q_(R"(
+ REPLACE INTO `/Root/RSJ_SimpleKey_3` (Key, SubKey, Value) VALUES
+ (1, "2.One", "Payload1"), -- SubKey contains in the `/Root/RSJ_SimpleKey_2`.Value
+ (5, "2.Five", "Payload2"), -- SubKey contains in the `/Root/RSJ_SimpleKey_2`.Value
+ (7, "2.Seven", "Payload3"),
+ (8, "2.Eight", "Payload4"),
+ (NULL, NULL, "Payload5")
+ )"), TTxControl::BeginTx().CommitTx()).GetValueSync().IsSuccess());
+
+ /* join with parameters */
+ {
const TString query = Q_(R"(
- DECLARE $in AS List<Struct<v: String?>>;
- SELECT *
- FROM AS_TABLE($in) AS l RIGHT SEMI JOIN `/Root/RSJ_SimpleKey_3` VIEW SubKeyIndex AS r
- ON l.v = r.SubKey
- ORDER BY Key
+ DECLARE $in AS List<Struct<v: String?>>;
+ SELECT *
+ FROM AS_TABLE($in) AS l RIGHT SEMI JOIN `/Root/RSJ_SimpleKey_3` VIEW SubKeyIndex AS r
+ ON l.v = r.SubKey
+ ORDER BY Key
)");
-
- auto params = TParamsBuilder().AddParam("$in").BeginList()
- .AddListItem().BeginStruct().AddMember("v").OptionalString("2.One").EndStruct()
- .AddListItem().BeginStruct().AddMember("v").OptionalString("2.One").EndStruct() // dup
- .AddListItem().BeginStruct().AddMember("v").OptionalString("2.Five").EndStruct()
- .AddListItem().BeginStruct().AddMember("v").OptionalString("Any").EndStruct() // not exists
- .AddListItem().BeginStruct().AddMember("v").OptionalString(Nothing()).EndStruct() // null
- .EndList().Build().Build();
-
- auto result = ExecQuery(session, query, params, R"([[[1];["2.One"];["Payload1"]];[[5];["2.Five"];["Payload2"]]])");
- // todo: reading of absent |Any| key is not visible in statistics :(
- AssertTableReads(result, "/Root/RSJ_SimpleKey_3/SubKeyIndex/indexImplTable", 2 /* [2.One, 2.Five, |Any|] */);
- AssertTableReads(result, "/Root/RSJ_SimpleKey_3", 2 /* read .Value field */);
- }
-
- /* join with real table */
- {
+
+ auto params = TParamsBuilder().AddParam("$in").BeginList()
+ .AddListItem().BeginStruct().AddMember("v").OptionalString("2.One").EndStruct()
+ .AddListItem().BeginStruct().AddMember("v").OptionalString("2.One").EndStruct() // dup
+ .AddListItem().BeginStruct().AddMember("v").OptionalString("2.Five").EndStruct()
+ .AddListItem().BeginStruct().AddMember("v").OptionalString("Any").EndStruct() // not exists
+ .AddListItem().BeginStruct().AddMember("v").OptionalString(Nothing()).EndStruct() // null
+ .EndList().Build().Build();
+
+ auto result = ExecQuery(session, query, params, R"([[[1];["2.One"];["Payload1"]];[[5];["2.Five"];["Payload2"]]])");
+ // todo: reading of absent |Any| key is not visible in statistics :(
+ AssertTableReads(result, "/Root/RSJ_SimpleKey_3/SubKeyIndex/indexImplTable", 2 /* [2.One, 2.Five, |Any|] */);
+ AssertTableReads(result, "/Root/RSJ_SimpleKey_3", 2 /* read .Value field */);
+ }
+
+ /* join with real table */
+ {
const TString query = Q_(R"(
- SELECT *
- FROM `/Root/RSJ_SimpleKey_2` AS l RIGHT SEMI JOIN `/Root/RSJ_SimpleKey_3` VIEW SubKeyIndex AS r
- ON l.Value = r.SubKey
- ORDER BY Key
+ SELECT *
+ FROM `/Root/RSJ_SimpleKey_2` AS l RIGHT SEMI JOIN `/Root/RSJ_SimpleKey_3` VIEW SubKeyIndex AS r
+ ON l.Value = r.SubKey
+ ORDER BY Key
)");
-
- auto result = ExecQuery(session, query, NoParams, R"([[[1];["2.One"];["Payload1"]];[[5];["2.Five"];["Payload2"]]])");
- AssertTableReads(result, "/Root/RSJ_SimpleKey_2", 5 /* all keys */);
- // todo: reading of absent |Any| key is not visible in statistics :(
- AssertTableReads(result, "/Root/RSJ_SimpleKey_3/SubKeyIndex/indexImplTable", 2 /* [2.One, 2.Five, |Any|] */);
- AssertTableReads(result, "/Root/RSJ_SimpleKey_3", 2 /* [1, 5] */);
- }
- }
-
- // join on complex secondary index => index-lookup
+
+ auto result = ExecQuery(session, query, NoParams, R"([[[1];["2.One"];["Payload1"]];[[5];["2.Five"];["Payload2"]]])");
+ AssertTableReads(result, "/Root/RSJ_SimpleKey_2", 5 /* all keys */);
+ // todo: reading of absent |Any| key is not visible in statistics :(
+ AssertTableReads(result, "/Root/RSJ_SimpleKey_3/SubKeyIndex/indexImplTable", 2 /* [2.One, 2.Five, |Any|] */);
+ AssertTableReads(result, "/Root/RSJ_SimpleKey_3", 2 /* [1, 5] */);
+ }
+ }
+
+ // join on complex secondary index => index-lookup
Y_UNIT_TEST_NEW_ENGINE(RightSemiJoin_ComplexSecondaryIndex) {
- TKikimrRunner kikimr(SyntaxV1Settings());
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- CreateRightSemiJoinSampleTables(session);
-
- /* join with parameters */
- {
+ TKikimrRunner kikimr(SyntaxV1Settings());
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ CreateRightSemiJoinSampleTables(session);
+
+ /* join with parameters */
+ {
const TString query = Q_(R"(
- DECLARE $in AS List<Struct<k: Int32?, v: String?>>;
- SELECT *
- FROM AS_TABLE($in) AS l RIGHT SEMI JOIN `/Root/RSJ_SecondaryKeys_1` VIEW Index AS r
- ON l.k = r.SubKey1 AND l.v = r.SubKey2
- ORDER BY Key
+ DECLARE $in AS List<Struct<k: Int32?, v: String?>>;
+ SELECT *
+ FROM AS_TABLE($in) AS l RIGHT SEMI JOIN `/Root/RSJ_SecondaryKeys_1` VIEW Index AS r
+ ON l.k = r.SubKey1 AND l.v = r.SubKey2
+ ORDER BY Key
)");
-
- auto params = TParamsBuilder().AddParam("$in").BeginList()
- .AddListItem().BeginStruct().AddMember("k").OptionalInt32(1)
- .AddMember("v").OptionalString("2.One").EndStruct()
- .AddListItem().BeginStruct().AddMember("k").OptionalInt32(1)
- .AddMember("v").OptionalString("2.One").EndStruct() // dup
- .AddListItem().BeginStruct().AddMember("k").OptionalInt32(5)
- .AddMember("v").OptionalString("2.Five").EndStruct()
- .AddListItem().BeginStruct().AddMember("k").OptionalInt32(42)
- .AddMember("v").OptionalString("Any").EndStruct() // not exists
- .AddListItem().BeginStruct().AddMember("k").OptionalInt32(Nothing())
- .AddMember("v").OptionalString(Nothing()).EndStruct() // null
- .EndList().Build().Build();
-
- auto result = ExecQuery(session, query, params, R"([[[1];[1];["2.One"];["Payload1"]];[[5];[5];["2.Five"];["Payload2"]]])");
- // todo: reading of absent |Any| key is not visible in statistics
- AssertTableReads(result, "/Root/RSJ_SecondaryKeys_1/Index/indexImplTable", 2 /* [2.One, 2.Five, |Any|] */);
- AssertTableReads(result, "/Root/RSJ_SecondaryKeys_1", 2 /* read .Value field */);
- }
-
- /* join with real table */
- {
+
+ auto params = TParamsBuilder().AddParam("$in").BeginList()
+ .AddListItem().BeginStruct().AddMember("k").OptionalInt32(1)
+ .AddMember("v").OptionalString("2.One").EndStruct()
+ .AddListItem().BeginStruct().AddMember("k").OptionalInt32(1)
+ .AddMember("v").OptionalString("2.One").EndStruct() // dup
+ .AddListItem().BeginStruct().AddMember("k").OptionalInt32(5)
+ .AddMember("v").OptionalString("2.Five").EndStruct()
+ .AddListItem().BeginStruct().AddMember("k").OptionalInt32(42)
+ .AddMember("v").OptionalString("Any").EndStruct() // not exists
+ .AddListItem().BeginStruct().AddMember("k").OptionalInt32(Nothing())
+ .AddMember("v").OptionalString(Nothing()).EndStruct() // null
+ .EndList().Build().Build();
+
+ auto result = ExecQuery(session, query, params, R"([[[1];[1];["2.One"];["Payload1"]];[[5];[5];["2.Five"];["Payload2"]]])");
+ // todo: reading of absent |Any| key is not visible in statistics
+ AssertTableReads(result, "/Root/RSJ_SecondaryKeys_1/Index/indexImplTable", 2 /* [2.One, 2.Five, |Any|] */);
+ AssertTableReads(result, "/Root/RSJ_SecondaryKeys_1", 2 /* read .Value field */);
+ }
+
+ /* join with real table */
+ {
const TString query = Q_(R"(
- SELECT *
- FROM `/Root/RSJ_SimpleKey_2` AS l RIGHT SEMI JOIN `/Root/RSJ_SecondaryKeys_1` VIEW Index AS r
- ON l.Key = r.SubKey1 AND l.Value = r.SubKey2
- ORDER BY Key
+ SELECT *
+ FROM `/Root/RSJ_SimpleKey_2` AS l RIGHT SEMI JOIN `/Root/RSJ_SecondaryKeys_1` VIEW Index AS r
+ ON l.Key = r.SubKey1 AND l.Value = r.SubKey2
+ ORDER BY Key
)");
-
- auto result = ExecQuery(session, query, NoParams, R"([[[1];[1];["2.One"];["Payload1"]];[[5];[5];["2.Five"];["Payload2"]]])");
- AssertTableReads(result, "/Root/RSJ_SimpleKey_2", 5 /* all keys */);
- // todo: reading of absent |Any| key is not visible in statistics
- AssertTableReads(result, "/Root/RSJ_SecondaryKeys_1/Index/indexImplTable", 2 /* [2.One, 2.Five, |Any|] */);
- AssertTableReads(result, "/Root/RSJ_SecondaryKeys_1", 2 /* [1, 5] */);
- }
-
- }
-
- // join on secondary index prefix => index-lookup
+
+ auto result = ExecQuery(session, query, NoParams, R"([[[1];[1];["2.One"];["Payload1"]];[[5];[5];["2.Five"];["Payload2"]]])");
+ AssertTableReads(result, "/Root/RSJ_SimpleKey_2", 5 /* all keys */);
+ // todo: reading of absent |Any| key is not visible in statistics
+ AssertTableReads(result, "/Root/RSJ_SecondaryKeys_1/Index/indexImplTable", 2 /* [2.One, 2.Five, |Any|] */);
+ AssertTableReads(result, "/Root/RSJ_SecondaryKeys_1", 2 /* [1, 5] */);
+ }
+
+ }
+
+ // join on secondary index prefix => index-lookup
Y_UNIT_TEST_NEW_ENGINE(RightSemiJoin_ComplexSecondaryIndexPrefix) {
- TKikimrRunner kikimr(SyntaxV1Settings());
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- CreateRightSemiJoinSampleTables(session);
-
- /* join with parameters */
- {
+ TKikimrRunner kikimr(SyntaxV1Settings());
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ CreateRightSemiJoinSampleTables(session);
+
+ /* join with parameters */
+ {
const TString query = Q_(R"(
- DECLARE $in AS List<Struct<k: Int32?>>;
- SELECT *
- FROM AS_TABLE($in) AS l RIGHT SEMI JOIN `/Root/RSJ_SecondaryKeys_1` VIEW Index AS r
- ON l.k = r.SubKey1
- ORDER BY Key
+ DECLARE $in AS List<Struct<k: Int32?>>;
+ SELECT *
+ FROM AS_TABLE($in) AS l RIGHT SEMI JOIN `/Root/RSJ_SecondaryKeys_1` VIEW Index AS r
+ ON l.k = r.SubKey1
+ ORDER BY Key
)");
-
- auto params = TParamsBuilder().AddParam("$in").BeginList()
- .AddListItem().BeginStruct().AddMember("k").OptionalInt32(1).EndStruct()
- .AddListItem().BeginStruct().AddMember("k").OptionalInt32(1).EndStruct() // dup
- .AddListItem().BeginStruct().AddMember("k").OptionalInt32(5).EndStruct()
- .AddListItem().BeginStruct().AddMember("k").OptionalInt32(42).EndStruct() // not exists
- .AddListItem().BeginStruct().AddMember("k").OptionalInt32(Nothing()).EndStruct() // null
- .EndList().Build().Build();
-
- auto result = ExecQuery(session, query, params, R"([[[1];[1];["2.One"];["Payload1"]];[[5];[5];["2.Five"];["Payload2"]]])");
- // todo: reading of absent |Any| key is not visible in statistics
- AssertTableReads(result, "/Root/RSJ_SecondaryKeys_1/Index/indexImplTable", 2 /* [2.One, 2.Five, |Any|] */);
- AssertTableReads(result, "/Root/RSJ_SecondaryKeys_1", 2 /* read .Value field */);
- }
-
- /* join with real table */
- {
+
+ auto params = TParamsBuilder().AddParam("$in").BeginList()
+ .AddListItem().BeginStruct().AddMember("k").OptionalInt32(1).EndStruct()
+ .AddListItem().BeginStruct().AddMember("k").OptionalInt32(1).EndStruct() // dup
+ .AddListItem().BeginStruct().AddMember("k").OptionalInt32(5).EndStruct()
+ .AddListItem().BeginStruct().AddMember("k").OptionalInt32(42).EndStruct() // not exists
+ .AddListItem().BeginStruct().AddMember("k").OptionalInt32(Nothing()).EndStruct() // null
+ .EndList().Build().Build();
+
+ auto result = ExecQuery(session, query, params, R"([[[1];[1];["2.One"];["Payload1"]];[[5];[5];["2.Five"];["Payload2"]]])");
+ // todo: reading of absent |Any| key is not visible in statistics
+ AssertTableReads(result, "/Root/RSJ_SecondaryKeys_1/Index/indexImplTable", 2 /* [2.One, 2.Five, |Any|] */);
+ AssertTableReads(result, "/Root/RSJ_SecondaryKeys_1", 2 /* read .Value field */);
+ }
+
+ /* join with real table */
+ {
const TString query = Q_(R"(
- SELECT *
- FROM `/Root/RSJ_SimpleKey_2` AS l RIGHT SEMI JOIN `/Root/RSJ_SecondaryKeys_1` VIEW Index AS r
- ON l.Key = r.SubKey1
- -- WHERE r.Key > 1
- ORDER BY Key
+ SELECT *
+ FROM `/Root/RSJ_SimpleKey_2` AS l RIGHT SEMI JOIN `/Root/RSJ_SecondaryKeys_1` VIEW Index AS r
+ ON l.Key = r.SubKey1
+ -- WHERE r.Key > 1
+ ORDER BY Key
)");
-
- auto result = ExecQuery(session, query, NoParams, R"([[[1];[1];["2.One"];["Payload1"]];[[5];[5];["2.Five"];["Payload2"]]])");
- AssertTableReads(result, "/Root/RSJ_SimpleKey_2", 5 /* all keys */);
- // todo: reading of absent |Any| key is not visible in statistics
- AssertTableReads(result, "/Root/RSJ_SecondaryKeys_1/Index/indexImplTable", 2 /* [2.One, 2.Five, |Any|] */);
- AssertTableReads(result, "/Root/RSJ_SecondaryKeys_1", 2 /* [1, 5] */);
- }
- }
-
- template<bool UseNewEngine = false>
+
+ auto result = ExecQuery(session, query, NoParams, R"([[[1];[1];["2.One"];["Payload1"]];[[5];[5];["2.Five"];["Payload2"]]])");
+ AssertTableReads(result, "/Root/RSJ_SimpleKey_2", 5 /* all keys */);
+ // todo: reading of absent |Any| key is not visible in statistics
+ AssertTableReads(result, "/Root/RSJ_SecondaryKeys_1/Index/indexImplTable", 2 /* [2.One, 2.Five, |Any|] */);
+ AssertTableReads(result, "/Root/RSJ_SecondaryKeys_1", 2 /* [1, 5] */);
+ }
+ }
+
+ template<bool UseNewEngine = false>
void TestInnerJoinWithPredicate(const TString& predicate, const TString& expected) {
- TKikimrRunner kikimr(SyntaxV1Settings());
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- UNIT_ASSERT(session.ExecuteSchemeQuery(R"(
- CREATE TABLE `/Root/SecondaryKeys` (
- Key Int32,
- Fk Int32,
- Value String,
- PRIMARY KEY (Key),
- INDEX Index GLOBAL ON (Fk)
- )
- )").GetValueSync().IsSuccess());
-
- auto result = session.ExecuteDataQuery(Q_(R"(
- REPLACE INTO `/Root/SecondaryKeys` (Key, Fk, Value) VALUES
- (1, 101, "Payload1"),
- (5, 105, "Payload2")
- )"), TTxControl::BeginTx().CommitTx()).GetValueSync();
-
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
+ TKikimrRunner kikimr(SyntaxV1Settings());
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ UNIT_ASSERT(session.ExecuteSchemeQuery(R"(
+ CREATE TABLE `/Root/SecondaryKeys` (
+ Key Int32,
+ Fk Int32,
+ Value String,
+ PRIMARY KEY (Key),
+ INDEX Index GLOBAL ON (Fk)
+ )
+ )").GetValueSync().IsSuccess());
+
+ auto result = session.ExecuteDataQuery(Q_(R"(
+ REPLACE INTO `/Root/SecondaryKeys` (Key, Fk, Value) VALUES
+ (1, 101, "Payload1"),
+ (5, 105, "Payload2")
+ )"), TTxControl::BeginTx().CommitTx()).GetValueSync();
+
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
const TString query = Sprintf(R"(
- DECLARE $in AS List<Struct<k: Int32?>>;
- SELECT *
- FROM AS_TABLE($in) AS l INNER JOIN `/Root/SecondaryKeys` VIEW Index AS r
- ON l.k = r.Fk
+ DECLARE $in AS List<Struct<k: Int32?>>;
+ SELECT *
+ FROM AS_TABLE($in) AS l INNER JOIN `/Root/SecondaryKeys` VIEW Index AS r
+ ON l.k = r.Fk
WHERE %s
- ORDER BY Key
+ ORDER BY Key
)", predicate.c_str());
-
- auto params = TParamsBuilder().AddParam("$in").BeginList()
+
+ auto params = TParamsBuilder().AddParam("$in").BeginList()
.AddListItem().BeginStruct().AddMember("k").OptionalInt32(105).EndStruct()
- .EndList().Build().Build();
-
- result = session.ExecuteDataQuery(Q_(query), TTxControl::BeginTx().CommitTx(), params)
- .ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ .EndList().Build().Build();
+
+ result = session.ExecuteDataQuery(Q_(query), TTxControl::BeginTx().CommitTx(), params)
+ .ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
CompareYson(expected, FormatResultSetYson(result.GetResultSet(0)));
- }
+ }
- Y_UNIT_TEST_NEW_ENGINE(RightTableKeyPredicate) {
- TestInnerJoinWithPredicate<UseNewEngine>("r.Key > 1", "[[[105];[5];[\"Payload2\"];[105]]]");
+ Y_UNIT_TEST_NEW_ENGINE(RightTableKeyPredicate) {
+ TestInnerJoinWithPredicate<UseNewEngine>("r.Key > 1", "[[[105];[5];[\"Payload2\"];[105]]]");
}
- Y_UNIT_TEST_NEW_ENGINE(RightTableIndexPredicate) {
- TestInnerJoinWithPredicate<UseNewEngine>("r.Fk > 1", "[[[105];[5];[\"Payload2\"];[105]]]");
+ Y_UNIT_TEST_NEW_ENGINE(RightTableIndexPredicate) {
+ TestInnerJoinWithPredicate<UseNewEngine>("r.Fk > 1", "[[[105];[5];[\"Payload2\"];[105]]]");
}
- Y_UNIT_TEST_NEW_ENGINE(RightTableValuePredicate) {
- TestInnerJoinWithPredicate<UseNewEngine>("r.Value = \"Payload2\"", "[[[105];[5];[\"Payload2\"];[105]]]");
+ Y_UNIT_TEST_NEW_ENGINE(RightTableValuePredicate) {
+ TestInnerJoinWithPredicate<UseNewEngine>("r.Value = \"Payload2\"", "[[[105];[5];[\"Payload2\"];[105]]]");
}
- Y_UNIT_TEST_NEW_ENGINE(JoinAggregateSingleRow) {
+ Y_UNIT_TEST_NEW_ENGINE(JoinAggregateSingleRow) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -742,7 +742,7 @@ Y_UNIT_TEST_SUITE(KqpJoin) {
.Build()
.Build();
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
DECLARE $key1 AS Int32;
DECLARE $key2 AS String;
@@ -750,12 +750,12 @@ Y_UNIT_TEST_SUITE(KqpJoin) {
j2.Key2 AS Key,
SOME(j2.Value),
SOME(j3.Value)
- FROM `/Root/Join1_2` AS j2
- LEFT JOIN `/Root/Join1_3` AS j3
+ FROM `/Root/Join1_2` AS j2
+ LEFT JOIN `/Root/Join1_3` AS j3
ON j3.Key = j2.Fk3
WHERE j2.Key1 = $key1 AND j2.Key2 = $key2
GROUP BY j2.Key2;
- )"), TTxControl::BeginTx().CommitTx(), params).ExtractValueSync();
+ )"), TTxControl::BeginTx().CommitTx(), params).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
CompareYson(R"([[["Two"];["Value22"];[1001]]])",
@@ -771,10 +771,10 @@ Y_UNIT_TEST_SUITE(KqpJoin) {
{
auto result = session.ExecuteDataQuery(Q_(R"(
SELECT t1.Value, SUM(t3.Value)
- FROM `/Root/Join1_1` AS t1
- INNER JOIN `/Root/Join1_2` AS t2
+ FROM `/Root/Join1_1` AS t1
+ INNER JOIN `/Root/Join1_2` AS t2
ON t1.Fk21 == t2.Key1
- LEFT JOIN `/Root/Join1_3` AS t3
+ LEFT JOIN `/Root/Join1_3` AS t3
ON t2.Fk3 = t3.Key
GROUP BY t1.Value;
)"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
@@ -785,15 +785,15 @@ Y_UNIT_TEST_SUITE(KqpJoin) {
}
{
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT t1.Value, SUM(t3.Value)
- FROM `/Root/Join1_1` AS t1
- INNER JOIN `/Root/Join1_2` AS t2
+ FROM `/Root/Join1_1` AS t1
+ INNER JOIN `/Root/Join1_2` AS t2
ON t1.Fk21 == t2.Key1
- LEFT JOIN `/Root/Join1_3` AS t3
+ LEFT JOIN `/Root/Join1_3` AS t3
ON t2.Fk3 = t3.Key
GROUP BY t1.Value LIMIT 3;
- )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
CompareYson(R"([[["Value1"];[3004]];[["Value2"];[1001]];[["Value3"];[2006]]])",
@@ -801,33 +801,33 @@ Y_UNIT_TEST_SUITE(KqpJoin) {
}
}
- Y_UNIT_TEST_NEW_ENGINE(JoinConvert) {
+ Y_UNIT_TEST_NEW_ENGINE(JoinConvert) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
CreateSampleTables(session);
UNIT_ASSERT(session.ExecuteSchemeQuery(R"(
- CREATE TABLE `/Root/Join1_3_ui64` (
+ CREATE TABLE `/Root/Join1_3_ui64` (
Key String,
Value Int64,
PRIMARY KEY (Key)
);
)").GetValueSync().IsSuccess());
- UNIT_ASSERT(session.ExecuteDataQuery(Q_(R"(
- REPLACE INTO `/Root/Join1_3_ui64` (Key, Value) VALUES
+ UNIT_ASSERT(session.ExecuteDataQuery(Q_(R"(
+ REPLACE INTO `/Root/Join1_3_ui64` (Key, Value) VALUES
("Name1", 108);
- )"), TTxControl::BeginTx().CommitTx()).GetValueSync().IsSuccess());
+ )"), TTxControl::BeginTx().CommitTx()).GetValueSync().IsSuccess());
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT t1.Value, t2.Value, t3.Value FROM `/Root/Join1_1` AS t1
INNER JOIN `/Root/Join1_2` AS t2
ON t1.Fk21 == t2.Key1 AND t1.Fk22 == t2.Key2
LEFT JOIN `/Root/Join1_3_ui64` AS t3
ON t2.Key1 = t3.Value
WHERE t1.Value == "Value5";
- )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
CompareYson(R"([[["Value5"];["Value31"];[108]]])",
@@ -840,12 +840,12 @@ Y_UNIT_TEST_SUITE(KqpJoin) {
auto session = db.CreateSession().GetValueSync().GetSession();
CreateSampleTables(session);
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT left.Fk21, left.Key, left.Value, right.Key1, right.Value
FROM `/Root/Join1_1` as left
EXCLUSION JOIN `/Root/Join1_2` as right
ON left.Fk21 = right.Key1
- )"), TTxControl::BeginTx().CommitTx()).GetValueSync();
+ )"), TTxControl::BeginTx().CommitTx()).GetValueSync();
UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
CompareYson(
@@ -853,7 +853,7 @@ Y_UNIT_TEST_SUITE(KqpJoin) {
[107];[7];["Value4"];#;#];
[#;#;#;[109];["Value41"]
]])",
- FormatResultSetYson(result.GetResultSet(0))
+ FormatResultSetYson(result.GetResultSet(0))
);
}
@@ -863,65 +863,65 @@ Y_UNIT_TEST_SUITE(KqpJoin) {
auto session = db.CreateSession().GetValueSync().GetSession();
CreateSampleTables(session);
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT left.Fk21, left.Key, left.Value, right.Key1, right.Value
FROM `/Root/Join1_1` as left
FULL OUTER JOIN `/Root/Join1_2` as right
ON left.Fk21 = right.Key1
- ORDER BY left.Fk21, left.Key, left.Value, right.Key1, right.Value
- )"), TTxControl::BeginTx().CommitTx()).GetValueSync();
+ ORDER BY left.Fk21, left.Key, left.Value, right.Key1, right.Value
+ )"), TTxControl::BeginTx().CommitTx()).GetValueSync();
UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
CompareYson(
R"([
- [#;#;#;[109];["Value41"]];
- [[101];[1];["Value1"];[101];["Value21"]];
- [[101];[1];["Value1"];[101];["Value22"]];
- [[101];[1];["Value1"];[101];["Value23"]];
+ [#;#;#;[109];["Value41"]];
+ [[101];[1];["Value1"];[101];["Value21"]];
+ [[101];[1];["Value1"];[101];["Value22"]];
+ [[101];[1];["Value1"];[101];["Value23"]];
[[102];[2];["Value1"];[102];["Value24"]];
[[103];[3];["Value2"];[103];["Value25"]];
- [[104];[4];["Value2"];[104];["Value26"]];
+ [[104];[4];["Value2"];[104];["Value26"]];
[[105];[5];["Value3"];[105];["Value27"]];
[[105];[5];["Value3"];[105];["Value28"]];
- [[106];[6];["Value3"];[106];["Value29"]];
+ [[106];[6];["Value3"];[106];["Value29"]];
[[107];[7];["Value4"];#;#];
- [[108];[8];["Value5"];[108];["Value31"]]
+ [[108];[8];["Value5"];[108];["Value31"]]
])",
- FormatResultSetYson(result.GetResultSet(0))
+ FormatResultSetYson(result.GetResultSet(0))
);
}
- Y_UNIT_TEST_NEW_ENGINE(FullOuterJoin2) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
- CreateSampleTables(session);
-
- auto result = session.ExecuteDataQuery(Q_(R"(
- SELECT left.Key, left.Value, right.Value
- FROM `/Root/Join1_1` AS left
- FULL OUTER JOIN `/Root/Join1_2` AS right
- ON left.Fk21 = right.Key1 AND left.Fk22 = right.Key2
- WHERE left.Key < 5
- ORDER BY left.Key
- )"), TTxControl::BeginTx().CommitTx()).GetValueSync();
-
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
- CompareYson(R"([
- [[1];["Value1"];["Value21"]];
- [[2];["Value1"];#];
- [[3];["Value2"];["Value25"]];
- [[4];["Value2"];#]
- ])", FormatResultSetYson(result.GetResultSet(0)));
- }
-
+ Y_UNIT_TEST_NEW_ENGINE(FullOuterJoin2) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+ CreateSampleTables(session);
+
+ auto result = session.ExecuteDataQuery(Q_(R"(
+ SELECT left.Key, left.Value, right.Value
+ FROM `/Root/Join1_1` AS left
+ FULL OUTER JOIN `/Root/Join1_2` AS right
+ ON left.Fk21 = right.Key1 AND left.Fk22 = right.Key2
+ WHERE left.Key < 5
+ ORDER BY left.Key
+ )"), TTxControl::BeginTx().CommitTx()).GetValueSync();
+
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ CompareYson(R"([
+ [[1];["Value1"];["Value21"]];
+ [[2];["Value1"];#];
+ [[3];["Value2"];["Value25"]];
+ [[4];["Value2"];#]
+ ])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+
Y_UNIT_TEST_NEW_ENGINE(FullOuterJoinSizeCheck) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
CreateSampleTables(session);
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT COUNT(*)
FROM `/Root/Join1_1` as left
FULL OUTER JOIN `/Root/Join1_2` as right
@@ -931,12 +931,12 @@ Y_UNIT_TEST_SUITE(KqpJoin) {
FROM `/Root/Join1_2` as right
FULL OUTER JOIN `/Root/Join1_1` as left
ON left.Fk21 = right.Key1
- )"), TTxControl::BeginTx().CommitTx()).GetValueSync();
+ )"), TTxControl::BeginTx().CommitTx()).GetValueSync();
UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
CompareYson(
"[[12u];[12u]]", // numbers MUST be same
- FormatResultSetYson(result.GetResultSet(0))
+ FormatResultSetYson(result.GetResultSet(0))
);
}
@@ -955,27 +955,27 @@ Y_UNIT_TEST_SUITE(KqpJoin) {
UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
CompareYson(R"([[88u]])", FormatResultSetYson(result.GetResultSet(0)));
}
-
- Y_UNIT_TEST_NEW_ENGINE(JoinDupColumnRight) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
- CreateSampleTables(session);
-
- auto result = session.ExecuteDataQuery(Q1_(R"(
- SELECT Key, Key1, Key2
- FROM `/Root/Join1_1` AS t1 LEFT JOIN `/Root/Join1_2` AS t2
- ON t1.Key = t2.Key1 AND t1.Fk21 = t2.Key1
- WHERE t1.Value == "Value1"
- ORDER BY Key;
- )"), TTxControl::BeginTx().CommitTx()).GetValueSync();
-
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
- CompareYson(R"([
- [[1];#;#];
- [[2];#;#]
- ])", FormatResultSetYson(result.GetResultSet(0)));
- }
+
+ Y_UNIT_TEST_NEW_ENGINE(JoinDupColumnRight) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+ CreateSampleTables(session);
+
+ auto result = session.ExecuteDataQuery(Q1_(R"(
+ SELECT Key, Key1, Key2
+ FROM `/Root/Join1_1` AS t1 LEFT JOIN `/Root/Join1_2` AS t2
+ ON t1.Key = t2.Key1 AND t1.Fk21 = t2.Key1
+ WHERE t1.Value == "Value1"
+ ORDER BY Key;
+ )"), TTxControl::BeginTx().CommitTx()).GetValueSync();
+
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ CompareYson(R"([
+ [[1];#;#];
+ [[2];#;#]
+ ])", FormatResultSetYson(result.GetResultSet(0)));
+ }
Y_UNIT_TEST_NEW_ENGINE(JoinLeftPureInner) {
TKikimrRunner kikimr;
@@ -1049,5 +1049,5 @@ Y_UNIT_TEST_SUITE(KqpJoin) {
}
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/ut/kqp_limits_ut.cpp b/ydb/core/kqp/ut/kqp_limits_ut.cpp
index de3b637f86..0e2478dad7 100644
--- a/ydb/core/kqp/ut/kqp_limits_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_limits_ut.cpp
@@ -63,15 +63,15 @@ static void CreateLargeTable(TKikimrRunner& kikimr, ui32 rowsPerShard, ui32 keyT
}
Y_UNIT_TEST_SUITE(KqpLimits) {
- Y_UNIT_TEST_NEW_ENGINE(DatashardProgramSize) {
- auto app = NKikimrConfig::TAppConfig();
- app.MutableTableServiceConfig()->MutableResourceManager()->SetMkqlLightProgramMemoryLimit(1'000'000'000);
-
- TKikimrRunner kikimr(app);
+ Y_UNIT_TEST_NEW_ENGINE(DatashardProgramSize) {
+ auto app = NKikimrConfig::TAppConfig();
+ app.MutableTableServiceConfig()->MutableResourceManager()->SetMkqlLightProgramMemoryLimit(1'000'000'000);
+
+ TKikimrRunner kikimr(app);
CreateLargeTable(kikimr, 0, 0, 0);
- kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_SLOW_LOG, NActors::NLog::PRI_ERROR);
-
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_SLOW_LOG, NActors::NLog::PRI_ERROR);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -95,12 +95,12 @@ Y_UNIT_TEST_SUITE(KqpLimits) {
rowsParam.EndList();
rowsParam.Build();
- auto result = session.ExecuteDataQuery(Q1_(R"(
- DECLARE $rows AS List<Struct<Key: Uint64?, KeyText: String?, Data: Int64?, DataText: String?>>;
+ auto result = session.ExecuteDataQuery(Q1_(R"(
+ DECLARE $rows AS List<Struct<Key: Uint64?, KeyText: String?, Data: Int64?, DataText: String?>>;
- UPSERT INTO `/Root/LargeTable`
+ UPSERT INTO `/Root/LargeTable`
SELECT * FROM AS_TABLE($rows);
- )"), TTxControl::BeginTx().CommitTx(), paramsBuilder.Build()).ExtractValueSync();
+ )"), TTxControl::BeginTx().CommitTx(), paramsBuilder.Build()).ExtractValueSync();
result.GetIssues().PrintTo(Cerr);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR);
UNIT_ASSERT(HasIssue(result.GetIssues(), NKikimrIssues::TIssuesIds::SHARD_PROGRAM_SIZE_EXCEEDED));
@@ -143,48 +143,48 @@ Y_UNIT_TEST_SUITE(KqpLimits) {
UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_RESULT_UNAVAILABLE));
}
- Y_UNIT_TEST(TooBigQuery) {
- auto app = NKikimrConfig::TAppConfig();
- app.MutableTableServiceConfig()->MutableResourceManager()->SetMkqlLightProgramMemoryLimit(1'000'000'000);
- app.MutableTableServiceConfig()->SetCompileTimeoutMs(TDuration::Minutes(5).MilliSeconds());
-
- TKikimrRunner kikimr(app);
- CreateLargeTable(kikimr, 0, 0, 0);
- kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_SLOW_LOG, NActors::NLog::PRI_ERROR);
-
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- TStringBuilder query;
- query << R"(
- --!syntax_v1
- PRAGMA Kikimr.UseNewEngine = 'true';
-
- UPSERT INTO `/Root/LargeTable`
- SELECT * FROM AS_TABLE(AsList(
- )";
-
- ui32 count = 5000;
- for (ui32 i = 0; i < count; ++i) {
- query << "AsStruct("
- << i << "UL AS Key, "
- << "'" << CreateGuidAsString() << TString(5000, '0' + i % 10) << "' AS KeyText, "
- << count + i << "L AS Data, "
- << "'" << CreateGuidAsString() << "' AS DataText"
- << ")";
- if (i + 1 != count) {
- query << ", ";
- }
- }
- query << "))";
-
- auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- result.GetIssues().PrintTo(Cerr);
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString());
- UNIT_ASSERT(HasIssue(result.GetIssues(), NKikimrIssues::TIssuesIds::SHARD_PROGRAM_SIZE_EXCEEDED));
- }
-
- Y_UNIT_TEST_NEW_ENGINE(AffectedShardsLimit) {
+ Y_UNIT_TEST(TooBigQuery) {
+ auto app = NKikimrConfig::TAppConfig();
+ app.MutableTableServiceConfig()->MutableResourceManager()->SetMkqlLightProgramMemoryLimit(1'000'000'000);
+ app.MutableTableServiceConfig()->SetCompileTimeoutMs(TDuration::Minutes(5).MilliSeconds());
+
+ TKikimrRunner kikimr(app);
+ CreateLargeTable(kikimr, 0, 0, 0);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_SLOW_LOG, NActors::NLog::PRI_ERROR);
+
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ TStringBuilder query;
+ query << R"(
+ --!syntax_v1
+ PRAGMA Kikimr.UseNewEngine = 'true';
+
+ UPSERT INTO `/Root/LargeTable`
+ SELECT * FROM AS_TABLE(AsList(
+ )";
+
+ ui32 count = 5000;
+ for (ui32 i = 0; i < count; ++i) {
+ query << "AsStruct("
+ << i << "UL AS Key, "
+ << "'" << CreateGuidAsString() << TString(5000, '0' + i % 10) << "' AS KeyText, "
+ << count + i << "L AS Data, "
+ << "'" << CreateGuidAsString() << "' AS DataText"
+ << ")";
+ if (i + 1 != count) {
+ query << ", ";
+ }
+ }
+ query << "))";
+
+ auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ result.GetIssues().PrintTo(Cerr);
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString());
+ UNIT_ASSERT(HasIssue(result.GetIssues(), NKikimrIssues::TIssuesIds::SHARD_PROGRAM_SIZE_EXCEEDED));
+ }
+
+ Y_UNIT_TEST_NEW_ENGINE(AffectedShardsLimit) {
NKikimrConfig::TAppConfig appConfig;
auto& queryLimits = *appConfig.MutableTableServiceConfig()->MutableQueryLimits();
queryLimits.MutablePhaseLimits()->SetAffectedShardsLimit(20);
@@ -212,25 +212,25 @@ Y_UNIT_TEST_SUITE(KqpLimits) {
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- auto result = session.ExecuteDataQuery(Q_(R"(
- SELECT COUNT(*) FROM `/Root/ManyShard20`
- )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ auto result = session.ExecuteDataQuery(Q_(R"(
+ SELECT COUNT(*) FROM `/Root/ManyShard20`
+ )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- result = session.ExecuteDataQuery(Q_(R"(
- SELECT COUNT(*) FROM `/Root/ManyShard21`
- )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ result = session.ExecuteDataQuery(Q_(R"(
+ SELECT COUNT(*) FROM `/Root/ManyShard21`
+ )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
result.GetIssues().PrintTo(Cerr);
- if (UseNewEngine) {
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::PRECONDITION_FAILED);
- UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_PRECONDITION_FAILED));
- } else {
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
- UNIT_ASSERT(HasIssue(result.GetIssues(), NKikimrIssues::TIssuesIds::ENGINE_ERROR));
- }
+ if (UseNewEngine) {
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::PRECONDITION_FAILED);
+ UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_PRECONDITION_FAILED));
+ } else {
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
+ UNIT_ASSERT(HasIssue(result.GetIssues(), NKikimrIssues::TIssuesIds::ENGINE_ERROR));
+ }
}
- Y_UNIT_TEST_NEW_ENGINE(ReadsetCountLimit) {
+ Y_UNIT_TEST_NEW_ENGINE(ReadsetCountLimit) {
NKikimrConfig::TAppConfig appConfig;
auto& queryLimits = *appConfig.MutableTableServiceConfig()->MutableQueryLimits();
queryLimits.MutablePhaseLimits()->SetReadsetCountLimit(50);
@@ -241,27 +241,27 @@ Y_UNIT_TEST_SUITE(KqpLimits) {
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- auto result = session.ExecuteDataQuery(Q_(R"(
- UPDATE `/Root/LargeTable`
+ auto result = session.ExecuteDataQuery(Q_(R"(
+ UPDATE `/Root/LargeTable`
SET Data = CAST(Key AS Int64) + 10
WHERE Key < 7000000;
- )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- result = session.ExecuteDataQuery(Q_(R"(
- UPDATE `/Root/LargeTable`
+ result = session.ExecuteDataQuery(Q_(R"(
+ UPDATE `/Root/LargeTable`
SET Data = CAST(Key AS Int64) + 10;
- )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- if (UseNewEngine) {
- // TODO: ???
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- } else {
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
- UNIT_ASSERT(HasIssue(result.GetIssues(), NKikimrIssues::TIssuesIds::ENGINE_ERROR));
- }
+ )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ if (UseNewEngine) {
+ // TODO: ???
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+ } else {
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
+ UNIT_ASSERT(HasIssue(result.GetIssues(), NKikimrIssues::TIssuesIds::ENGINE_ERROR));
+ }
}
- Y_UNIT_TEST_NEW_ENGINE(TotalReadSizeLimit) {
+ Y_UNIT_TEST_NEW_ENGINE(TotalReadSizeLimit) {
NKikimrConfig::TAppConfig appConfig;
auto& queryLimits = *appConfig.MutableTableServiceConfig()->MutableQueryLimits();
queryLimits.MutablePhaseLimits()->SetTotalReadSizeLimitBytes(100'000'000);
@@ -272,37 +272,37 @@ Y_UNIT_TEST_SUITE(KqpLimits) {
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT Key, KeyText, SUBSTRING(DataText, 0, 10) AS DataText
- FROM `/Root/LargeTable`;
- )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ FROM `/Root/LargeTable`;
+ )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
result.GetIssues().PrintTo(Cerr);
- if (UseNewEngine) {
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::PRECONDITION_FAILED);
- UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_PRECONDITION_FAILED,
- [] (const NYql::TIssue& issue) {
- return issue.Message.Contains("Transaction total read size");
- }));
-
- } else {
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR);
- UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::DEFAULT_ERROR,
- [] (const NYql::TIssue& issue) {
- return issue.Message.Contains("Transaction total read size");
+ if (UseNewEngine) {
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::PRECONDITION_FAILED);
+ UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_PRECONDITION_FAILED,
+ [] (const NYql::TIssue& issue) {
+ return issue.Message.Contains("Transaction total read size");
+ }));
+
+ } else {
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR);
+ UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::DEFAULT_ERROR,
+ [] (const NYql::TIssue& issue) {
+ return issue.Message.Contains("Transaction total read size");
}));
- }
+ }
- result = session.ExecuteDataQuery(Q_(R"(
+ result = session.ExecuteDataQuery(Q_(R"(
SELECT Key, KeyText, SUBSTRING(DataText, 0, 10) AS DataText
- FROM `/Root/LargeTable`
+ FROM `/Root/LargeTable`
WHERE Key < 4000000;
- )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
}
- Y_UNIT_TEST_NEW_ENGINE(ComputeNodeMemoryLimit) {
+ Y_UNIT_TEST_NEW_ENGINE(ComputeNodeMemoryLimit) {
NKikimrConfig::TAppConfig appConfig;
- appConfig.MutableTableServiceConfig()->MutableResourceManager()->SetMkqlLightProgramMemoryLimit(1'000'000'000);
+ appConfig.MutableTableServiceConfig()->MutableResourceManager()->SetMkqlLightProgramMemoryLimit(1'000'000'000);
auto& queryLimits = *appConfig.MutableTableServiceConfig()->MutableQueryLimits();
queryLimits.MutablePhaseLimits()->SetComputeNodeMemoryLimitBytes(100'000'000);
@@ -311,14 +311,14 @@ Y_UNIT_TEST_SUITE(KqpLimits) {
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT ToDict(
ListMap(
ListFromRange(0ul, 5000000ul),
($x) -> { RETURN AsTuple($x, $x + 1); }
)
);
- )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
result.GetIssues().PrintTo(Cerr);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(),
@@ -331,16 +331,16 @@ Y_UNIT_TEST_SUITE(KqpLimits) {
}));
}
- Y_UNIT_TEST_NEW_ENGINE(QueryExecCancel) {
+ Y_UNIT_TEST_NEW_ENGINE(QueryExecCancel) {
TKikimrRunner kikimr;
CreateLargeTable(kikimr, 500000, 10, 100, 5000, 1);
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- auto prepareResult = session.PrepareDataQuery(Q_(R"(
- SELECT COUNT(*) FROM `/Root/LargeTable` WHERE SUBSTRING(DataText, 50, 5) = "11111";
- )")).GetValueSync();
+ auto prepareResult = session.PrepareDataQuery(Q_(R"(
+ SELECT COUNT(*) FROM `/Root/LargeTable` WHERE SUBSTRING(DataText, 50, 5) = "11111";
+ )")).GetValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(prepareResult.GetStatus(), EStatus::SUCCESS, prepareResult.GetIssues().ToString());
auto dataQuery = prepareResult.GetQuery();
@@ -353,23 +353,23 @@ Y_UNIT_TEST_SUITE(KqpLimits) {
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::CANCELLED);
}
- Y_UNIT_TEST_NEW_ENGINE(QueryExecTimeout) {
+ Y_UNIT_TEST_NEW_ENGINE(QueryExecTimeout) {
NKikimrConfig::TAppConfig appConfig;
- appConfig.MutableTableServiceConfig()->MutableResourceManager()->SetMkqlLightProgramMemoryLimit(10'000'000'000);
+ appConfig.MutableTableServiceConfig()->MutableResourceManager()->SetMkqlLightProgramMemoryLimit(10'000'000'000);
TKikimrRunner kikimr(appConfig);
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- auto prepareResult = session.PrepareDataQuery(Q_(R"(
+ auto prepareResult = session.PrepareDataQuery(Q_(R"(
SELECT ToDict(
ListMap(
ListFromRange(0ul, 10000000ul),
($x) -> { RETURN AsTuple($x, $x + 1); }
)
);
- )")).GetValueSync();
+ )")).GetValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(prepareResult.GetStatus(), EStatus::SUCCESS, prepareResult.GetIssues().ToString());
auto dataQuery = prepareResult.GetQuery();
@@ -382,5 +382,5 @@ Y_UNIT_TEST_SUITE(KqpLimits) {
}
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/ut/kqp_locks_ut.cpp b/ydb/core/kqp/ut/kqp_locks_ut.cpp
index c9e41d5931..bd9d9ede5b 100644
--- a/ydb/core/kqp/ut/kqp_locks_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_locks_ut.cpp
@@ -7,34 +7,34 @@ using namespace NYdb;
using namespace NYdb::NTable;
Y_UNIT_TEST_SUITE(KqpLocks) {
- Y_UNIT_TEST_NEW_ENGINE(Invalidate) {
+ Y_UNIT_TEST_NEW_ENGINE(Invalidate) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session1 = db.CreateSession().GetValueSync().GetSession();
auto session2 = db.CreateSession().GetValueSync().GetSession();
- auto result = session1.ExecuteDataQuery(Q_(R"(
- UPSERT INTO `/Root/Test`
+ auto result = session1.ExecuteDataQuery(Q_(R"(
+ UPSERT INTO `/Root/Test`
SELECT Group + 10U AS Group, Name, Amount, Comment ?? "" || "Updated" AS Comment
- FROM `/Root/Test`
+ FROM `/Root/Test`
WHERE Group == 1U AND Name == "Paul";
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW())).ExtractValueSync();
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW())).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
auto tx1 = result.GetTransaction();
UNIT_ASSERT(tx1);
- result = session2.ExecuteDataQuery(Q_(R"(
- UPSERT INTO `/Root/Test` (Group, Name, Comment)
+ result = session2.ExecuteDataQuery(Q_(R"(
+ UPSERT INTO `/Root/Test` (Group, Name, Comment)
VALUES (1U, "Paul", "Changed");
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- result = session1.ExecuteDataQuery(Q_(R"(
- UPSERT INTO `/Root/Test` (Group, Name, Comment)
+ result = session1.ExecuteDataQuery(Q_(R"(
+ UPSERT INTO `/Root/Test` (Group, Name, Comment)
VALUES (11U, "Sergey", "BadRow");
- )"), TTxControl::Tx(*tx1).CommitTx()).ExtractValueSync();
+ )"), TTxControl::Tx(*tx1).CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::ABORTED, result.GetIssues().ToString());
result.GetIssues().PrintTo(Cerr);
UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_LOCKS_INVALIDATED,
@@ -42,35 +42,35 @@ Y_UNIT_TEST_SUITE(KqpLocks) {
return issue.Message.Contains("/Root/Test");
}));
- result = session2.ExecuteDataQuery(Q_(R"(
- SELECT * FROM `/Root/Test` WHERE Name == "Paul" ORDER BY Group, Name;
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ result = session2.ExecuteDataQuery(Q_(R"(
+ SELECT * FROM `/Root/Test` WHERE Name == "Paul" ORDER BY Group, Name;
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
CompareYson(R"([[[300u];["Changed"];[1u];["Paul"]]])", FormatResultSetYson(result.GetResultSet(0)));
}
- Y_UNIT_TEST_NEW_ENGINE(InvalidateOnCommit) {
+ Y_UNIT_TEST_NEW_ENGINE(InvalidateOnCommit) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session1 = db.CreateSession().GetValueSync().GetSession();
auto session2 = db.CreateSession().GetValueSync().GetSession();
- auto result = session1.ExecuteDataQuery(Q_(R"(
- UPSERT INTO `/Root/Test`
+ auto result = session1.ExecuteDataQuery(Q_(R"(
+ UPSERT INTO `/Root/Test`
SELECT Group + 10U AS Group, Name, Amount, Comment ?? "" || "Updated" AS Comment
- FROM `/Root/Test`
+ FROM `/Root/Test`
WHERE Group == 1U AND Name == "Paul";
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW())).ExtractValueSync();
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW())).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
auto tx1 = result.GetTransaction();
UNIT_ASSERT(tx1);
- result = session2.ExecuteDataQuery(Q_(R"(
- UPSERT INTO `/Root/Test` (Group, Name, Comment)
+ result = session2.ExecuteDataQuery(Q_(R"(
+ UPSERT INTO `/Root/Test` (Group, Name, Comment)
VALUES (1U, "Paul", "Changed");
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
auto commitResult = tx1->Commit().GetValueSync();
@@ -81,40 +81,40 @@ Y_UNIT_TEST_SUITE(KqpLocks) {
return issue.Message.Contains("/Root/Test");
}));
- result = session2.ExecuteDataQuery(Q_(R"(
- SELECT * FROM `/Root/Test` WHERE Name == "Paul" ORDER BY Group, Name;
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ result = session2.ExecuteDataQuery(Q_(R"(
+ SELECT * FROM `/Root/Test` WHERE Name == "Paul" ORDER BY Group, Name;
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
CompareYson(R"([[[300u];["Changed"];[1u];["Paul"]]])", FormatResultSetYson(result.GetResultSet(0)));
}
- Y_UNIT_TEST_NEW_ENGINE(DifferentKeyUpdate) {
+ Y_UNIT_TEST_NEW_ENGINE(DifferentKeyUpdate) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session1 = db.CreateSession().GetValueSync().GetSession();
auto session2 = db.CreateSession().GetValueSync().GetSession();
- auto result = session1.ExecuteDataQuery(Q_(R"(
+ auto result = session1.ExecuteDataQuery(Q_(R"(
SELECT * FROM `/Root/Test` WHERE Group = 1;
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW())).ExtractValueSync();
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW())).ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
auto tx1 = result.GetTransaction();
UNIT_ASSERT(tx1);
- result = session2.ExecuteDataQuery(Q_(R"(
+ result = session2.ExecuteDataQuery(Q_(R"(
UPSERT INTO `/Root/Test` (Group, Name, Comment)
VALUES (2U, "Paul", "Changed");
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
- result = session1.ExecuteDataQuery(Q_(R"(
+ result = session1.ExecuteDataQuery(Q_(R"(
SELECT "Nothing";
- )"), TTxControl::Tx(*tx1).CommitTx()).ExtractValueSync();
+ )"), TTxControl::Tx(*tx1).CommitTx()).ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
}
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/ut/kqp_merge_connection_ut.cpp b/ydb/core/kqp/ut/kqp_merge_connection_ut.cpp
index bd40fd6e5c..8ba8607165 100644
--- a/ydb/core/kqp/ut/kqp_merge_connection_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_merge_connection_ut.cpp
@@ -87,7 +87,7 @@ void CheckPlanForMergeCn(const TMaybe<TString>& planJson, bool hasChildSort, con
UNIT_ASSERT(childSort.IsDefined());
}
- // Check that TopSort has no Merge Connection in children
+ // Check that TopSort has no Merge Connection in children
auto topSort = FindPlanNodeByKv(plan, "Name", sortOp);
if (topSort.IsDefined()) {
mergeCn = FindPlanNodeByKv(topSort, "Node Type", "DqCnMerge");
diff --git a/ydb/core/kqp/ut/kqp_mvcc_ut.cpp b/ydb/core/kqp/ut/kqp_mvcc_ut.cpp
index ce0f4419c7..571b39df3a 100644
--- a/ydb/core/kqp/ut/kqp_mvcc_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_mvcc_ut.cpp
@@ -9,22 +9,22 @@ using namespace NYdb;
using namespace NYdb::NTable;
Y_UNIT_TEST_SUITE(KqpSnapshotRead) {
- Y_UNIT_TEST_NEW_ENGINE(TestSnapshotExpiration) {
+ Y_UNIT_TEST_NEW_ENGINE(TestSnapshotExpiration) {
TKikimrRunner kikimr(TKikimrSettings()
.SetEnableMvcc(true)
.SetEnableMvccSnapshotReads(true)
.SetKeepSnapshotTimeout(TDuration::Seconds(1)));
-// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_BLOBS_STORAGE, NActors::NLog::PRI_DEBUG);
-// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG);
+// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_BLOBS_STORAGE, NActors::NLog::PRI_DEBUG);
+// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG);
auto db = kikimr.GetTableClient();
auto session1 = db.CreateSession().GetValueSync().GetSession();
auto session2 = db.CreateSession().GetValueSync().GetSession();
- auto result = session1.ExecuteDataQuery(Q_(R"(
+ auto result = session1.ExecuteDataQuery(Q_(R"(
SELECT * FROM `/Root/TwoShard` WHERE Key = 1u OR Key = 4000000001u;
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW())).ExtractValueSync();
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW())).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
CompareYson(R"([
@@ -34,19 +34,19 @@ Y_UNIT_TEST_SUITE(KqpSnapshotRead) {
auto tx = result.GetTransaction();
- result = session2.ExecuteDataQuery(Q_(R"(
+ result = session2.ExecuteDataQuery(Q_(R"(
UPSERT INTO `/Root/TwoShard` (Key, Value1, Value2) VALUES (1u, "ChangedOne", 1);
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
auto deadline = TInstant::Now() + TDuration::Seconds(30);
- auto caught = false;
+ auto caught = false;
do {
Sleep(TDuration::Seconds(1));
- auto result = session1.ExecuteDataQuery(Q_(R"(
+ auto result = session1.ExecuteDataQuery(Q_(R"(
SELECT * FROM `/Root/TwoShard` WHERE Key = 1u OR Key = 4000000001u;
- )"), TTxControl::Tx(*tx)).ExtractValueSync();
+ )"), TTxControl::Tx(*tx)).ExtractValueSync();
if (result.GetStatus() == EStatus::SUCCESS)
continue;
@@ -55,25 +55,25 @@ Y_UNIT_TEST_SUITE(KqpSnapshotRead) {
return issue.Message.Contains("stale snapshot");
}), result.GetIssues().ToString());
- caught = true;
+ caught = true;
break;
} while (TInstant::Now() < deadline);
- UNIT_ASSERT_C(caught, "Failed to wait for snapshot expiration.");
+ UNIT_ASSERT_C(caught, "Failed to wait for snapshot expiration.");
}
Y_UNIT_TEST_NEW_ENGINE(ReadOnlyTxCommitsOnConcurrentWrite) {
TKikimrRunner kikimr(TKikimrSettings().SetEnableMvcc(true).SetEnableMvccSnapshotReads(true));
-// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_COMPUTE, NActors::NLog::PRI_DEBUG);
-// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_BLOBS_STORAGE, NActors::NLog::PRI_DEBUG);
-// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG);
+// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_COMPUTE, NActors::NLog::PRI_DEBUG);
+// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_BLOBS_STORAGE, NActors::NLog::PRI_DEBUG);
+// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG);
auto db = kikimr.GetTableClient();
auto session1 = db.CreateSession().GetValueSync().GetSession();
auto session2 = db.CreateSession().GetValueSync().GetSession();
auto result = session1.ExecuteDataQuery(Q_(R"(
- SELECT * FROM `/Root/TwoShard` WHERE Key = 1u OR Key = 4000000001u;
+ SELECT * FROM `/Root/TwoShard` WHERE Key = 1u OR Key = 4000000001u;
)"), TTxControl::BeginTx(TTxSettings::SerializableRW())).ExtractValueSync();
auto tx = result.GetTransaction();
@@ -85,13 +85,13 @@ Y_UNIT_TEST_SUITE(KqpSnapshotRead) {
])", FormatResultSetYson(result.GetResultSet(0)));
result = session2.ExecuteDataQuery(Q_(R"(
- UPSERT INTO `/Root/TwoShard` (Key, Value1, Value2) VALUES (1u, "ChangedOne", 1);
+ UPSERT INTO `/Root/TwoShard` (Key, Value1, Value2) VALUES (1u, "ChangedOne", 1);
)"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
result = session2.ExecuteDataQuery(Q_(R"(
- SELECT * FROM `/Root/TwoShard` WHERE Key = 1u;
+ SELECT * FROM `/Root/TwoShard` WHERE Key = 1u;
)"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
@@ -100,7 +100,7 @@ Y_UNIT_TEST_SUITE(KqpSnapshotRead) {
])", FormatResultSetYson(result.GetResultSet(0)));
result = session1.ExecuteDataQuery(Q_(R"(
- SELECT * FROM `/Root/TwoShard` WHERE Key = 1u;
+ SELECT * FROM `/Root/TwoShard` WHERE Key = 1u;
)"), TTxControl::Tx(*tx)).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
@@ -109,7 +109,7 @@ Y_UNIT_TEST_SUITE(KqpSnapshotRead) {
])", FormatResultSetYson(result.GetResultSet(0)));
result = session1.ExecuteDataQuery(Q_(R"(
- SELECT * FROM `/Root/TwoShard` WHERE Key = 2u OR Key = 4000000002u;
+ SELECT * FROM `/Root/TwoShard` WHERE Key = 2u OR Key = 4000000002u;
)"), TTxControl::Tx(*tx).CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
@@ -119,12 +119,12 @@ Y_UNIT_TEST_SUITE(KqpSnapshotRead) {
])", FormatResultSetYson(result.GetResultSet(0)));
}
- Y_UNIT_TEST_NEW_ENGINE(ReadOnlyTxWithIndexCommitsOnConcurrentWrite) {
+ Y_UNIT_TEST_NEW_ENGINE(ReadOnlyTxWithIndexCommitsOnConcurrentWrite) {
TKikimrRunner kikimr(TKikimrSettings().SetEnableMvcc(true).SetEnableMvccSnapshotReads(true));
-// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_COMPUTE, NActors::NLog::PRI_DEBUG);
-// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_BLOBS_STORAGE, NActors::NLog::PRI_DEBUG);
-// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG);
+// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_COMPUTE, NActors::NLog::PRI_DEBUG);
+// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_BLOBS_STORAGE, NActors::NLog::PRI_DEBUG);
+// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG);
auto db = kikimr.GetTableClient();
auto session1 = db.CreateSession().GetValueSync().GetSession();
@@ -132,21 +132,21 @@ Y_UNIT_TEST_SUITE(KqpSnapshotRead) {
CreateSampleTablesWithIndex(session1);
- auto result = session1.ExecuteDataQuery(Q_(R"(
- UPSERT INTO `/Root/SecondaryWithDataColumns` (Key, Index2, Value) VALUES
+ auto result = session1.ExecuteDataQuery(Q_(R"(
+ UPSERT INTO `/Root/SecondaryWithDataColumns` (Key, Index2, Value) VALUES
("Pk1", "Fk1", "One"),
("Pk2", "Fk2", "Two"),
("Pk3", "Fk3", "Three"),
("Pk4", "Fk4", "Four"),
("Pk5", "Fk5", "Five"),
("Pk6", "Fk6", "Six");
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- result = session1.ExecuteDataQuery(Q1_(R"(
- SELECT Value FROM `/Root/SecondaryWithDataColumns` view Index WHERE Index2 = "Fk1";
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW())).ExtractValueSync();
+ result = session1.ExecuteDataQuery(Q1_(R"(
+ SELECT Value FROM `/Root/SecondaryWithDataColumns` view Index WHERE Index2 = "Fk1";
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW())).ExtractValueSync();
auto tx = result.GetTransaction();
@@ -155,24 +155,24 @@ Y_UNIT_TEST_SUITE(KqpSnapshotRead) {
[["One"]]
])", FormatResultSetYson(result.GetResultSet(0)));
- result = session2.ExecuteDataQuery(Q_(R"(
- UPSERT INTO `/Root/SecondaryWithDataColumns` (Key, Index2, Value) VALUES ("Pk1", "Fk1", "ChangedOne");
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ result = session2.ExecuteDataQuery(Q_(R"(
+ UPSERT INTO `/Root/SecondaryWithDataColumns` (Key, Index2, Value) VALUES ("Pk1", "Fk1", "ChangedOne");
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- result = session2.ExecuteDataQuery(Q1_(R"(
- SELECT Value FROM `/Root/SecondaryWithDataColumns` view Index WHERE Index2 = "Fk1";
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ result = session2.ExecuteDataQuery(Q1_(R"(
+ SELECT Value FROM `/Root/SecondaryWithDataColumns` view Index WHERE Index2 = "Fk1";
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
CompareYson(R"([
[["ChangedOne"]]
])", FormatResultSetYson(result.GetResultSet(0)));
- result = session1.ExecuteDataQuery(Q1_(R"(
- SELECT Value FROM `/Root/SecondaryWithDataColumns` view Index WHERE Index2 = "Fk1";
- )"), TTxControl::Tx(*tx).CommitTx()).ExtractValueSync();
+ result = session1.ExecuteDataQuery(Q1_(R"(
+ SELECT Value FROM `/Root/SecondaryWithDataColumns` view Index WHERE Index2 = "Fk1";
+ )"), TTxControl::Tx(*tx).CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
CompareYson(R"([
@@ -183,16 +183,16 @@ Y_UNIT_TEST_SUITE(KqpSnapshotRead) {
Y_UNIT_TEST_NEW_ENGINE(ReadWriteTxFailsOnConcurrentWrite1) {
TKikimrRunner kikimr(TKikimrSettings().SetEnableMvcc(true).SetEnableMvccSnapshotReads(true));
-// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_COMPUTE, NActors::NLog::PRI_DEBUG);
-// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_BLOBS_STORAGE, NActors::NLog::PRI_DEBUG);
-// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG);
+// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_COMPUTE, NActors::NLog::PRI_DEBUG);
+// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_BLOBS_STORAGE, NActors::NLog::PRI_DEBUG);
+// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG);
auto db = kikimr.GetTableClient();
auto session1 = db.CreateSession().GetValueSync().GetSession();
auto session2 = db.CreateSession().GetValueSync().GetSession();
auto result = session1.ExecuteDataQuery(Q_(R"(
- SELECT * FROM `/Root/TwoShard` WHERE Key = 1u OR Key = 4000000001u;
+ SELECT * FROM `/Root/TwoShard` WHERE Key = 1u OR Key = 4000000001u;
)"), TTxControl::BeginTx(TTxSettings::SerializableRW())).ExtractValueSync();
auto tx = result.GetTransaction();
@@ -204,13 +204,13 @@ Y_UNIT_TEST_SUITE(KqpSnapshotRead) {
])", FormatResultSetYson(result.GetResultSet(0)));
result = session2.ExecuteDataQuery(Q_(R"(
- UPSERT INTO `/Root/TwoShard` (Key, Value1, Value2) VALUES (1u, "ChangedOne", 1);
+ UPSERT INTO `/Root/TwoShard` (Key, Value1, Value2) VALUES (1u, "ChangedOne", 1);
)"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
result = session1.ExecuteDataQuery(Q_(R"(
- UPSERT INTO `/Root/TwoShard` (Key, Value1, Value2) VALUES (1u, "TwiceChangedOne", 2);
+ UPSERT INTO `/Root/TwoShard` (Key, Value1, Value2) VALUES (1u, "TwiceChangedOne", 2);
)"), TTxControl::Tx(*tx).CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::ABORTED, result.GetIssues().ToString());
@@ -220,16 +220,16 @@ Y_UNIT_TEST_SUITE(KqpSnapshotRead) {
Y_UNIT_TEST_NEW_ENGINE(ReadWriteTxFailsOnConcurrentWrite2) {
TKikimrRunner kikimr(TKikimrSettings().SetEnableMvcc(true).SetEnableMvccSnapshotReads(true));
-// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_COMPUTE, NActors::NLog::PRI_DEBUG);
-// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_BLOBS_STORAGE, NActors::NLog::PRI_DEBUG);
-// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG);
+// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_COMPUTE, NActors::NLog::PRI_DEBUG);
+// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_BLOBS_STORAGE, NActors::NLog::PRI_DEBUG);
+// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG);
auto db = kikimr.GetTableClient();
auto session1 = db.CreateSession().GetValueSync().GetSession();
auto session2 = db.CreateSession().GetValueSync().GetSession();
auto result = session1.ExecuteDataQuery(Q_(R"(
- SELECT * FROM `/Root/TwoShard` WHERE Key = 1u OR Key = 4000000001u;
+ SELECT * FROM `/Root/TwoShard` WHERE Key = 1u OR Key = 4000000001u;
)"), TTxControl::BeginTx(TTxSettings::SerializableRW())).ExtractValueSync();
auto tx = result.GetTransaction();
@@ -241,14 +241,14 @@ Y_UNIT_TEST_SUITE(KqpSnapshotRead) {
])", FormatResultSetYson(result.GetResultSet(0)));
result = session2.ExecuteDataQuery(Q_(R"(
- UPSERT INTO `/Root/EightShard` (Key, Text) VALUES (101u, "SomeText");
+ UPSERT INTO `/Root/EightShard` (Key, Text) VALUES (101u, "SomeText");
)"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
result = session1.ExecuteDataQuery(Q_(R"(
- UPDATE `/Root/TwoShard` SET Value2 = 2 WHERE Key = 1u;
- UPDATE `/Root/EightShard` SET Text = "AnotherString" WHERE Key = 101u;
+ UPDATE `/Root/TwoShard` SET Value2 = 2 WHERE Key = 1u;
+ UPDATE `/Root/EightShard` SET Text = "AnotherString" WHERE Key = 101u;
)"), TTxControl::Tx(*tx).CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::ABORTED, result.GetIssues().ToString());
@@ -258,16 +258,16 @@ Y_UNIT_TEST_SUITE(KqpSnapshotRead) {
Y_UNIT_TEST_NEW_ENGINE(ReadWriteTxFailsOnConcurrentWrite3) {
TKikimrRunner kikimr(TKikimrSettings().SetEnableMvcc(true).SetEnableMvccSnapshotReads(true));
-// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_COMPUTE, NActors::NLog::PRI_DEBUG);
-// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_BLOBS_STORAGE, NActors::NLog::PRI_DEBUG);
-// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG);
+// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_COMPUTE, NActors::NLog::PRI_DEBUG);
+// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_BLOBS_STORAGE, NActors::NLog::PRI_DEBUG);
+// kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG);
auto db = kikimr.GetTableClient();
auto session1 = db.CreateSession().GetValueSync().GetSession();
auto session2 = db.CreateSession().GetValueSync().GetSession();
auto result = session1.ExecuteDataQuery(Q_(R"(
- SELECT * FROM `/Root/TwoShard` WHERE Key = 1u OR Key = 4000000001u;
+ SELECT * FROM `/Root/TwoShard` WHERE Key = 1u OR Key = 4000000001u;
)"), TTxControl::BeginTx(TTxSettings::SerializableRW())).ExtractValueSync();
auto tx = result.GetTransaction();
@@ -279,13 +279,13 @@ Y_UNIT_TEST_SUITE(KqpSnapshotRead) {
])", FormatResultSetYson(result.GetResultSet(0)));
result = session2.ExecuteDataQuery(Q_(R"(
- UPSERT INTO `/Root/TwoShard` (Key, Value1, Value2) VALUES (2u, "ChangedTwo", 1);
+ UPSERT INTO `/Root/TwoShard` (Key, Value1, Value2) VALUES (2u, "ChangedTwo", 1);
)"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
result = session1.ExecuteDataQuery(Q_(R"(
- SELECT * FROM `/Root/TwoShard` WHERE Key = 2u OR Key = 4000000002u;
+ SELECT * FROM `/Root/TwoShard` WHERE Key = 2u OR Key = 4000000002u;
)"), TTxControl::Tx(*tx)).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
@@ -295,7 +295,7 @@ Y_UNIT_TEST_SUITE(KqpSnapshotRead) {
])", FormatResultSetYson(result.GetResultSet(0)));
result = session1.ExecuteDataQuery(Q_(R"(
- UPSERT INTO `/Root/TwoShard` (Key, Value1, Value2) VALUES (2u, "TwiceChangedTwo", 1);
+ UPSERT INTO `/Root/TwoShard` (Key, Value1, Value2) VALUES (2u, "TwiceChangedTwo", 1);
)"), TTxControl::Tx(*tx).CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::ABORTED, result.GetIssues().ToString());
diff --git a/ydb/core/kqp/ut/kqp_newengine_effects_ut.cpp b/ydb/core/kqp/ut/kqp_newengine_effects_ut.cpp
index e9ea355146..17b315ac4d 100644
--- a/ydb/core/kqp/ut/kqp_newengine_effects_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_newengine_effects_ut.cpp
@@ -1,511 +1,511 @@
#include <ydb/core/kqp/ut/common/kqp_ut_common.h>
-
+
#include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h>
-
-namespace NKikimr {
-namespace NKqp {
-
-using namespace NYdb;
-using namespace NYdb::NTable;
-
-Y_UNIT_TEST_SUITE(KqpNewEngineEffects) {
- Y_UNIT_TEST(InsertAbort_Literal_Success) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- INSERT INTO `/Root/TwoShard` (Value1, Key) VALUES
- ("foo", 10u), ("bar", 11u)
- )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- SELECT * FROM `/Root/TwoShard` WHERE Key > 5 AND Key < 100 ORDER BY Key;
- )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- CompareYson(R"([
- [[10u];["foo"];#];
- [[11u];["bar"];#]
- ])", FormatResultSetYson(result.GetResultSet(0)));
- }
-
- Y_UNIT_TEST(InsertAbort_Literal_Duplicates) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- INSERT INTO `/Root/TwoShard` (Value1, Key) VALUES
- ("foo", 10u), ("bar", 11u), ("baz", 10u)
- )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::PRECONDITION_FAILED, result.GetIssues().ToString());
- UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION, [](const NYql::TIssue& issue) {
- return issue.Message.Contains("Duplicated keys found.");
- }));
-
- result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- SELECT * FROM `/Root/TwoShard` WHERE Key > 5 AND Key < 100 ORDER BY Key;
- )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- CompareYson(R"([])", FormatResultSetYson(result.GetResultSet(0)));
- }
-
- Y_UNIT_TEST(InsertAbort_Literal_Conflict) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- INSERT INTO `/Root/TwoShard` (Value1, Key) VALUES
- ("foo", 1u), ("bar", 11u)
- )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::PRECONDITION_FAILED, result.GetIssues().ToString());
- UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION, [](const NYql::TIssue& issue) {
- return issue.Message.Contains("Conflict with existing key.");
- }), result.GetIssues().ToString());
-
- result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- SELECT * FROM `/Root/TwoShard` WHERE Key = 1 OR Key = 11 ORDER BY Key;
- )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- CompareYson(R"([
- [[1u];["One"];[-1]]
- ])", FormatResultSetYson(result.GetResultSet(0)));
- }
-
- Y_UNIT_TEST(InsertAbort_Params_Success) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto params = TParamsBuilder().AddParam("$in")
- .BeginList()
- .AddListItem().BeginStruct()
- .AddMember("Key").Uint32(10)
- .AddMember("Value1").String("foo")
- .AddMember("Value2").Int32(5)
- .EndStruct()
- .AddListItem().BeginStruct()
- .AddMember("Key").Uint32(11)
- .AddMember("Value1").String("bar")
- .AddMember("Value2").Int32(7)
- .EndStruct()
- .EndList().Build().Build();
-
- auto result = session.ExecuteDataQuery(R"(
- --!syntax_v1
- PRAGMA kikimr.UseNewEngine = "true";
- DECLARE $in AS List<Struct<Key: Uint32, Value1: String, Value2:Int32>>;
- INSERT INTO `/Root/TwoShard` SELECT * FROM AS_TABLE($in)
- )", TTxControl::BeginTx().CommitTx(), std::move(params)).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- SELECT * FROM `/Root/TwoShard` WHERE Key > 5 AND Key < 100 ORDER BY Key;
- )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- CompareYson(R"([
- [[10u];["foo"];[5]];
- [[11u];["bar"];[7]]
- ])", FormatResultSetYson(result.GetResultSet(0)));
- }
-
- Y_UNIT_TEST(InsertAbort_Params_Duplicates) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto params = TParamsBuilder().AddParam("$in")
- .BeginList()
- .AddListItem().BeginStruct()
- .AddMember("Key").Uint32(1)
- .AddMember("Value1").String("foo")
- .AddMember("Value2").Int32(5)
- .EndStruct()
- .AddListItem().BeginStruct()
- .AddMember("Key").Uint32(11)
- .AddMember("Value1").String("bar")
- .AddMember("Value2").Int32(7)
- .EndStruct()
- .AddListItem().BeginStruct()
- .AddMember("Key").Uint32(1)
- .AddMember("Value1").String("baz")
- .AddMember("Value2").Int32(9)
- .EndStruct()
- .EndList().Build().Build();
-
- auto result = session.ExecuteDataQuery(R"(
- --!syntax_v1
- PRAGMA kikimr.UseNewEngine = "true";
- DECLARE $in AS List<Struct<Key: Uint32, Value1: String, Value2:Int32>>;
- INSERT INTO `/Root/TwoShard` SELECT * FROM AS_TABLE($in)
- )", TTxControl::BeginTx().CommitTx(), std::move(params)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::PRECONDITION_FAILED, result.GetIssues().ToString());
- UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION, [](const NYql::TIssue& issue) {
- return issue.Message.Contains("Duplicated keys found.");
- }));
-
- result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- SELECT * FROM `/Root/TwoShard` WHERE Key = 1 OR Key = 11 ORDER BY Key;
- )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- CompareYson(R"([
- [[1u];["One"];[-1]]
- ])", FormatResultSetYson(result.GetResultSet(0)));
- }
-
- Y_UNIT_TEST(InsertAbort_Params_Conflict) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto params = TParamsBuilder().AddParam("$in")
- .BeginList()
- .AddListItem().BeginStruct()
- .AddMember("Key").Uint32(1)
- .AddMember("Value1").String("foo")
- .AddMember("Value2").Int32(5)
- .EndStruct()
- .AddListItem().BeginStruct()
- .AddMember("Key").Uint32(11)
- .AddMember("Value1").String("bar")
- .AddMember("Value2").Int32(7)
- .EndStruct()
- .EndList().Build().Build();
-
- auto result = session.ExecuteDataQuery(R"(
- --!syntax_v1
- PRAGMA kikimr.UseNewEngine = "true";
- DECLARE $in AS List<Struct<Key: Uint32, Value1: String, Value2:Int32>>;
- INSERT INTO `/Root/TwoShard` SELECT * FROM AS_TABLE($in)
- )", TTxControl::BeginTx().CommitTx(), std::move(params)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::PRECONDITION_FAILED, result.GetIssues().ToString());
- UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION, [](const NYql::TIssue& issue) {
- return issue.Message.Contains("Conflict with existing key.");
- }));
-
- result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- SELECT * FROM `/Root/TwoShard` WHERE Key = 1 OR Key = 11 ORDER BY Key;
- )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- CompareYson(R"([
- [[1u];["One"];[-1]]
- ])", FormatResultSetYson(result.GetResultSet(0)));
- }
-
- Y_UNIT_TEST(InsertAbort_Select_Success) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto ret = session.ExecuteSchemeQuery(R"(
- CREATE TABLE `Foo` (
- Key Uint32,
- Value1 String,
- Value2 Int32,
- PRIMARY KEY (Key)
- )
- )").ExtractValueSync();
- UNIT_ASSERT_C(ret.IsSuccess(), ret.GetIssues().ToString());
-
- auto result = session.ExecuteDataQuery(R"(
- REPLACE INTO `/Root/Foo` (Key, Value1, Value2) VALUES
- (10u, "foo", 5), (11u, "bar", 7)
- )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- INSERT INTO `/Root/TwoShard` SELECT * FROM `/Root/Foo`
- )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- SELECT * FROM `/Root/TwoShard` WHERE Key > 5 AND Key < 100 ORDER BY Key;
- )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- CompareYson(R"([
- [[10u];["foo"];[5]];
- [[11u];["bar"];[7]]
- ])", FormatResultSetYson(result.GetResultSet(0)));
- }
-
- Y_UNIT_TEST(InsertAbort_Select_Duplicates) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto ret = session.ExecuteSchemeQuery(R"(
- CREATE TABLE `Foo` (
- Key Uint32,
- Value1 String,
- Value2 Int32,
- Value3 Uint32,
- PRIMARY KEY (Key)
- )
- )").ExtractValueSync();
- UNIT_ASSERT_C(ret.IsSuccess(), ret.GetIssues().ToString());
-
- auto result = session.ExecuteDataQuery(R"(
- REPLACE INTO `/Root/Foo` (Key, Value1, Value2, Value3) VALUES
- (1u, "foo", 5, 10), (2u, "bar", 7, 11), (3u, "baz", 9, 10)
- )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- INSERT INTO `/Root/TwoShard` SELECT Value3 as Key, Value1, Value2 FROM `/Root/Foo`
- )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::PRECONDITION_FAILED, result.GetIssues().ToString());
- UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION, [](const NYql::TIssue& issue) {
- return issue.Message.Contains("Duplicated keys found.");
- }));
-
- result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- SELECT * FROM `/Root/TwoShard` WHERE Key = 1 OR Key = 10 OR Key = 11 ORDER BY Key;
- )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- CompareYson(R"([
- [[1u];["One"];[-1]]
- ])", FormatResultSetYson(result.GetResultSet(0)));
- }
-
- Y_UNIT_TEST(InsertAbort_Select_Conflict) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto ret = session.ExecuteSchemeQuery(R"(
- CREATE TABLE `Foo` (
- Key Uint32,
- Value1 String,
- Value2 Int32,
- PRIMARY KEY (Key)
- )
- )").ExtractValueSync();
- UNIT_ASSERT_C(ret.IsSuccess(), ret.GetIssues().ToString());
-
- auto result = session.ExecuteDataQuery(R"(
- REPLACE INTO `/Root/Foo` (Key, Value1, Value2) VALUES
- (1u, "foo", 5), (11u, "bar", 7)
- )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- INSERT INTO `/Root/TwoShard` SELECT * FROM `/Root/Foo`
- )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::PRECONDITION_FAILED, result.GetIssues().ToString());
- UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION, [](const NYql::TIssue& issue) {
- return issue.Message.Contains("Conflict with existing key.");
- }));
-
- result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- SELECT * FROM `/Root/TwoShard` WHERE Key = 1 OR Key = 11 ORDER BY Key;
- )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- CompareYson(R"([
- [[1u];["One"];[-1]]
- ])", FormatResultSetYson(result.GetResultSet(0)));
- }
-
- Y_UNIT_TEST(InsertRevert_Literal_Success) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- INSERT OR REVERT INTO `/Root/TwoShard` (Value1, Key) VALUES
- ("foo", 10u), ("bar", 11u)
- )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- SELECT * FROM `/Root/TwoShard` WHERE Key > 5 AND Key < 100 ORDER BY Key;
- )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- CompareYson(R"([
- [[10u];["foo"];#];
- [[11u];["bar"];#]
- ])", FormatResultSetYson(result.GetResultSet(0)));
- }
-
- Y_UNIT_TEST(InsertRevert_Literal_Duplicates) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- INSERT OR REVERT INTO `/Root/TwoShard` (Value1, Key) VALUES
- ("foo", 10u), ("bar", 11u), ("baz", 10u)
- )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetIssues().Size(), 0, result.GetIssues().ToString());
-
- result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- SELECT * FROM `/Root/TwoShard` WHERE Key > 5 AND Key < 100 ORDER BY Key;
- )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- CompareYson(R"([])", FormatResultSetYson(result.GetResultSet(0)));
- }
-
- Y_UNIT_TEST(InsertRevert_Literal_Conflict) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- INSERT OR REVERT INTO `/Root/TwoShard` (Value1, Key) VALUES
- ("foo", 1u), ("bar", 11u)
- )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetIssues().Size(), 0, result.GetIssues().ToString());
-
- result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- SELECT * FROM `/Root/TwoShard` WHERE Key = 1 OR Key = 11 ORDER BY Key;
- )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- CompareYson(R"([
- [[1u];["One"];[-1]]
- ])", FormatResultSetYson(result.GetResultSet(0)));
- }
-
- Y_UNIT_TEST(UpdateOn_Literal) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
-
- UPDATE `/Root/TwoShard` ON (Key, Value1) VALUES
- (1u, "Updated"), (4000000010u, "Updated")
- )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
-
- result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- SELECT * FROM `/Root/TwoShard` ORDER BY Key;
- )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- CompareYson(R"([
- [[1u];["Updated"];[-1]];
- [[2u];["Two"];[0]];
- [[3u];["Three"];[1]];
- [[4000000001u];["BigOne"];[-1]];
- [[4000000002u];["BigTwo"];[0]];
- [[4000000003u];["BigThree"];[1]]
- ])", FormatResultSetYson(result.GetResultSet(0)));
- }
-
- Y_UNIT_TEST(UpdateOn_Params) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto params = TParamsBuilder().AddParam("$in")
- .BeginList()
- .AddListItem().BeginStruct()
- .AddMember("Key").Uint32(1)
- .AddMember("Value1").String("Updated")
- .AddMember("Value2").Int32(-10)
- .EndStruct()
- .AddListItem().BeginStruct()
- .AddMember("Key").Uint32(4000000010)
- .AddMember("Value1").String("Updated")
- .AddMember("Value2").Int32(7)
- .EndStruct()
- .EndList().Build().Build();
-
- auto result = session.ExecuteDataQuery(R"(
- --!syntax_v1
- PRAGMA kikimr.UseNewEngine = "true";
- DECLARE $in AS List<Struct<Key: Uint32, Value1: String, Value2:Int32>>;
- UPDATE `/Root/TwoShard` ON (Key, Value1, Value2) SELECT Key, Value1, Value2 FROM AS_TABLE($in)
- )", TTxControl::BeginTx().CommitTx(), std::move(params)).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- SELECT * FROM `/Root/TwoShard` ORDER BY Key;
- )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- CompareYson(R"([
- [[1u];["Updated"];[-10]];
- [[2u];["Two"];[0]];
- [[3u];["Three"];[1]];
- [[4000000001u];["BigOne"];[-1]];
- [[4000000002u];["BigTwo"];[0]];
- [[4000000003u];["BigThree"];[1]]
- ])", FormatResultSetYson(result.GetResultSet(0)));
- }
-
- Y_UNIT_TEST(UpdateOn_Select) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
-
- $to_update = (
- SELECT Key, "Updated" as Value1, (Value2 + 5) as Value2
- FROM `/Root/TwoShard`
- WHERE Value2 >= 0
- );
- UPDATE `/Root/TwoShard` ON SELECT * FROM $to_update;
- )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
-
- result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- SELECT * FROM `/Root/TwoShard` ORDER BY Key;
- )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- CompareYson(R"([
- [[1u];["One"];[-1]];
- [[2u];["Updated"];[5]];
- [[3u];["Updated"];[6]];
- [[4000000001u];["BigOne"];[-1]];
- [[4000000002u];["Updated"];[5]];
- [[4000000003u];["Updated"];[6]]
- ])", FormatResultSetYson(result.GetResultSet(0)));
- }
-}
-
-} // namespace NKqp
-} // namespace NKikimr
+
+namespace NKikimr {
+namespace NKqp {
+
+using namespace NYdb;
+using namespace NYdb::NTable;
+
+Y_UNIT_TEST_SUITE(KqpNewEngineEffects) {
+ Y_UNIT_TEST(InsertAbort_Literal_Success) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ INSERT INTO `/Root/TwoShard` (Value1, Key) VALUES
+ ("foo", 10u), ("bar", 11u)
+ )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ SELECT * FROM `/Root/TwoShard` WHERE Key > 5 AND Key < 100 ORDER BY Key;
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ CompareYson(R"([
+ [[10u];["foo"];#];
+ [[11u];["bar"];#]
+ ])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+
+ Y_UNIT_TEST(InsertAbort_Literal_Duplicates) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ INSERT INTO `/Root/TwoShard` (Value1, Key) VALUES
+ ("foo", 10u), ("bar", 11u), ("baz", 10u)
+ )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::PRECONDITION_FAILED, result.GetIssues().ToString());
+ UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION, [](const NYql::TIssue& issue) {
+ return issue.Message.Contains("Duplicated keys found.");
+ }));
+
+ result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ SELECT * FROM `/Root/TwoShard` WHERE Key > 5 AND Key < 100 ORDER BY Key;
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ CompareYson(R"([])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+
+ Y_UNIT_TEST(InsertAbort_Literal_Conflict) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ INSERT INTO `/Root/TwoShard` (Value1, Key) VALUES
+ ("foo", 1u), ("bar", 11u)
+ )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::PRECONDITION_FAILED, result.GetIssues().ToString());
+ UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION, [](const NYql::TIssue& issue) {
+ return issue.Message.Contains("Conflict with existing key.");
+ }), result.GetIssues().ToString());
+
+ result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ SELECT * FROM `/Root/TwoShard` WHERE Key = 1 OR Key = 11 ORDER BY Key;
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ CompareYson(R"([
+ [[1u];["One"];[-1]]
+ ])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+
+ Y_UNIT_TEST(InsertAbort_Params_Success) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto params = TParamsBuilder().AddParam("$in")
+ .BeginList()
+ .AddListItem().BeginStruct()
+ .AddMember("Key").Uint32(10)
+ .AddMember("Value1").String("foo")
+ .AddMember("Value2").Int32(5)
+ .EndStruct()
+ .AddListItem().BeginStruct()
+ .AddMember("Key").Uint32(11)
+ .AddMember("Value1").String("bar")
+ .AddMember("Value2").Int32(7)
+ .EndStruct()
+ .EndList().Build().Build();
+
+ auto result = session.ExecuteDataQuery(R"(
+ --!syntax_v1
+ PRAGMA kikimr.UseNewEngine = "true";
+ DECLARE $in AS List<Struct<Key: Uint32, Value1: String, Value2:Int32>>;
+ INSERT INTO `/Root/TwoShard` SELECT * FROM AS_TABLE($in)
+ )", TTxControl::BeginTx().CommitTx(), std::move(params)).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ SELECT * FROM `/Root/TwoShard` WHERE Key > 5 AND Key < 100 ORDER BY Key;
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ CompareYson(R"([
+ [[10u];["foo"];[5]];
+ [[11u];["bar"];[7]]
+ ])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+
+ Y_UNIT_TEST(InsertAbort_Params_Duplicates) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto params = TParamsBuilder().AddParam("$in")
+ .BeginList()
+ .AddListItem().BeginStruct()
+ .AddMember("Key").Uint32(1)
+ .AddMember("Value1").String("foo")
+ .AddMember("Value2").Int32(5)
+ .EndStruct()
+ .AddListItem().BeginStruct()
+ .AddMember("Key").Uint32(11)
+ .AddMember("Value1").String("bar")
+ .AddMember("Value2").Int32(7)
+ .EndStruct()
+ .AddListItem().BeginStruct()
+ .AddMember("Key").Uint32(1)
+ .AddMember("Value1").String("baz")
+ .AddMember("Value2").Int32(9)
+ .EndStruct()
+ .EndList().Build().Build();
+
+ auto result = session.ExecuteDataQuery(R"(
+ --!syntax_v1
+ PRAGMA kikimr.UseNewEngine = "true";
+ DECLARE $in AS List<Struct<Key: Uint32, Value1: String, Value2:Int32>>;
+ INSERT INTO `/Root/TwoShard` SELECT * FROM AS_TABLE($in)
+ )", TTxControl::BeginTx().CommitTx(), std::move(params)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::PRECONDITION_FAILED, result.GetIssues().ToString());
+ UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION, [](const NYql::TIssue& issue) {
+ return issue.Message.Contains("Duplicated keys found.");
+ }));
+
+ result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ SELECT * FROM `/Root/TwoShard` WHERE Key = 1 OR Key = 11 ORDER BY Key;
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ CompareYson(R"([
+ [[1u];["One"];[-1]]
+ ])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+
+ Y_UNIT_TEST(InsertAbort_Params_Conflict) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto params = TParamsBuilder().AddParam("$in")
+ .BeginList()
+ .AddListItem().BeginStruct()
+ .AddMember("Key").Uint32(1)
+ .AddMember("Value1").String("foo")
+ .AddMember("Value2").Int32(5)
+ .EndStruct()
+ .AddListItem().BeginStruct()
+ .AddMember("Key").Uint32(11)
+ .AddMember("Value1").String("bar")
+ .AddMember("Value2").Int32(7)
+ .EndStruct()
+ .EndList().Build().Build();
+
+ auto result = session.ExecuteDataQuery(R"(
+ --!syntax_v1
+ PRAGMA kikimr.UseNewEngine = "true";
+ DECLARE $in AS List<Struct<Key: Uint32, Value1: String, Value2:Int32>>;
+ INSERT INTO `/Root/TwoShard` SELECT * FROM AS_TABLE($in)
+ )", TTxControl::BeginTx().CommitTx(), std::move(params)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::PRECONDITION_FAILED, result.GetIssues().ToString());
+ UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION, [](const NYql::TIssue& issue) {
+ return issue.Message.Contains("Conflict with existing key.");
+ }));
+
+ result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ SELECT * FROM `/Root/TwoShard` WHERE Key = 1 OR Key = 11 ORDER BY Key;
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ CompareYson(R"([
+ [[1u];["One"];[-1]]
+ ])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+
+ Y_UNIT_TEST(InsertAbort_Select_Success) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto ret = session.ExecuteSchemeQuery(R"(
+ CREATE TABLE `Foo` (
+ Key Uint32,
+ Value1 String,
+ Value2 Int32,
+ PRIMARY KEY (Key)
+ )
+ )").ExtractValueSync();
+ UNIT_ASSERT_C(ret.IsSuccess(), ret.GetIssues().ToString());
+
+ auto result = session.ExecuteDataQuery(R"(
+ REPLACE INTO `/Root/Foo` (Key, Value1, Value2) VALUES
+ (10u, "foo", 5), (11u, "bar", 7)
+ )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ INSERT INTO `/Root/TwoShard` SELECT * FROM `/Root/Foo`
+ )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ SELECT * FROM `/Root/TwoShard` WHERE Key > 5 AND Key < 100 ORDER BY Key;
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ CompareYson(R"([
+ [[10u];["foo"];[5]];
+ [[11u];["bar"];[7]]
+ ])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+
+ Y_UNIT_TEST(InsertAbort_Select_Duplicates) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto ret = session.ExecuteSchemeQuery(R"(
+ CREATE TABLE `Foo` (
+ Key Uint32,
+ Value1 String,
+ Value2 Int32,
+ Value3 Uint32,
+ PRIMARY KEY (Key)
+ )
+ )").ExtractValueSync();
+ UNIT_ASSERT_C(ret.IsSuccess(), ret.GetIssues().ToString());
+
+ auto result = session.ExecuteDataQuery(R"(
+ REPLACE INTO `/Root/Foo` (Key, Value1, Value2, Value3) VALUES
+ (1u, "foo", 5, 10), (2u, "bar", 7, 11), (3u, "baz", 9, 10)
+ )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ INSERT INTO `/Root/TwoShard` SELECT Value3 as Key, Value1, Value2 FROM `/Root/Foo`
+ )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::PRECONDITION_FAILED, result.GetIssues().ToString());
+ UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION, [](const NYql::TIssue& issue) {
+ return issue.Message.Contains("Duplicated keys found.");
+ }));
+
+ result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ SELECT * FROM `/Root/TwoShard` WHERE Key = 1 OR Key = 10 OR Key = 11 ORDER BY Key;
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ CompareYson(R"([
+ [[1u];["One"];[-1]]
+ ])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+
+ Y_UNIT_TEST(InsertAbort_Select_Conflict) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto ret = session.ExecuteSchemeQuery(R"(
+ CREATE TABLE `Foo` (
+ Key Uint32,
+ Value1 String,
+ Value2 Int32,
+ PRIMARY KEY (Key)
+ )
+ )").ExtractValueSync();
+ UNIT_ASSERT_C(ret.IsSuccess(), ret.GetIssues().ToString());
+
+ auto result = session.ExecuteDataQuery(R"(
+ REPLACE INTO `/Root/Foo` (Key, Value1, Value2) VALUES
+ (1u, "foo", 5), (11u, "bar", 7)
+ )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ INSERT INTO `/Root/TwoShard` SELECT * FROM `/Root/Foo`
+ )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::PRECONDITION_FAILED, result.GetIssues().ToString());
+ UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION, [](const NYql::TIssue& issue) {
+ return issue.Message.Contains("Conflict with existing key.");
+ }));
+
+ result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ SELECT * FROM `/Root/TwoShard` WHERE Key = 1 OR Key = 11 ORDER BY Key;
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ CompareYson(R"([
+ [[1u];["One"];[-1]]
+ ])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+
+ Y_UNIT_TEST(InsertRevert_Literal_Success) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ INSERT OR REVERT INTO `/Root/TwoShard` (Value1, Key) VALUES
+ ("foo", 10u), ("bar", 11u)
+ )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ SELECT * FROM `/Root/TwoShard` WHERE Key > 5 AND Key < 100 ORDER BY Key;
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ CompareYson(R"([
+ [[10u];["foo"];#];
+ [[11u];["bar"];#]
+ ])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+
+ Y_UNIT_TEST(InsertRevert_Literal_Duplicates) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ INSERT OR REVERT INTO `/Root/TwoShard` (Value1, Key) VALUES
+ ("foo", 10u), ("bar", 11u), ("baz", 10u)
+ )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetIssues().Size(), 0, result.GetIssues().ToString());
+
+ result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ SELECT * FROM `/Root/TwoShard` WHERE Key > 5 AND Key < 100 ORDER BY Key;
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ CompareYson(R"([])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+
+ Y_UNIT_TEST(InsertRevert_Literal_Conflict) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ INSERT OR REVERT INTO `/Root/TwoShard` (Value1, Key) VALUES
+ ("foo", 1u), ("bar", 11u)
+ )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetIssues().Size(), 0, result.GetIssues().ToString());
+
+ result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ SELECT * FROM `/Root/TwoShard` WHERE Key = 1 OR Key = 11 ORDER BY Key;
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ CompareYson(R"([
+ [[1u];["One"];[-1]]
+ ])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+
+ Y_UNIT_TEST(UpdateOn_Literal) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+
+ UPDATE `/Root/TwoShard` ON (Key, Value1) VALUES
+ (1u, "Updated"), (4000000010u, "Updated")
+ )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+
+ result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ SELECT * FROM `/Root/TwoShard` ORDER BY Key;
+ )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ CompareYson(R"([
+ [[1u];["Updated"];[-1]];
+ [[2u];["Two"];[0]];
+ [[3u];["Three"];[1]];
+ [[4000000001u];["BigOne"];[-1]];
+ [[4000000002u];["BigTwo"];[0]];
+ [[4000000003u];["BigThree"];[1]]
+ ])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+
+ Y_UNIT_TEST(UpdateOn_Params) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto params = TParamsBuilder().AddParam("$in")
+ .BeginList()
+ .AddListItem().BeginStruct()
+ .AddMember("Key").Uint32(1)
+ .AddMember("Value1").String("Updated")
+ .AddMember("Value2").Int32(-10)
+ .EndStruct()
+ .AddListItem().BeginStruct()
+ .AddMember("Key").Uint32(4000000010)
+ .AddMember("Value1").String("Updated")
+ .AddMember("Value2").Int32(7)
+ .EndStruct()
+ .EndList().Build().Build();
+
+ auto result = session.ExecuteDataQuery(R"(
+ --!syntax_v1
+ PRAGMA kikimr.UseNewEngine = "true";
+ DECLARE $in AS List<Struct<Key: Uint32, Value1: String, Value2:Int32>>;
+ UPDATE `/Root/TwoShard` ON (Key, Value1, Value2) SELECT Key, Value1, Value2 FROM AS_TABLE($in)
+ )", TTxControl::BeginTx().CommitTx(), std::move(params)).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ SELECT * FROM `/Root/TwoShard` ORDER BY Key;
+ )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ CompareYson(R"([
+ [[1u];["Updated"];[-10]];
+ [[2u];["Two"];[0]];
+ [[3u];["Three"];[1]];
+ [[4000000001u];["BigOne"];[-1]];
+ [[4000000002u];["BigTwo"];[0]];
+ [[4000000003u];["BigThree"];[1]]
+ ])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+
+ Y_UNIT_TEST(UpdateOn_Select) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+
+ $to_update = (
+ SELECT Key, "Updated" as Value1, (Value2 + 5) as Value2
+ FROM `/Root/TwoShard`
+ WHERE Value2 >= 0
+ );
+ UPDATE `/Root/TwoShard` ON SELECT * FROM $to_update;
+ )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+
+ result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ SELECT * FROM `/Root/TwoShard` ORDER BY Key;
+ )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ CompareYson(R"([
+ [[1u];["One"];[-1]];
+ [[2u];["Updated"];[5]];
+ [[3u];["Updated"];[6]];
+ [[4000000001u];["BigOne"];[-1]];
+ [[4000000002u];["Updated"];[5]];
+ [[4000000003u];["Updated"];[6]]
+ ])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+}
+
+} // namespace NKqp
+} // namespace NKikimr
diff --git a/ydb/core/kqp/ut/kqp_newengine_flowcontrol_ut.cpp b/ydb/core/kqp/ut/kqp_newengine_flowcontrol_ut.cpp
index 9448e02204..16c89eceb5 100644
--- a/ydb/core/kqp/ut/kqp_newengine_flowcontrol_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_newengine_flowcontrol_ut.cpp
@@ -1,190 +1,190 @@
#include <ydb/core/kqp/ut/common/kqp_ut_common.h>
#include <ydb/core/protos/kqp.pb.h>
#include <ydb/core/kqp/counters/kqp_counters.h>
-
+
#include <ydb/public/lib/experimental/ydb_experimental.h>
-
+
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h>
-namespace NKikimr::NKqp {
-
-using namespace NYdb;
-using namespace NYdb::NTable;
-
-namespace {
-
-void CreateSampleTables(TKikimrRunner& kikimr) {
- kikimr.GetTestClient().CreateTable("/Root", R"(
- Name: "FourShard"
- Columns { Name: "Key", Type: "Uint64" }
- Columns { Name: "Value1", Type: "String" }
- Columns { Name: "Value2", Type: "String" }
- KeyColumnNames: ["Key"],
- SplitBoundary { KeyPrefix { Tuple { Optional { Uint64: 100 } } } }
- SplitBoundary { KeyPrefix { Tuple { Optional { Uint64: 200 } } } }
- SplitBoundary { KeyPrefix { Tuple { Optional { Uint64: 300 } } } }
- )");
-
- TTableClient tableClient{kikimr.GetDriver()};
- auto session = tableClient.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExecuteDataQuery(R"(
+namespace NKikimr::NKqp {
+
+using namespace NYdb;
+using namespace NYdb::NTable;
+
+namespace {
+
+void CreateSampleTables(TKikimrRunner& kikimr) {
+ kikimr.GetTestClient().CreateTable("/Root", R"(
+ Name: "FourShard"
+ Columns { Name: "Key", Type: "Uint64" }
+ Columns { Name: "Value1", Type: "String" }
+ Columns { Name: "Value2", Type: "String" }
+ KeyColumnNames: ["Key"],
+ SplitBoundary { KeyPrefix { Tuple { Optional { Uint64: 100 } } } }
+ SplitBoundary { KeyPrefix { Tuple { Optional { Uint64: 200 } } } }
+ SplitBoundary { KeyPrefix { Tuple { Optional { Uint64: 300 } } } }
+ )");
+
+ TTableClient tableClient{kikimr.GetDriver()};
+ auto session = tableClient.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExecuteDataQuery(R"(
REPLACE INTO `/Root/FourShard` (Key, Value1, Value2) VALUES
- (1u, "Value-001", "1"),
- (2u, "Value-002", "2"),
- (101u, "Value-101", "101"),
- (102u, "Value-102", "102"),
- (201u, "Value-201", "201"),
- (202u, "Value-202", "202"),
- (301u, "Value-301", "301"),
- (302u, "Value-302", "302")
- )", TTxControl::BeginTx().CommitTx()).GetValueSync();
-
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- session.Close();
-}
-
-} // anonymous namespace
-
-Y_UNIT_TEST_SUITE(KqpFlowControl) {
-
-#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 9u
-
-void DoFlowControlTest(ui64 limit, bool hasBlockedByCapacity) {
- NKikimrConfig::TAppConfig appCfg;
- appCfg.MutableTableServiceConfig()->MutableResourceManager()->SetChannelBufferSize(limit);
- appCfg.MutableTableServiceConfig()->MutableResourceManager()->SetMinChannelBufferSize(limit);
- appCfg.MutableTableServiceConfig()->MutableResourceManager()->SetScanBufferSize(limit);
- appCfg.MutableTableServiceConfig()->MutableResourceManager()->SetMkqlHeavyProgramMemoryLimit(200ul << 20);
- appCfg.MutableTableServiceConfig()->MutableResourceManager()->SetQueryMemoryLimit(20ul << 30);
-
- TKikimrRunner kikimr{appCfg, "", KikimrDefaultUtDomainRoot};
-
- CreateSampleTables(kikimr);
- NExperimental::TStreamQueryClient db(kikimr.GetDriver());
-
- Y_DEFER {
- NYql::NDq::GetDqExecutionSettingsForTests().Reset();
- };
-
- NYql::NDq::GetDqExecutionSettingsForTests().FlowControl.MaxOutputChunkSize = limit;
- NYql::NDq::GetDqExecutionSettingsForTests().FlowControl.InFlightBytesOvercommit = 1.0f;
-
- auto settings = NExperimental::TExecuteStreamQuerySettings()
+ (1u, "Value-001", "1"),
+ (2u, "Value-002", "2"),
+ (101u, "Value-101", "101"),
+ (102u, "Value-102", "102"),
+ (201u, "Value-201", "201"),
+ (202u, "Value-202", "202"),
+ (301u, "Value-301", "301"),
+ (302u, "Value-302", "302")
+ )", TTxControl::BeginTx().CommitTx()).GetValueSync();
+
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ session.Close();
+}
+
+} // anonymous namespace
+
+Y_UNIT_TEST_SUITE(KqpFlowControl) {
+
+#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 9u
+
+void DoFlowControlTest(ui64 limit, bool hasBlockedByCapacity) {
+ NKikimrConfig::TAppConfig appCfg;
+ appCfg.MutableTableServiceConfig()->MutableResourceManager()->SetChannelBufferSize(limit);
+ appCfg.MutableTableServiceConfig()->MutableResourceManager()->SetMinChannelBufferSize(limit);
+ appCfg.MutableTableServiceConfig()->MutableResourceManager()->SetScanBufferSize(limit);
+ appCfg.MutableTableServiceConfig()->MutableResourceManager()->SetMkqlHeavyProgramMemoryLimit(200ul << 20);
+ appCfg.MutableTableServiceConfig()->MutableResourceManager()->SetQueryMemoryLimit(20ul << 30);
+
+ TKikimrRunner kikimr{appCfg, "", KikimrDefaultUtDomainRoot};
+
+ CreateSampleTables(kikimr);
+ NExperimental::TStreamQueryClient db(kikimr.GetDriver());
+
+ Y_DEFER {
+ NYql::NDq::GetDqExecutionSettingsForTests().Reset();
+ };
+
+ NYql::NDq::GetDqExecutionSettingsForTests().FlowControl.MaxOutputChunkSize = limit;
+ NYql::NDq::GetDqExecutionSettingsForTests().FlowControl.InFlightBytesOvercommit = 1.0f;
+
+ auto settings = NExperimental::TExecuteStreamQuerySettings()
.ProfileMode(NExperimental::EStreamQueryProfileMode::Full);
-
- auto result = db.ExecuteStreamQuery(R"(
+
+ auto result = db.ExecuteStreamQuery(R"(
$r = (select * from `/Root/FourShard` where Key > 201);
-
- SELECT l.Key as key, l.Text as text, r.Value1 as value
+
+ SELECT l.Key as key, l.Text as text, r.Value1 as value
FROM `/Root/EightShard` AS l JOIN $r AS r ON l.Key = r.Key
- ORDER BY key, text, value
- )", settings).GetValueSync();
-
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- TVector<TString> profiles;
- CompareYson(R"([
- [[202u];["Value2"];["Value-202"]];
- [[301u];["Value1"];["Value-301"]];
- [[302u];["Value2"];["Value-302"]]
- ])", StreamResultToYson(result, &profiles));
-
- UNIT_ASSERT_EQUAL(2, profiles.size());
-
- NYql::NDqProto::TDqExecutionStats stats;
- google::protobuf::TextFormat::ParseFromString(profiles[1], &stats);
- UNIT_ASSERT(stats.IsInitialized());
-
- ui32 blockedByCapacity = 0;
- for (const auto& stage : stats.GetStages()) {
- for (const auto& ca : stage.GetComputeActors()) {
- for (const auto& task : ca.GetTasks()) {
- for (const auto& output : task.GetOutputChannels()) {
- blockedByCapacity += output.GetBlockedByCapacity();
- }
- }
- }
- }
-
- UNIT_ASSERT_EQUAL(hasBlockedByCapacity, blockedByCapacity > 0);
-}
-
-Y_UNIT_TEST(FlowControl_Unlimited) {
- DoFlowControlTest(100ul << 20, false);
-}
-
-Y_UNIT_TEST(FlowControl_BigLimit) {
- DoFlowControlTest(1ul << 10, false);
-}
-
-Y_UNIT_TEST(FlowControl_SmallLimit) {
- DoFlowControlTest(1ul, true);
-}
-
-//Y_UNIT_TEST(SlowClient) {
-void SlowClient() {
- NKikimrConfig::TAppConfig appCfg;
- appCfg.MutableTableServiceConfig()->MutableResourceManager()->SetChannelBufferSize(1);
-
- TKikimrRunner kikimr(appCfg);
-
- {
- TTableClient tableClient{kikimr.GetDriver()};
- auto session = tableClient.CreateSession().GetValueSync().GetSession();
- auto value = std::string(1000, 'a');
-
- for (int q = 0; q < 100; ++q) {
- TStringBuilder query;
+ ORDER BY key, text, value
+ )", settings).GetValueSync();
+
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ TVector<TString> profiles;
+ CompareYson(R"([
+ [[202u];["Value2"];["Value-202"]];
+ [[301u];["Value1"];["Value-301"]];
+ [[302u];["Value2"];["Value-302"]]
+ ])", StreamResultToYson(result, &profiles));
+
+ UNIT_ASSERT_EQUAL(2, profiles.size());
+
+ NYql::NDqProto::TDqExecutionStats stats;
+ google::protobuf::TextFormat::ParseFromString(profiles[1], &stats);
+ UNIT_ASSERT(stats.IsInitialized());
+
+ ui32 blockedByCapacity = 0;
+ for (const auto& stage : stats.GetStages()) {
+ for (const auto& ca : stage.GetComputeActors()) {
+ for (const auto& task : ca.GetTasks()) {
+ for (const auto& output : task.GetOutputChannels()) {
+ blockedByCapacity += output.GetBlockedByCapacity();
+ }
+ }
+ }
+ }
+
+ UNIT_ASSERT_EQUAL(hasBlockedByCapacity, blockedByCapacity > 0);
+}
+
+Y_UNIT_TEST(FlowControl_Unlimited) {
+ DoFlowControlTest(100ul << 20, false);
+}
+
+Y_UNIT_TEST(FlowControl_BigLimit) {
+ DoFlowControlTest(1ul << 10, false);
+}
+
+Y_UNIT_TEST(FlowControl_SmallLimit) {
+ DoFlowControlTest(1ul, true);
+}
+
+//Y_UNIT_TEST(SlowClient) {
+void SlowClient() {
+ NKikimrConfig::TAppConfig appCfg;
+ appCfg.MutableTableServiceConfig()->MutableResourceManager()->SetChannelBufferSize(1);
+
+ TKikimrRunner kikimr(appCfg);
+
+ {
+ TTableClient tableClient{kikimr.GetDriver()};
+ auto session = tableClient.CreateSession().GetValueSync().GetSession();
+ auto value = std::string(1000, 'a');
+
+ for (int q = 0; q < 100; ++q) {
+ TStringBuilder query;
query << "REPLACE INTO [/Root/KeyValue] (Key, Value) VALUES (" << q << ", \"" << value << "\")";
-
- auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).GetValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
- }
- }
-
- NExperimental::TStreamQueryClient db(kikimr.GetDriver());
-
+
+ auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).GetValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ }
+ }
+
+ NExperimental::TStreamQueryClient db(kikimr.GetDriver());
+
auto it = db.ExecuteStreamQuery("SELECT Key, Value FROM `/Root/KeyValue`").GetValueSync();
- auto part = it.ReadNext().GetValueSync();
-
- auto counters = kikimr.GetTestServer().GetRuntime()->GetAppData(0).Counters;
- TKqpCounters kqpCounters(counters);
-
- UNIT_ASSERT_EQUAL(kqpCounters.RmComputeActors->Val(), 2);
-
- Cerr << "-- got value and go sleep...\n";
- ::Sleep(TDuration::Seconds(3));
- Cerr << "-- go on...\n";
-
- UNIT_ASSERT_EQUAL(kqpCounters.RmComputeActors->Val(), 2);
-
- // consume 990 elements
- int remains = 990;
- while (remains > 0) {
- if (part.HasResultSet()) {
- part.ExtractResultSet();
- --remains;
- ::Sleep(TDuration::MilliSeconds(10));
- Cerr << "-- remains: " << remains << Endl;
- }
- part = it.ReadNext().GetValueSync();
- UNIT_ASSERT(!part.EOS());
- }
-
- UNIT_ASSERT_EQUAL(kqpCounters.RmComputeActors->Val(), 2);
-
- while (!part.EOS()) {
- part = it.ReadNext().GetValueSync();
- }
-
- UNIT_ASSERT_EQUAL(kqpCounters.RmComputeActors->Val(), 0);
- UNIT_ASSERT_EQUAL(kqpCounters.RmMemory->Val(), 0);
-}
-
-#endif
-
-} // suite
-
-} // namespace NKikimr::NKqp
+ auto part = it.ReadNext().GetValueSync();
+
+ auto counters = kikimr.GetTestServer().GetRuntime()->GetAppData(0).Counters;
+ TKqpCounters kqpCounters(counters);
+
+ UNIT_ASSERT_EQUAL(kqpCounters.RmComputeActors->Val(), 2);
+
+ Cerr << "-- got value and go sleep...\n";
+ ::Sleep(TDuration::Seconds(3));
+ Cerr << "-- go on...\n";
+
+ UNIT_ASSERT_EQUAL(kqpCounters.RmComputeActors->Val(), 2);
+
+ // consume 990 elements
+ int remains = 990;
+ while (remains > 0) {
+ if (part.HasResultSet()) {
+ part.ExtractResultSet();
+ --remains;
+ ::Sleep(TDuration::MilliSeconds(10));
+ Cerr << "-- remains: " << remains << Endl;
+ }
+ part = it.ReadNext().GetValueSync();
+ UNIT_ASSERT(!part.EOS());
+ }
+
+ UNIT_ASSERT_EQUAL(kqpCounters.RmComputeActors->Val(), 2);
+
+ while (!part.EOS()) {
+ part = it.ReadNext().GetValueSync();
+ }
+
+ UNIT_ASSERT_EQUAL(kqpCounters.RmComputeActors->Val(), 0);
+ UNIT_ASSERT_EQUAL(kqpCounters.RmMemory->Val(), 0);
+}
+
+#endif
+
+} // suite
+
+} // namespace NKikimr::NKqp
diff --git a/ydb/core/kqp/ut/kqp_newengine_ut.cpp b/ydb/core/kqp/ut/kqp_newengine_ut.cpp
index fc8e3d7b8c..3737687d8d 100644
--- a/ydb/core/kqp/ut/kqp_newengine_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_newengine_ut.cpp
@@ -19,7 +19,7 @@ TKikimrRunner KikimrRunnerWithSessionActor() {
}
Y_UNIT_TEST_SUITE(KqpNewEngine) {
-
+
void TestSimpleSelect(const TKikimrRunner& kikimr) {
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -28,14 +28,14 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
PRAGMA kikimr.UseNewEngine = "true";
SELECT Value1, Value2, Key FROM [/Root/TwoShard] WHERE Value2 != 0 ORDER BY Key DESC;
)", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- AssertSuccessResult(result);
+ AssertSuccessResult(result);
CompareYson(R"(
[
- [["BigThree"];[1];[4000000003u]];
- [["BigOne"];[-1];[4000000001u]];
- [["Three"];[1];[3u]];
- [["One"];[-1];[1u]]
+ [["BigThree"];[1];[4000000003u]];
+ [["BigOne"];[-1];[4000000001u]];
+ [["Three"];[1];[3u]];
+ [["One"];[-1];[1u]]
]
)", FormatResultSetYson(result.GetResultSet(0)));
}
@@ -71,36 +71,36 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
.AddParam("$key").Uint64(302).Build()
.Build();
- NYdb::NTable::TExecDataQuerySettings execSettings;
- execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
-
+ NYdb::NTable::TExecDataQuerySettings execSettings;
+ execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
+
auto result = session.ExecuteDataQuery(query,
- TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params, execSettings).ExtractValueSync();
+ TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params, execSettings).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
CompareYson(R"([[[2];[302u];["Value2"]]])", FormatResultSetYson(result.GetResultSet(0)));
- auto stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
- UNIT_ASSERT_VALUES_EQUAL(1, stats.query_phases().size()); // no LiteralExecuter phase
- UNIT_ASSERT_VALUES_EQUAL(1, stats.query_phases()[0].table_access().size());
- UNIT_ASSERT_VALUES_EQUAL(1, stats.query_phases()[0].affected_shards());
- UNIT_ASSERT_VALUES_EQUAL("/Root/EightShard", stats.query_phases()[0].table_access()[0].name());
- UNIT_ASSERT_VALUES_EQUAL(1, stats.query_phases()[0].table_access()[0].partitions_count());
-
+ auto stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
+ UNIT_ASSERT_VALUES_EQUAL(1, stats.query_phases().size()); // no LiteralExecuter phase
+ UNIT_ASSERT_VALUES_EQUAL(1, stats.query_phases()[0].table_access().size());
+ UNIT_ASSERT_VALUES_EQUAL(1, stats.query_phases()[0].affected_shards());
+ UNIT_ASSERT_VALUES_EQUAL("/Root/EightShard", stats.query_phases()[0].table_access()[0].name());
+ UNIT_ASSERT_VALUES_EQUAL(1, stats.query_phases()[0].table_access()[0].partitions_count());
+
params = kikimr.GetTableClient().GetParamsBuilder()
.AddParam("$key").Uint64(330).Build()
.Build();
result = session.ExecuteDataQuery(query,
- TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params, execSettings).ExtractValueSync();
+ TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params, execSettings).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
CompareYson(R"([])", FormatResultSetYson(result.GetResultSet(0)));
-
- stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
- UNIT_ASSERT_VALUES_EQUAL(1, stats.query_phases().size()); // no LiteralExecuter phase
- UNIT_ASSERT_VALUES_EQUAL(1, stats.query_phases()[0].table_access().size());
- UNIT_ASSERT_VALUES_EQUAL(1, stats.query_phases()[0].affected_shards());
- UNIT_ASSERT_VALUES_EQUAL("/Root/EightShard", stats.query_phases()[0].table_access()[0].name());
- UNIT_ASSERT_VALUES_EQUAL(1, stats.query_phases()[0].table_access()[0].partitions_count());
+
+ stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
+ UNIT_ASSERT_VALUES_EQUAL(1, stats.query_phases().size()); // no LiteralExecuter phase
+ UNIT_ASSERT_VALUES_EQUAL(1, stats.query_phases()[0].table_access().size());
+ UNIT_ASSERT_VALUES_EQUAL(1, stats.query_phases()[0].affected_shards());
+ UNIT_ASSERT_VALUES_EQUAL("/Root/EightShard", stats.query_phases()[0].table_access()[0].name());
+ UNIT_ASSERT_VALUES_EQUAL(1, stats.query_phases()[0].table_access()[0].partitions_count());
}
Y_UNIT_TEST(PkSelect2) {
@@ -165,7 +165,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
SELECT * FROM [/Root/EightShard] WHERE Key > $low AND Key < $high ORDER BY Key;
)", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params).ExtractValueSync();
result.GetIssues().PrintTo(Cerr);
- AssertSuccessResult(result);
+ AssertSuccessResult(result);
CompareYson(R"([
[[3];[203u];["Value3"]];
@@ -199,7 +199,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
SELECT * FROM [/Root/EightShard] WHERE Key >= $low AND Key <= $high ORDER BY Key;
)", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params).ExtractValueSync();
result.GetIssues().PrintTo(Cerr);
- AssertSuccessResult(result);
+ AssertSuccessResult(result);
CompareYson(R"([
[[1];[202u];["Value2"]];
@@ -231,7 +231,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
SELECT * FROM [/Root/EightShard] WHERE Key > $low ORDER BY Key;
)", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params).ExtractValueSync();
result.GetIssues().PrintTo(Cerr);
- AssertSuccessResult(result);
+ AssertSuccessResult(result);
CompareYson(R"([
[[1];[603u];["Value3"]];
@@ -263,7 +263,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
SELECT * FROM [/Root/EightShard] WHERE Key < $high ORDER BY Key;
)", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params).ExtractValueSync();
result.GetIssues().PrintTo(Cerr);
- AssertSuccessResult(result);
+ AssertSuccessResult(result);
CompareYson(R"([
[[1];[101u];["Value1"]];
@@ -306,31 +306,31 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
])", FormatResultSetYson(result.GetResultSet(0)));
}
- Y_UNIT_TEST(MultiOutput) {
+ Y_UNIT_TEST(MultiOutput) {
TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
$left = (select Key, Value1, Value2 from [/Root/TwoShard] where Value2 = 1);
$right = (select Key, Value1, Value2 from [/Root/TwoShard] where Value2 = -1);
- select Key, Value1, Value2 from $left order by Key;
- select Key, Value1, Value2 from $right order by Key;
- )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- AssertSuccessResult(result);
-
- CompareYson(R"([
- [[3u];["Three"];[1]];
- [[4000000003u];["BigThree"];[1]];
- ])", FormatResultSetYson(result.GetResultSet(0)));
-
- CompareYson(R"([
- [[1u];["One"];[-1]];
- [[4000000001u];["BigOne"];[-1]]
- ])", FormatResultSetYson(result.GetResultSet(1)));
- }
-
+ select Key, Value1, Value2 from $left order by Key;
+ select Key, Value1, Value2 from $right order by Key;
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ AssertSuccessResult(result);
+
+ CompareYson(R"([
+ [[3u];["Three"];[1]];
+ [[4000000003u];["BigThree"];[1]];
+ ])", FormatResultSetYson(result.GetResultSet(0)));
+
+ CompareYson(R"([
+ [[1u];["One"];[-1]];
+ [[4000000001u];["BigOne"];[-1]]
+ ])", FormatResultSetYson(result.GetResultSet(1)));
+ }
+
Y_UNIT_TEST(InShardsWrite) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
@@ -341,13 +341,13 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
UPSERT INTO [/Root/TwoShard]
SELECT Key, Value1, Value2 + 1 AS Value2 FROM [/Root/TwoShard];
)", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- AssertSuccessResult(result);
+ AssertSuccessResult(result);
result = session.ExecuteDataQuery(R"(
PRAGMA kikimr.UseNewEngine = "true";
SELECT * FROM [/Root/TwoShard] ORDER BY Key;
)", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- AssertSuccessResult(result);
+ AssertSuccessResult(result);
CompareYson(R"(
[
@@ -371,13 +371,13 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
UPSERT INTO [/Root/TwoShard]
SELECT Key - 3u AS Key, Value1, Value2 + 100 AS Value2 FROM [/Root/TwoShard];
)", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- AssertSuccessResult(result);
+ AssertSuccessResult(result);
result = session.ExecuteDataQuery(R"(
PRAGMA kikimr.UseNewEngine = "true";
SELECT * FROM [/Root/TwoShard] WHERE Value2 > 10 ORDER BY Key;
)", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- AssertSuccessResult(result);
+ AssertSuccessResult(result);
CompareYson(R"(
[
@@ -482,13 +482,13 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
(10u, "One", -10),
(20u, "Two", -20);
)", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- AssertSuccessResult(result);
+ AssertSuccessResult(result);
result = session.ExecuteDataQuery(R"(
PRAGMA kikimr.UseNewEngine = "true";
SELECT * FROM [/Root/TwoShard] WHERE Value2 <= -10 ORDER BY Key;
)", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- AssertSuccessResult(result);
+ AssertSuccessResult(result);
CompareYson(R"(
[
@@ -532,13 +532,13 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
)", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(),
std::move(params)).ExtractValueSync();
- AssertSuccessResult(result);
+ AssertSuccessResult(result);
result = session.ExecuteDataQuery(R"(
PRAGMA kikimr.UseNewEngine = "true";
SELECT * FROM [/Root/TwoShard] WHERE Value1 = "New" ORDER BY Key;
)", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- AssertSuccessResult(result);
+ AssertSuccessResult(result);
CompareYson(R"(
[
@@ -582,13 +582,13 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
UPSERT INTO [/Root/TwoShard]
SELECT * FROM AS_TABLE($items);
)", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), std::move(params)).ExtractValueSync();
- AssertSuccessResult(result);
+ AssertSuccessResult(result);
result = session.ExecuteDataQuery(R"(
PRAGMA kikimr.UseNewEngine = "true";
SELECT * FROM [/Root/TwoShard] WHERE Value1 = "New" ORDER BY Key;
)", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- AssertSuccessResult(result);
+ AssertSuccessResult(result);
CompareYson(R"(
[
@@ -666,7 +666,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
auto result = query.Execute(TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(),
paramsBuilder.Build()).ExtractValueSync();
- AssertSuccessResult(result);
+ AssertSuccessResult(result);
auto batchLatency = TInstant::Now() - beginTime;
totalLatency += batchLatency;
@@ -684,7 +684,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
SELECT Data, SUM(Key) AS Total FROM [/Root/EightShard] GROUP BY Data ORDER BY Data;
)", TTxControl::BeginTx(TTxSettings::OnlineRO()).CommitTx()).ExtractValueSync();
result.GetIssues().PrintTo(Cerr);
- AssertSuccessResult(result);
+ AssertSuccessResult(result);
CompareYson(R"(
[[[1];[3615u]];[[2];[3616u]];[[3];[3617u]]]
@@ -700,7 +700,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
SELECT Data, Text, COUNT(Key) AS Total FROM [/Root/EightShard] GROUP BY Data, Text ORDER BY Data, Text;
)", TTxControl::BeginTx(TTxSettings::OnlineRO()).CommitTx()).ExtractValueSync();
result.GetIssues().PrintTo(Cerr);
- AssertSuccessResult(result);
+ AssertSuccessResult(result);
CompareYson(R"([
[[1];["Value1"];3u];[[1];["Value2"];3u];[[1];["Value3"];2u];
@@ -714,44 +714,44 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
-
- NYdb::NTable::TExecDataQuerySettings execSettings;
- execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
-
+
+ NYdb::NTable::TExecDataQuerySettings execSettings;
+ execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
+
auto result = session.ExecuteDataQuery(R"(
PRAGMA kikimr.UseNewEngine = "true";
SELECT '42', 42, 5*5;
- )", TTxControl::BeginTx(TTxSettings::OnlineRO()).CommitTx(), execSettings).ExtractValueSync();
+ )", TTxControl::BeginTx(TTxSettings::OnlineRO()).CommitTx(), execSettings).ExtractValueSync();
CompareYson(R"([["42";42;25]])", FormatResultSetYson(result.GetResultSet(0)));
-
- auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
- UNIT_ASSERT_EQUAL(stats.query_phases().size(), 1);
- UNIT_ASSERT(stats.query_phases(0).table_access().size() == 0);
+
+ auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
+ UNIT_ASSERT_EQUAL(stats.query_phases().size(), 1);
+ UNIT_ASSERT(stats.query_phases(0).table_access().size() == 0);
}
Y_UNIT_TEST(MultiStatement) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
-
- NYdb::NTable::TExecDataQuerySettings execSettings;
- execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
-
+
+ NYdb::NTable::TExecDataQuerySettings execSettings;
+ execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
+
auto result = session.ExecuteDataQuery(R"(
PRAGMA kikimr.UseNewEngine = "true";
SELECT 1;
SELECT 2;
SELECT 3;
- )", TTxControl::BeginTx(TTxSettings::OnlineRO()).CommitTx(), execSettings).ExtractValueSync();
+ )", TTxControl::BeginTx(TTxSettings::OnlineRO()).CommitTx(), execSettings).ExtractValueSync();
CompareYson(R"([[1]])", FormatResultSetYson(result.GetResultSet(0)));
CompareYson(R"([[2]])", FormatResultSetYson(result.GetResultSet(1)));
CompareYson(R"([[3]])", FormatResultSetYson(result.GetResultSet(2)));
-
- auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
- UNIT_ASSERT_EQUAL(stats.query_phases().size(), 1);
- UNIT_ASSERT(stats.query_phases(0).table_access().size() == 0);
+
+ auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
+ UNIT_ASSERT_EQUAL(stats.query_phases().size(), 1);
+ UNIT_ASSERT(stats.query_phases(0).table_access().size() == 0);
}
Y_UNIT_TEST(MultiStatementMixPure) {
@@ -781,7 +781,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
PRAGMA kikimr.UseNewEngine = "true";
SELECT * FROM [/Root/TwoShard] WHERE Key = 1;
)", TTxControl::BeginTx(TTxSettings::SerializableRW())).GetValueSync();
- AssertSuccessResult(result);
+ AssertSuccessResult(result);
auto tx = result.GetTransaction();
@@ -790,7 +790,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
PRAGMA kikimr.UseNewEngine = "true";
UPSERT INTO [/Root/TwoShard] (Key, Value1) VALUES(1, "NewValue");
)", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).GetValueSync();
- AssertSuccessResult(result);
+ AssertSuccessResult(result);
result = session1.ExecuteDataQuery(R"(
PRAGMA kikimr.UseNewEngine = "true";
@@ -816,7 +816,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
PRAGMA kikimr.UseNewEngine = "true";
SELECT * FROM [/Root/TwoShard];
)", TTxControl::BeginTx(TTxSettings::SerializableRW())).GetValueSync();
- AssertSuccessResult(result);
+ AssertSuccessResult(result);
auto tx = result.GetTransaction();
@@ -825,7 +825,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
PRAGMA kikimr.UseNewEngine = "true";
UPSERT INTO [/Root/TwoShard] (Key, Value1) VALUES(101, "NewValue");
)", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).GetValueSync();
- AssertSuccessResult(result);
+ AssertSuccessResult(result);
result = session1.ExecuteDataQuery(R"(
PRAGMA kikimr.UseNewEngine = "true";
@@ -851,7 +851,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
PRAGMA kikimr.UseNewEngine = "true";
SELECT * FROM [/Root/TwoShard];
)", TTxControl::BeginTx(TTxSettings::SerializableRW())).GetValueSync();
- AssertSuccessResult(result);
+ AssertSuccessResult(result);
auto tx = result.GetTransaction();
@@ -859,7 +859,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
PRAGMA kikimr.UseNewEngine = "true";
SELECT * FROM [/Root/EightShard];
)", TTxControl::Tx(*tx).CommitTx()).GetValueSync();
- AssertSuccessResult(result);
+ AssertSuccessResult(result);
}
Y_UNIT_TEST(LocksEffects) {
@@ -871,7 +871,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
PRAGMA kikimr.UseNewEngine = "true";
SELECT * FROM [/Root/TwoShard] WHERE Key = 1;
)", TTxControl::BeginTx(TTxSettings::SerializableRW())).GetValueSync();
- AssertSuccessResult(result);
+ AssertSuccessResult(result);
auto tx = result.GetTransaction();
@@ -880,7 +880,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
PRAGMA kikimr.UseNewEngine = "true";
UPSERT INTO [/Root/TwoShard] (Key, Value1) VALUES(1, "NewValue");
)", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).GetValueSync();
- AssertSuccessResult(result);
+ AssertSuccessResult(result);
result = session1.ExecuteDataQuery(R"(
PRAGMA kikimr.UseNewEngine = "true";
@@ -895,84 +895,84 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
PRAGMA kikimr.UseNewEngine = "true";
SELECT * FROM [/Root/TwoShard] WHERE Key <= 2;
)", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).GetValueSync();
- AssertSuccessResult(result);
+ AssertSuccessResult(result);
CompareYson(R"([[[1u];["NewValue"];[-1]];[[2u];["Two"];[0]]])",
FormatResultSetYson(result.GetResultSet(0)));
}
- Y_UNIT_TEST(LocksNoMutations) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
+ Y_UNIT_TEST(LocksNoMutations) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
SELECT * FROM `/Root/KeyValue`
- )", TTxControl::BeginTx(TTxSettings::SerializableRW())).GetValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- auto tx = result.GetTransaction();
-
- result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- SELECT 42;
- )", TTxControl::Tx(*tx).CommitTx()).GetValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- CompareYson(R"([[42]])", FormatResultSetYson(result.GetResultSet(0)));
- }
-
- Y_UNIT_TEST(LocksNoMutationsSharded) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW())).GetValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ auto tx = result.GetTransaction();
+
+ result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ SELECT 42;
+ )", TTxControl::Tx(*tx).CommitTx()).GetValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ CompareYson(R"([[42]])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+
+ Y_UNIT_TEST(LocksNoMutationsSharded) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
SELECT * FROM [/Root/TwoShard]
- )", TTxControl::BeginTx(TTxSettings::SerializableRW())).GetValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- auto tx = result.GetTransaction();
-
- result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- SELECT 42;
- )", TTxControl::Tx(*tx).CommitTx()).GetValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- CompareYson(R"([[42]])", FormatResultSetYson(result.GetResultSet(0)));
- }
-
- Y_UNIT_TEST(BrokenLocksAtROTx) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW())).GetValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ auto tx = result.GetTransaction();
+
+ result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ SELECT 42;
+ )", TTxControl::Tx(*tx).CommitTx()).GetValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ CompareYson(R"([[42]])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+
+ Y_UNIT_TEST(BrokenLocksAtROTx) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
SELECT * FROM `/Root/KeyValue`
- )", TTxControl::BeginTx(TTxSettings::SerializableRW())).GetValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- auto tx = result.GetTransaction();
-
- {
- auto session2 = db.CreateSession().GetValueSync().GetSession();
- result = session2.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW())).GetValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ auto tx = result.GetTransaction();
+
+ {
+ auto session2 = db.CreateSession().GetValueSync().GetSession();
+ result = session2.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
UPSERT INTO `/Root/KeyValue` (Key, Value)
- VALUES (3u, "Three")
- )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- }
-
- result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- SELECT 42;
- )", TTxControl::Tx(*tx).CommitTx()).GetValueSync();
-
+ VALUES (3u, "Three")
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+ }
+
+ result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ SELECT 42;
+ )", TTxControl::Tx(*tx).CommitTx()).GetValueSync();
+
if (kikimr.IsUsingSnapshotReads()) {
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
} else {
@@ -980,36 +980,36 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::ABORTED, result.GetIssues().ToString());
UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_LOCKS_INVALIDATED));
}
- }
-
- Y_UNIT_TEST(BrokenLocksAtROTxSharded) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
+ }
+
+ Y_UNIT_TEST(BrokenLocksAtROTxSharded) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
SELECT * FROM `/Root/TwoShard`
- )", TTxControl::BeginTx(TTxSettings::SerializableRW())).GetValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- auto tx = result.GetTransaction();
-
- {
- auto session2 = db.CreateSession().GetValueSync().GetSession();
- result = session2.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW())).GetValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ auto tx = result.GetTransaction();
+
+ {
+ auto session2 = db.CreateSession().GetValueSync().GetSession();
+ result = session2.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
UPSERT INTO `/Root/TwoShard` (Key, Value1, Value2)
- VALUES (4u, "Four", 4)
- )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- }
-
- result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- SELECT 42;
- )", TTxControl::Tx(*tx).CommitTx()).GetValueSync();
-
+ VALUES (4u, "Four", 4)
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+ }
+
+ result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ SELECT 42;
+ )", TTxControl::Tx(*tx).CommitTx()).GetValueSync();
+
if (kikimr.IsUsingSnapshotReads()) {
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
} else {
@@ -1017,47 +1017,47 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::ABORTED, result.GetIssues().ToString());
UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_LOCKS_INVALIDATED));
}
- }
-
- Y_UNIT_TEST(BrokenLocksOnUpdate) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
-
+ }
+
+ Y_UNIT_TEST(BrokenLocksOnUpdate) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+
SELECT * FROM `/Root/TwoShard` WHERE Key = 4000000001u; -- read 2nd shard
-
+
UPSERT INTO `/Root/TwoShard` (Key, Value1, Value2) VALUES -- write 1st shard
- (11u, "Eleven", 11);
- )", TTxControl::BeginTx()).GetValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- auto tx = result.GetTransaction();
-
- {
- auto session2 = db.CreateSession().GetValueSync().GetSession();
- result = session2.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
+ (11u, "Eleven", 11);
+ )", TTxControl::BeginTx()).GetValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ auto tx = result.GetTransaction();
+
+ {
+ auto session2 = db.CreateSession().GetValueSync().GetSession();
+ result = session2.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
UPSERT INTO `/Root/TwoShard` (Key, Value1, Value2) VALUES -- write 2nd shard
- (4000000001u, "XXX", -101)
- )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- }
-
- auto txResult = tx->Commit().GetValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(txResult.GetStatus(), EStatus::ABORTED, txResult.GetIssues().ToString());
- UNIT_ASSERT(HasIssue(txResult.GetIssues(), NYql::TIssuesIds::KIKIMR_LOCKS_INVALIDATED));
-
- result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
+ (4000000001u, "XXX", -101)
+ )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+ }
+
+ auto txResult = tx->Commit().GetValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(txResult.GetStatus(), EStatus::ABORTED, txResult.GetIssues().ToString());
+ UNIT_ASSERT(HasIssue(txResult.GetIssues(), NYql::TIssuesIds::KIKIMR_LOCKS_INVALIDATED));
+
+ result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
SELECT Key, Value1, Value2 FROM `/Root/TwoShard` WHERE Key = 11u
- )", TTxControl::BeginTx().CommitTx()).GetValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
- CompareYson(R"([])", FormatResultSetYson(result.GetResultSet(0)));
- }
-
+ )", TTxControl::BeginTx().CommitTx()).GetValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ CompareYson(R"([])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+
Y_UNIT_TEST(DeferredEffects) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
@@ -1123,36 +1123,36 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
])", FormatResultSetYson(result.GetResultSet(0)));
}
- Y_UNIT_TEST(PrunePartitionsByLiteral) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- NYdb::NTable::TExecDataQuerySettings execSettings;
- execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
-
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
+ Y_UNIT_TEST(PrunePartitionsByLiteral) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ NYdb::NTable::TExecDataQuerySettings execSettings;
+ execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
+
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
SELECT * FROM [/Root/EightShard] WHERE Key = 101 OR Key = 301
- ORDER BY Key;
- )", TTxControl::BeginTx().CommitTx(), execSettings).GetValueSync();
-
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
-
- CompareYson(R"([[[1];[101u];["Value1"]];[[3];[301u];["Value1"]]])", FormatResultSetYson(result.GetResultSet(0)));
-
- auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 1); // no literal phase
-
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).affected_shards(), 2);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).partitions_count(), 2);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/EightShard");
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 2);
- UNIT_ASSERT(stats.query_phases(0).table_access(0).reads().bytes() > 0);
- UNIT_ASSERT(stats.query_phases(0).duration_us() > 0);
- }
-
+ ORDER BY Key;
+ )", TTxControl::BeginTx().CommitTx(), execSettings).GetValueSync();
+
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+
+ CompareYson(R"([[[1];[101u];["Value1"]];[[3];[301u];["Value1"]]])", FormatResultSetYson(result.GetResultSet(0)));
+
+ auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 1); // no literal phase
+
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).affected_shards(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).partitions_count(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/EightShard");
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 2);
+ UNIT_ASSERT(stats.query_phases(0).table_access(0).reads().bytes() > 0);
+ UNIT_ASSERT(stats.query_phases(0).duration_us() > 0);
+ }
+
Y_UNIT_TEST(PrunePartitionsByExpr) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
@@ -1179,63 +1179,63 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2);
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 0);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).affected_shards(), 0);
- UNIT_ASSERT(stats.query_phases(0).table_access().size() == 0);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).affected_shards(), 0);
+ UNIT_ASSERT(stats.query_phases(0).table_access().size() == 0);
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).affected_shards(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).partitions_count(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).affected_shards(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).partitions_count(), 1);
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/EightShard");
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 1);
UNIT_ASSERT(stats.query_phases(1).table_access(0).reads().bytes() > 0);
UNIT_ASSERT(stats.query_phases(1).duration_us() > 0);
}
-
- Y_UNIT_TEST(PruneWritePartitions) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- NYdb::NTable::TExecDataQuerySettings execSettings;
- execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
-
- auto params = TParamsBuilder()
- .AddParam("$Key").Uint32(10).Build()
- .Build();
-
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- DECLARE $Key AS UInt32;
-
+
+ Y_UNIT_TEST(PruneWritePartitions) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ NYdb::NTable::TExecDataQuerySettings execSettings;
+ execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
+
+ auto params = TParamsBuilder()
+ .AddParam("$Key").Uint32(10).Build()
+ .Build();
+
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ DECLARE $Key AS UInt32;
+
UPSERT INTO [/Root/TwoShard] (Key, Value1, Value2) VALUES
- ($Key, "One", -10)
- )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params, execSettings).ExtractValueSync();
- AssertSuccessResult(result);
-
- auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2);
-
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 0);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).affected_shards(), 0);
- UNIT_ASSERT(stats.query_phases(0).table_access().size() == 0);
-
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).affected_shards(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).partitions_count(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TwoShard");
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).updates().rows(), 1);
-
- result = session.ExecuteDataQuery(R"(
+ ($Key, "One", -10)
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params, execSettings).ExtractValueSync();
+ AssertSuccessResult(result);
+
+ auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2);
+
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).affected_shards(), 0);
+ UNIT_ASSERT(stats.query_phases(0).table_access().size() == 0);
+
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).affected_shards(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).partitions_count(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TwoShard");
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).updates().rows(), 1);
+
+ result = session.ExecuteDataQuery(R"(
PRAGMA kikimr.UseNewEngine = "true";
SELECT * FROM [/Root/TwoShard] WHERE Value2 <= -10 ORDER BY Key;
- )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- AssertSuccessResult(result);
-
- CompareYson(R"([
- [[10u];["One"];[-10]];
- ])", FormatResultSetYson(result.GetResultSet(0)));
- }
-
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ AssertSuccessResult(result);
+
+ CompareYson(R"([
+ [[10u];["One"];[-10]];
+ ])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+
Y_UNIT_TEST(Truncated) {
TVector<NKikimrKqp::TKqpSetting> settings;
NKikimrKqp::TKqpSetting setting;
@@ -1270,14 +1270,14 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
("Newvalue 1", 1u),
("Newvalue 5", 5u);
)", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
-
+ PRAGMA kikimr.UseNewEngine = "true";
+
SELECT * FROM [/Root/TwoShard] WHERE Key <= 5 ORDER BY Key;
)", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
CompareYson(R"([
[[1u];["Newvalue 1"];#];
@@ -1293,14 +1293,14 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
("new_app_1", 100, "new_app_host_1.search.yandex.net", "Initialize"),
("new_app_1", 200, "new_app_host_2.search.yandex.net", "Initialized");
)", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
-
+ PRAGMA kikimr.UseNewEngine = "true";
+
SELECT * FROM `/Root/Logs` WHERE App = "new_app_1" ORDER BY App, Ts, Host;
)", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
CompareYson(R"([
[["new_app_1"];["new_app_host_1.search.yandex.net"];["Initialize"];[100]];
@@ -1341,38 +1341,38 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
])", FormatResultSetYson(result.GetResultSet(0)));
}
- Y_UNIT_TEST(JoinWithParams) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto params = TParamsBuilder()
- .AddParam("$data")
- .BeginList()
- .AddListItem().BeginStruct().AddMember("Key").Uint64(1).EndStruct()
- .AddListItem().BeginStruct().AddMember("Key").Uint64(5).EndStruct()
- .EndList()
- .Build()
- .Build();
-
- auto result = session.ExecuteDataQuery(R"(
- --!syntax_v1
- PRAGMA kikimr.UseNewEngine = "true";
-
- DECLARE $data AS List<Struct<Key: Uint64>>;
-
- SELECT t.Key AS Key, t.Value AS Value
- FROM AS_TABLE($data) AS d
- INNER JOIN `/Root/KeyValue` AS t ON t.Key = d.Key
- ORDER BY Key, Value;
- )", TTxControl::BeginTx().CommitTx(), params).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
-
- CompareYson(R"([
- [[1u];["One"]];
- ])", FormatResultSetYson(result.GetResultSet(0)));
- }
-
+ Y_UNIT_TEST(JoinWithParams) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto params = TParamsBuilder()
+ .AddParam("$data")
+ .BeginList()
+ .AddListItem().BeginStruct().AddMember("Key").Uint64(1).EndStruct()
+ .AddListItem().BeginStruct().AddMember("Key").Uint64(5).EndStruct()
+ .EndList()
+ .Build()
+ .Build();
+
+ auto result = session.ExecuteDataQuery(R"(
+ --!syntax_v1
+ PRAGMA kikimr.UseNewEngine = "true";
+
+ DECLARE $data AS List<Struct<Key: Uint64>>;
+
+ SELECT t.Key AS Key, t.Value AS Value
+ FROM AS_TABLE($data) AS d
+ INNER JOIN `/Root/KeyValue` AS t ON t.Key = d.Key
+ ORDER BY Key, Value;
+ )", TTxControl::BeginTx().CommitTx(), params).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+
+ CompareYson(R"([
+ [[1u];["One"]];
+ ])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+
Y_UNIT_TEST(JoinIdxLookup) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
@@ -1420,49 +1420,49 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(2).table_access(0).reads().rows(), 4);
}
- Y_UNIT_TEST(LeftSemiJoin) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- // add nulls
- auto result = session.ExecuteDataQuery(R"(
+ Y_UNIT_TEST(LeftSemiJoin) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ // add nulls
+ auto result = session.ExecuteDataQuery(R"(
PRAGMA kikimr.UseNewEngine = "true";
REPLACE INTO `/Root/KeyValue` (Key, Value) VALUES (4u, "Four"), (NULL, "Null");
REPLACE INTO `/Root/Join2` (Key1, Key2, Name, Value2) VALUES (1, NULL, "Name Null", "Value Null");
- )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
-
- result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
-
- SELECT Key1, Key2, Name, Value2
+ )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+
+ result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+
+ SELECT Key1, Key2, Name, Value2
FROM `/Root/Join2` AS t1
LEFT SEMI JOIN `/Root/KeyValue` AS t2
- ON t1.Key2 == t2.Value
- ORDER BY Key1, Key2, Name;
- )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
-
- CompareYson(R"([
- [[101u];["One"];["Name1"];["Value21"]];
- [[101u];["Two"];["Name1"];["Value22"]];
- [[102u];["One"];["Name2"];["Value24"]];
- [[103u];["One"];["Name1"];["Value25"]];
- [[104u];["One"];["Name3"];["Value26"]];
- [[105u];["One"];["Name2"];["Value27"]];
- [[105u];["Two"];["Name4"];["Value28"]];
- [[106u];["One"];["Name3"];["Value29"]];
- [[108u];["One"];#;["Value31"]]
- ])", FormatResultSetYson(result.GetResultSet(0)));
- }
-
- Y_UNIT_TEST_NEW_ENGINE(JoinPure) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExecuteDataQuery(Q1_(R"(
+ ON t1.Key2 == t2.Value
+ ORDER BY Key1, Key2, Name;
+ )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+
+ CompareYson(R"([
+ [[101u];["One"];["Name1"];["Value21"]];
+ [[101u];["Two"];["Name1"];["Value22"]];
+ [[102u];["One"];["Name2"];["Value24"]];
+ [[103u];["One"];["Name1"];["Value25"]];
+ [[104u];["One"];["Name3"];["Value26"]];
+ [[105u];["One"];["Name2"];["Value27"]];
+ [[105u];["Two"];["Name4"];["Value28"]];
+ [[106u];["One"];["Name3"];["Value29"]];
+ [[108u];["One"];#;["Value31"]]
+ ])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+
+ Y_UNIT_TEST_NEW_ENGINE(JoinPure) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExecuteDataQuery(Q1_(R"(
$list1 = AsList(
AsStruct("One" AS Key1, 1 AS Value1),
AsStruct("Two" AS Key1, 2 AS Value1),
@@ -1478,54 +1478,54 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
AsStruct("v2" AS Value3),
);
- $list4 = AsList(
- AsStruct((Cast ("One" AS String?)) AS Key4, 100 AS Value4),
- AsStruct((Cast ("Two" AS String?)) AS Key4, 200 AS Value4),
- AsStruct((Cast (Null AS String?)) AS Key4, 300 AS Value4),
- );
-
+ $list4 = AsList(
+ AsStruct((Cast ("One" AS String?)) AS Key4, 100 AS Value4),
+ AsStruct((Cast ("Two" AS String?)) AS Key4, 200 AS Value4),
+ AsStruct((Cast (Null AS String?)) AS Key4, 300 AS Value4),
+ );
+
$table1 = SELECT * FROM AS_TABLE($list1);
$table2 = SELECT * FROM AS_TABLE($list2);
$table3 = SELECT * FROM AS_TABLE($list3);
- $table4 = SELECT * FROM AS_TABLE($list4);
+ $table4 = SELECT * FROM AS_TABLE($list4);
SELECT * FROM $table1 as t1
JOIN $table2 as t2 ON t1.Key1 = t2.Key2
- JOIN $table4 as t4 ON t1.Key1 = t4.Key4
+ JOIN $table4 as t4 ON t1.Key1 = t4.Key4
CROSS JOIN $table3 as t3
ORDER BY Value3;
- )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
CompareYson(R"([
- ["One";"One";["One"];1;10;"v1";100];
- ["One";"One";["One"];1;10;"v2";100]
+ ["One";"One";["One"];1;10;"v1";100];
+ ["One";"One";["One"];1;10;"v2";100]
])", FormatResultSetYson(result.GetResultSet(0)));
}
- Y_UNIT_TEST_NEW_ENGINE(JoinPureUncomparableKeys) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExecuteDataQuery(Q1_(R"(
- $l = AsList(
- AsStruct(AsTuple(1,2,3) as Key, "1,2,3" as Payload),
- AsStruct(AsTuple(1,2,4) as Key, "1,2,4" as Payload)
- );
- $r = AsList(
- AsStruct(AsTuple(1,2) as Key, "1,2" as Payload),
- AsStruct(AsTuple(2,3) as Key, "2,3" as Payload)
- );
-
- select l.Key, l.Payload, r.Key, r.Payload
- from AS_TABLE($l) as l join AS_TABLE($r) as r on l.Key = r.Key;
- )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
-
- CompareYson(R"([])", FormatResultSetYson(result.GetResultSet(0)));
- }
-
+ Y_UNIT_TEST_NEW_ENGINE(JoinPureUncomparableKeys) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExecuteDataQuery(Q1_(R"(
+ $l = AsList(
+ AsStruct(AsTuple(1,2,3) as Key, "1,2,3" as Payload),
+ AsStruct(AsTuple(1,2,4) as Key, "1,2,4" as Payload)
+ );
+ $r = AsList(
+ AsStruct(AsTuple(1,2) as Key, "1,2" as Payload),
+ AsStruct(AsTuple(2,3) as Key, "2,3" as Payload)
+ );
+
+ select l.Key, l.Payload, r.Key, r.Payload
+ from AS_TABLE($l) as l join AS_TABLE($r) as r on l.Key = r.Key;
+ )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+
+ CompareYson(R"([])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+
Y_UNIT_TEST_NEW_ENGINE(SelfJoin) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
@@ -1563,53 +1563,53 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2);
-
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/TwoShard");
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).updates().rows(), 0);
-
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TwoShard");
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).updates().rows(), 2);
- UNIT_ASSERT(stats.query_phases(1).table_access(0).updates().bytes() > 0);
- UNIT_ASSERT(stats.query_phases(1).duration_us() > 0);
-
- result = session.ExecuteDataQuery(R"(
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2);
+
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/TwoShard");
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).updates().rows(), 0);
+
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TwoShard");
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).updates().rows(), 2);
+ UNIT_ASSERT(stats.query_phases(1).table_access(0).updates().bytes() > 0);
+ UNIT_ASSERT(stats.query_phases(1).duration_us() > 0);
+
+ result = session.ExecuteDataQuery(R"(
PRAGMA kikimr.UseNewEngine = "true";
SELECT * FROM [/Root/TwoShard] ORDER BY Key;
- )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
-
- CompareYson(R"([
- [[1u];["One"];[-1]];
- [[2u];["Two"];[0]];
- [[3u];["Updated"];[1]];
- [[4000000001u];["BigOne"];[-1]];
- [[4000000002u];["BigTwo"];[0]];
- [[4000000003u];["Updated"];[1]]
- ])", FormatResultSetYson(result.GetResultSet(0)));
- }
-
- Y_UNIT_TEST(InplaceUpdate) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- NYdb::NTable::TExecDataQuerySettings execSettings;
- execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
-
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- PRAGMA kikimr.OptEnableInplaceUpdate = "true";
-
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+
+ CompareYson(R"([
+ [[1u];["One"];[-1]];
+ [[2u];["Two"];[0]];
+ [[3u];["Updated"];[1]];
+ [[4000000001u];["BigOne"];[-1]];
+ [[4000000002u];["BigTwo"];[0]];
+ [[4000000003u];["Updated"];[1]]
+ ])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+
+ Y_UNIT_TEST(InplaceUpdate) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ NYdb::NTable::TExecDataQuerySettings execSettings;
+ execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
+
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ PRAGMA kikimr.OptEnableInplaceUpdate = "true";
+
UPDATE [/Root/TwoShard]
- SET Value1 = "Updated"
- WHERE Value2 = 1;
- )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), execSettings).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
-
- auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
+ SET Value1 = "Updated"
+ WHERE Value2 = 1;
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), execSettings).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+
+ auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 1);
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1);
@@ -1651,15 +1651,15 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2);
- // Phase reading rows to delete
+ // Phase reading rows to delete
UNIT_ASSERT(stats.query_phases(0).duration_us() > 0);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/TwoShard");
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/TwoShard");
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 6);
- // Phase deleting rows
+ // Phase deleting rows
UNIT_ASSERT(stats.query_phases(1).duration_us() > 0);
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1);
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TwoShard");
@@ -1697,15 +1697,15 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2);
- // Phase reading rows to delete
+ // Phase reading rows to delete
UNIT_ASSERT(stats.query_phases(0).duration_us() > 0);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/TwoShard");
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/TwoShard");
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 6);
- // Phase deleting rows
+ // Phase deleting rows
UNIT_ASSERT(stats.query_phases(1).duration_us() > 0);
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1);
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/TwoShard");
@@ -1726,265 +1726,265 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
])", FormatResultSetYson(result.GetResultSet(0)));
}
- Y_UNIT_TEST(MultiEffects) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- NYdb::NTable::TExecDataQuerySettings execSettings;
- execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
-
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
-
+ Y_UNIT_TEST(MultiEffects) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ NYdb::NTable::TExecDataQuerySettings execSettings;
+ execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
+
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+
UPDATE [/Root/TwoShard] SET Value1 = "Updated" WHERE Value2 = 1;
UPSERT INTO [/Root/TwoShard] (Key, Value1, Value2) VALUES
- (4u, "Four", 4);
- )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), execSettings).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
-
- result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
-
+ (4u, "Four", 4);
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), execSettings).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+
+ result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+
SELECT * FROM [/Root/TwoShard] ORDER BY Key;
- )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
-
- CompareYson(R"([
- [[1u];["One"];[-1]];
- [[2u];["Two"];[0]];
- [[3u];["Updated"];[1]];
- [[4u];["Four"];[4]];
- [[4000000001u];["BigOne"];[-1]];
- [[4000000002u];["BigTwo"];[0]];
- [[4000000003u];["Updated"];[1]]
- ])", FormatResultSetYson(result.GetResultSet(0)));
- }
-
- Y_UNIT_TEST(UpdateFromParams) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- NYdb::NTable::TExecDataQuerySettings execSettings;
- execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
-
- auto params = TParamsBuilder()
- .AddParam("$data")
- .BeginList()
- .AddListItem().BeginStruct()
- .AddMember("Key").Uint32(10)
- .AddMember("Value1").String("Ten")
- .AddMember("Value2").Int32(-10)
- .EndStruct()
- .AddListItem().BeginStruct()
- .AddMember("Key").Uint32(11)
- .AddMember("Value1").String("Eleven")
- .AddMember("Value2").Int32(-11)
- .EndStruct()
- .EndList()
- .Build()
- .Build();
-
- auto result = session.ExecuteDataQuery(R"(
- --!syntax_v1
- PRAGMA kikimr.UseNewEngine = "true";
-
- DECLARE $data AS List<Struct<Key: Uint32, Value1: String, Value2: Int32>>;
-
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+
+ CompareYson(R"([
+ [[1u];["One"];[-1]];
+ [[2u];["Two"];[0]];
+ [[3u];["Updated"];[1]];
+ [[4u];["Four"];[4]];
+ [[4000000001u];["BigOne"];[-1]];
+ [[4000000002u];["BigTwo"];[0]];
+ [[4000000003u];["Updated"];[1]]
+ ])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+
+ Y_UNIT_TEST(UpdateFromParams) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ NYdb::NTable::TExecDataQuerySettings execSettings;
+ execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
+
+ auto params = TParamsBuilder()
+ .AddParam("$data")
+ .BeginList()
+ .AddListItem().BeginStruct()
+ .AddMember("Key").Uint32(10)
+ .AddMember("Value1").String("Ten")
+ .AddMember("Value2").Int32(-10)
+ .EndStruct()
+ .AddListItem().BeginStruct()
+ .AddMember("Key").Uint32(11)
+ .AddMember("Value1").String("Eleven")
+ .AddMember("Value2").Int32(-11)
+ .EndStruct()
+ .EndList()
+ .Build()
+ .Build();
+
+ auto result = session.ExecuteDataQuery(R"(
+ --!syntax_v1
+ PRAGMA kikimr.UseNewEngine = "true";
+
+ DECLARE $data AS List<Struct<Key: Uint32, Value1: String, Value2: Int32>>;
+
UPSERT INTO `/Root/TwoShard`
- SELECT Key, Value1, Value2 FROM AS_TABLE($data)
- )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params, execSettings).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
-
- result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
-
+ SELECT Key, Value1, Value2 FROM AS_TABLE($data)
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params, execSettings).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+
+ result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+
SELECT * FROM [/Root/TwoShard] WHERE Key > 5 AND Key < 12 ORDER BY Key;
- )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
-
- CompareYson(R"([
- [[10u];["Ten"];[-10]];
- [[11u];["Eleven"];[-11]]
- ])", FormatResultSetYson(result.GetResultSet(0)));
- }
-
- Y_UNIT_TEST(PruneEffectPartitions) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto query = R"(
- --!syntax_v1
- pragma kikimr.UseNewEngine = "true";
-
- declare $key as UInt64;
- declare $text as String;
-
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+
+ CompareYson(R"([
+ [[10u];["Ten"];[-10]];
+ [[11u];["Eleven"];[-11]]
+ ])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+
+ Y_UNIT_TEST(PruneEffectPartitions) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto query = R"(
+ --!syntax_v1
+ pragma kikimr.UseNewEngine = "true";
+
+ declare $key as UInt64;
+ declare $text as String;
+
update `/Root/EightShard` set Text = $text where Key = $key
- )";
-
- auto params = TParamsBuilder()
- .AddParam("$key").Uint64(501).Build()
- .AddParam("$text").String("foo").Build()
- .Build();
-
- auto settings = TExecDataQuerySettings()
- .CollectQueryStats(ECollectQueryStatsMode::Basic);
-
- auto it = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), std::move(params), settings).GetValueSync();
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- auto& stats = NYdb::TProtoAccessor::GetProto(*it.GetStats());
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2);
-
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).affected_shards(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).partitions_count(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/EightShard");
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).updates().rows(), 0);
-
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).affected_shards(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).partitions_count(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/EightShard");
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 0);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).updates().rows(), 1);
-
- it = session.ExecuteDataQuery(R"(
- pragma kikimr.UseNewEngine = "true";
-
+ )";
+
+ auto params = TParamsBuilder()
+ .AddParam("$key").Uint64(501).Build()
+ .AddParam("$text").String("foo").Build()
+ .Build();
+
+ auto settings = TExecDataQuerySettings()
+ .CollectQueryStats(ECollectQueryStatsMode::Basic);
+
+ auto it = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), std::move(params), settings).GetValueSync();
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ auto& stats = NYdb::TProtoAccessor::GetProto(*it.GetStats());
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2);
+
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).affected_shards(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).partitions_count(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/EightShard");
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).updates().rows(), 0);
+
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).affected_shards(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).partitions_count(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/EightShard");
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).updates().rows(), 1);
+
+ it = session.ExecuteDataQuery(R"(
+ pragma kikimr.UseNewEngine = "true";
+
select Key, Text, Data from `/Root/EightShard` where Text = "foo" order by Key
- )",TTxControl::BeginTx().CommitTx()).GetValueSync();
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- CompareYson(R"([
- [[501u];["foo"];[2]];
- ])", FormatResultSetYson(it.GetResultSet(0)));
- }
-
- Y_UNIT_TEST(DecimalColumn) {
- TKikimrRunner kikimr;
-
- TTableClient client{kikimr.GetDriver()};
- auto session = client.CreateSession().GetValueSync().GetSession();
-
- UNIT_ASSERT(session.CreateTable("/Root/DecimalTest",
- TTableBuilder()
- .AddNullableColumn("Key", EPrimitiveType::Uint64)
- .AddNullableColumn("Value", TDecimalType(22, 9))
- .SetPrimaryKeyColumn("Key")
- .Build()).GetValueSync().IsSuccess());
-
- auto params = TParamsBuilder().AddParam("$in").BeginList()
- .AddListItem().BeginStruct()
- .AddMember("Key").Uint64(1)
- .AddMember("Value").Decimal(TDecimalValue("10.123456789"))
- .EndStruct()
- .AddListItem().BeginStruct()
- .AddMember("Key").Uint64(2)
- .AddMember("Value").Decimal(TDecimalValue("20.987654321"))
- .EndStruct()
- .EndList().Build().Build();
-
- auto result = session.ExecuteDataQuery(R"(
- --!syntax_v1
- PRAGMA Kikimr.UseNewEngine = "true";
- DECLARE $in AS List<Struct<Key: Uint64, Value: Decimal(22, 9)>>;
-
- REPLACE INTO `/Root/DecimalTest`
- SELECT Key, Value FROM AS_TABLE($in);
- )", TTxControl::BeginTx().CommitTx(), params).GetValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- result = session.ExecuteDataQuery(R"(
- PRAGMA Kikimr.UseNewEngine = "true";
- SELECT Key, Value FROM `/Root/DecimalTest` ORDER BY Key
- )", TTxControl::BeginTx().CommitTx(), params).GetValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
- CompareYson(R"([
- [[1u];["10.123456789"]];
- [[2u];["20.987654321"]]
- ])", FormatResultSetYson(result.GetResultSet(0)));
- }
-
- Y_UNIT_TEST(LocksInRoTx) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
-
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExecuteDataQuery(R"(
- pragma kikimr.UseNewEngine = "true";
- select * from `/Root/TwoShard` where Key = 1;
- )", TTxControl::BeginTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- auto tx = result.GetTransaction();
- UNIT_ASSERT(tx && tx->IsActive());
-
- for (ui64 i : {2u, 3u, 4u, 4000000001u, 4000000002u, 4000000003u}) {
- result = session.ExecuteDataQuery(Sprintf(R"(
- pragma kikimr.UseNewEngine = "true";
- select * from `/Root/TwoShard` where Key = %ld;
- )", i), TTxControl::Tx(*tx)).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
- }
-
- auto ret = tx->Commit().ExtractValueSync();
- UNIT_ASSERT_C(ret.IsSuccess(), ret.GetIssues().ToString());
- }
-
- Y_UNIT_TEST(ItemsLimit) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- enum EType {
- None, Literal, Parameter, Expression, /* QueryResult */
- };
-
- for (auto takeType : {EType::Literal, EType::Parameter, EType::Expression}) {
- for (auto skipType : {EType::None, EType::Literal, EType::Parameter, EType::Expression}) {
- for (auto withSort : {false, true}) { // TODO: passthrough FlatMap
- TStringBuilder query;
- query << "--!syntax_v1" << Endl
- << "pragma kikimr.UseNewEngine = 'true';" << Endl
- << "declare $limit as Uint64;" << Endl
- << "declare $offset as Uint64;" << Endl
- << "select Key, Value from `/Root/KeyValue`";
-
- if (withSort) {
- query << " order by Key"; // redundant sort by PK
- }
-
+ )",TTxControl::BeginTx().CommitTx()).GetValueSync();
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ CompareYson(R"([
+ [[501u];["foo"];[2]];
+ ])", FormatResultSetYson(it.GetResultSet(0)));
+ }
+
+ Y_UNIT_TEST(DecimalColumn) {
+ TKikimrRunner kikimr;
+
+ TTableClient client{kikimr.GetDriver()};
+ auto session = client.CreateSession().GetValueSync().GetSession();
+
+ UNIT_ASSERT(session.CreateTable("/Root/DecimalTest",
+ TTableBuilder()
+ .AddNullableColumn("Key", EPrimitiveType::Uint64)
+ .AddNullableColumn("Value", TDecimalType(22, 9))
+ .SetPrimaryKeyColumn("Key")
+ .Build()).GetValueSync().IsSuccess());
+
+ auto params = TParamsBuilder().AddParam("$in").BeginList()
+ .AddListItem().BeginStruct()
+ .AddMember("Key").Uint64(1)
+ .AddMember("Value").Decimal(TDecimalValue("10.123456789"))
+ .EndStruct()
+ .AddListItem().BeginStruct()
+ .AddMember("Key").Uint64(2)
+ .AddMember("Value").Decimal(TDecimalValue("20.987654321"))
+ .EndStruct()
+ .EndList().Build().Build();
+
+ auto result = session.ExecuteDataQuery(R"(
+ --!syntax_v1
+ PRAGMA Kikimr.UseNewEngine = "true";
+ DECLARE $in AS List<Struct<Key: Uint64, Value: Decimal(22, 9)>>;
+
+ REPLACE INTO `/Root/DecimalTest`
+ SELECT Key, Value FROM AS_TABLE($in);
+ )", TTxControl::BeginTx().CommitTx(), params).GetValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ result = session.ExecuteDataQuery(R"(
+ PRAGMA Kikimr.UseNewEngine = "true";
+ SELECT Key, Value FROM `/Root/DecimalTest` ORDER BY Key
+ )", TTxControl::BeginTx().CommitTx(), params).GetValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ CompareYson(R"([
+ [[1u];["10.123456789"]];
+ [[2u];["20.987654321"]]
+ ])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+
+ Y_UNIT_TEST(LocksInRoTx) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExecuteDataQuery(R"(
+ pragma kikimr.UseNewEngine = "true";
+ select * from `/Root/TwoShard` where Key = 1;
+ )", TTxControl::BeginTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ auto tx = result.GetTransaction();
+ UNIT_ASSERT(tx && tx->IsActive());
+
+ for (ui64 i : {2u, 3u, 4u, 4000000001u, 4000000002u, 4000000003u}) {
+ result = session.ExecuteDataQuery(Sprintf(R"(
+ pragma kikimr.UseNewEngine = "true";
+ select * from `/Root/TwoShard` where Key = %ld;
+ )", i), TTxControl::Tx(*tx)).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ }
+
+ auto ret = tx->Commit().ExtractValueSync();
+ UNIT_ASSERT_C(ret.IsSuccess(), ret.GetIssues().ToString());
+ }
+
+ Y_UNIT_TEST(ItemsLimit) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ enum EType {
+ None, Literal, Parameter, Expression, /* QueryResult */
+ };
+
+ for (auto takeType : {EType::Literal, EType::Parameter, EType::Expression}) {
+ for (auto skipType : {EType::None, EType::Literal, EType::Parameter, EType::Expression}) {
+ for (auto withSort : {false, true}) { // TODO: passthrough FlatMap
+ TStringBuilder query;
+ query << "--!syntax_v1" << Endl
+ << "pragma kikimr.UseNewEngine = 'true';" << Endl
+ << "declare $limit as Uint64;" << Endl
+ << "declare $offset as Uint64;" << Endl
+ << "select Key, Value from `/Root/KeyValue`";
+
+ if (withSort) {
+ query << " order by Key"; // redundant sort by PK
+ }
+
const int limitValue = 5;
const int offsetValue = 1;
- switch (takeType) {
- case None: UNIT_FAIL("Unexpected"); break;
+ switch (takeType) {
+ case None: UNIT_FAIL("Unexpected"); break;
case Literal: query << " limit " << limitValue; break;
- case Parameter: query << " limit $limit"; break;
+ case Parameter: query << " limit $limit"; break;
case Expression: query << " limit ($limit + 1)"; break;
- }
-
- switch (skipType) {
- case None: break;
+ }
+
+ switch (skipType) {
+ case None: break;
case Literal: query << " offset " << offsetValue; break;
- case Parameter: query << " offset $offset"; break;
- case Expression: query << " offset ($offset + 1)"; break;
- }
-
- auto result = session.ExplainDataQuery(query).GetValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS,
- TStringBuilder() << query << Endl << "Failed with: " << result.GetIssues().ToString());
- UNIT_ASSERT_C(result.GetAst().Contains("('('\"ItemsLimit\" %kqp%tx_result_binding_0_0))"),
- TStringBuilder() << query << Endl << "Failed with AST: " << result.GetAst());
-
- NJson::TJsonValue plan;
- NJson::ReadJsonTree(result.GetPlan(), &plan, true);
+ case Parameter: query << " offset $offset"; break;
+ case Expression: query << " offset ($offset + 1)"; break;
+ }
+
+ auto result = session.ExplainDataQuery(query).GetValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS,
+ TStringBuilder() << query << Endl << "Failed with: " << result.GetIssues().ToString());
+ UNIT_ASSERT_C(result.GetAst().Contains("('('\"ItemsLimit\" %kqp%tx_result_binding_0_0))"),
+ TStringBuilder() << query << Endl << "Failed with AST: " << result.GetAst());
+
+ NJson::TJsonValue plan;
+ NJson::ReadJsonTree(result.GetPlan(), &plan, true);
auto node = FindPlanNodeByKv(plan, "Node Type", "Limit-TableFullScan");
UNIT_ASSERT(node.IsDefined());
@@ -1999,19 +1999,19 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
auto readLimit = FindPlanNodeByKv(node, "ReadLimit", readLimitValue);
UNIT_ASSERT(readLimit.IsDefined());
- }
- }
- }
-
- // TODO: Take(query result) . ReadTable
-// test(R"(
-// pragma kikimr.UseNewEngine = "true";
-// $limit = (
-// select Key + 1 from `/Root/KeyValue` where Value = 'Four'
-// );
-// select Key, Value from `/Root/KeyValue` limit $limit ?? 7;
-// )");
- }
+ }
+ }
+ }
+
+ // TODO: Take(query result) . ReadTable
+// test(R"(
+// pragma kikimr.UseNewEngine = "true";
+// $limit = (
+// select Key + 1 from `/Root/KeyValue` where Value = 'Four'
+// );
+// select Key, Value from `/Root/KeyValue` limit $limit ?? 7;
+// )");
+ }
Y_UNIT_TEST(OnlineRO_Consistent) {
TKikimrRunner kikimr;
@@ -2426,34 +2426,34 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
UNIT_ASSERT_EQUAL(stats.query_phases().size(), 2);
}
-
- Y_UNIT_TEST(MultiEffectsOnSameTable) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- NYdb::NTable::TExecDataQuerySettings execSettings;
- execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
-
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- UPSERT INTO `/Root/EightShard` (Key, Data) VALUES (100, 100500), (100500, 100);
- DELETE FROM `/Root/EightShard` ON (Key) VALUES (100);
- )", TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
-
-// auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
-// Cerr << "!!!\n" << stats.DebugString() << Endl;
-
- result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- SELECT Key, Data
- FROM `/Root/EightShard`
- WHERE Key=100 or Key=100500
- )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- CompareYson("[[[100500u];[100]]]", FormatResultSetYson(result.GetResultSet(0)));
- }
+
+ Y_UNIT_TEST(MultiEffectsOnSameTable) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ NYdb::NTable::TExecDataQuerySettings execSettings;
+ execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
+
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ UPSERT INTO `/Root/EightShard` (Key, Data) VALUES (100, 100500), (100500, 100);
+ DELETE FROM `/Root/EightShard` ON (Key) VALUES (100);
+ )", TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+
+// auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
+// Cerr << "!!!\n" << stats.DebugString() << Endl;
+
+ result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ SELECT Key, Data
+ FROM `/Root/EightShard`
+ WHERE Key=100 or Key=100500
+ )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+ CompareYson("[[[100500u];[100]]]", FormatResultSetYson(result.GetResultSet(0)));
+ }
Y_UNIT_TEST(MultiUsagePrecompute) {
TKikimrRunner kikimr;
@@ -2597,39 +2597,39 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
[[100u];["NewValue"]]
])", FormatResultSetYson(result.GetResultSet(0)));
}
-
- Y_UNIT_TEST(JoinWithPrecompute) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- NYdb::NTable::TExecDataQuerySettings execSettings;
- execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
-
- auto result = session.ExecuteDataQuery(R"(
- --!syntax_v1
- PRAGMA kikimr.UseNewEngine = "true";
-
- $data =
- SELECT Key1, Key2, Name, Value2
- FROM `/Root/Join2`
- WHERE Name IS NOT NULL;
-
- $max = SELECT max(Key1) AS k FROM $data WHERE Name != "foo";
-
- $to_delete =
- SELECT l.Key AS Key, l.Fk21 AS Key1, l.Fk22 AS Key2
- FROM `/Root/Join1` AS l JOIN $data AS r ON l.Fk21 = r.Key1;
-
- SELECT $max ?? 0, Key1, Key2 FROM $to_delete ORDER BY Key1, Key2 LIMIT 1;
-
- DELETE FROM `/Root/Join1` ON SELECT Key FROM $to_delete;
- DELETE FROM `/Root/Join2` ON SELECT Key1, Key2 FROM $data;
- )", TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
-
- CompareYson(R"([[106u;[101u];["One"]]])", FormatResultSetYson(result.GetResultSet(0)));
- }
+
+ Y_UNIT_TEST(JoinWithPrecompute) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ NYdb::NTable::TExecDataQuerySettings execSettings;
+ execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
+
+ auto result = session.ExecuteDataQuery(R"(
+ --!syntax_v1
+ PRAGMA kikimr.UseNewEngine = "true";
+
+ $data =
+ SELECT Key1, Key2, Name, Value2
+ FROM `/Root/Join2`
+ WHERE Name IS NOT NULL;
+
+ $max = SELECT max(Key1) AS k FROM $data WHERE Name != "foo";
+
+ $to_delete =
+ SELECT l.Key AS Key, l.Fk21 AS Key1, l.Fk22 AS Key2
+ FROM `/Root/Join1` AS l JOIN $data AS r ON l.Fk21 = r.Key1;
+
+ SELECT $max ?? 0, Key1, Key2 FROM $to_delete ORDER BY Key1, Key2 LIMIT 1;
+
+ DELETE FROM `/Root/Join1` ON SELECT Key FROM $to_delete;
+ DELETE FROM `/Root/Join2` ON SELECT Key1, Key2 FROM $data;
+ )", TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+
+ CompareYson(R"([[106u;[101u];["One"]]])", FormatResultSetYson(result.GetResultSet(0)));
+ }
Y_UNIT_TEST(JoinProjectMulti) {
TKikimrRunner kikimr;
@@ -2922,53 +2922,53 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
[[3500u];[1u];["Anna"]]
])", FormatResultSetYson(result.GetResultSet(0)));
}
-
- // https://st.yandex-team.ru/KIKIMR-14022
- Y_UNIT_TEST(JoinSameKey) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
-
- SELECT *
- FROM `Join1` AS l JOIN `Join2` AS r ON l.Fk21 = r.Key1 AND r.Key1 = l.Fk21
- WHERE l.Key = 1
- ORDER BY r.Value2
- )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
-
- CompareYson(R"([
- [[101u];["One"];[1];["Value1"];[101u];["One"];["Name1"];["Value21"]];
- [[101u];["One"];[1];["Value1"];[101u];["Two"];["Name1"];["Value22"]];
- [[101u];["One"];[1];["Value1"];[101u];["Three"];["Name3"];["Value23"]]
- ])", FormatResultSetYson(result.GetResultSet(0)));
- }
-
- Y_UNIT_TEST_NEW_ENGINE(JoinDictWithPure) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExecuteDataQuery(Q1_(R"(
- PRAGMA kikimr.UseNewEngine = 'true';
-
- SELECT xx.id AS id, Key1, Key2, Name, Value2
- FROM
- (SELECT 101 AS id) AS xx
- LEFT JOIN
- (SELECT * FROM `/Root/Join2` AS a1 FULL JOIN (SELECT 101 AS id) AS yy ON (a1.Key1 = yy.id)) AS yy
- ON (xx.id = Coalesce(yy.id));
- )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
-
- CompareYson(R"([
- [101;[101u];["One"];["Name1"];["Value21"]];
- [101;[101u];["Three"];["Name3"];["Value23"]];
- [101;[101u];["Two"];["Name1"];["Value22"]]
- ])", FormatResultSetYson(result.GetResultSet(0)));
- }
+
+ // https://st.yandex-team.ru/KIKIMR-14022
+ Y_UNIT_TEST(JoinSameKey) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+
+ SELECT *
+ FROM `Join1` AS l JOIN `Join2` AS r ON l.Fk21 = r.Key1 AND r.Key1 = l.Fk21
+ WHERE l.Key = 1
+ ORDER BY r.Value2
+ )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+
+ CompareYson(R"([
+ [[101u];["One"];[1];["Value1"];[101u];["One"];["Name1"];["Value21"]];
+ [[101u];["One"];[1];["Value1"];[101u];["Two"];["Name1"];["Value22"]];
+ [[101u];["One"];[1];["Value1"];[101u];["Three"];["Name3"];["Value23"]]
+ ])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+
+ Y_UNIT_TEST_NEW_ENGINE(JoinDictWithPure) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExecuteDataQuery(Q1_(R"(
+ PRAGMA kikimr.UseNewEngine = 'true';
+
+ SELECT xx.id AS id, Key1, Key2, Name, Value2
+ FROM
+ (SELECT 101 AS id) AS xx
+ LEFT JOIN
+ (SELECT * FROM `/Root/Join2` AS a1 FULL JOIN (SELECT 101 AS id) AS yy ON (a1.Key1 = yy.id)) AS yy
+ ON (xx.id = Coalesce(yy.id));
+ )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+
+ CompareYson(R"([
+ [101;[101u];["One"];["Name1"];["Value21"]];
+ [101;[101u];["Three"];["Name3"];["Value23"]];
+ [101;[101u];["Two"];["Name1"];["Value22"]]
+ ])", FormatResultSetYson(result.GetResultSet(0)));
+ }
Y_UNIT_TEST(AsyncIndexUpdate) {
TKikimrRunner kikimr;
@@ -2996,29 +2996,29 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
)", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
}
-
- Y_UNIT_TEST(DuplicatedResults) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA Kikimr.UseNewEngine = 'true';
- SELECT * FROM `/Root/KeyValue`;
- SELECT * FROM `/Root/KeyValue`;
- SELECT * FROM `/Root/Test`;
- SELECT * FROM `/Root/KeyValue`;
- SELECT * FROM `/Root/Test`;
- )", TTxControl::BeginTx().CommitTx()).GetValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- UNIT_ASSERT_VALUES_EQUAL(result.GetResultSets().size(), 5);
-
- CompareYson(R"([[[1u];["One"]];[[2u];["Two"]]])", FormatResultSetYson(result.GetResultSet(0)));
- CompareYson(R"([[[1u];["One"]];[[2u];["Two"]]])", FormatResultSetYson(result.GetResultSet(1)));
- CompareYson(R"([[[3500u];["None"];[1u];["Anna"]];[[300u];["None"];[1u];["Paul"]];[[7200u];["None"];[2u];["Tony"]]])", FormatResultSetYson(result.GetResultSet(2)));
- CompareYson(R"([[[1u];["One"]];[[2u];["Two"]]])", FormatResultSetYson(result.GetResultSet(3)));
- CompareYson(R"([[[3500u];["None"];[1u];["Anna"]];[[300u];["None"];[1u];["Paul"]];[[7200u];["None"];[2u];["Tony"]]])", FormatResultSetYson(result.GetResultSet(4)));
- }
+
+ Y_UNIT_TEST(DuplicatedResults) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA Kikimr.UseNewEngine = 'true';
+ SELECT * FROM `/Root/KeyValue`;
+ SELECT * FROM `/Root/KeyValue`;
+ SELECT * FROM `/Root/Test`;
+ SELECT * FROM `/Root/KeyValue`;
+ SELECT * FROM `/Root/Test`;
+ )", TTxControl::BeginTx().CommitTx()).GetValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResultSets().size(), 5);
+
+ CompareYson(R"([[[1u];["One"]];[[2u];["Two"]]])", FormatResultSetYson(result.GetResultSet(0)));
+ CompareYson(R"([[[1u];["One"]];[[2u];["Two"]]])", FormatResultSetYson(result.GetResultSet(1)));
+ CompareYson(R"([[[3500u];["None"];[1u];["Anna"]];[[300u];["None"];[1u];["Paul"]];[[7200u];["None"];[2u];["Tony"]]])", FormatResultSetYson(result.GetResultSet(2)));
+ CompareYson(R"([[[1u];["One"]];[[2u];["Two"]]])", FormatResultSetYson(result.GetResultSet(3)));
+ CompareYson(R"([[[3500u];["None"];[1u];["Anna"]];[[300u];["None"];[1u];["Paul"]];[[7200u];["None"];[2u];["Tony"]]])", FormatResultSetYson(result.GetResultSet(4)));
+ }
}
} // namespace NKikimr::NKqp
diff --git a/ydb/core/kqp/ut/kqp_olap_ut.cpp b/ydb/core/kqp/ut/kqp_olap_ut.cpp
index 30823a0399..908fc34a13 100644
--- a/ydb/core/kqp/ut/kqp_olap_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_olap_ut.cpp
@@ -805,7 +805,7 @@ Y_UNIT_TEST_SUITE(KqpOlap) {
Cerr << Endl;
Cerr << "Expected result: " << pushdownPresent << Endl;
}
-
+
if (viaSettings.has_value()) {
auto setting = NKikimrKqp::TKqpSetting();
setting.SetName("_KqpPushOlapProcess");
@@ -1919,5 +1919,5 @@ Y_UNIT_TEST_SUITE(KqpOlap) {
}
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/ut/kqp_params_ut.cpp b/ydb/core/kqp/ut/kqp_params_ut.cpp
index 61beba54d7..1eee111329 100644
--- a/ydb/core/kqp/ut/kqp_params_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_params_ut.cpp
@@ -7,17 +7,17 @@ using namespace NYdb;
using namespace NYdb::NTable;
Y_UNIT_TEST_SUITE(KqpParams) {
- Y_UNIT_TEST_NEW_ENGINE(RowsList) {
+ Y_UNIT_TEST_NEW_ENGINE(RowsList) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- auto query = session.PrepareDataQuery(Q1_(R"(
- DECLARE $rows AS List<Struct<Group: Uint32?, Name: String?, Amount: Uint64?, Comment: String?>>;
+ auto query = session.PrepareDataQuery(Q1_(R"(
+ DECLARE $rows AS List<Struct<Group: Uint32?, Name: String?, Amount: Uint64?, Comment: String?>>;
- UPSERT INTO `/Root/Test`
+ UPSERT INTO `/Root/Test`
SELECT Group, Name, Amount FROM AS_TABLE($rows);
- )")).ExtractValueSync().GetQuery();
+ )")).ExtractValueSync().GetQuery();
auto params = query.GetParamsBuilder()
.AddParam("$rows")
@@ -45,9 +45,9 @@ Y_UNIT_TEST_SUITE(KqpParams) {
std::move(params)).ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
- result = session.ExecuteDataQuery(Q_(R"(
- SELECT * FROM `/Root/Test` WHERE Group = 137;
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ result = session.ExecuteDataQuery(Q_(R"(
+ SELECT * FROM `/Root/Test` WHERE Group = 137;
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
CompareYson(R"([
@@ -67,14 +67,14 @@ Y_UNIT_TEST_SUITE(KqpParams) {
.Build()
.Build();
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
DECLARE $group AS Uint32;
DECLARE $name AS String;
SELECT * FROM `/Root/Test` WHERE Group = $group AND Name = $name;
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params).ExtractValueSync();
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString());
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString());
}
Y_UNIT_TEST_NEW_ENGINE(BadParameterType) {
@@ -91,14 +91,14 @@ Y_UNIT_TEST_SUITE(KqpParams) {
.Build()
.Build();
- auto result = session.ExecuteDataQuery(Q1_(R"(
+ auto result = session.ExecuteDataQuery(Q1_(R"(
DECLARE $group AS Uint32;
DECLARE $name AS String;
SELECT * FROM `/Root/Test` WHERE Group = $group AND Name = $name;
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params).ExtractValueSync();
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString());
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString());
}
Y_UNIT_TEST_NEW_ENGINE(DefaultParameterValue) {
@@ -112,12 +112,12 @@ Y_UNIT_TEST_SUITE(KqpParams) {
.Build()
.Build();
- auto result = session.ExecuteDataQuery(Q1_(R"(
- DECLARE $value1 AS Uint32?;
- DECLARE $value2 AS String?;
+ auto result = session.ExecuteDataQuery(Q1_(R"(
+ DECLARE $value1 AS Uint32?;
+ DECLARE $value2 AS String?;
SELECT $value1, $value2;
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params).ExtractValueSync();
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
CompareYson(R"([[[11u];#]])", FormatResultSetYson(result.GetResultSet(0)));
@@ -179,7 +179,7 @@ Y_UNIT_TEST_SUITE(KqpParams) {
.Build()
.Build();
- auto result = session.ExecuteDataQuery(Q1_(R"(
+ auto result = session.ExecuteDataQuery(Q1_(R"(
DECLARE $ParamBool AS Bool;
DECLARE $ParamByte AS Uint8;
DECLARE $ParamInt32 AS Int32;
@@ -197,12 +197,12 @@ Y_UNIT_TEST_SUITE(KqpParams) {
DECLARE $ParamTimestamp AS Timestamp;
DECLARE $ParamInterval AS Interval;
- DECLARE $ParamOpt AS String?;
- DECLARE $ParamTuple AS Tuple<Utf8, Int32>;
- DECLARE $ParamList AS List<Uint64>;
- DECLARE $ParamEmptyList AS List<Uint64>;
- DECLARE $ParamStruct AS Struct<Name:Utf8,Value:Int64>;
- DECLARE $ParamDict AS Dict<String,Uint32>;
+ DECLARE $ParamOpt AS String?;
+ DECLARE $ParamTuple AS Tuple<Utf8, Int32>;
+ DECLARE $ParamList AS List<Uint64>;
+ DECLARE $ParamEmptyList AS List<Uint64>;
+ DECLARE $ParamStruct AS Struct<Name:Utf8,Value:Int64>;
+ DECLARE $ParamDict AS Dict<String,Uint32>;
SELECT
$ParamBool AS ValueBool,
@@ -227,23 +227,23 @@ Y_UNIT_TEST_SUITE(KqpParams) {
$ParamEmptyList AS ValueEmptyList,
$ParamStruct AS ValueStruct,
$ParamDict AS ValueDict;
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params).ExtractValueSync();
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
-
- auto actual = ReformatYson(FormatResultSetYson(result.GetResultSet(0)));
- auto expected1 = ReformatYson(R"([[
- %true;5u;-10;10u;-20;20u;30.5;40.5;"StringValue";"Utf8Value";"[{Value=50}]";"[{\"Value\":60}]";18271u;
- 1578755093u;1578863917000000u;3600;["Opt"];["Tuple0";1];[17u;19u];[];["Paul";-5];
- [["Key2";20u];["Key1";10u]]
- ]])");
- auto expected2 = ReformatYson(R"([[
- %true;5u;-10;10u;-20;20u;30.5;40.5;"StringValue";"Utf8Value";"[{Value=50}]";"[{\"Value\":60}]";18271u;
- 1578755093u;1578863917000000u;3600;["Opt"];["Tuple0";1];[17u;19u];[];["Paul";-5];
- [["Key1";10u];["Key2";20u]]
- ]])");
-
- UNIT_ASSERT_C(actual == expected1 || actual == expected2, "expected: " << expected1 << ", got: " << actual);
+
+ auto actual = ReformatYson(FormatResultSetYson(result.GetResultSet(0)));
+ auto expected1 = ReformatYson(R"([[
+ %true;5u;-10;10u;-20;20u;30.5;40.5;"StringValue";"Utf8Value";"[{Value=50}]";"[{\"Value\":60}]";18271u;
+ 1578755093u;1578863917000000u;3600;["Opt"];["Tuple0";1];[17u;19u];[];["Paul";-5];
+ [["Key2";20u];["Key1";10u]]
+ ]])");
+ auto expected2 = ReformatYson(R"([[
+ %true;5u;-10;10u;-20;20u;30.5;40.5;"StringValue";"Utf8Value";"[{Value=50}]";"[{\"Value\":60}]";18271u;
+ 1578755093u;1578863917000000u;3600;["Opt"];["Tuple0";1];[17u;19u];[];["Paul";-5];
+ [["Key1";10u];["Key2";20u]]
+ ]])");
+
+ UNIT_ASSERT_C(actual == expected1 || actual == expected2, "expected: " << expected1 << ", got: " << actual);
}
Y_UNIT_TEST_NEW_ENGINE(InvalidJson) {
@@ -286,5 +286,5 @@ Y_UNIT_TEST_SUITE(KqpParams) {
}
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/ut/kqp_pragma_ut.cpp b/ydb/core/kqp/ut/kqp_pragma_ut.cpp
index 3e335e76d1..cdf99e4043 100644
--- a/ydb/core/kqp/ut/kqp_pragma_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_pragma_ut.cpp
@@ -23,29 +23,29 @@ Y_UNIT_TEST_SUITE(KqpPragma) {
CompareYson(R"([[1u;"One"]])", FormatResultSetYson(result.GetResultSet(0)));
}
- Y_UNIT_TEST_NEW_ENGINE(Runtime) {
+ Y_UNIT_TEST_NEW_ENGINE(Runtime) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
PRAGMA kikimr.IsolationLevel = "ReadCommitted";
- SELECT * FROM `/Root/KeyValue` WHERE Key = 1;
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ SELECT * FROM `/Root/KeyValue` WHERE Key = 1;
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::GENERIC_ERROR);
UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_PRAGMA_NOT_SUPPORTED));
}
- Y_UNIT_TEST_NEW_ENGINE(Auth) {
+ Y_UNIT_TEST_NEW_ENGINE(Auth) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
PRAGMA kikimr.Auth = "default_kikimr";
- SELECT * FROM `/Root/KeyValue` WHERE Key = 1;
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ SELECT * FROM `/Root/KeyValue` WHERE Key = 1;
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::GENERIC_ERROR);
UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_PRAGMA_NOT_SUPPORTED));
diff --git a/ydb/core/kqp/ut/kqp_query_ut.cpp b/ydb/core/kqp/ut/kqp_query_ut.cpp
index e9c25cb802..5ef6d18f38 100644
--- a/ydb/core/kqp/ut/kqp_query_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_query_ut.cpp
@@ -18,14 +18,14 @@ using namespace NYdb;
using namespace NYdb::NTable;
Y_UNIT_TEST_SUITE(KqpQuery) {
- Y_UNIT_TEST_NEW_ENGINE(PreparedQueryInvalidate) {
+ Y_UNIT_TEST_NEW_ENGINE(PreparedQueryInvalidate) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- auto prepareResult = session.PrepareDataQuery(Q_(R"(
- SELECT * FROM `/Root/Test`
- )")).GetValueSync();
+ auto prepareResult = session.PrepareDataQuery(Q_(R"(
+ SELECT * FROM `/Root/Test`
+ )")).GetValueSync();
UNIT_ASSERT_VALUES_EQUAL(prepareResult.GetStatus(), EStatus::SUCCESS);
auto query = prepareResult.GetQuery();
@@ -56,14 +56,14 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
UNIT_ASSERT_VALUES_EQUAL(counters.RecompileRequestGet()->Val(), 1);
}
- Y_UNIT_TEST_NEW_ENGINE(QueryCache) {
+ Y_UNIT_TEST_NEW_ENGINE(QueryCache) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- auto query = Q_(R"(
- SELECT * FROM `/Root/Test`;
- )");
+ auto query = Q_(R"(
+ SELECT * FROM `/Root/Test`;
+ )");
auto txControl = TTxControl::BeginTx().CommitTx();
@@ -83,7 +83,7 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
UNIT_ASSERT_VALUES_EQUAL(stats.compilation().from_cache(), true);
}
- Y_UNIT_TEST_NEW_ENGINE(QueryCacheTtl) {
+ Y_UNIT_TEST_NEW_ENGINE(QueryCacheTtl) {
NKikimrConfig::TAppConfig appConfig;
appConfig.MutableTableServiceConfig()->SetCompileQueryCacheTTLSec(2);
@@ -91,9 +91,9 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- auto query = Q_(R"(
- SELECT * FROM `/Root/Test`;
- )");
+ auto query = Q_(R"(
+ SELECT * FROM `/Root/Test`;
+ )");
auto txControl = TTxControl::BeginTx().CommitTx();
@@ -131,14 +131,14 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
UNIT_ASSERT_VALUES_EQUAL(stats.compilation().from_cache(), false);
}
- Y_UNIT_TEST_NEW_ENGINE(QueryCacheInvalidate) {
+ Y_UNIT_TEST_NEW_ENGINE(QueryCacheInvalidate) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- auto query = Q_(R"(
- SELECT * FROM `/Root/Test`;
- )");
+ auto query = Q_(R"(
+ SELECT * FROM `/Root/Test`;
+ )");
auto txControl = TTxControl::BeginTx().CommitTx();
@@ -163,7 +163,7 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
UNIT_ASSERT_VALUES_EQUAL(counters.RecompileRequestGet()->Val(), 1);
}
- Y_UNIT_TEST_NEW_ENGINE(QueryCachePermissionsLoss) {
+ Y_UNIT_TEST_NEW_ENGINE(QueryCachePermissionsLoss) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -182,9 +182,9 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
AssertSuccessResult(result);
};
- auto query = Q_(R"(
- SELECT * FROM `/Root/Test`
- )");
+ auto query = Q_(R"(
+ SELECT * FROM `/Root/Test`
+ )");
setPermissions(true);
{
@@ -215,14 +215,14 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
}
}
- Y_UNIT_TEST_NEW_ENGINE(QueryTimeout) {
+ Y_UNIT_TEST_NEW_ENGINE(QueryTimeout) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- auto query = Q_(R"(
+ auto query = Q_(R"(
SELECT * FROM `/Root/TwoShard`;
- )");
+ )");
auto txControl = TTxControl::BeginTx().CommitTx();
@@ -245,14 +245,14 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS || result.GetStatus() == EStatus::SESSION_BUSY, result.GetIssues().ToString());
}
- Y_UNIT_TEST_NEW_ENGINE(QueryTimeoutImmediate) {
+ Y_UNIT_TEST_NEW_ENGINE(QueryTimeoutImmediate) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- auto query = Q_(R"(
+ auto query = Q_(R"(
SELECT * FROM `/Root/TwoShard` WHERE Key == 1;
- )");
+ )");
auto txControl = TTxControl::BeginTx().CommitTx();
@@ -275,7 +275,7 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus() == EStatus::SUCCESS || result.GetStatus() == EStatus::SESSION_BUSY, true, result.GetIssues().ToString());
}
- Y_UNIT_TEST_NEW_ENGINE(QueryClientTimeout) {
+ Y_UNIT_TEST_NEW_ENGINE(QueryClientTimeout) {
TStringStream logStream;
{
@@ -312,8 +312,8 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
TString line;
int v = 0;
while (logStream.ReadLine(line)) {
- if (line.find("ExecuteDataQuery") == line.npos)
- continue;
+ if (line.find("ExecuteDataQuery") == line.npos)
+ continue;
const TString pattern("timeout# ");
auto start = line.find(pattern);
if (start == line.npos)
@@ -327,17 +327,17 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
}
UNIT_ASSERT(v);
- UNIT_ASSERT_C(v <= 3, "got " << v);
+ UNIT_ASSERT_C(v <= 3, "got " << v);
}
- Y_UNIT_TEST_NEW_ENGINE(QueryCancel) {
+ Y_UNIT_TEST_NEW_ENGINE(QueryCancel) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- auto query = Q_(R"(
+ auto query = Q_(R"(
SELECT * FROM `/Root/TwoShard`
- )");
+ )");
auto txControl = TTxControl::BeginTx().CommitTx();
@@ -358,14 +358,14 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
}
- Y_UNIT_TEST_NEW_ENGINE(QueryCancelImmediate) {
+ Y_UNIT_TEST_NEW_ENGINE(QueryCancelImmediate) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- auto query = Q_(R"(
+ auto query = Q_(R"(
SELECT * FROM `/Root/TwoShard` WHERE Key == 1;
- )");
+ )");
auto txControl = TTxControl::BeginTx().CommitTx();
@@ -386,16 +386,16 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
}
- Y_UNIT_TEST_NEW_ENGINE(QueryCancelWrite) {
+ Y_UNIT_TEST_NEW_ENGINE(QueryCancelWrite) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- auto query = Q_(R"(
+ auto query = Q_(R"(
UPSERT INTO `/Root/TwoShard` (Key, Value1) VALUES
(4, "Four"),
(4000000000u, "BigZero");
- )");
+ )");
auto txControl = TTxControl::BeginTx().CommitTx();
@@ -410,15 +410,15 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
NDataShard::gCancelTxFailPoint.Disable();
}
- Y_UNIT_TEST_NEW_ENGINE(QueryCancelWriteImmediate) {
+ Y_UNIT_TEST_NEW_ENGINE(QueryCancelWriteImmediate) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- auto query = Q_(R"(
+ auto query = Q_(R"(
UPSERT INTO `/Root/TwoShard` (Key, Value1) VALUES
(4, "Four");
- )");
+ )");
auto txControl = TTxControl::BeginTx().CommitTx();
@@ -433,7 +433,7 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
NDataShard::gCancelTxFailPoint.Disable();
}
- Y_UNIT_TEST_NEW_ENGINE(QueryResultsTruncated) {
+ Y_UNIT_TEST_NEW_ENGINE(QueryResultsTruncated) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -448,16 +448,16 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
const ui32 RowsCount = 1000;
- auto replaceQuery = Q1_(R"(
+ auto replaceQuery = Q1_(R"(
DECLARE $rows AS
- List<Struct<
+ List<Struct<
Key: Uint64?,
Value: String?
- >>;
+ >>;
- REPLACE INTO `/Root/Tmp`
+ REPLACE INTO `/Root/Tmp`
SELECT * FROM AS_TABLE($rows);
- )");
+ )");
{
auto paramsBuilder = session.GetParamsBuilder();
@@ -481,9 +481,9 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
}
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT * FROM `/Root/Tmp`;
- )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
UNIT_ASSERT(!result.GetResultSet(0).Truncated());
UNIT_ASSERT(result.GetResultSet(0).RowsCount() == RowsCount);
@@ -508,9 +508,9 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
}
- result = session.ExecuteDataQuery(Q_(R"(
+ result = session.ExecuteDataQuery(Q_(R"(
SELECT * FROM `/Root/Tmp`;
- )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
UNIT_ASSERT(result.GetResultSet(0).Truncated());
UNIT_ASSERT(result.GetResultSet(0).RowsCount() == RowsCount);
@@ -538,28 +538,28 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
NYdb::NTable::TExecDataQuerySettings execSettings;
execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
- result = session.ExecuteDataQuery(Q_(R"(
+ result = session.ExecuteDataQuery(Q_(R"(
SELECT * FROM `/Root/Tmp`;
- )"), TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync();
+ )"), TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
UNIT_ASSERT(result.GetResultSet(0).Truncated());
UNIT_ASSERT(result.GetResultSet(0).RowsCount() == RowsCount);
auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
- if (UseNewEngine) {
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 0);
-
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/Tmp");
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 1001);
- } else {
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/Tmp");
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 1001);
- }
+ if (UseNewEngine) {
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 0);
+
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/Tmp");
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 1001);
+ } else {
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/Tmp");
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 1001);
+ }
}
Y_UNIT_TEST(YqlSyntaxV0) {
@@ -612,7 +612,7 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
UNIT_ASSERT(result.IsSuccess());
}
- Y_UNIT_TEST_NEW_ENGINE(YqlTableSample) {
+ Y_UNIT_TEST_NEW_ENGINE(YqlTableSample) {
auto setting = NKikimrKqp::TKqpSetting();
setting.SetName("_KqpYqlSyntaxVersion");
setting.SetValue("1");
@@ -621,7 +621,7 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- const TString query(Q_(R"(SELECT * FROM `/Root/Test` TABLESAMPLE SYSTEM(1.0);)"));
+ const TString query(Q_(R"(SELECT * FROM `/Root/Test` TABLESAMPLE SYSTEM(1.0);)"));
auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED);
UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_UNSUPPORTED, [](const NYql::TIssue& issue) {
@@ -665,7 +665,7 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
auto session = db.CreateSession().GetValueSync().GetSession();
auto result = session.ExplainDataQuery(R"(
- PRAGMA Kikimr.UseNewEngine = 'false';
+ PRAGMA Kikimr.UseNewEngine = 'false';
SELECT * FROM [/Root/Test] WHERE Group = 1 AND Name > "Name";
)").GetValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
@@ -686,46 +686,46 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
UNIT_ASSERT(NJson::ValidateJson(result.GetPlan()));
}
-
- Y_UNIT_TEST_NEW_ENGINE(RewriteIfPresentToMap) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- const TString query = Q_(R"(
- declare $key as Uint64;
- declare $text as String;
- declare $data as Int32;
-
+
+ Y_UNIT_TEST_NEW_ENGINE(RewriteIfPresentToMap) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ const TString query = Q_(R"(
+ declare $key as Uint64;
+ declare $text as String;
+ declare $data as Int32;
+
update [/Root/EightShard]
- set Text = $text, Data = $data
- where Length(Text) != 7 and Data = $data and Key = $key;
-
+ set Text = $text, Data = $data
+ where Length(Text) != 7 and Data = $data and Key = $key;
+
upsert into [/Root/EightShard] (Key, Text, Data) values
- ($key, $text || "_10", $data + 100);
- )");
-
- auto params = TParamsBuilder()
- .AddParam("$key").Uint64(1).Build()
- .AddParam("$text").String("foo").Build()
- .AddParam("$data").Int32(100).Build()
- .Build();
-
- auto prepareResult = session.PrepareDataQuery(query).ExtractValueSync();
- UNIT_ASSERT_C(prepareResult.IsSuccess(), prepareResult.GetIssues().ToString());
-
- auto result = prepareResult.GetQuery().Execute(TTxControl::BeginTx().CommitTx(), params).GetValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
- }
-
- Y_UNIT_TEST_NEW_ENGINE(Pure) {
+ ($key, $text || "_10", $data + 100);
+ )");
+
+ auto params = TParamsBuilder()
+ .AddParam("$key").Uint64(1).Build()
+ .AddParam("$text").String("foo").Build()
+ .AddParam("$data").Int32(100).Build()
+ .Build();
+
+ auto prepareResult = session.PrepareDataQuery(query).ExtractValueSync();
+ UNIT_ASSERT_C(prepareResult.IsSuccess(), prepareResult.GetIssues().ToString());
+
+ auto result = prepareResult.GetQuery().Execute(TTxControl::BeginTx().CommitTx(), params).GetValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ }
+
+ Y_UNIT_TEST_NEW_ENGINE(Pure) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- auto query = Q_(R"(
+ auto query = Q_(R"(
SELECT 1 + 1;
- )");
+ )");
auto result = session.ExecuteDataQuery(
query,
@@ -894,7 +894,7 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
}
- Y_UNIT_TEST_NEW_ENGINE(UnsafeTimestampCastV1) {
+ Y_UNIT_TEST_NEW_ENGINE(UnsafeTimestampCastV1) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -907,13 +907,13 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
);
)").GetValueSync().IsSuccess());
- const TString query = Q1_(R"(
+ const TString query = Q1_(R"(
DECLARE $key AS Uint64;
DECLARE $value AS String;
UPSERT INTO `/Root/TsTest` (Key, Value) VALUES
($key, $value);
- )");
+ )");
auto params = TParamsBuilder()
.AddParam("$key").Uint64(1).Build()
@@ -930,7 +930,7 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString());
}
- Y_UNIT_TEST_NEW_ENGINE(QueryStats) {
+ Y_UNIT_TEST_NEW_ENGINE(QueryStats) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -938,15 +938,15 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
NYdb::NTable::TExecDataQuerySettings execSettings;
execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
- auto result = session.ExecuteDataQuery(Q_(R"(
- UPSERT INTO `/Root/EightShard`
+ auto result = session.ExecuteDataQuery(Q_(R"(
+ UPSERT INTO `/Root/EightShard`
SELECT
Key,
Value1 AS Text,
Value2 AS Data
- FROM `/Root/TwoShard`
+ FROM `/Root/TwoShard`
WHERE Key < 10;
- )"), TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync();
+ )"), TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
@@ -965,19 +965,19 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
UNIT_ASSERT(compile.duration_us() > 0);
UNIT_ASSERT(compile.cpu_time_us() > 0);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), UseNewEngine ? 3 : 2);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), UseNewEngine ? 3 : 2);
uint64_t totalDurationUs = 0;
uint64_t totalCpuTimeUs = 0;
- if (UseNewEngine) {
- auto& phase0 = stats.query_phases(0);
- UNIT_ASSERT(phase0.table_access().size() == 0);
+ if (UseNewEngine) {
+ auto& phase0 = stats.query_phases(0);
+ UNIT_ASSERT(phase0.table_access().size() == 0);
totalDurationUs += phase0.duration_us();
totalCpuTimeUs += phase0.cpu_time_us();
- }
-
- auto& phase0 = stats.query_phases(UseNewEngine ? 1 : 0);
+ }
+
+ auto& phase0 = stats.query_phases(UseNewEngine ? 1 : 0);
UNIT_ASSERT(phase0.duration_us() > 0);
UNIT_ASSERT(phase0.cpu_time_us() > 0);
totalDurationUs += phase0.duration_us();
@@ -990,18 +990,18 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
UNIT_ASSERT(!phase0.table_access(0).has_updates());
UNIT_ASSERT(!phase0.table_access(0).has_deletes());
- auto& phase1 = stats.query_phases(UseNewEngine ? 2 : 1);
+ auto& phase1 = stats.query_phases(UseNewEngine ? 2 : 1);
UNIT_ASSERT(phase1.duration_us() > 0);
UNIT_ASSERT(phase1.cpu_time_us() > 0);
totalDurationUs += phase1.duration_us();
totalCpuTimeUs += phase1.cpu_time_us();
- UNIT_ASSERT_VALUES_EQUAL(phase1.table_access().size(), UseNewEngine ? 1 : 2);
- if (!UseNewEngine) {
- UNIT_ASSERT_VALUES_EQUAL(phase1.table_access(1).name(), "/Root/TwoShard");
- UNIT_ASSERT(!phase1.table_access(1).has_reads());
- UNIT_ASSERT(!phase1.table_access(1).has_updates());
- UNIT_ASSERT(!phase1.table_access(1).has_deletes());
- }
+ UNIT_ASSERT_VALUES_EQUAL(phase1.table_access().size(), UseNewEngine ? 1 : 2);
+ if (!UseNewEngine) {
+ UNIT_ASSERT_VALUES_EQUAL(phase1.table_access(1).name(), "/Root/TwoShard");
+ UNIT_ASSERT(!phase1.table_access(1).has_reads());
+ UNIT_ASSERT(!phase1.table_access(1).has_updates());
+ UNIT_ASSERT(!phase1.table_access(1).has_deletes());
+ }
UNIT_ASSERT_VALUES_EQUAL(phase1.table_access(0).name(), "/Root/EightShard");
UNIT_ASSERT(!phase1.table_access(0).has_reads());
UNIT_ASSERT_VALUES_EQUAL(phase1.table_access(0).updates().rows(), 3);
@@ -1012,7 +1012,7 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
UNIT_ASSERT_VALUES_EQUAL(stats.total_cpu_time_us(), totalCpuTimeUs);
}
- Y_UNIT_TEST_NEW_ENGINE(RowsLimit) {
+ Y_UNIT_TEST_NEW_ENGINE(RowsLimit) {
auto setting = NKikimrKqp::TKqpSetting();
setting.SetName("_ResultRowsLimit");
setting.SetValue("5");
@@ -1021,16 +1021,16 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- auto result = session.ExecuteDataQuery(Q_(R"(
- SELECT * FROM `/Root/EightShard` WHERE Text = "Value2";
- )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ auto result = session.ExecuteDataQuery(Q_(R"(
+ SELECT * FROM `/Root/EightShard` WHERE Text = "Value2";
+ )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
UNIT_ASSERT(result.GetResultSet(0).Truncated());
UNIT_ASSERT_VALUES_EQUAL(result.GetResultSet(0).RowsCount(), 5);
}
- Y_UNIT_TEST_NEW_ENGINE(RowsLimitServiceOverride) {
+ Y_UNIT_TEST_NEW_ENGINE(RowsLimitServiceOverride) {
auto setting = NKikimrKqp::TKqpSetting();
setting.SetName("_ResultRowsLimit");
setting.SetValue("5");
@@ -1042,46 +1042,46 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- auto result = session.ExecuteDataQuery(Q_(R"(
- SELECT * FROM `/Root/EightShard` WHERE Text = "Value2";
- )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ auto result = session.ExecuteDataQuery(Q_(R"(
+ SELECT * FROM `/Root/EightShard` WHERE Text = "Value2";
+ )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
UNIT_ASSERT(result.GetResultSet(0).Truncated());
UNIT_ASSERT_VALUES_EQUAL(result.GetResultSet(0).RowsCount(), 6);
}
-
- Y_UNIT_TEST_NEW_ENGINE(NoEvaluate) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- {
- auto result = session.ExecuteDataQuery(Q_(R"(
- DEFINE ACTION $hello() AS
- SELECT "Hello!";
- END DEFINE;
-
- EVALUATE IF true
- DO $hello()
- )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+
+ Y_UNIT_TEST_NEW_ENGINE(NoEvaluate) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ {
+ auto result = session.ExecuteDataQuery(Q_(R"(
+ DEFINE ACTION $hello() AS
+ SELECT "Hello!";
+ END DEFINE;
+
+ EVALUATE IF true
+ DO $hello()
+ )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::UNSUPPORTED, result.GetIssues().ToString());
UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_UNSUPPORTED, [](const NYql::TIssue& issue) {
return issue.Message.Contains("EVALUATE IF is not supported in YDB queries.");
- }));
- }
-
- {
- auto result = session.ExecuteDataQuery(Q1_(R"(
- EVALUATE FOR $i IN AsList(1, 2, 3) DO BEGIN
- SELECT $i;
- END DO;
- )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ }));
+ }
+
+ {
+ auto result = session.ExecuteDataQuery(Q1_(R"(
+ EVALUATE FOR $i IN AsList(1, 2, 3) DO BEGIN
+ SELECT $i;
+ END DO;
+ )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::UNSUPPORTED, result.GetIssues().ToString());
UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_UNSUPPORTED, [](const NYql::TIssue& issue) {
return issue.Message.Contains("EVALUATE is not supported in YDB queries.");
- }));
- }
+ }));
+ }
{
auto params = db.GetParamsBuilder()
@@ -1120,17 +1120,17 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
return issue.Message.Contains("ATOM evaluation is not supported in YDB queries.");
}));
}
- }
+ }
- Y_UNIT_TEST_NEW_ENGINE(UdfTerminate) {
+ Y_UNIT_TEST_NEW_ENGINE(UdfTerminate) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT * FROM `/Root/KeyValue`
WHERE TestUdfs::TestFilterTerminate(Cast(Key as Int64) ?? 0, 10)
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
result.GetIssues().PrintTo(Cerr);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR);
UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::DEFAULT_ERROR, [](const NYql::TIssue& issue) {
@@ -1138,16 +1138,16 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
}));
}
- Y_UNIT_TEST_NEW_ENGINE(UdfMemoryLimit) {
+ Y_UNIT_TEST_NEW_ENGINE(UdfMemoryLimit) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
// NOTE: 10MB is greater than default memory allocation for datashard tx.
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT * FROM `/Root/KeyValue`
WHERE Value < TestUdfs::RandString(10000000);
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
result.GetIssues().PrintTo(Cerr);
}
@@ -1248,33 +1248,33 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
result.GetIssues().PrintTo(Cerr);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR);
}
-
- Y_UNIT_TEST_NEW_ENGINE(SelectNull) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExecuteDataQuery(Q_(R"(
- SELECT Null
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- AssertSuccessResult(result);
-
- auto rs = TResultSetParser(result.GetResultSet(0));
- UNIT_ASSERT(rs.TryNextRow());
-
- auto& cp = rs.ColumnParser(0);
-
- if constexpr (UseNewEngine) {
- UNIT_ASSERT_VALUES_EQUAL(TTypeParser::ETypeKind::Null, cp.GetKind());
- } else {
- UNIT_ASSERT_VALUES_EQUAL(TTypeParser::ETypeKind::Optional, cp.GetKind());
- cp.OpenOptional();
- UNIT_ASSERT_VALUES_EQUAL(TTypeParser::ETypeKind::Void, cp.GetKind());
- UNIT_ASSERT(cp.IsNull());
- }
- }
-
+
+ Y_UNIT_TEST_NEW_ENGINE(SelectNull) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExecuteDataQuery(Q_(R"(
+ SELECT Null
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ AssertSuccessResult(result);
+
+ auto rs = TResultSetParser(result.GetResultSet(0));
+ UNIT_ASSERT(rs.TryNextRow());
+
+ auto& cp = rs.ColumnParser(0);
+
+ if constexpr (UseNewEngine) {
+ UNIT_ASSERT_VALUES_EQUAL(TTypeParser::ETypeKind::Null, cp.GetKind());
+ } else {
+ UNIT_ASSERT_VALUES_EQUAL(TTypeParser::ETypeKind::Optional, cp.GetKind());
+ cp.OpenOptional();
+ UNIT_ASSERT_VALUES_EQUAL(TTypeParser::ETypeKind::Void, cp.GetKind());
+ UNIT_ASSERT(cp.IsNull());
+ }
+ }
+
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/ut/kqp_scan_spilling_ut.cpp b/ydb/core/kqp/ut/kqp_scan_spilling_ut.cpp
index 4a4327c41b..613724e640 100644
--- a/ydb/core/kqp/ut/kqp_scan_spilling_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_scan_spilling_ut.cpp
@@ -1,96 +1,96 @@
#include <ydb/core/kqp/ut/common/kqp_ut_common.h>
#include <ydb/core/kqp/counters/kqp_counters.h>
-
+
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h>
-#include <util/system/fs.h>
-
-namespace NKikimr {
-namespace NKqp {
-
-using namespace NYdb;
-using namespace NYdb::NTable;
-
-
-namespace {
-
-NKikimrConfig::TAppConfig AppCfg() {
- NKikimrConfig::TAppConfig appCfg;
-
- auto* rm = appCfg.MutableTableServiceConfig()->MutableResourceManager();
- rm->SetChannelBufferSize(100);
- rm->SetMinChannelBufferSize(100);
- rm->SetScanBufferSize(1 << 20);
- rm->SetMkqlLightProgramMemoryLimit(100 << 20);
- rm->SetMkqlHeavyProgramMemoryLimit(100 << 20);
-
- auto* spilling = appCfg.MutableTableServiceConfig()->MutableSpillingServiceConfig()->MutableLocalFileConfig();
- spilling->SetEnable(true);
- spilling->SetRoot("./spilling/");
-
- return appCfg;
-}
-
-} // anonymous namespace
-
+#include <util/system/fs.h>
+
+namespace NKikimr {
+namespace NKqp {
+
+using namespace NYdb;
+using namespace NYdb::NTable;
+
+
+namespace {
+
+NKikimrConfig::TAppConfig AppCfg() {
+ NKikimrConfig::TAppConfig appCfg;
+
+ auto* rm = appCfg.MutableTableServiceConfig()->MutableResourceManager();
+ rm->SetChannelBufferSize(100);
+ rm->SetMinChannelBufferSize(100);
+ rm->SetScanBufferSize(1 << 20);
+ rm->SetMkqlLightProgramMemoryLimit(100 << 20);
+ rm->SetMkqlHeavyProgramMemoryLimit(100 << 20);
+
+ auto* spilling = appCfg.MutableTableServiceConfig()->MutableSpillingServiceConfig()->MutableLocalFileConfig();
+ spilling->SetEnable(true);
+ spilling->SetRoot("./spilling/");
+
+ return appCfg;
+}
+
+} // anonymous namespace
+
Y_UNIT_TEST_SUITE(KqpScanSpilling) {
-
-Y_UNIT_TEST(SelfJoin) {
- Cerr << "cwd: " << NFs::CurrentWorkingDirectory() << Endl;
-
- NYql::NDq::GetDqExecutionSettingsForTests().FlowControl.MaxOutputChunkSize = 20;
- NYql::NDq::GetDqExecutionSettingsForTests().FlowControl.InFlightBytesOvercommit = 1;
-
- Y_DEFER {
- NYql::NDq::GetDqExecutionSettingsForTests().Reset();
- };
-
- TKikimrRunner kikimr(AppCfg());
- kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_COMPUTE, NActors::NLog::PRI_DEBUG);
- kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_BLOBS_STORAGE, NActors::NLog::PRI_DEBUG);
-
- auto db = kikimr.GetTableClient();
-
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- for (ui32 i = 0; i < 10; ++i) {
- auto result = session.ExecuteDataQuery(Sprintf(R"(
- --!syntax_v1
- REPLACE INTO `/Root/KeyValue` (Key, Value) VALUES (%d, "%s")
- )", i, TString(20, 'a' + i).c_str()), TTxControl::BeginTx().CommitTx()).GetValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
- }
-
- auto query = R"(
- --!syntax_v1
- select t1.Key, t1.Value, t2.Key, t2.Value
- from `/Root/KeyValue` as t1 join `/Root/KeyValue` as t2 on t1.Key = t2.Key
- order by t1.Key
- )";
-
+
+Y_UNIT_TEST(SelfJoin) {
+ Cerr << "cwd: " << NFs::CurrentWorkingDirectory() << Endl;
+
+ NYql::NDq::GetDqExecutionSettingsForTests().FlowControl.MaxOutputChunkSize = 20;
+ NYql::NDq::GetDqExecutionSettingsForTests().FlowControl.InFlightBytesOvercommit = 1;
+
+ Y_DEFER {
+ NYql::NDq::GetDqExecutionSettingsForTests().Reset();
+ };
+
+ TKikimrRunner kikimr(AppCfg());
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_COMPUTE, NActors::NLog::PRI_DEBUG);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_BLOBS_STORAGE, NActors::NLog::PRI_DEBUG);
+
+ auto db = kikimr.GetTableClient();
+
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ for (ui32 i = 0; i < 10; ++i) {
+ auto result = session.ExecuteDataQuery(Sprintf(R"(
+ --!syntax_v1
+ REPLACE INTO `/Root/KeyValue` (Key, Value) VALUES (%d, "%s")
+ )", i, TString(20, 'a' + i).c_str()), TTxControl::BeginTx().CommitTx()).GetValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ }
+
+ auto query = R"(
+ --!syntax_v1
+ select t1.Key, t1.Value, t2.Key, t2.Value
+ from `/Root/KeyValue` as t1 join `/Root/KeyValue` as t2 on t1.Key = t2.Key
+ order by t1.Key
+ )";
+
auto it = db.StreamExecuteScanQuery(query).GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- CompareYson(R"([
- [[0u];["aaaaaaaaaaaaaaaaaaaa"];[0u];["aaaaaaaaaaaaaaaaaaaa"]];
- [[1u];["bbbbbbbbbbbbbbbbbbbb"];[1u];["bbbbbbbbbbbbbbbbbbbb"]];
- [[2u];["cccccccccccccccccccc"];[2u];["cccccccccccccccccccc"]];
- [[3u];["dddddddddddddddddddd"];[3u];["dddddddddddddddddddd"]];
- [[4u];["eeeeeeeeeeeeeeeeeeee"];[4u];["eeeeeeeeeeeeeeeeeeee"]];
- [[5u];["ffffffffffffffffffff"];[5u];["ffffffffffffffffffff"]];
- [[6u];["gggggggggggggggggggg"];[6u];["gggggggggggggggggggg"]];
- [[7u];["hhhhhhhhhhhhhhhhhhhh"];[7u];["hhhhhhhhhhhhhhhhhhhh"]];
- [[8u];["iiiiiiiiiiiiiiiiiiii"];[8u];["iiiiiiiiiiiiiiiiiiii"]];
- [[9u];["jjjjjjjjjjjjjjjjjjjj"];[9u];["jjjjjjjjjjjjjjjjjjjj"]]
- ])", StreamResultToYson(it));
-
- TKqpCounters counters(kikimr.GetTestServer().GetRuntime()->GetAppData().Counters);
- UNIT_ASSERT_VALUES_EQUAL(18, counters.SpillingWriteBlobs->Val());
- UNIT_ASSERT_VALUES_EQUAL(18, counters.SpillingReadBlobs->Val());
- UNIT_ASSERT(0 == counters.SpillingStoredBlobs->Val() || 18 == counters.SpillingStoredBlobs->Val());
-}
-
-} // suite
-
-} // namespace NKqp
-} // namespace NKikimr
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ CompareYson(R"([
+ [[0u];["aaaaaaaaaaaaaaaaaaaa"];[0u];["aaaaaaaaaaaaaaaaaaaa"]];
+ [[1u];["bbbbbbbbbbbbbbbbbbbb"];[1u];["bbbbbbbbbbbbbbbbbbbb"]];
+ [[2u];["cccccccccccccccccccc"];[2u];["cccccccccccccccccccc"]];
+ [[3u];["dddddddddddddddddddd"];[3u];["dddddddddddddddddddd"]];
+ [[4u];["eeeeeeeeeeeeeeeeeeee"];[4u];["eeeeeeeeeeeeeeeeeeee"]];
+ [[5u];["ffffffffffffffffffff"];[5u];["ffffffffffffffffffff"]];
+ [[6u];["gggggggggggggggggggg"];[6u];["gggggggggggggggggggg"]];
+ [[7u];["hhhhhhhhhhhhhhhhhhhh"];[7u];["hhhhhhhhhhhhhhhhhhhh"]];
+ [[8u];["iiiiiiiiiiiiiiiiiiii"];[8u];["iiiiiiiiiiiiiiiiiiii"]];
+ [[9u];["jjjjjjjjjjjjjjjjjjjj"];[9u];["jjjjjjjjjjjjjjjjjjjj"]]
+ ])", StreamResultToYson(it));
+
+ TKqpCounters counters(kikimr.GetTestServer().GetRuntime()->GetAppData().Counters);
+ UNIT_ASSERT_VALUES_EQUAL(18, counters.SpillingWriteBlobs->Val());
+ UNIT_ASSERT_VALUES_EQUAL(18, counters.SpillingReadBlobs->Val());
+ UNIT_ASSERT(0 == counters.SpillingStoredBlobs->Val() || 18 == counters.SpillingStoredBlobs->Val());
+}
+
+} // suite
+
+} // namespace NKqp
+} // namespace NKikimr
diff --git a/ydb/core/kqp/ut/kqp_scan_ut.cpp b/ydb/core/kqp/ut/kqp_scan_ut.cpp
index 92407675da..347ca35441 100644
--- a/ydb/core/kqp/ut/kqp_scan_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_scan_ut.cpp
@@ -5,240 +5,240 @@
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h>
-#include <util/generic/size_literals.h>
-
+#include <util/generic/size_literals.h>
+
namespace NKikimr {
namespace NKqp {
using namespace NYdb;
using namespace NYdb::NTable;
-
-namespace {
-
-NKikimrConfig::TAppConfig AppCfg() {
- NKikimrConfig::TAppConfig appCfg;
- appCfg.MutableTableServiceConfig()->MutableResourceManager()->SetChannelBufferSize(1_MB);
- appCfg.MutableTableServiceConfig()->MutableResourceManager()->SetMinChannelBufferSize(1_MB);
- appCfg.MutableTableServiceConfig()->MutableResourceManager()->SetMaxTotalChannelBuffersSize(100_GB);
- appCfg.MutableTableServiceConfig()->MutableResourceManager()->SetScanBufferSize(1_MB);
- appCfg.MutableTableServiceConfig()->MutableResourceManager()->SetMkqlLightProgramMemoryLimit(1_MB);
- appCfg.MutableTableServiceConfig()->MutableResourceManager()->SetMkqlHeavyProgramMemoryLimit(1_MB);
- return appCfg;
-}
-
-void CreateSampleTables(TKikimrRunner& kikimr) {
- kikimr.GetTestClient().CreateTable("/Root", R"(
- Name: "FourShard"
- Columns { Name: "Key", Type: "Uint64" }
- Columns { Name: "Value1", Type: "String" }
- Columns { Name: "Value2", Type: "String" }
- KeyColumnNames: ["Key"],
- SplitBoundary { KeyPrefix { Tuple { Optional { Uint64: 100 } } } }
- SplitBoundary { KeyPrefix { Tuple { Optional { Uint64: 200 } } } }
- SplitBoundary { KeyPrefix { Tuple { Optional { Uint64: 300 } } } }
- )");
-
- TTableClient tableClient{kikimr.GetDriver()};
- auto session = tableClient.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExecuteDataQuery(R"(
+
+namespace {
+
+NKikimrConfig::TAppConfig AppCfg() {
+ NKikimrConfig::TAppConfig appCfg;
+ appCfg.MutableTableServiceConfig()->MutableResourceManager()->SetChannelBufferSize(1_MB);
+ appCfg.MutableTableServiceConfig()->MutableResourceManager()->SetMinChannelBufferSize(1_MB);
+ appCfg.MutableTableServiceConfig()->MutableResourceManager()->SetMaxTotalChannelBuffersSize(100_GB);
+ appCfg.MutableTableServiceConfig()->MutableResourceManager()->SetScanBufferSize(1_MB);
+ appCfg.MutableTableServiceConfig()->MutableResourceManager()->SetMkqlLightProgramMemoryLimit(1_MB);
+ appCfg.MutableTableServiceConfig()->MutableResourceManager()->SetMkqlHeavyProgramMemoryLimit(1_MB);
+ return appCfg;
+}
+
+void CreateSampleTables(TKikimrRunner& kikimr) {
+ kikimr.GetTestClient().CreateTable("/Root", R"(
+ Name: "FourShard"
+ Columns { Name: "Key", Type: "Uint64" }
+ Columns { Name: "Value1", Type: "String" }
+ Columns { Name: "Value2", Type: "String" }
+ KeyColumnNames: ["Key"],
+ SplitBoundary { KeyPrefix { Tuple { Optional { Uint64: 100 } } } }
+ SplitBoundary { KeyPrefix { Tuple { Optional { Uint64: 200 } } } }
+ SplitBoundary { KeyPrefix { Tuple { Optional { Uint64: 300 } } } }
+ )");
+
+ TTableClient tableClient{kikimr.GetDriver()};
+ auto session = tableClient.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExecuteDataQuery(R"(
REPLACE INTO `/Root/FourShard` (Key, Value1, Value2) VALUES
- (1u, "Value-001", "1"),
- (2u, "Value-002", "2"),
- (101u, "Value-101", "101"),
- (102u, "Value-102", "102"),
- (201u, "Value-201", "201"),
- (202u, "Value-202", "202"),
- (301u, "Value-301", "301"),
- (302u, "Value-302", "302")
- )", TTxControl::BeginTx().CommitTx()).GetValueSync();
-
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- session.Close();
-}
-
-void CreateNullSampleTables(TKikimrRunner& kikimr) {
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- UNIT_ASSERT(session.ExecuteSchemeQuery(R"(
+ (1u, "Value-001", "1"),
+ (2u, "Value-002", "2"),
+ (101u, "Value-101", "101"),
+ (102u, "Value-102", "102"),
+ (201u, "Value-201", "201"),
+ (202u, "Value-202", "202"),
+ (301u, "Value-301", "301"),
+ (302u, "Value-302", "302")
+ )", TTxControl::BeginTx().CommitTx()).GetValueSync();
+
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ session.Close();
+}
+
+void CreateNullSampleTables(TKikimrRunner& kikimr) {
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ UNIT_ASSERT(session.ExecuteSchemeQuery(R"(
CREATE TABLE [/Root/TestNulls] (
- Key1 Uint32,
- Key2 Uint32,
- Value String,
- PRIMARY KEY (Key1, Key2)
- );
- )").GetValueSync().IsSuccess());
-
- UNIT_ASSERT(session.ExecuteDataQuery(R"(
+ Key1 Uint32,
+ Key2 Uint32,
+ Value String,
+ PRIMARY KEY (Key1, Key2)
+ );
+ )").GetValueSync().IsSuccess());
+
+ UNIT_ASSERT(session.ExecuteDataQuery(R"(
REPLACE INTO [/Root/TestNulls] (Key1, Key2, Value) VALUES
- (NULL, NULL, "One"),
- (NULL, 100u, "Two"),
- (NULL, 200u, "Three"),
- (1u, NULL, "Four"),
- (1u, 100u, "Five"),
- (1u, 200u, "Six"),
- (2u, NULL, "Seven"),
- (2u, 100u, "Eight"),
- (2u, 200u, "Nine"),
- (3u, 100u, "Ten"),
- (3u, 200u, "Eleven"),
- (3u, 300u, "Twelve"),
- (3u, 400u, "Thirteen"),
- (3u, 500u, "Fourteen");
- )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).GetValueSync().IsSuccess());
-}
-
-} // namespace
-
+ (NULL, NULL, "One"),
+ (NULL, 100u, "Two"),
+ (NULL, 200u, "Three"),
+ (1u, NULL, "Four"),
+ (1u, 100u, "Five"),
+ (1u, 200u, "Six"),
+ (2u, NULL, "Seven"),
+ (2u, 100u, "Eight"),
+ (2u, 200u, "Nine"),
+ (3u, 100u, "Ten"),
+ (3u, 200u, "Eleven"),
+ (3u, 300u, "Twelve"),
+ (3u, 400u, "Thirteen"),
+ (3u, 500u, "Fourteen");
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).GetValueSync().IsSuccess());
+}
+
+} // namespace
+
Y_UNIT_TEST_SUITE(KqpScan) {
- Y_UNIT_TEST(IsNull) {
- TKikimrRunner kikimr;
- CreateNullSampleTables(kikimr);
-
- auto it = kikimr.GetTableClient().StreamExecuteScanQuery(R"(
+ Y_UNIT_TEST(IsNull) {
+ TKikimrRunner kikimr;
+ CreateNullSampleTables(kikimr);
+
+ auto it = kikimr.GetTableClient().StreamExecuteScanQuery(R"(
SELECT Value FROM `/Root/TestNulls`
- WHERE Key1 IS NULL AND Key2 IS NULL
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- CompareYson(R"([[["One"]]])", StreamResultToYson(it));
- }
-
- Y_UNIT_TEST(IsNullPartial) {
- TKikimrRunner kikimr;
- CreateNullSampleTables(kikimr);
-
- auto it = kikimr.GetTableClient().StreamExecuteScanQuery(R"(
+ WHERE Key1 IS NULL AND Key2 IS NULL
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ CompareYson(R"([[["One"]]])", StreamResultToYson(it));
+ }
+
+ Y_UNIT_TEST(IsNullPartial) {
+ TKikimrRunner kikimr;
+ CreateNullSampleTables(kikimr);
+
+ auto it = kikimr.GetTableClient().StreamExecuteScanQuery(R"(
SELECT * FROM `/Root/Test`
- WHERE Group == 1 AND Name IS NULL
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- CompareYson(R"([])", StreamResultToYson(it));
- }
-
- Y_UNIT_TEST(NullInKey) {
- TKikimrRunner kikimr;
- CreateNullSampleTables(kikimr);
-
- auto it = kikimr.GetTableClient().StreamExecuteScanQuery(R"(
+ WHERE Group == 1 AND Name IS NULL
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ CompareYson(R"([])", StreamResultToYson(it));
+ }
+
+ Y_UNIT_TEST(NullInKey) {
+ TKikimrRunner kikimr;
+ CreateNullSampleTables(kikimr);
+
+ auto it = kikimr.GetTableClient().StreamExecuteScanQuery(R"(
SELECT Value FROM `/Root/TestNulls`
- WHERE Key1 <= 1
- ORDER BY Value
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- CompareYson(R"([[["Five"]];[["Four"]];[["Six"]]])", StreamResultToYson(it));
- }
-
- Y_UNIT_TEST(NullInKeySuffix) {
- TKikimrRunner kikimr;
- CreateNullSampleTables(kikimr);
-
- auto it = kikimr.GetTableClient().StreamExecuteScanQuery(R"(
- DECLARE $key1 AS Uint32?;
+ WHERE Key1 <= 1
+ ORDER BY Value
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ CompareYson(R"([[["Five"]];[["Four"]];[["Six"]]])", StreamResultToYson(it));
+ }
+
+ Y_UNIT_TEST(NullInKeySuffix) {
+ TKikimrRunner kikimr;
+ CreateNullSampleTables(kikimr);
+
+ auto it = kikimr.GetTableClient().StreamExecuteScanQuery(R"(
+ DECLARE $key1 AS Uint32?;
SELECT Value FROM `/Root/TestNulls`
- WHERE Key1 > 1
- )").GetValueSync();
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- CompareYson(R"([
- [["Seven"]];[["Eight"]];[["Nine"]];[["Ten"]];[["Eleven"]];[["Twelve"]];[["Thirteen"]];[["Fourteen"]]
- ])", StreamResultToYson(it));
- }
-
- Y_UNIT_TEST(DecimalColumn) {
- TKikimrRunner kikimr(AppCfg());
-
- TTableClient client{kikimr.GetDriver()};
- auto session = client.CreateSession().GetValueSync().GetSession();
-
- auto partitions = TExplicitPartitions()
- .AppendSplitPoints(TValueBuilder()
- .BeginTuple().AddElement().BeginOptional().Decimal(TDecimalValue("1.5", 22, 9)).EndOptional().EndTuple()
- .Build());
-
- auto ret = session.CreateTable("/Root/DecimalTest",
- TTableBuilder()
- .AddNullableColumn("Key", TDecimalType(22, 9))
- .AddNullableColumn("Value", TDecimalType(22, 9))
- .SetPrimaryKeyColumn("Key")
- // .SetPartitionAtKeys(partitions) // Error at split boundary 0: Unsupported typeId 4865 at index 0
- .Build()).GetValueSync();
- UNIT_ASSERT_C(ret.IsSuccess(), ret.GetIssues().ToString());
-
- auto params = TParamsBuilder().AddParam("$in").BeginList()
- .AddListItem().BeginStruct()
- .AddMember("Key").Decimal(TDecimalValue("1.0"))
- .AddMember("Value").Decimal(TDecimalValue("10.123456789"))
- .EndStruct()
- .AddListItem().BeginStruct()
- .AddMember("Key").Decimal(TDecimalValue("2.0"))
- .AddMember("Value").Decimal(TDecimalValue("20.987654321"))
- .EndStruct()
- .EndList().Build().Build();
-
- auto result = session.ExecuteDataQuery(R"(
- --!syntax_v1
- DECLARE $in AS List<Struct<Key: Decimal(22, 9), Value: Decimal(22, 9)>>;
- REPLACE INTO `/Root/DecimalTest`
- SELECT Key, Value FROM AS_TABLE($in);
- )", TTxControl::BeginTx().CommitTx(), params).GetValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- auto db = kikimr.GetTableClient();
- auto it = db.StreamExecuteScanQuery("select Key, max(Value) from `/Root/DecimalTest` group by Key order by Key").GetValueSync();
- CompareYson(R"([
- [["1"];["10.123456789"]];
- [["2"];["20.987654321"]]
- ])", StreamResultToYson(it));
- }
-
- Y_UNIT_TEST(Offset) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
-
- auto it = db.StreamExecuteScanQuery(R"(
+ WHERE Key1 > 1
+ )").GetValueSync();
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ CompareYson(R"([
+ [["Seven"]];[["Eight"]];[["Nine"]];[["Ten"]];[["Eleven"]];[["Twelve"]];[["Thirteen"]];[["Fourteen"]]
+ ])", StreamResultToYson(it));
+ }
+
+ Y_UNIT_TEST(DecimalColumn) {
+ TKikimrRunner kikimr(AppCfg());
+
+ TTableClient client{kikimr.GetDriver()};
+ auto session = client.CreateSession().GetValueSync().GetSession();
+
+ auto partitions = TExplicitPartitions()
+ .AppendSplitPoints(TValueBuilder()
+ .BeginTuple().AddElement().BeginOptional().Decimal(TDecimalValue("1.5", 22, 9)).EndOptional().EndTuple()
+ .Build());
+
+ auto ret = session.CreateTable("/Root/DecimalTest",
+ TTableBuilder()
+ .AddNullableColumn("Key", TDecimalType(22, 9))
+ .AddNullableColumn("Value", TDecimalType(22, 9))
+ .SetPrimaryKeyColumn("Key")
+ // .SetPartitionAtKeys(partitions) // Error at split boundary 0: Unsupported typeId 4865 at index 0
+ .Build()).GetValueSync();
+ UNIT_ASSERT_C(ret.IsSuccess(), ret.GetIssues().ToString());
+
+ auto params = TParamsBuilder().AddParam("$in").BeginList()
+ .AddListItem().BeginStruct()
+ .AddMember("Key").Decimal(TDecimalValue("1.0"))
+ .AddMember("Value").Decimal(TDecimalValue("10.123456789"))
+ .EndStruct()
+ .AddListItem().BeginStruct()
+ .AddMember("Key").Decimal(TDecimalValue("2.0"))
+ .AddMember("Value").Decimal(TDecimalValue("20.987654321"))
+ .EndStruct()
+ .EndList().Build().Build();
+
+ auto result = session.ExecuteDataQuery(R"(
+ --!syntax_v1
+ DECLARE $in AS List<Struct<Key: Decimal(22, 9), Value: Decimal(22, 9)>>;
+ REPLACE INTO `/Root/DecimalTest`
+ SELECT Key, Value FROM AS_TABLE($in);
+ )", TTxControl::BeginTx().CommitTx(), params).GetValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ auto db = kikimr.GetTableClient();
+ auto it = db.StreamExecuteScanQuery("select Key, max(Value) from `/Root/DecimalTest` group by Key order by Key").GetValueSync();
+ CompareYson(R"([
+ [["1"];["10.123456789"]];
+ [["2"];["20.987654321"]]
+ ])", StreamResultToYson(it));
+ }
+
+ Y_UNIT_TEST(Offset) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+
+ auto it = db.StreamExecuteScanQuery(R"(
SELECT Key, Text, Data FROM `/Root/EightShard` ORDER BY Key LIMIT 3 OFFSET 6
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- CompareYson(R"([
- [[301u];["Value1"];[3]];
- [[302u];["Value2"];[2]];
- [[303u];["Value3"];[1]]
- ])", StreamResultToYson(it));
- }
-
- Y_UNIT_TEST(Limit) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
-
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ CompareYson(R"([
+ [[301u];["Value1"];[3]];
+ [[302u];["Value2"];[2]];
+ [[303u];["Value3"];[1]]
+ ])", StreamResultToYson(it));
+ }
+
+ Y_UNIT_TEST(Limit) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+
auto it = db.StreamExecuteScanQuery(R"(
SELECT * FROM `/Root/KeyValue` LIMIT 10
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- CompareYson(R"([[[1u];["One"]];[[2u];["Two"]]])", StreamResultToYson(it));
- }
-
- Y_UNIT_TEST(TopSort) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
-
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ CompareYson(R"([[[1u];["One"]];[[2u];["Two"]]])", StreamResultToYson(it));
+ }
+
+ Y_UNIT_TEST(TopSort) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+
auto it = db.StreamExecuteScanQuery(R"(
SELECT * FROM `/Root/KeyValue` ORDER BY Key LIMIT 1
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- CompareYson(R"([[[1u];["One"]]])", StreamResultToYson(it));
- }
-
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ CompareYson(R"([[[1u];["One"]]])", StreamResultToYson(it));
+ }
+
Y_UNIT_TEST(Grep) {
- TKikimrRunner kikimr(AppCfg());
+ TKikimrRunner kikimr(AppCfg());
auto db = kikimr.GetTableClient();
auto it = db.StreamExecuteScanQuery(R"(
@@ -259,7 +259,7 @@ Y_UNIT_TEST_SUITE(KqpScan) {
}
Y_UNIT_TEST(GrepByString) {
- TKikimrRunner kikimr(AppCfg());
+ TKikimrRunner kikimr(AppCfg());
auto db = kikimr.GetTableClient();
auto it = db.StreamExecuteScanQuery(R"(
@@ -301,7 +301,7 @@ Y_UNIT_TEST_SUITE(KqpScan) {
}
Y_UNIT_TEST(GrepRange) {
- TKikimrRunner kikimr(AppCfg());
+ TKikimrRunner kikimr(AppCfg());
auto db = kikimr.GetTableClient();
auto params = db.GetParamsBuilder()
@@ -331,7 +331,7 @@ Y_UNIT_TEST_SUITE(KqpScan) {
}
Y_UNIT_TEST(GrepLimit) {
- TKikimrRunner kikimr(AppCfg());
+ TKikimrRunner kikimr(AppCfg());
auto db = kikimr.GetTableClient();
auto params = db.GetParamsBuilder()
@@ -369,7 +369,7 @@ Y_UNIT_TEST_SUITE(KqpScan) {
}
Y_UNIT_TEST(GrepNonKeyColumns) {
- TKikimrRunner kikimr(AppCfg());
+ TKikimrRunner kikimr(AppCfg());
auto db = kikimr.GetTableClient();
TParamsBuilder params;
@@ -403,7 +403,7 @@ Y_UNIT_TEST_SUITE(KqpScan) {
}
Y_UNIT_TEST(SingleKey) {
- TKikimrRunner kikimr(AppCfg());
+ TKikimrRunner kikimr(AppCfg());
auto db = kikimr.GetTableClient();
auto params = db.GetParamsBuilder()
@@ -425,8 +425,8 @@ Y_UNIT_TEST_SUITE(KqpScan) {
])", StreamResultToYson(it));
}
- Y_UNIT_TEST(AggregateByColumn) {
- TKikimrRunner kikimr(AppCfg());
+ Y_UNIT_TEST(AggregateByColumn) {
+ TKikimrRunner kikimr(AppCfg());
auto db = kikimr.GetTableClient();
auto it = db.StreamExecuteScanQuery(R"(
@@ -443,270 +443,270 @@ Y_UNIT_TEST_SUITE(KqpScan) {
[["Value1"];[3608u]]
])", StreamResultToYson(it));
}
-
- Y_UNIT_TEST(AggregateNoColumn) {
- TKikimrRunner kikimr(AppCfg());
+
+ Y_UNIT_TEST(AggregateNoColumn) {
+ TKikimrRunner kikimr(AppCfg());
auto db = kikimr.GetTableClient();
-
+
auto it = db.StreamExecuteScanQuery(R"(
- SELECT SUM(Data), AVG(Data), COUNT(*), MAX(Data), MIN(Data), SUM(Data * 3 + Key * 2) as foo
+ SELECT SUM(Data), AVG(Data), COUNT(*), MAX(Data), MIN(Data), SUM(Data * 3 + Key * 2) as foo
FROM `/Root/EightShard`
- WHERE Key > 300
- )").GetValueSync();
-
- UNIT_ASSERT(it.IsSuccess());
-
- CompareYson(R"([[[36];[2.];18u;[3];[1];[19980u]]])", StreamResultToYson(it));
- }
-
- Y_UNIT_TEST(AggregateNoColumnNoRemaps) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
-
+ WHERE Key > 300
+ )").GetValueSync();
+
+ UNIT_ASSERT(it.IsSuccess());
+
+ CompareYson(R"([[[36];[2.];18u;[3];[1];[19980u]]])", StreamResultToYson(it));
+ }
+
+ Y_UNIT_TEST(AggregateNoColumnNoRemaps) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+
auto it = db.StreamExecuteScanQuery(R"(
- SELECT SUM(Data), AVG(Data), COUNT(*)
+ SELECT SUM(Data), AVG(Data), COUNT(*)
FROM `/Root/EightShard`
- WHERE Key > 300
- )").GetValueSync();
-
- UNIT_ASSERT(it.IsSuccess());
-
- CompareYson(R"([[[36];[2.];18u]])", StreamResultToYson(it));
- }
-
- Y_UNIT_TEST(AggregateWithFunction) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
-
+ WHERE Key > 300
+ )").GetValueSync();
+
+ UNIT_ASSERT(it.IsSuccess());
+
+ CompareYson(R"([[[36];[2.];18u]])", StreamResultToYson(it));
+ }
+
+ Y_UNIT_TEST(AggregateWithFunction) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+
auto it = db.StreamExecuteScanQuery(R"(
- SELECT (SUM(Data) * 100) / (MIN(Data) + 10)
+ SELECT (SUM(Data) * 100) / (MIN(Data) + 10)
FROM `/Root/EightShard`
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- CompareYson(R"([[[436]]])", StreamResultToYson(it));
- }
-
- Y_UNIT_TEST(AggregateCountStar) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
-
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ CompareYson(R"([[[436]]])", StreamResultToYson(it));
+ }
+
+ Y_UNIT_TEST(AggregateCountStar) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+
auto it = db.StreamExecuteScanQuery("SELECT COUNT(*) FROM `/Root/EightShard`").GetValueSync();
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- CompareYson(R"([[24u]])", StreamResultToYson(it));
- }
-
- Y_UNIT_TEST(AggregateEmptyCountStar) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
-
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ CompareYson(R"([[24u]])", StreamResultToYson(it));
+ }
+
+ Y_UNIT_TEST(AggregateEmptyCountStar) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+
auto it = db.StreamExecuteScanQuery("SELECT COUNT(*) FROM `/Root/EightShard` WHERE Key < 10").GetValueSync();
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- CompareYson(R"([[0u]])", StreamResultToYson(it));
- }
-
- Y_UNIT_TEST(AggregateEmptySum) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
-
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ CompareYson(R"([[0u]])", StreamResultToYson(it));
+ }
+
+ Y_UNIT_TEST(AggregateEmptySum) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+
auto it = db.StreamExecuteScanQuery("SELECT SUM(Data) FROM `/Root/EightShard` WHERE Key < 10").GetValueSync();
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- CompareYson(R"([[#]])", StreamResultToYson(it));
- }
-
- Y_UNIT_TEST(JoinSimple) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
- CreateSampleTables(kikimr);
-
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ CompareYson(R"([[#]])", StreamResultToYson(it));
+ }
+
+ Y_UNIT_TEST(JoinSimple) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+ CreateSampleTables(kikimr);
+
auto it = db.StreamExecuteScanQuery(R"(
- SELECT *
+ SELECT *
FROM `/Root/EightShard` AS l JOIN `/Root/FourShard` AS r ON l.Key = r.Key
ORDER BY Key, Text, Data, Value1, Value2
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- CompareYson(R"([
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ CompareYson(R"([
[[1];[101u];["Value1"];["Value-101"];["101"]];
[[3];[102u];["Value2"];["Value-102"];["102"]];
[[2];[201u];["Value1"];["Value-201"];["201"]];
[[1];[202u];["Value2"];["Value-202"];["202"]];
[[3];[301u];["Value1"];["Value-301"];["301"]];
[[2];[302u];["Value2"];["Value-302"];["302"]]
- ])", StreamResultToYson(it));
- }
-
- Y_UNIT_TEST(Join) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
- CreateSampleTables(kikimr);
-
- auto test = [&](bool simpleColumns) {
+ ])", StreamResultToYson(it));
+ }
+
+ Y_UNIT_TEST(Join) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+ CreateSampleTables(kikimr);
+
+ auto test = [&](bool simpleColumns) {
auto it = db.StreamExecuteScanQuery(Sprintf(R"(
- PRAGMA %sSimpleColumns;
+ PRAGMA %sSimpleColumns;
$r = (select * from `/Root/FourShard` where Key > 201);
-
- SELECT l.Key as key, l.Text as text, r.Value1 as value
+
+ SELECT l.Key as key, l.Text as text, r.Value1 as value
FROM `/Root/EightShard` AS l JOIN $r AS r ON l.Key = r.Key
- ORDER BY key, text, value
- )", simpleColumns ? "" : "Disable")).GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- CompareYson(R"([
- [[202u];["Value2"];["Value-202"]];
- [[301u];["Value1"];["Value-301"]];
- [[302u];["Value2"];["Value-302"]]
- ])", StreamResultToYson(it));
- };
-
- test(true);
- test(false);
- }
-
- Y_UNIT_TEST(Join2) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
- CreateSampleTables(kikimr);
-
+ ORDER BY key, text, value
+ )", simpleColumns ? "" : "Disable")).GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ CompareYson(R"([
+ [[202u];["Value2"];["Value-202"]];
+ [[301u];["Value1"];["Value-301"]];
+ [[302u];["Value2"];["Value-302"]]
+ ])", StreamResultToYson(it));
+ };
+
+ test(true);
+ test(false);
+ }
+
+ Y_UNIT_TEST(Join2) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+ CreateSampleTables(kikimr);
+
auto it = db.StreamExecuteScanQuery(R"(
- SELECT *
+ SELECT *
FROM `/Root/EightShard` AS l
JOIN `/Root/FourShard` AS r
- ON l.Key = r.Key
- WHERE l.Text != "Value1" AND r.Value2 > "1"
+ ON l.Key = r.Key
+ WHERE l.Text != "Value1" AND r.Value2 > "1"
ORDER BY Key, Text, Data, Value1, Value2
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- CompareYson(R"([
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ CompareYson(R"([
[[3];[102u];["Value2"];["Value-102"];["102"]];
[[1];[202u];["Value2"];["Value-202"];["202"]];
[[2];[302u];["Value2"];["Value-302"];["302"]]
- ])", StreamResultToYson(it));
- }
-
- Y_UNIT_TEST(Join3) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
- CreateSampleTables(kikimr);
-
- auto test = [&](bool simpleColumns) {
+ ])", StreamResultToYson(it));
+ }
+
+ Y_UNIT_TEST(Join3) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+ CreateSampleTables(kikimr);
+
+ auto test = [&](bool simpleColumns) {
auto it = db.StreamExecuteScanQuery(Sprintf(R"(
- PRAGMA %sSimpleColumns;
- $join = (
- SELECT l.Key as Key, l.Text as Text, l.Data as Data, r.Value1 as Value1, r.Value2 as Value2
+ PRAGMA %sSimpleColumns;
+ $join = (
+ SELECT l.Key as Key, l.Text as Text, l.Data as Data, r.Value1 as Value1, r.Value2 as Value2
FROM `/Root/EightShard` AS l JOIN `/Root/FourShard` AS r ON l.Key = r.Key
- );
-
- SELECT Key, COUNT(*) AS Cnt
- FROM $join
- WHERE Cast(Data As Int64) < (Key - 100) and Value1 != 'Value-101'
- GROUP BY Key
- ORDER BY Key, Cnt
- )", simpleColumns ? "" : "Disable")).GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- CompareYson(R"([
- [[201u];1u];
- [[202u];1u];
- [[301u];1u];
- [[302u];1u]
- ])", StreamResultToYson(it));
- };
-
- test(true);
- test(false);
- }
-
- Y_UNIT_TEST(LeftSemiJoinSimple) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
- CreateSampleTables(kikimr);
-
- auto it = db.StreamExecuteScanQuery(R"(
- SELECT *
+ );
+
+ SELECT Key, COUNT(*) AS Cnt
+ FROM $join
+ WHERE Cast(Data As Int64) < (Key - 100) and Value1 != 'Value-101'
+ GROUP BY Key
+ ORDER BY Key, Cnt
+ )", simpleColumns ? "" : "Disable")).GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ CompareYson(R"([
+ [[201u];1u];
+ [[202u];1u];
+ [[301u];1u];
+ [[302u];1u]
+ ])", StreamResultToYson(it));
+ };
+
+ test(true);
+ test(false);
+ }
+
+ Y_UNIT_TEST(LeftSemiJoinSimple) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+ CreateSampleTables(kikimr);
+
+ auto it = db.StreamExecuteScanQuery(R"(
+ SELECT *
FROM `/Root/FourShard` AS l LEFT SEMI JOIN `/Root/EightShard` AS r ON l.Key = r.Key
- ORDER BY Key
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- CompareYson(R"([
- [[101u];["Value-101"];["101"]];
- [[102u];["Value-102"];["102"]];
- [[201u];["Value-201"];["201"]];
- [[202u];["Value-202"];["202"]];
- [[301u];["Value-301"];["301"]];
- [[302u];["Value-302"];["302"]]])", StreamResultToYson(it));
- }
-
- Y_UNIT_TEST(RightJoinSimple) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
- CreateSampleTables(kikimr);
-
- auto it = db.StreamExecuteScanQuery(R"(
- SELECT l.Key, l.Text, l.Data, r.Key, r.Value1, r.Value2
+ ORDER BY Key
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ CompareYson(R"([
+ [[101u];["Value-101"];["101"]];
+ [[102u];["Value-102"];["102"]];
+ [[201u];["Value-201"];["201"]];
+ [[202u];["Value-202"];["202"]];
+ [[301u];["Value-301"];["301"]];
+ [[302u];["Value-302"];["302"]]])", StreamResultToYson(it));
+ }
+
+ Y_UNIT_TEST(RightJoinSimple) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+ CreateSampleTables(kikimr);
+
+ auto it = db.StreamExecuteScanQuery(R"(
+ SELECT l.Key, l.Text, l.Data, r.Key, r.Value1, r.Value2
FROM `/Root/EightShard` AS l RIGHT JOIN `/Root/FourShard` AS r ON l.Key = r.Key
- WHERE r.Key < 200
- ORDER BY l.Key, l.Text, l.Data, r.Key, r.Value1, r.Value2
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- CompareYson(R"([
- [#;#;#;[1u];["Value-001"];["1"]];
- [#;#;#;[2u];["Value-002"];["2"]];
- [[101u];["Value1"];[1];[101u];["Value-101"];["101"]];
- [[102u];["Value2"];[3];[102u];["Value-102"];["102"]]]
- )", StreamResultToYson(it));
- }
-
- Y_UNIT_TEST(RightOnlyJoinSimple) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
- CreateSampleTables(kikimr);
-
- auto it = db.StreamExecuteScanQuery(R"(
- SELECT Key, Value1, Value2
+ WHERE r.Key < 200
+ ORDER BY l.Key, l.Text, l.Data, r.Key, r.Value1, r.Value2
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ CompareYson(R"([
+ [#;#;#;[1u];["Value-001"];["1"]];
+ [#;#;#;[2u];["Value-002"];["2"]];
+ [[101u];["Value1"];[1];[101u];["Value-101"];["101"]];
+ [[102u];["Value2"];[3];[102u];["Value-102"];["102"]]]
+ )", StreamResultToYson(it));
+ }
+
+ Y_UNIT_TEST(RightOnlyJoinSimple) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+ CreateSampleTables(kikimr);
+
+ auto it = db.StreamExecuteScanQuery(R"(
+ SELECT Key, Value1, Value2
FROM `/Root/EightShard` AS l RIGHT ONLY JOIN `/Root/FourShard` AS r ON l.Key = r.Key
- WHERE Key < 200
- ORDER BY Key, Value1, Value2
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- CompareYson(R"([
- [[1u];["Value-001"];["1"]];
- [[2u];["Value-002"];["2"]]
- ])", StreamResultToYson(it));
- }
-
- Y_UNIT_TEST(RightSemiJoinSimple) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
- CreateSampleTables(kikimr);
-
- auto it = db.StreamExecuteScanQuery(R"(
- SELECT Key, Value1, Value2
+ WHERE Key < 200
+ ORDER BY Key, Value1, Value2
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ CompareYson(R"([
+ [[1u];["Value-001"];["1"]];
+ [[2u];["Value-002"];["2"]]
+ ])", StreamResultToYson(it));
+ }
+
+ Y_UNIT_TEST(RightSemiJoinSimple) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+ CreateSampleTables(kikimr);
+
+ auto it = db.StreamExecuteScanQuery(R"(
+ SELECT Key, Value1, Value2
FROM `/Root/EightShard` AS l RIGHT SEMI JOIN `/Root/FourShard` AS r ON l.Key = r.Key
- WHERE Key < 200
- ORDER BY Key, Value1, Value2
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- CompareYson(R"([
- [[101u];["Value-101"];["101"]];
- [[102u];["Value-102"];["102"]]]
- )", StreamResultToYson(it));
- }
-
+ WHERE Key < 200
+ ORDER BY Key, Value1, Value2
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ CompareYson(R"([
+ [[101u];["Value-101"];["101"]];
+ [[102u];["Value-102"];["102"]]]
+ )", StreamResultToYson(it));
+ }
+
Y_UNIT_TEST(JoinWithParams) {
TKikimrRunner kikimr(AppCfg());
auto db = kikimr.GetTableClient();
@@ -734,396 +734,396 @@ Y_UNIT_TEST_SUITE(KqpScan) {
}
}
- Y_UNIT_TEST(NoTruncate) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- UNIT_ASSERT(session.ExecuteSchemeQuery(R"(
+ Y_UNIT_TEST(NoTruncate) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ UNIT_ASSERT(session.ExecuteSchemeQuery(R"(
CREATE TABLE `/Root/Tmp` (
- Key Uint64,
- Value String,
- PRIMARY KEY (Key)
- );
- )").GetValueSync().IsSuccess());
-
- const ui64 RowsCount = 1005;
-
- auto replaceQuery = R"(
- DECLARE $rows AS
- 'List<Struct<
- Key: Uint64?,
- Value: String?
- >>';
-
- REPLACE INTO [/Root/Tmp]
- SELECT * FROM AS_TABLE($rows);
- )";
-
- {
- auto paramsBuilder = session.GetParamsBuilder();
- auto& rowsParam = paramsBuilder.AddParam("$rows");
-
- rowsParam.BeginList();
- for (ui64 i = 0; i < RowsCount; ++i) {
- rowsParam.AddListItem()
- .BeginStruct()
- .AddMember("Key")
- .OptionalUint64(i)
- .AddMember("Value")
- .OptionalString(ToString(i))
- .EndStruct();
- }
- rowsParam.EndList();
- rowsParam.Build();
-
- auto result = session.ExecuteDataQuery(replaceQuery, TTxControl::BeginTx().CommitTx(),
- paramsBuilder.Build()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- }
-
+ Key Uint64,
+ Value String,
+ PRIMARY KEY (Key)
+ );
+ )").GetValueSync().IsSuccess());
+
+ const ui64 RowsCount = 1005;
+
+ auto replaceQuery = R"(
+ DECLARE $rows AS
+ 'List<Struct<
+ Key: Uint64?,
+ Value: String?
+ >>';
+
+ REPLACE INTO [/Root/Tmp]
+ SELECT * FROM AS_TABLE($rows);
+ )";
+
+ {
+ auto paramsBuilder = session.GetParamsBuilder();
+ auto& rowsParam = paramsBuilder.AddParam("$rows");
+
+ rowsParam.BeginList();
+ for (ui64 i = 0; i < RowsCount; ++i) {
+ rowsParam.AddListItem()
+ .BeginStruct()
+ .AddMember("Key")
+ .OptionalUint64(i)
+ .AddMember("Value")
+ .OptionalString(ToString(i))
+ .EndStruct();
+ }
+ rowsParam.EndList();
+ rowsParam.Build();
+
+ auto result = session.ExecuteDataQuery(replaceQuery, TTxControl::BeginTx().CommitTx(),
+ paramsBuilder.Build()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+ }
+
auto it = db.StreamExecuteScanQuery(R"(
SELECT * FROM `/Root/Tmp`;
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- ui64 totalRows = 0;
- for (;;) {
- auto streamPart = it.ReadNext().GetValueSync();
- if (!streamPart.IsSuccess()) {
- UNIT_ASSERT_C(streamPart.EOS(), streamPart.GetIssues().ToString());
- break;
- }
-
- if (streamPart.HasResultSet()) {
- auto result = streamPart.ExtractResultSet();
- UNIT_ASSERT(!result.Truncated());
-
- totalRows += result.RowsCount();
- }
- }
-
- UNIT_ASSERT_VALUES_EQUAL(totalRows, RowsCount);
- }
-
- Y_UNIT_TEST(Join3TablesNoRemap) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
- CreateSampleTables(kikimr);
-
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ ui64 totalRows = 0;
+ for (;;) {
+ auto streamPart = it.ReadNext().GetValueSync();
+ if (!streamPart.IsSuccess()) {
+ UNIT_ASSERT_C(streamPart.EOS(), streamPart.GetIssues().ToString());
+ break;
+ }
+
+ if (streamPart.HasResultSet()) {
+ auto result = streamPart.ExtractResultSet();
+ UNIT_ASSERT(!result.Truncated());
+
+ totalRows += result.RowsCount();
+ }
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(totalRows, RowsCount);
+ }
+
+ Y_UNIT_TEST(Join3TablesNoRemap) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+ CreateSampleTables(kikimr);
+
auto it = db.StreamExecuteScanQuery(R"(
PRAGMA DisableSimpleColumns;
- SELECT *
+ SELECT *
FROM `/Root/EightShard` AS t8
JOIN `/Root/FourShard` AS t4
- ON t8.Key = t4.Key
+ ON t8.Key = t4.Key
JOIN `/Root/TwoShard` AS t2
- ON t8.Data = t2.Key
- ORDER BY t8.Key, t8.Text, t8.Data, t4.Value1, t4.Value2, t2.Value1, t2.Value2
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- CompareYson(R"(
- [
- [[1u];["One"];[-1];[101u];["Value-101"];["101"];[1];[101u];["Value1"]];
- [[3u];["Three"];[1];[102u];["Value-102"];["102"];[3];[102u];["Value2"]];
- [[2u];["Two"];[0];[201u];["Value-201"];["201"];[2];[201u];["Value1"]];
- [[1u];["One"];[-1];[202u];["Value-202"];["202"];[1];[202u];["Value2"]];
- [[3u];["Three"];[1];[301u];["Value-301"];["301"];[3];[301u];["Value1"]];
- [[2u];["Two"];[0];[302u];["Value-302"];["302"];[2];[302u];["Value2"]]
- ])", StreamResultToYson(it));
- }
-
- Y_UNIT_TEST(Join3Tables) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
- CreateSampleTables(kikimr);
-
- auto test = [&](bool simpleColumns) {
+ ON t8.Data = t2.Key
+ ORDER BY t8.Key, t8.Text, t8.Data, t4.Value1, t4.Value2, t2.Value1, t2.Value2
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ CompareYson(R"(
+ [
+ [[1u];["One"];[-1];[101u];["Value-101"];["101"];[1];[101u];["Value1"]];
+ [[3u];["Three"];[1];[102u];["Value-102"];["102"];[3];[102u];["Value2"]];
+ [[2u];["Two"];[0];[201u];["Value-201"];["201"];[2];[201u];["Value1"]];
+ [[1u];["One"];[-1];[202u];["Value-202"];["202"];[1];[202u];["Value2"]];
+ [[3u];["Three"];[1];[301u];["Value-301"];["301"];[3];[301u];["Value1"]];
+ [[2u];["Two"];[0];[302u];["Value-302"];["302"];[2];[302u];["Value2"]]
+ ])", StreamResultToYson(it));
+ }
+
+ Y_UNIT_TEST(Join3Tables) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+ CreateSampleTables(kikimr);
+
+ auto test = [&](bool simpleColumns) {
auto it = db.StreamExecuteScanQuery(Sprintf(R"(
- PRAGMA %sSimpleColumns;
- SELECT t8.Key as key, t8.Text as text, t4.Value1, t2.Value2
+ PRAGMA %sSimpleColumns;
+ SELECT t8.Key as key, t8.Text as text, t4.Value1, t2.Value2
FROM `/Root/EightShard` AS t8
JOIN `/Root/FourShard` AS t4
- ON t8.Key = t4.Key
+ ON t8.Key = t4.Key
JOIN `/Root/TwoShard` AS t2
- ON t8.Data = t2.Key
- WHERE t8.Key > 200 AND t2.Value2 >= 0
- ORDER BY key, text, t4.Value1, t2.Value2
- )", simpleColumns ? "" : "Disable")).GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- CompareYson(R"(
- [
- [[201u];["Value1"];["Value-201"];[0]];
- [[301u];["Value1"];["Value-301"];[1]];
- [[302u];["Value2"];["Value-302"];[0]]
- ])", StreamResultToYson(it));
- };
-
- test(true /* SimpleColumns */);
- test(false /* DisableSimpleColumns */);
- }
-
- Y_UNIT_TEST(JoinLeftOnly) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
- CreateSampleTables(kikimr);
-
- auto test = [&](bool simpleColumns) {
+ ON t8.Data = t2.Key
+ WHERE t8.Key > 200 AND t2.Value2 >= 0
+ ORDER BY key, text, t4.Value1, t2.Value2
+ )", simpleColumns ? "" : "Disable")).GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ CompareYson(R"(
+ [
+ [[201u];["Value1"];["Value-201"];[0]];
+ [[301u];["Value1"];["Value-301"];[1]];
+ [[302u];["Value2"];["Value-302"];[0]]
+ ])", StreamResultToYson(it));
+ };
+
+ test(true /* SimpleColumns */);
+ test(false /* DisableSimpleColumns */);
+ }
+
+ Y_UNIT_TEST(JoinLeftOnly) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+ CreateSampleTables(kikimr);
+
+ auto test = [&](bool simpleColumns) {
auto it = db.StreamExecuteScanQuery(Sprintf(R"(
- PRAGMA %sSimpleColumns;
- SELECT *
+ PRAGMA %sSimpleColumns;
+ SELECT *
FROM `/Root/EightShard` AS l
LEFT ONLY JOIN `/Root/FourShard` AS r
- ON l.Key = r.Key
- WHERE Data = 1
- ORDER BY Key, Text, Data
- )", simpleColumns ? "" : "Disable")).GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- CompareYson(R"(
- [
- [[1];[303u];["Value3"]];
- [[1];[401u];["Value1"]];
- [[1];[502u];["Value2"]];
- [[1];[603u];["Value3"]];
- [[1];[701u];["Value1"]];
- [[1];[802u];["Value2"]]
- ])", StreamResultToYson(it));
- };
-
- test(true /* SimpleColums */);
- test(false /* DisableSimpleColumns */);
- }
-
- Y_UNIT_TEST(CrossJoin) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
- CreateSampleTables(kikimr);
-
+ ON l.Key = r.Key
+ WHERE Data = 1
+ ORDER BY Key, Text, Data
+ )", simpleColumns ? "" : "Disable")).GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ CompareYson(R"(
+ [
+ [[1];[303u];["Value3"]];
+ [[1];[401u];["Value1"]];
+ [[1];[502u];["Value2"]];
+ [[1];[603u];["Value3"]];
+ [[1];[701u];["Value1"]];
+ [[1];[802u];["Value2"]]
+ ])", StreamResultToYson(it));
+ };
+
+ test(true /* SimpleColums */);
+ test(false /* DisableSimpleColumns */);
+ }
+
+ Y_UNIT_TEST(CrossJoin) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+ CreateSampleTables(kikimr);
+
auto it = db.StreamExecuteScanQuery(R"(
- SELECT l.Key, r.Key, l.Text, r.Value1
+ SELECT l.Key, r.Key, l.Text, r.Value1
FROM `/Root/EightShard` AS l CROSS JOIN `/Root/FourShard` AS r
- WHERE l.Key > r.Key AND l.Data = 1 AND r.Value2 > "200"
- ORDER BY l.Key, l.Text, r.Value1
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- CompareYson(R"([
- [[202u];[201u];["Value2"];["Value-201"]];
- [[303u];[201u];["Value3"];["Value-201"]];
- [[303u];[202u];["Value3"];["Value-202"]];
- [[303u];[301u];["Value3"];["Value-301"]];
- [[303u];[302u];["Value3"];["Value-302"]];
- [[401u];[201u];["Value1"];["Value-201"]];
- [[401u];[202u];["Value1"];["Value-202"]];
- [[401u];[301u];["Value1"];["Value-301"]];
- [[401u];[302u];["Value1"];["Value-302"]];
- [[502u];[201u];["Value2"];["Value-201"]];
- [[502u];[202u];["Value2"];["Value-202"]];
- [[502u];[301u];["Value2"];["Value-301"]];
- [[502u];[302u];["Value2"];["Value-302"]];
- [[603u];[201u];["Value3"];["Value-201"]];
- [[603u];[202u];["Value3"];["Value-202"]];
- [[603u];[301u];["Value3"];["Value-301"]];
- [[603u];[302u];["Value3"];["Value-302"]];
- [[701u];[201u];["Value1"];["Value-201"]];
- [[701u];[202u];["Value1"];["Value-202"]];
- [[701u];[301u];["Value1"];["Value-301"]];
- [[701u];[302u];["Value1"];["Value-302"]];
- [[802u];[201u];["Value2"];["Value-201"]];
- [[802u];[202u];["Value2"];["Value-202"]];
- [[802u];[301u];["Value2"];["Value-301"]];
- [[802u];[302u];["Value2"];["Value-302"]]
- ])", StreamResultToYson(it));
- }
-
- Y_UNIT_TEST(SelfJoin3xSameLabels) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
- CreateSampleTables(kikimr);
-
+ WHERE l.Key > r.Key AND l.Data = 1 AND r.Value2 > "200"
+ ORDER BY l.Key, l.Text, r.Value1
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ CompareYson(R"([
+ [[202u];[201u];["Value2"];["Value-201"]];
+ [[303u];[201u];["Value3"];["Value-201"]];
+ [[303u];[202u];["Value3"];["Value-202"]];
+ [[303u];[301u];["Value3"];["Value-301"]];
+ [[303u];[302u];["Value3"];["Value-302"]];
+ [[401u];[201u];["Value1"];["Value-201"]];
+ [[401u];[202u];["Value1"];["Value-202"]];
+ [[401u];[301u];["Value1"];["Value-301"]];
+ [[401u];[302u];["Value1"];["Value-302"]];
+ [[502u];[201u];["Value2"];["Value-201"]];
+ [[502u];[202u];["Value2"];["Value-202"]];
+ [[502u];[301u];["Value2"];["Value-301"]];
+ [[502u];[302u];["Value2"];["Value-302"]];
+ [[603u];[201u];["Value3"];["Value-201"]];
+ [[603u];[202u];["Value3"];["Value-202"]];
+ [[603u];[301u];["Value3"];["Value-301"]];
+ [[603u];[302u];["Value3"];["Value-302"]];
+ [[701u];[201u];["Value1"];["Value-201"]];
+ [[701u];[202u];["Value1"];["Value-202"]];
+ [[701u];[301u];["Value1"];["Value-301"]];
+ [[701u];[302u];["Value1"];["Value-302"]];
+ [[802u];[201u];["Value2"];["Value-201"]];
+ [[802u];[202u];["Value2"];["Value-202"]];
+ [[802u];[301u];["Value2"];["Value-301"]];
+ [[802u];[302u];["Value2"];["Value-302"]]
+ ])", StreamResultToYson(it));
+ }
+
+ Y_UNIT_TEST(SelfJoin3xSameLabels) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+ CreateSampleTables(kikimr);
+
auto it = db.StreamExecuteScanQuery(R"(
- $foo = (
- SELECT t1.Key AS Key
+ $foo = (
+ SELECT t1.Key AS Key
FROM `/Root/KeyValue` AS t1
JOIN `/Root/KeyValue` AS t2
- ON t1.Key = t2.Key
- GROUP BY t1.Key
- );
-
- SELECT t1.Key AS Key
- FROM $foo AS Foo
+ ON t1.Key = t2.Key
+ GROUP BY t1.Key
+ );
+
+ SELECT t1.Key AS Key
+ FROM $foo AS Foo
JOIN `/Root/KeyValue` AS t1
- ON t1.Key = Foo.Key
- ORDER BY Key
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- CompareYson("[[[1u]];[[2u]]]", StreamResultToYson(it));
- }
-
- Y_UNIT_TEST(SelfJoin3x) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
- CreateSampleTables(kikimr);
-
+ ON t1.Key = Foo.Key
+ ORDER BY Key
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ CompareYson("[[[1u]];[[2u]]]", StreamResultToYson(it));
+ }
+
+ Y_UNIT_TEST(SelfJoin3x) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+ CreateSampleTables(kikimr);
+
auto it = db.StreamExecuteScanQuery(R"(
- $foo = (
- SELECT t1.Key AS Key
+ $foo = (
+ SELECT t1.Key AS Key
FROM `/Root/KeyValue` AS t1
JOIN `/Root/KeyValue` AS t2
- ON t1.Key = t2.Key
- GROUP BY t1.Key
- );
-
- SELECT t3.Key AS Key
- FROM $foo AS Foo
+ ON t1.Key = t2.Key
+ GROUP BY t1.Key
+ );
+
+ SELECT t3.Key AS Key
+ FROM $foo AS Foo
JOIN `/Root/KeyValue` AS t3
- ON t3.Key = Foo.Key
- ORDER BY Key
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- CompareYson("[[[1u]];[[2u]]]", StreamResultToYson(it));
- }
-
-#if 0
- Y_UNIT_TEST(JoinParams) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
-
- auto params = TParamsBuilder().AddParam("$in").BeginList()
- .AddListItem().BeginStruct().AddMember("k").Uint64(1).AddMember("v").String("v1").EndStruct()
- .AddListItem().BeginStruct().AddMember("k").Uint64(2).AddMember("v").String("v2").EndStruct()
- .EndList().Build().Build();
-
+ ON t3.Key = Foo.Key
+ ORDER BY Key
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ CompareYson("[[[1u]];[[2u]]]", StreamResultToYson(it));
+ }
+
+#if 0
+ Y_UNIT_TEST(JoinParams) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+
+ auto params = TParamsBuilder().AddParam("$in").BeginList()
+ .AddListItem().BeginStruct().AddMember("k").Uint64(1).AddMember("v").String("v1").EndStruct()
+ .AddListItem().BeginStruct().AddMember("k").Uint64(2).AddMember("v").String("v2").EndStruct()
+ .EndList().Build().Build();
+
auto it = db.StreamExecuteScanQuery(R"(
- DECLARE $in AS 'List<Struct<k: Uint64, v: String>>';
- SELECT *
+ DECLARE $in AS 'List<Struct<k: Uint64, v: String>>';
+ SELECT *
FROM `/Root/KeyValue` AS l
- JOIN AS_TABLE($in) AS r
- ON l.Key = r.k
- )", params).GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- Cerr << StreamResultToYson(it) << Endl;
- }
-
- Y_UNIT_TEST(JoinParams2) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
-
- auto params = TParamsBuilder()
- .AddParam("$in1").BeginList()
- .AddListItem().BeginStruct().AddMember("k").Uint64(1).AddMember("v").String("v1").EndStruct()
- .AddListItem().BeginStruct().AddMember("k").Uint64(2).AddMember("v").String("v2").EndStruct()
- .EndList().Build()
- .AddParam("$in2").BeginList()
- .AddListItem().BeginStruct().AddMember("k").Uint64(1).AddMember("v").String("v1").EndStruct()
- .AddListItem().BeginStruct().AddMember("k").Uint64(2).AddMember("v").String("v2").EndStruct()
- .EndList().Build()
- .Build();
-
+ JOIN AS_TABLE($in) AS r
+ ON l.Key = r.k
+ )", params).GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ Cerr << StreamResultToYson(it) << Endl;
+ }
+
+ Y_UNIT_TEST(JoinParams2) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+
+ auto params = TParamsBuilder()
+ .AddParam("$in1").BeginList()
+ .AddListItem().BeginStruct().AddMember("k").Uint64(1).AddMember("v").String("v1").EndStruct()
+ .AddListItem().BeginStruct().AddMember("k").Uint64(2).AddMember("v").String("v2").EndStruct()
+ .EndList().Build()
+ .AddParam("$in2").BeginList()
+ .AddListItem().BeginStruct().AddMember("k").Uint64(1).AddMember("v").String("v1").EndStruct()
+ .AddListItem().BeginStruct().AddMember("k").Uint64(2).AddMember("v").String("v2").EndStruct()
+ .EndList().Build()
+ .Build();
+
auto it = db.StreamExecuteScanQuery(R"(
- DECLARE $in1 AS 'List<Struct<k: Uint64, v: String>>';
- DECLARE $in2 AS 'List<Struct<k: Uint64, v: String>>';
- SELECT *
- FROM AS_TABLE($in1) AS l
- JOIN AS_TABLE($in2) AS r
- ON l.k = r.k;
+ DECLARE $in1 AS 'List<Struct<k: Uint64, v: String>>';
+ DECLARE $in2 AS 'List<Struct<k: Uint64, v: String>>';
+ SELECT *
+ FROM AS_TABLE($in1) AS l
+ JOIN AS_TABLE($in2) AS r
+ ON l.k = r.k;
UPSERT INTO [/Root/KeyValue] (Key, Value) Values (1, "test");
- )", params).GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- Cerr << StreamResultToYson(it) << Endl;
- }
-
- Y_UNIT_TEST(JoinParams3) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
-
- auto params = TParamsBuilder()
- .AddParam("$in1").BeginList()
- .AddListItem().BeginStruct().AddMember("k").Uint64(1).AddMember("v").String("v1").EndStruct()
- .AddListItem().BeginStruct().AddMember("k").Uint64(2).AddMember("v").String("v2").EndStruct()
- .EndList().Build()
- .AddParam("$in2").BeginList()
- .AddListItem().BeginStruct().AddMember("k").Uint64(1).AddMember("v").String("v1").EndStruct()
- .AddListItem().BeginStruct().AddMember("k").Uint64(2).AddMember("v").String("v2").EndStruct()
- .EndList().Build()
- .Build();
-
+ )", params).GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ Cerr << StreamResultToYson(it) << Endl;
+ }
+
+ Y_UNIT_TEST(JoinParams3) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+
+ auto params = TParamsBuilder()
+ .AddParam("$in1").BeginList()
+ .AddListItem().BeginStruct().AddMember("k").Uint64(1).AddMember("v").String("v1").EndStruct()
+ .AddListItem().BeginStruct().AddMember("k").Uint64(2).AddMember("v").String("v2").EndStruct()
+ .EndList().Build()
+ .AddParam("$in2").BeginList()
+ .AddListItem().BeginStruct().AddMember("k").Uint64(1).AddMember("v").String("v1").EndStruct()
+ .AddListItem().BeginStruct().AddMember("k").Uint64(2).AddMember("v").String("v2").EndStruct()
+ .EndList().Build()
+ .Build();
+
auto it = db.StreamExecuteScanQuery(R"(
- DECLARE $in1 AS 'List<Struct<k: Uint64, v: String>>';
- DECLARE $in2 AS 'List<Struct<k: Uint64, v: String>>';
-
- $l = (select * from AS_TABLE($in1) where k > 0);
- $r = (select * from AS_TABLE($in2) where k > 10);
-
- SELECT *
- FROM $l AS l
- JOIN $r AS r
- ON l.k = r.k
- )", params).GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- Cerr << StreamResultToYson(it) << Endl;
- }
-#endif
-
- Y_UNIT_TEST(PrunePartitionsByLiteral) {
- TKikimrRunner kikimr(AppCfg());
- NExperimental::TStreamQueryClient db(kikimr.GetDriver());
-
- auto settings = NExperimental::TExecuteStreamQuerySettings()
- .ProfileMode(NExperimental::EStreamQueryProfileMode::Basic);
-
- auto it = db.ExecuteStreamQuery(R"(
+ DECLARE $in1 AS 'List<Struct<k: Uint64, v: String>>';
+ DECLARE $in2 AS 'List<Struct<k: Uint64, v: String>>';
+
+ $l = (select * from AS_TABLE($in1) where k > 0);
+ $r = (select * from AS_TABLE($in2) where k > 10);
+
+ SELECT *
+ FROM $l AS l
+ JOIN $r AS r
+ ON l.k = r.k
+ )", params).GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ Cerr << StreamResultToYson(it) << Endl;
+ }
+#endif
+
+ Y_UNIT_TEST(PrunePartitionsByLiteral) {
+ TKikimrRunner kikimr(AppCfg());
+ NExperimental::TStreamQueryClient db(kikimr.GetDriver());
+
+ auto settings = NExperimental::TExecuteStreamQuerySettings()
+ .ProfileMode(NExperimental::EStreamQueryProfileMode::Basic);
+
+ auto it = db.ExecuteStreamQuery(R"(
SELECT * FROM `/Root/EightShard` WHERE Key = 301;
- )", settings).GetValueSync();
-
- UNIT_ASSERT(it.IsSuccess());
-
- TVector<TString> profiles;
- CompareYson(R"([[[3];[301u];["Value1"]]])", StreamResultToYson(it, &profiles));
-
- UNIT_ASSERT_EQUAL(2, profiles.size());
-
- {
- NYql::NDqProto::TDqExecutionStats stats;
- google::protobuf::TextFormat::ParseFromString(profiles[0], &stats);
- UNIT_ASSERT(stats.IsInitialized());
-
- NKqpProto::TKqpExecutionExtraStats extraStats;
- UNIT_ASSERT(stats.GetExtra().UnpackTo(&extraStats));
- UNIT_ASSERT_VALUES_EQUAL(extraStats.GetAffectedShards(), 0);
- }
-
- {
- NYql::NDqProto::TDqExecutionStats stats;
- google::protobuf::TextFormat::ParseFromString(profiles[1], &stats);
- UNIT_ASSERT(stats.IsInitialized());
-
- NKqpProto::TKqpExecutionExtraStats extraStats;
- UNIT_ASSERT(stats.GetExtra().UnpackTo(&extraStats));
- UNIT_ASSERT_VALUES_EQUAL_C(extraStats.GetAffectedShards(), 1, "" << stats.DebugString());
- }
- }
-
+ )", settings).GetValueSync();
+
+ UNIT_ASSERT(it.IsSuccess());
+
+ TVector<TString> profiles;
+ CompareYson(R"([[[3];[301u];["Value1"]]])", StreamResultToYson(it, &profiles));
+
+ UNIT_ASSERT_EQUAL(2, profiles.size());
+
+ {
+ NYql::NDqProto::TDqExecutionStats stats;
+ google::protobuf::TextFormat::ParseFromString(profiles[0], &stats);
+ UNIT_ASSERT(stats.IsInitialized());
+
+ NKqpProto::TKqpExecutionExtraStats extraStats;
+ UNIT_ASSERT(stats.GetExtra().UnpackTo(&extraStats));
+ UNIT_ASSERT_VALUES_EQUAL(extraStats.GetAffectedShards(), 0);
+ }
+
+ {
+ NYql::NDqProto::TDqExecutionStats stats;
+ google::protobuf::TextFormat::ParseFromString(profiles[1], &stats);
+ UNIT_ASSERT(stats.IsInitialized());
+
+ NKqpProto::TKqpExecutionExtraStats extraStats;
+ UNIT_ASSERT(stats.GetExtra().UnpackTo(&extraStats));
+ UNIT_ASSERT_VALUES_EQUAL_C(extraStats.GetAffectedShards(), 1, "" << stats.DebugString());
+ }
+ }
+
Y_UNIT_TEST(PrunePartitionsByExpr) {
- TKikimrRunner kikimr(AppCfg());
+ TKikimrRunner kikimr(AppCfg());
NExperimental::TStreamQueryClient db(kikimr.GetDriver());
auto settings = NExperimental::TExecuteStreamQuerySettings()
@@ -1140,105 +1140,105 @@ Y_UNIT_TEST_SUITE(KqpScan) {
UNIT_ASSERT(it.IsSuccess());
- TVector<TString> profiles;
- CompareYson(R"([[[3];[301u];["Value1"]]])", StreamResultToYson(it, &profiles));
-
- UNIT_ASSERT_EQUAL(2, profiles.size());
-
- {
- NYql::NDqProto::TDqExecutionStats stats;
- google::protobuf::TextFormat::ParseFromString(profiles[0], &stats);
- UNIT_ASSERT(stats.IsInitialized());
-
- NKqpProto::TKqpExecutionExtraStats extraStats;
- UNIT_ASSERT(stats.GetExtra().UnpackTo(&extraStats));
- UNIT_ASSERT_VALUES_EQUAL(extraStats.GetAffectedShards(), 0);
- }
-
- {
- NYql::NDqProto::TDqExecutionStats stats;
- google::protobuf::TextFormat::ParseFromString(profiles[1], &stats);
- UNIT_ASSERT(stats.IsInitialized());
-
- NKqpProto::TKqpExecutionExtraStats extraStats;
- UNIT_ASSERT(stats.GetExtra().UnpackTo(&extraStats));
- UNIT_ASSERT_VALUES_EQUAL(extraStats.GetAffectedShards(), 1);
- }
- }
-
- Y_UNIT_TEST(TooManyComputeActors) {
- TVector<NKikimrKqp::TKqpSetting> settings;
- auto setting = NKikimrKqp::TKqpSetting();
- setting.SetName("_KqpMaxComputeActors");
- setting.SetValue("5");
- settings.push_back(setting);
-
- TKikimrRunner kikimr{settings};
- auto db = kikimr.GetTableClient();
- CreateSampleTables(kikimr);
-
+ TVector<TString> profiles;
+ CompareYson(R"([[[3];[301u];["Value1"]]])", StreamResultToYson(it, &profiles));
+
+ UNIT_ASSERT_EQUAL(2, profiles.size());
+
+ {
+ NYql::NDqProto::TDqExecutionStats stats;
+ google::protobuf::TextFormat::ParseFromString(profiles[0], &stats);
+ UNIT_ASSERT(stats.IsInitialized());
+
+ NKqpProto::TKqpExecutionExtraStats extraStats;
+ UNIT_ASSERT(stats.GetExtra().UnpackTo(&extraStats));
+ UNIT_ASSERT_VALUES_EQUAL(extraStats.GetAffectedShards(), 0);
+ }
+
+ {
+ NYql::NDqProto::TDqExecutionStats stats;
+ google::protobuf::TextFormat::ParseFromString(profiles[1], &stats);
+ UNIT_ASSERT(stats.IsInitialized());
+
+ NKqpProto::TKqpExecutionExtraStats extraStats;
+ UNIT_ASSERT(stats.GetExtra().UnpackTo(&extraStats));
+ UNIT_ASSERT_VALUES_EQUAL(extraStats.GetAffectedShards(), 1);
+ }
+ }
+
+ Y_UNIT_TEST(TooManyComputeActors) {
+ TVector<NKikimrKqp::TKqpSetting> settings;
+ auto setting = NKikimrKqp::TKqpSetting();
+ setting.SetName("_KqpMaxComputeActors");
+ setting.SetValue("5");
+ settings.push_back(setting);
+
+ TKikimrRunner kikimr{settings};
+ auto db = kikimr.GetTableClient();
+ CreateSampleTables(kikimr);
+
auto it = db.StreamExecuteScanQuery(R"(
- $join = (
- SELECT l.Key as Key, l.Text as Text, l.Data as Data, r.Value1 as Value1, r.Value2 as Value2
+ $join = (
+ SELECT l.Key as Key, l.Text as Text, l.Data as Data, r.Value1 as Value1, r.Value2 as Value2
FROM `/Root/EightShard` AS l JOIN `/Root/FourShard` AS r ON l.Key = r.Key
- );
-
- SELECT Key, COUNT(*) AS Cnt
- FROM $join
- WHERE Cast(Data As Int64) < (Key - 100) and Value1 != 'Value-101'
- GROUP BY Key
- ORDER BY Key, Cnt
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- auto part = it.ReadNext().GetValueSync();
-
- UNIT_ASSERT_EQUAL_C(part.GetStatus(), EStatus::PRECONDITION_FAILED, part.GetStatus());
- UNIT_ASSERT_STRINGS_EQUAL(part.GetIssues().back().GetSubIssues().back()->Message, "Requested too many execution units: 12");
-
- part = it.ReadNext().GetValueSync();
- UNIT_ASSERT(part.EOS());
- UNIT_ASSERT_EQUAL(part.GetStatus(), EStatus::CLIENT_OUT_OF_RANGE);
- }
-
- Y_UNIT_TEST(EarlyFinish) {
- auto appCfg = AppCfg();
- appCfg.MutableTableServiceConfig()->MutableResourceManager()->SetChannelBufferSize(5);
- appCfg.MutableTableServiceConfig()->MutableResourceManager()->SetMinChannelBufferSize(5);
-
- NYql::NDq::GetDqExecutionSettingsForTests().FlowControl.MaxOutputChunkSize = 1;
- Y_DEFER {
- NYql::NDq::GetDqExecutionSettingsForTests().Reset();
- };
-
- TKikimrRunner kikimr(appCfg);
-
- NYdb::NTable::TTableClient client(kikimr.GetDriver());
- auto session = client.CreateSession().GetValueSync().GetSession();
-
- for (int i = 0; i < 100; ++i) {
- AssertSuccessResult(session.ExecuteDataQuery(
- Sprintf(R"(
+ );
+
+ SELECT Key, COUNT(*) AS Cnt
+ FROM $join
+ WHERE Cast(Data As Int64) < (Key - 100) and Value1 != 'Value-101'
+ GROUP BY Key
+ ORDER BY Key, Cnt
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ auto part = it.ReadNext().GetValueSync();
+
+ UNIT_ASSERT_EQUAL_C(part.GetStatus(), EStatus::PRECONDITION_FAILED, part.GetStatus());
+ UNIT_ASSERT_STRINGS_EQUAL(part.GetIssues().back().GetSubIssues().back()->Message, "Requested too many execution units: 12");
+
+ part = it.ReadNext().GetValueSync();
+ UNIT_ASSERT(part.EOS());
+ UNIT_ASSERT_EQUAL(part.GetStatus(), EStatus::CLIENT_OUT_OF_RANGE);
+ }
+
+ Y_UNIT_TEST(EarlyFinish) {
+ auto appCfg = AppCfg();
+ appCfg.MutableTableServiceConfig()->MutableResourceManager()->SetChannelBufferSize(5);
+ appCfg.MutableTableServiceConfig()->MutableResourceManager()->SetMinChannelBufferSize(5);
+
+ NYql::NDq::GetDqExecutionSettingsForTests().FlowControl.MaxOutputChunkSize = 1;
+ Y_DEFER {
+ NYql::NDq::GetDqExecutionSettingsForTests().Reset();
+ };
+
+ TKikimrRunner kikimr(appCfg);
+
+ NYdb::NTable::TTableClient client(kikimr.GetDriver());
+ auto session = client.CreateSession().GetValueSync().GetSession();
+
+ for (int i = 0; i < 100; ++i) {
+ AssertSuccessResult(session.ExecuteDataQuery(
+ Sprintf(R"(
REPLACE INTO `/Root/EightShard` (Key, Text, Data) VALUES
- (%d, "Value1", 0),
- (%d, "Value2", 1),
- (%d, "Value3", 2),
- (%d, "Value4", 3),
- (%d, "Value5", 4),
- (%d, "Value6", 5),
- (%d, "Value7", 6),
- (%d, "Value8", 7)
- )", i, 100 + i, 200 + i, 300 + i, 400 + i, 500 + i, 600 + i, 700 + i),
- TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).GetValueSync());
- }
-
+ (%d, "Value1", 0),
+ (%d, "Value2", 1),
+ (%d, "Value3", 2),
+ (%d, "Value4", 3),
+ (%d, "Value5", 4),
+ (%d, "Value6", 5),
+ (%d, "Value7", 6),
+ (%d, "Value8", 7)
+ )", i, 100 + i, 200 + i, 300 + i, 400 + i, 500 + i, 600 + i, 700 + i),
+ TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).GetValueSync());
+ }
+
auto db = kikimr.GetTableClient();
auto it = db.StreamExecuteScanQuery("SELECT * FROM `/Root/EightShard` LIMIT 2").GetValueSync();
-
- Cerr << StreamResultToYson(it) << Endl;
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- }
-
+
+ Cerr << StreamResultToYson(it) << Endl;
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ }
+
Y_UNIT_TEST(MultipleResults) {
TKikimrRunner kikimr(AppCfg());
auto db = kikimr.GetTableClient();
@@ -1548,96 +1548,96 @@ Y_UNIT_TEST_SUITE(KqpScan) {
CompareYson(R"([[[" GET /index.html HTTP/1.1"];["some very very very very long string"]]])",res);
}
- Y_UNIT_TEST(SqlInParameter) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
-
- auto result = db.CreateSession().GetValueSync().GetSession().ExecuteDataQuery(R"(
+ Y_UNIT_TEST(SqlInParameter) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+
+ auto result = db.CreateSession().GetValueSync().GetSession().ExecuteDataQuery(R"(
REPLACE INTO `/Root/KeyValue` (Key, Value) VALUES
- (3u, "Three"),
- (4u, "Four"),
- (10u, "Ten"),
- (NULL, "Null Value");
- )", TTxControl::BeginTx().CommitTx()).GetValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- auto query = R"(
- DECLARE $in AS List<Uint64?>;
- SELECT Key, Value FROM `/Root/KeyValue` WHERE Key IN $in
- ORDER BY Key
- )";
-
- auto params = TParamsBuilder();
- auto& pl = params.AddParam("$in").BeginList();
- for (auto v : {1, 2, 3, 42, 50, 100}) {
- pl.AddListItem().OptionalUint64(v);
- }
- pl.AddListItem().OptionalUint64(Nothing());
- pl.EndList().Build();
-
- auto it = db.StreamExecuteScanQuery(query, params.Build()).GetValueSync();
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- CompareYson(R"([[[1u];["One"]];
- [[2u];["Two"]];
- [[3u];["Three"]]])", StreamResultToYson(it));
- }
-
- Y_UNIT_TEST(SqlInLiteral) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
-
- auto result = db.CreateSession().GetValueSync().GetSession().ExecuteDataQuery(R"(
+ (3u, "Three"),
+ (4u, "Four"),
+ (10u, "Ten"),
+ (NULL, "Null Value");
+ )", TTxControl::BeginTx().CommitTx()).GetValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ auto query = R"(
+ DECLARE $in AS List<Uint64?>;
+ SELECT Key, Value FROM `/Root/KeyValue` WHERE Key IN $in
+ ORDER BY Key
+ )";
+
+ auto params = TParamsBuilder();
+ auto& pl = params.AddParam("$in").BeginList();
+ for (auto v : {1, 2, 3, 42, 50, 100}) {
+ pl.AddListItem().OptionalUint64(v);
+ }
+ pl.AddListItem().OptionalUint64(Nothing());
+ pl.EndList().Build();
+
+ auto it = db.StreamExecuteScanQuery(query, params.Build()).GetValueSync();
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ CompareYson(R"([[[1u];["One"]];
+ [[2u];["Two"]];
+ [[3u];["Three"]]])", StreamResultToYson(it));
+ }
+
+ Y_UNIT_TEST(SqlInLiteral) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+
+ auto result = db.CreateSession().GetValueSync().GetSession().ExecuteDataQuery(R"(
REPLACE INTO `/Root/KeyValue` (Key, Value) VALUES
- (3u, "Three"),
- (4u, "Four"),
- (10u, "Ten"),
- (NULL, "Null Value");
- )", TTxControl::BeginTx().CommitTx()).GetValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- auto query = R"(
- SELECT Key, Value FROM `/Root/KeyValue` WHERE Key IN (1, 2, 3, 42)
- ORDER BY Key
- )";
-
- auto it = db.StreamExecuteScanQuery(query).GetValueSync();
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- CompareYson(R"([[[1u];["One"]];
- [[2u];["Two"]];
- [[3u];["Three"]]])", StreamResultToYson(it));
- }
-
- Y_UNIT_TEST(UdfFailure) {
- auto testFn = [](bool disableLlvmForUdfStages) {
- TVector<NKikimrKqp::TKqpSetting> settings;
- auto setting = NKikimrKqp::TKqpSetting();
- setting.SetName("_KqpDisableLlvmForUdfStages");
- setting.SetValue(disableLlvmForUdfStages ? "true" : "false");
- settings.push_back(setting);
-
- TKikimrRunner kikimr{settings};
- auto db = kikimr.GetTableClient();
-
- auto it = db.StreamExecuteScanQuery(R"(
- SELECT * FROM `/Root/KeyValue`
- WHERE TestUdfs::TestFilterTerminate(Cast(Key as Int64) ?? 0, 10)
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- auto streamPart = it.ReadNext().GetValueSync();
- UNIT_ASSERT_C(!streamPart.IsSuccess(), streamPart.GetIssues().ToString());
- UNIT_ASSERT_C(!streamPart.EOS(), streamPart.GetIssues().ToString());
- UNIT_ASSERT(
- HasIssue(streamPart.GetIssues(), NYql::TIssuesIds::DEFAULT_ERROR, [](const NYql::TIssue& issue) {
- return issue.Message.Contains("Terminate was called") // general termination prefix
- && issue.Message.Contains("Bad filter value."); // test specific UDF exception
- }));
- };
-
- testFn(false);
- testFn(true);
- }
+ (3u, "Three"),
+ (4u, "Four"),
+ (10u, "Ten"),
+ (NULL, "Null Value");
+ )", TTxControl::BeginTx().CommitTx()).GetValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ auto query = R"(
+ SELECT Key, Value FROM `/Root/KeyValue` WHERE Key IN (1, 2, 3, 42)
+ ORDER BY Key
+ )";
+
+ auto it = db.StreamExecuteScanQuery(query).GetValueSync();
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ CompareYson(R"([[[1u];["One"]];
+ [[2u];["Two"]];
+ [[3u];["Three"]]])", StreamResultToYson(it));
+ }
+
+ Y_UNIT_TEST(UdfFailure) {
+ auto testFn = [](bool disableLlvmForUdfStages) {
+ TVector<NKikimrKqp::TKqpSetting> settings;
+ auto setting = NKikimrKqp::TKqpSetting();
+ setting.SetName("_KqpDisableLlvmForUdfStages");
+ setting.SetValue(disableLlvmForUdfStages ? "true" : "false");
+ settings.push_back(setting);
+
+ TKikimrRunner kikimr{settings};
+ auto db = kikimr.GetTableClient();
+
+ auto it = db.StreamExecuteScanQuery(R"(
+ SELECT * FROM `/Root/KeyValue`
+ WHERE TestUdfs::TestFilterTerminate(Cast(Key as Int64) ?? 0, 10)
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ auto streamPart = it.ReadNext().GetValueSync();
+ UNIT_ASSERT_C(!streamPart.IsSuccess(), streamPart.GetIssues().ToString());
+ UNIT_ASSERT_C(!streamPart.EOS(), streamPart.GetIssues().ToString());
+ UNIT_ASSERT(
+ HasIssue(streamPart.GetIssues(), NYql::TIssuesIds::DEFAULT_ERROR, [](const NYql::TIssue& issue) {
+ return issue.Message.Contains("Terminate was called") // general termination prefix
+ && issue.Message.Contains("Bad filter value."); // test specific UDF exception
+ }));
+ };
+
+ testFn(false);
+ testFn(true);
+ }
Y_UNIT_TEST(SecondaryIndex) {
TKikimrRunner kikimr(AppCfg());
@@ -1678,40 +1678,40 @@ Y_UNIT_TEST_SUITE(KqpScan) {
UNIT_ASSERT_VALUES_EQUAL_C(status, EStatus::BAD_REQUEST, "ScanQuery with explicit index should fail");
}
- Y_UNIT_TEST(BoolFlag) {
- TKikimrRunner kikimr(AppCfg());
-
- TTableClient client{kikimr.GetDriver()};
- auto session = client.CreateSession().GetValueSync().GetSession();
-
- UNIT_ASSERT(session.CreateTable("/Root/BoolTest",
- TTableBuilder()
- .AddNullableColumn("Key", EPrimitiveType::Uint64)
- .AddNullableColumn("Value", EPrimitiveType::Uint64)
- .AddNullableColumn("Flag", EPrimitiveType::Bool)
- .SetPrimaryKeyColumn("Key")
- .Build()
- ).GetValueSync().IsSuccess());
-
- auto result = session.ExecuteDataQuery(R"(
- --!syntax_v1
- INSERT INTO `/Root/BoolTest` (Key, Value, Flag) VALUES
- (1, 100, true), (2, 200, false), (3, 300, true), (4, 400, false), (5, 500, null)
- )", TTxControl::BeginTx().CommitTx()).GetValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- auto db = kikimr.GetTableClient();
- auto it = db.StreamExecuteScanQuery(R"(
- select Key, Value, Flag
- from `/Root/BoolTest`
- where not Flag and Value < 1000
- )").GetValueSync();
-
- CompareYson(R"([
- [[2u];[200u];[%false]];
- [[4u];[400u];[%false]]
- ])", StreamResultToYson(it));
- }
+ Y_UNIT_TEST(BoolFlag) {
+ TKikimrRunner kikimr(AppCfg());
+
+ TTableClient client{kikimr.GetDriver()};
+ auto session = client.CreateSession().GetValueSync().GetSession();
+
+ UNIT_ASSERT(session.CreateTable("/Root/BoolTest",
+ TTableBuilder()
+ .AddNullableColumn("Key", EPrimitiveType::Uint64)
+ .AddNullableColumn("Value", EPrimitiveType::Uint64)
+ .AddNullableColumn("Flag", EPrimitiveType::Bool)
+ .SetPrimaryKeyColumn("Key")
+ .Build()
+ ).GetValueSync().IsSuccess());
+
+ auto result = session.ExecuteDataQuery(R"(
+ --!syntax_v1
+ INSERT INTO `/Root/BoolTest` (Key, Value, Flag) VALUES
+ (1, 100, true), (2, 200, false), (3, 300, true), (4, 400, false), (5, 500, null)
+ )", TTxControl::BeginTx().CommitTx()).GetValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ auto db = kikimr.GetTableClient();
+ auto it = db.StreamExecuteScanQuery(R"(
+ select Key, Value, Flag
+ from `/Root/BoolTest`
+ where not Flag and Value < 1000
+ )").GetValueSync();
+
+ CompareYson(R"([
+ [[2u];[200u];[%false]];
+ [[4u];[400u];[%false]]
+ ])", StreamResultToYson(it));
+ }
Y_UNIT_TEST(Counters) {
TKikimrRunner kikimr(AppCfg());
@@ -1727,49 +1727,49 @@ Y_UNIT_TEST_SUITE(KqpScan) {
UNIT_ASSERT_VALUES_EQUAL(1, counters.GetQueryTypeCounter(NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCAN)->Val());
}
- Y_UNIT_TEST(DropRedundantSortByPk) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
-
- auto settings = TStreamExecScanQuerySettings()
- .Explain(true);
-
+ Y_UNIT_TEST(DropRedundantSortByPk) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+
+ auto settings = TStreamExecScanQuerySettings()
+ .Explain(true);
+
auto test = [&](const TString& table, const TString& tableRlPath, const TVector<TString>& keys, bool top, bool expectSort) {
- auto query = TStringBuilder()
- << "SELECT * FROM `" << table << "` "
- << "ORDER BY " << JoinSeq(", ", keys)
- << (top ? " LIMIT 42" : "");
- auto result = db.StreamExecuteScanQuery(query, settings).GetValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- auto res = CollectStreamResult(result);
+ auto query = TStringBuilder()
+ << "SELECT * FROM `" << table << "` "
+ << "ORDER BY " << JoinSeq(", ", keys)
+ << (top ? " LIMIT 42" : "");
+ auto result = db.StreamExecuteScanQuery(query, settings).GetValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ auto res = CollectStreamResult(result);
UNIT_ASSERT(res.PlanJson);
- // Cerr << res.PlanJson << Endl;
-
- NJson::TJsonValue plan;
+ // Cerr << res.PlanJson << Endl;
+
+ NJson::TJsonValue plan;
NJson::ReadJsonTree(*res.PlanJson, &plan, /* throwOnError */ true);
-
+
auto node = FindPlanNodeByKv(plan, "Tables", NJson::TJsonArray({tableRlPath}).GetStringRobust());
UNIT_ASSERT_C(node.IsDefined(), query);
UNIT_ASSERT_EQUAL_C(node.GetMapSafe().at("Node Type").GetStringSafe().Contains("Sort"), expectSort, query);
- };
-
- // simple key
- for (bool top : {false, true}) {
+ };
+
+ // simple key
+ for (bool top : {false, true}) {
test("/Root/KeyValue", "KeyValue", {"Key"}, top, false); // key
test("/Root/KeyValue", "KeyValue", {"Value"}, top, true); // not key
- }
-
- // complex key
- for (bool top : {false, true}) {
+ }
+
+ // complex key
+ for (bool top : {false, true}) {
test("/Root/Logs", "Logs", {"App", "Ts", "Host"}, top, false); // full key
test("/Root/Logs", "Logs", {"App", "Ts"}, top, false); // key prefix
test("/Root/Logs", "Logs", {"App"}, top, false); // key prefix
test("/Root/Logs", "Logs", {"Ts", "Host"}, top, true); // not key prefix
test("/Root/Logs", "Logs", {"Message"}, top, true); // not key
- }
- }
-
+ }
+ }
+
Y_UNIT_TEST(LMapFunction) {
auto settings = TKikimrSettings()
.SetWithSampleTables(false)
@@ -1901,5 +1901,5 @@ Y_UNIT_TEST_SUITE(KqpScan) {
}
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/ut/kqp_scripting_ut.cpp b/ydb/core/kqp/ut/kqp_scripting_ut.cpp
index 6103bba9c7..65e75ac303 100644
--- a/ydb/core/kqp/ut/kqp_scripting_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_scripting_ut.cpp
@@ -3,8 +3,8 @@
#include <ydb/public/sdk/cpp/client/draft/ydb_scripting.h>
-#include <library/cpp/json/json_prettifier.h>
-
+#include <library/cpp/json/json_prettifier.h>
+
namespace NKikimr {
namespace NKqp {
@@ -42,11 +42,11 @@ Y_UNIT_TEST_SUITE(KqpScripting) {
UNIT_ASSERT_VALUES_EQUAL(rs0.ColumnParser(0).GetUint64(), 2u);
}
- Y_UNIT_TEST_NEW_ENGINE(UnsafeTimestampCast) {
+ Y_UNIT_TEST_NEW_ENGINE(UnsafeTimestampCast) {
TKikimrRunner kikimr;
TScriptingClient client(kikimr.GetDriver());
- auto result = client.ExecuteYqlScript(Q_(R"(
+ auto result = client.ExecuteYqlScript(Q_(R"(
CREATE TABLE [/Root/TsTest] (
Key Timestamp,
Value String,
@@ -56,13 +56,13 @@ Y_UNIT_TEST_SUITE(KqpScripting) {
UPSERT INTO [/Root/TsTest]
SELECT * FROM [/Root/KeyValue];
- )")).GetValueSync();
+ )")).GetValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- result = client.ExecuteYqlScript(Q1_(R"(
+ result = client.ExecuteYqlScript(Q1_(R"(
UPSERT INTO `/Root/TsTest`
SELECT * FROM `/Root/KeyValue`;
- )")).GetValueSync();
+ )")).GetValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString());
}
diff --git a/ydb/core/kqp/ut/kqp_sort_ut.cpp b/ydb/core/kqp/ut/kqp_sort_ut.cpp
index db7ee95901..0b7c65a3d5 100644
--- a/ydb/core/kqp/ut/kqp_sort_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_sort_ut.cpp
@@ -7,16 +7,16 @@ using namespace NYdb;
using namespace NYdb::NTable;
Y_UNIT_TEST_SUITE(KqpSort) {
- Y_UNIT_TEST_NEW_ENGINE(ReverseDefault) {
+ Y_UNIT_TEST_NEW_ENGINE(ReverseDefault) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- TString query = Q_(R"(
+ TString query = Q_(R"(
SELECT Group, Name, Amount, Comment
FROM `/Root/Test`
ORDER BY Group DESC, Name DESC;
- )");
+ )");
{
auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync();
@@ -30,7 +30,7 @@ Y_UNIT_TEST_SUITE(KqpSort) {
}
}
- Y_UNIT_TEST_NEW_ENGINE(ReverseOptimized) {
+ Y_UNIT_TEST_NEW_ENGINE(ReverseOptimized) {
auto setting = NKikimrKqp::TKqpSetting();
setting.SetName("_AllowReverseRange");
setting.SetValue("True");
@@ -39,27 +39,27 @@ Y_UNIT_TEST_SUITE(KqpSort) {
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- TString query = Q_(R"(
+ TString query = Q_(R"(
SELECT Group, Name, Amount, Comment
FROM `/Root/Test`
ORDER BY Group DESC, Name DESC;
- )");
-
- auto result = session.ExplainDataQuery(query).GetValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
-
- if (UseNewEngine) {
- UNIT_ASSERT_C(result.GetAst().Contains("('\"Reverse\")"), result.GetAst());
-
- NJson::TJsonValue plan;
- NJson::ReadJsonTree(result.GetPlan(), &plan, true);
+ )");
+
+ auto result = session.ExplainDataQuery(query).GetValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+
+ if (UseNewEngine) {
+ UNIT_ASSERT_C(result.GetAst().Contains("('\"Reverse\")"), result.GetAst());
+
+ NJson::TJsonValue plan;
+ NJson::ReadJsonTree(result.GetPlan(), &plan, true);
auto node = FindPlanNodeByKv(plan, "Node Type", "Limit-TableFullScan"); // without `Sort`
- UNIT_ASSERT(node.IsDefined());
+ UNIT_ASSERT(node.IsDefined());
auto read = FindPlanNodeByKv(node, "Name", "TableFullScan");
UNIT_ASSERT(read.IsDefined());
UNIT_ASSERT(read.GetMapSafe().contains("Reverse"));
- } else {
- UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst());
+ } else {
+ UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst());
UNIT_ASSERT_C(!result.GetAst().Contains("Sort"), result.GetAst());
}
@@ -74,7 +74,7 @@ Y_UNIT_TEST_SUITE(KqpSort) {
}
}
- Y_UNIT_TEST_NEW_ENGINE(ReverseOptimizedWithPredicate) {
+ Y_UNIT_TEST_NEW_ENGINE(ReverseOptimizedWithPredicate) {
auto setting = NKikimrKqp::TKqpSetting();
setting.SetName("_AllowReverseRange");
setting.SetValue("True");
@@ -83,27 +83,27 @@ Y_UNIT_TEST_SUITE(KqpSort) {
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- TString query = Q_(R"(
+ TString query = Q_(R"(
SELECT Group, Name, Amount, Comment
FROM `/Root/Test`
WHERE Group < 2u
ORDER BY Group DESC, Name DESC;
- )");
+ )");
- auto result = session.ExplainDataQuery(query).GetValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ auto result = session.ExplainDataQuery(query).GetValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- if (UseNewEngine) {
- UNIT_ASSERT_C(result.GetAst().Contains("('\"Reverse\")"), result.GetAst());
-
- NJson::TJsonValue plan;
- NJson::ReadJsonTree(result.GetPlan(), &plan, true);
+ if (UseNewEngine) {
+ UNIT_ASSERT_C(result.GetAst().Contains("('\"Reverse\")"), result.GetAst());
+
+ NJson::TJsonValue plan;
+ NJson::ReadJsonTree(result.GetPlan(), &plan, true);
auto node = FindPlanNodeByKv(plan, "Node Type", "Limit-TableRangeScan"); // without `Sort`
- UNIT_ASSERT_C(node.IsDefined(), result.GetPlan());
+ UNIT_ASSERT_C(node.IsDefined(), result.GetPlan());
auto read = FindPlanNodeByKv(node, "Name", "TableRangeScan");
UNIT_ASSERT(read.IsDefined());
UNIT_ASSERT(read.GetMapSafe().contains("Reverse"));
- } else {
+ } else {
UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst());
}
@@ -117,7 +117,7 @@ Y_UNIT_TEST_SUITE(KqpSort) {
}
}
- Y_UNIT_TEST_NEW_ENGINE(ReverseFirstKeyOptimized) {
+ Y_UNIT_TEST_NEW_ENGINE(ReverseFirstKeyOptimized) {
auto setting = NKikimrKqp::TKqpSetting();
setting.SetName("_AllowReverseRange");
setting.SetValue("True");
@@ -126,29 +126,29 @@ Y_UNIT_TEST_SUITE(KqpSort) {
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- TString query = Q_(R"(
+ TString query = Q_(R"(
SELECT Group, Name, Amount, Comment
FROM [/Root/Test]
ORDER BY Group DESC;
- )");
+ )");
{
auto result = session.ExplainDataQuery(query).GetValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- if (UseNewEngine) {
- UNIT_ASSERT_C(result.GetAst().Contains("('\"Reverse\")"), result.GetAst());
-
- NJson::TJsonValue plan;
- NJson::ReadJsonTree(result.GetPlan(), &plan, true);
+ if (UseNewEngine) {
+ UNIT_ASSERT_C(result.GetAst().Contains("('\"Reverse\")"), result.GetAst());
+
+ NJson::TJsonValue plan;
+ NJson::ReadJsonTree(result.GetPlan(), &plan, true);
auto node = FindPlanNodeByKv(plan, "Node Type", "Limit-TableFullScan"); // without `Sort`
- UNIT_ASSERT_C(node.IsDefined(), result.GetPlan());
+ UNIT_ASSERT_C(node.IsDefined(), result.GetPlan());
auto read = FindPlanNodeByKv(node, "Name", "TableFullScan");
UNIT_ASSERT(read.IsDefined());
UNIT_ASSERT(read.GetMapSafe().contains("Reverse"));
- } else {
- UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst());
- }
+ } else {
+ UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst());
+ }
}
{
@@ -162,7 +162,7 @@ Y_UNIT_TEST_SUITE(KqpSort) {
}
}
- Y_UNIT_TEST_NEW_ENGINE(ReverseMixedOrderNotOptimized) {
+ Y_UNIT_TEST_NEW_ENGINE(ReverseMixedOrderNotOptimized) {
auto setting = NKikimrKqp::TKqpSetting();
setting.SetName("_AllowReverseRange");
setting.SetValue("True");
@@ -171,27 +171,27 @@ Y_UNIT_TEST_SUITE(KqpSort) {
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- TString query = Q_(R"(
+ TString query = Q_(R"(
SELECT Group, Name, Amount, Comment
FROM `/Root/Test`
ORDER BY Group DESC, Name ASC;
- )");
+ )");
- {
+ {
auto result = session.ExplainDataQuery(query).GetValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- if (UseNewEngine) {
- UNIT_ASSERT_C(!result.GetAst().Contains("('\"Reverse\")"), result.GetAst());
-
- NJson::TJsonValue plan;
- NJson::ReadJsonTree(result.GetPlan(), &plan, true);
+ if (UseNewEngine) {
+ UNIT_ASSERT_C(!result.GetAst().Contains("('\"Reverse\")"), result.GetAst());
+
+ NJson::TJsonValue plan;
+ NJson::ReadJsonTree(result.GetPlan(), &plan, true);
auto node = FindPlanNodeByKv(plan, "Node Type", "TopSort-TableFullScan");
- UNIT_ASSERT_C(node.IsDefined(), result.GetPlan());
- UNIT_ASSERT(!node.GetMapSafe().contains("Reverse"));
- } else {
- UNIT_ASSERT_C(!result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst());
- }
+ UNIT_ASSERT_C(node.IsDefined(), result.GetPlan());
+ UNIT_ASSERT(!node.GetMapSafe().contains("Reverse"));
+ } else {
+ UNIT_ASSERT_C(!result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst());
+ }
}
{
@@ -205,7 +205,7 @@ Y_UNIT_TEST_SUITE(KqpSort) {
}
}
- Y_UNIT_TEST_NEW_ENGINE(ReverseRangeOptimized) {
+ Y_UNIT_TEST_NEW_ENGINE(ReverseRangeOptimized) {
auto setting = NKikimrKqp::TKqpSetting();
setting.SetName("_AllowReverseRange");
setting.SetValue("True");
@@ -214,30 +214,30 @@ Y_UNIT_TEST_SUITE(KqpSort) {
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- TString query = Q_(R"(
+ TString query = Q_(R"(
SELECT Group, Name, Amount, Comment
FROM `/Root/Test`
WHERE Group < 2
ORDER BY Group DESC, Name DESC;
- )");
+ )");
- {
+ {
auto result = session.ExplainDataQuery(query).GetValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- if (UseNewEngine) {
- UNIT_ASSERT_C(result.GetAst().Contains("('\"Reverse\")"), result.GetAst());
-
- NJson::TJsonValue plan;
- NJson::ReadJsonTree(result.GetPlan(), &plan, true);
+ if (UseNewEngine) {
+ UNIT_ASSERT_C(result.GetAst().Contains("('\"Reverse\")"), result.GetAst());
+
+ NJson::TJsonValue plan;
+ NJson::ReadJsonTree(result.GetPlan(), &plan, true);
auto node = FindPlanNodeByKv(plan, "Node Type", "Limit-TableRangeScan");
- UNIT_ASSERT_C(node.IsDefined(), result.GetPlan());
+ UNIT_ASSERT_C(node.IsDefined(), result.GetPlan());
auto read = FindPlanNodeByKv(node, "Name", "TableRangeScan");
UNIT_ASSERT(read.IsDefined());
UNIT_ASSERT(read.GetMapSafe().contains("Reverse"));
- } else {
- UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst());
- }
+ } else {
+ UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst());
+ }
}
{
@@ -250,7 +250,7 @@ Y_UNIT_TEST_SUITE(KqpSort) {
}
}
- Y_UNIT_TEST_NEW_ENGINE(ReverseLimitOptimized) {
+ Y_UNIT_TEST_NEW_ENGINE(ReverseLimitOptimized) {
auto setting = NKikimrKqp::TKqpSetting();
setting.SetName("_AllowReverseRange");
setting.SetValue("True");
@@ -259,35 +259,35 @@ Y_UNIT_TEST_SUITE(KqpSort) {
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- TString query = Q_(R"(
+ TString query = Q_(R"(
SELECT Group, Name, Amount, Comment
FROM `/Root/Test`
ORDER BY Group DESC, Name DESC
LIMIT 1;
- )");
+ )");
- {
+ {
auto result = session.ExplainDataQuery(query).GetValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
UNIT_ASSERT_C(result.GetAst().Contains("'\"ItemsLimit\""), result.GetAst());
-
- if (UseNewEngine) {
- UNIT_ASSERT_C(result.GetAst().Contains("('\"Reverse\")"), result.GetAst());
-
- NJson::TJsonValue plan;
- NJson::ReadJsonTree(result.GetPlan(), &plan, true);
+
+ if (UseNewEngine) {
+ UNIT_ASSERT_C(result.GetAst().Contains("('\"Reverse\")"), result.GetAst());
+
+ NJson::TJsonValue plan;
+ NJson::ReadJsonTree(result.GetPlan(), &plan, true);
auto node = FindPlanNodeByKv(plan, "Node Type", "Limit-TableFullScan");
- UNIT_ASSERT_C(node.IsDefined(), result.GetPlan());
+ UNIT_ASSERT_C(node.IsDefined(), result.GetPlan());
auto read = FindPlanNodeByKv(node, "Name", "TableFullScan");
UNIT_ASSERT(read.IsDefined());
UNIT_ASSERT(read.GetMapSafe().contains("Reverse"));
auto limit = FindPlanNodeByKv(node, "Name", "Limit");
UNIT_ASSERT(limit.IsDefined());
UNIT_ASSERT(limit.GetMapSafe().contains("Limit"));
- } else {
- UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst());
- }
+ } else {
+ UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst());
+ }
}
{
@@ -299,7 +299,7 @@ Y_UNIT_TEST_SUITE(KqpSort) {
}
}
- Y_UNIT_TEST_NEW_ENGINE(ReverseRangeLimitOptimized) {
+ Y_UNIT_TEST_NEW_ENGINE(ReverseRangeLimitOptimized) {
auto setting = NKikimrKqp::TKqpSetting();
setting.SetName("_AllowReverseRange");
setting.SetValue("True");
@@ -308,35 +308,35 @@ Y_UNIT_TEST_SUITE(KqpSort) {
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- TString query = Q_(R"(
+ TString query = Q_(R"(
SELECT Group, Name, Amount, Comment
FROM `/Root/Test`
WHERE Group < 2
ORDER BY Group DESC, Name DESC
LIMIT 1;
- )");
+ )");
- {
+ {
auto result = session.ExplainDataQuery(query).GetValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- if (UseNewEngine) {
- UNIT_ASSERT_C(result.GetAst().Contains("('\"Reverse\")"), result.GetAst());
-
- NJson::TJsonValue plan;
- NJson::ReadJsonTree(result.GetPlan(), &plan, true);
+ if (UseNewEngine) {
+ UNIT_ASSERT_C(result.GetAst().Contains("('\"Reverse\")"), result.GetAst());
+
+ NJson::TJsonValue plan;
+ NJson::ReadJsonTree(result.GetPlan(), &plan, true);
auto node = FindPlanNodeByKv(plan, "Node Type", "Limit-TableRangeScan");
- UNIT_ASSERT_C(node.IsDefined(), result.GetPlan());
+ UNIT_ASSERT_C(node.IsDefined(), result.GetPlan());
auto read = FindPlanNodeByKv(node, "Name", "TableRangeScan");
UNIT_ASSERT(read.IsDefined());
UNIT_ASSERT(read.GetMapSafe().contains("Reverse"));
auto limit = FindPlanNodeByKv(node, "Name", "Limit");
UNIT_ASSERT(limit.IsDefined());
UNIT_ASSERT(limit.GetMapSafe().contains("Limit"));
- } else {
- UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst());
- }
-
+ } else {
+ UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst());
+ }
+
UNIT_ASSERT_C(result.GetAst().Contains("'\"ItemsLimit\""), result.GetAst());
}
@@ -349,7 +349,7 @@ Y_UNIT_TEST_SUITE(KqpSort) {
}
}
- Y_UNIT_TEST_NEW_ENGINE(ReverseEightShardOptimized) {
+ Y_UNIT_TEST_NEW_ENGINE(ReverseEightShardOptimized) {
auto setting = NKikimrKqp::TKqpSetting();
setting.SetName("_AllowReverseRange");
setting.SetValue("True");
@@ -358,32 +358,32 @@ Y_UNIT_TEST_SUITE(KqpSort) {
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- TString query = Q_(R"(
+ TString query = Q_(R"(
SELECT Key, Text, Data
FROM `/Root/EightShard`
ORDER BY Key DESC
LIMIT 8;
- )");
+ )");
- {
+ {
auto result = session.ExplainDataQuery(query).GetValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
UNIT_ASSERT_C(result.GetAst().Contains("'\"ItemsLimit\""), result.GetAst());
-
- if (UseNewEngine) {
- UNIT_ASSERT_C(result.GetAst().Contains("('\"Reverse\")"), result.GetAst());
-
- NJson::TJsonValue plan;
- NJson::ReadJsonTree(result.GetPlan(), &plan, true);
+
+ if (UseNewEngine) {
+ UNIT_ASSERT_C(result.GetAst().Contains("('\"Reverse\")"), result.GetAst());
+
+ NJson::TJsonValue plan;
+ NJson::ReadJsonTree(result.GetPlan(), &plan, true);
auto node = FindPlanNodeByKv(plan, "Node Type", "Limit-TableFullScan");
- UNIT_ASSERT_C(node.IsDefined(), result.GetPlan());
+ UNIT_ASSERT_C(node.IsDefined(), result.GetPlan());
auto read = FindPlanNodeByKv(node, "Name", "TableFullScan");
UNIT_ASSERT(read.IsDefined());
UNIT_ASSERT(read.GetMapSafe().contains("Reverse"));
- } else {
- UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst());
- }
+ } else {
+ UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst());
+ }
}
{
@@ -402,12 +402,12 @@ Y_UNIT_TEST_SUITE(KqpSort) {
}
}
- Y_UNIT_TEST_NEW_ENGINE(TopSortParameter) {
+ Y_UNIT_TEST_NEW_ENGINE(TopSortParameter) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- TString query = Q_(R"(
+ TString query = Q_(R"(
DECLARE $limit AS Uint32;
DECLARE $offset AS Uint32;
DECLARE $minKey AS Uint64;
@@ -419,19 +419,19 @@ Y_UNIT_TEST_SUITE(KqpSort) {
LIMIT $limit OFFSET $offset;
)");
- {
+ {
auto result = session.ExplainDataQuery(query).GetValueSync();
result.GetIssues().PrintTo(Cerr);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
-
- if (UseNewEngine) {
- NJson::TJsonValue plan;
- NJson::ReadJsonTree(result.GetPlan(), &plan, true);
+
+ if (UseNewEngine) {
+ NJson::TJsonValue plan;
+ NJson::ReadJsonTree(result.GetPlan(), &plan, true);
auto node = FindPlanNodeByKv(plan, "Node Type", "TopSort-TableRangeScan");
- UNIT_ASSERT_C(node.IsDefined(), result.GetPlan());
- } else {
- UNIT_ASSERT_C(result.GetAst().Contains("KiPartialTake"), result.GetAst());
- }
+ UNIT_ASSERT_C(node.IsDefined(), result.GetPlan());
+ } else {
+ UNIT_ASSERT_C(result.GetAst().Contains("KiPartialTake"), result.GetAst());
+ }
}
{
@@ -460,12 +460,12 @@ Y_UNIT_TEST_SUITE(KqpSort) {
}
}
- Y_UNIT_TEST_NEW_ENGINE(TopSortExpr) {
+ Y_UNIT_TEST_NEW_ENGINE(TopSortExpr) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- TString query = Q_(R"(
+ TString query = Q_(R"(
DECLARE $limit AS Uint32;
DECLARE $offset AS Uint32;
DECLARE $minKey AS Uint64;
@@ -477,19 +477,19 @@ Y_UNIT_TEST_SUITE(KqpSort) {
LIMIT $limit + 1 OFFSET $offset - 1;
)");
- {
+ {
auto result = session.ExplainDataQuery(query).GetValueSync();
result.GetIssues().PrintTo(Cerr);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
-
- if (UseNewEngine) {
- NJson::TJsonValue plan;
- NJson::ReadJsonTree(result.GetPlan(), &plan, true);
+
+ if (UseNewEngine) {
+ NJson::TJsonValue plan;
+ NJson::ReadJsonTree(result.GetPlan(), &plan, true);
auto node = FindPlanNodeByKv(plan, "Node Type", "TopSort-TableRangeScan");
- UNIT_ASSERT_C(node.IsDefined(), result.GetPlan());
- } else {
- UNIT_ASSERT_C(result.GetAst().Contains("KiPartialTake"), result.GetAst());
- }
+ UNIT_ASSERT_C(node.IsDefined(), result.GetPlan());
+ } else {
+ UNIT_ASSERT_C(result.GetAst().Contains("KiPartialTake"), result.GetAst());
+ }
}
{
@@ -518,12 +518,12 @@ Y_UNIT_TEST_SUITE(KqpSort) {
}
}
- Y_UNIT_TEST_NEW_ENGINE(TopSortExprPk) {
+ Y_UNIT_TEST_NEW_ENGINE(TopSortExprPk) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- TString query = Q_(R"(
+ TString query = Q_(R"(
DECLARE $limit AS Uint32;
DECLARE $offset AS Uint32;
DECLARE $minKey AS Uint64;
@@ -535,7 +535,7 @@ Y_UNIT_TEST_SUITE(KqpSort) {
LIMIT $limit + 1 OFFSET $offset - 1;
)");
- {
+ {
auto result = session.ExplainDataQuery(query).GetValueSync();
result.GetIssues().PrintTo(Cerr);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
@@ -866,7 +866,7 @@ Y_UNIT_TEST_SUITE(KqpSort) {
}
}
- Y_UNIT_TEST_NEW_ENGINE(TopSortResults) {
+ Y_UNIT_TEST_NEW_ENGINE(TopSortResults) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -923,7 +923,7 @@ Y_UNIT_TEST_SUITE(KqpSort) {
UNIT_ASSERT(result.IsSuccess());
}
- auto queryTemplate = Q_(R"(
+ auto queryTemplate = Q_(R"(
PRAGMA kikimr.OptDisableTopSort = '%s';
DECLARE $filter AS Int32;
@@ -934,10 +934,10 @@ Y_UNIT_TEST_SUITE(KqpSort) {
WHERE Value1 != $filter
ORDER BY Value2 DESC, Value3, Key DESC
LIMIT $limit OFFSET $offset;
- )");
+ )");
- auto query = Sprintf(queryTemplate.c_str(), "False");
- auto queryDisabled = Sprintf(queryTemplate.c_str(), "True");
+ auto query = Sprintf(queryTemplate.c_str(), "False");
+ auto queryDisabled = Sprintf(queryTemplate.c_str(), "True");
const ui32 QueriesCount = 20;
for (ui32 i = 0; i < QueriesCount; ++i) {
@@ -968,34 +968,34 @@ Y_UNIT_TEST_SUITE(KqpSort) {
}
}
- Y_UNIT_TEST_NEW_ENGINE(TopParameter) {
+ Y_UNIT_TEST_NEW_ENGINE(TopParameter) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- TString query = Q_(R"(
+ TString query = Q_(R"(
DECLARE $limit AS Uint64;
SELECT *
FROM `/Root/TwoShard`
- ORDER BY Key
+ ORDER BY Key
LIMIT $limit;
)");
- {
+ {
auto result = session.ExplainDataQuery(query).GetValueSync();
result.GetIssues().PrintTo(Cerr);
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
-
- if (UseNewEngine) {
- NJson::TJsonValue plan;
- NJson::ReadJsonTree(result.GetPlan(), &plan, true);
+
+ if (UseNewEngine) {
+ NJson::TJsonValue plan;
+ NJson::ReadJsonTree(result.GetPlan(), &plan, true);
auto node = FindPlanNodeByKv(plan, "Node Type", "Limit-TableFullScan");
- UNIT_ASSERT_C(node.IsDefined(), result.GetPlan());
- } else {
- UNIT_ASSERT_C(!result.GetAst().Contains("KiPartialTake"), result.GetAst());
- }
+ UNIT_ASSERT_C(node.IsDefined(), result.GetPlan());
+ } else {
+ UNIT_ASSERT_C(!result.GetAst().Contains("KiPartialTake"), result.GetAst());
+ }
}
{
@@ -1015,12 +1015,12 @@ Y_UNIT_TEST_SUITE(KqpSort) {
}
}
- Y_UNIT_TEST_NEW_ENGINE(TopParameterFilter) {
+ Y_UNIT_TEST_NEW_ENGINE(TopParameterFilter) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- auto query = Q_(R"(
+ auto query = Q_(R"(
DECLARE $limit AS Uint64;
DECLARE $value AS Int32;
@@ -1037,17 +1037,17 @@ Y_UNIT_TEST_SUITE(KqpSort) {
Cerr << result.GetAst() << Endl;
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
-
- if (UseNewEngine) {
- NJson::TJsonValue plan;
- NJson::ReadJsonTree(result.GetPlan(), &plan, true);
+
+ if (UseNewEngine) {
+ NJson::TJsonValue plan;
+ NJson::ReadJsonTree(result.GetPlan(), &plan, true);
auto node = FindPlanNodeByKv(plan, "Node Type", "Filter-TableFullScan");
- UNIT_ASSERT_C(node.IsDefined(), result.GetPlan());
+ UNIT_ASSERT_C(node.IsDefined(), result.GetPlan());
auto limit = FindPlanNodeByKv(node, "Limit", "Min(1001,$limit)");
UNIT_ASSERT(limit.IsDefined());
- } else {
- UNIT_ASSERT_C(result.GetAst().Contains("KiPartialTake"), result.GetAst());
- }
+ } else {
+ UNIT_ASSERT_C(result.GetAst().Contains("KiPartialTake"), result.GetAst());
+ }
}
{
@@ -1063,91 +1063,91 @@ Y_UNIT_TEST_SUITE(KqpSort) {
auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), params).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- auto actual = ReformatYson(FormatResultSetYson(result.GetResultSet(0)));
-
- if (ReformatYson(R"([
- [[1u];["One"];[-1]];
- [[3u];["Three"];[1]]
- ])") == actual)
- {
- return;
- }
-
- auto expected = R"([
- [[4000000001u];["BigOne"];[-1]];
- [[4000000003u];["BigThree"];[1]]
- ])";
-
- UNIT_ASSERT_NO_DIFF(ReformatYson(expected), actual);
+ auto actual = ReformatYson(FormatResultSetYson(result.GetResultSet(0)));
+
+ if (ReformatYson(R"([
+ [[1u];["One"];[-1]];
+ [[3u];["Three"];[1]]
+ ])") == actual)
+ {
+ return;
+ }
+
+ auto expected = R"([
+ [[4000000001u];["BigOne"];[-1]];
+ [[4000000003u];["BigThree"];[1]]
+ ])";
+
+ UNIT_ASSERT_NO_DIFF(ReformatYson(expected), actual);
}
}
-
+
// https://st.yandex-team.ru/KIKIMR-11523
- Y_UNIT_TEST_NEW_ENGINE(PassLimit) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- {
- auto status = session.ExecuteSchemeQuery(R"(
- create table `/Root/index` (
- id Utf8,
- id2 Utf8,
- op_id Utf8,
- payload Utf8,
- primary key (id, id2, op_id)
- );
-
- create table `/Root/ops` (
- id Utf8,
- payload Utf8,
- primary key (id)
- );
- )").GetValueSync();
- UNIT_ASSERT_C(status.IsSuccess(), status.GetIssues().ToString());
- }
-
- auto result = session.ExplainDataQuery(Q1_(R"(
- declare $idx_id as Utf8;
- declare $limit as Uint64;
-
- $q = (
- SELECT id, id2, op_id, payload
- FROM `/Root/index`
- WHERE id = "1" AND (id2 > "a" OR (id2 = "a" AND op_id > ""U))
- ORDER BY id, id2, op_id
- LIMIT $limit
- );
-
- SELECT ops.id,
- idx.id, idx.id2, idx.op_id
- FROM $q AS idx
- LEFT JOIN `/Root/ops` AS ops ON idx.op_id = ops.id
- )")).GetValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- Cerr << result.GetPlan() << Endl << Endl;
- Cerr << result.GetAst() << Endl;
-
- if (UseNewEngine) {
- NJson::TJsonValue plan;
- NJson::ReadJsonTree(result.GetPlan(), &plan, true);
-
- auto tableLookup = FindPlanNodeByKv(plan, "Node Type", "Filter-TablePointLookup");
- UNIT_ASSERT(tableLookup.IsDefined());
-
- auto& filterOp = tableLookup.GetMapSafe().at("Operators").GetArraySafe().at(0).GetMapSafe();
- UNIT_ASSERT_VALUES_EQUAL("Filter", filterOp.at("Name").GetStringSafe());
- UNIT_ASSERT_VALUES_EQUAL("$limit", filterOp.at("Limit").GetStringSafe());
-
- auto& lookupOp = tableLookup.GetMapSafe().at("Operators").GetArraySafe().at(1).GetMapSafe();
- UNIT_ASSERT_VALUES_EQUAL("TablePointLookup", lookupOp.at("Name").GetStringSafe());
- UNIT_ASSERT_VALUES_EQUAL("index", lookupOp.at("Table").GetStringSafe());
- UNIT_ASSERT(!lookupOp.contains("ReadLimit"));
- } else {
- UNIT_ASSERT(result.GetAst().Contains("(Sort (KiPartialTake (Filter"));
- }
- }
+ Y_UNIT_TEST_NEW_ENGINE(PassLimit) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ {
+ auto status = session.ExecuteSchemeQuery(R"(
+ create table `/Root/index` (
+ id Utf8,
+ id2 Utf8,
+ op_id Utf8,
+ payload Utf8,
+ primary key (id, id2, op_id)
+ );
+
+ create table `/Root/ops` (
+ id Utf8,
+ payload Utf8,
+ primary key (id)
+ );
+ )").GetValueSync();
+ UNIT_ASSERT_C(status.IsSuccess(), status.GetIssues().ToString());
+ }
+
+ auto result = session.ExplainDataQuery(Q1_(R"(
+ declare $idx_id as Utf8;
+ declare $limit as Uint64;
+
+ $q = (
+ SELECT id, id2, op_id, payload
+ FROM `/Root/index`
+ WHERE id = "1" AND (id2 > "a" OR (id2 = "a" AND op_id > ""U))
+ ORDER BY id, id2, op_id
+ LIMIT $limit
+ );
+
+ SELECT ops.id,
+ idx.id, idx.id2, idx.op_id
+ FROM $q AS idx
+ LEFT JOIN `/Root/ops` AS ops ON idx.op_id = ops.id
+ )")).GetValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ Cerr << result.GetPlan() << Endl << Endl;
+ Cerr << result.GetAst() << Endl;
+
+ if (UseNewEngine) {
+ NJson::TJsonValue plan;
+ NJson::ReadJsonTree(result.GetPlan(), &plan, true);
+
+ auto tableLookup = FindPlanNodeByKv(plan, "Node Type", "Filter-TablePointLookup");
+ UNIT_ASSERT(tableLookup.IsDefined());
+
+ auto& filterOp = tableLookup.GetMapSafe().at("Operators").GetArraySafe().at(0).GetMapSafe();
+ UNIT_ASSERT_VALUES_EQUAL("Filter", filterOp.at("Name").GetStringSafe());
+ UNIT_ASSERT_VALUES_EQUAL("$limit", filterOp.at("Limit").GetStringSafe());
+
+ auto& lookupOp = tableLookup.GetMapSafe().at("Operators").GetArraySafe().at(1).GetMapSafe();
+ UNIT_ASSERT_VALUES_EQUAL("TablePointLookup", lookupOp.at("Name").GetStringSafe());
+ UNIT_ASSERT_VALUES_EQUAL("index", lookupOp.at("Table").GetStringSafe());
+ UNIT_ASSERT(!lookupOp.contains("ReadLimit"));
+ } else {
+ UNIT_ASSERT(result.GetAst().Contains("(Sort (KiPartialTake (Filter"));
+ }
+ }
}
} // namespace NKqp
diff --git a/ydb/core/kqp/ut/kqp_sqlin_ut.cpp b/ydb/core/kqp/ut/kqp_sqlin_ut.cpp
index 152209680b..b41df7c39e 100644
--- a/ydb/core/kqp/ut/kqp_sqlin_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_sqlin_ut.cpp
@@ -1,619 +1,619 @@
#include <ydb/core/kqp/ut/common/kqp_ut_common.h>
#include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h>
-
-namespace NKikimr {
-namespace NKqp {
-
-using namespace NYdb;
-using namespace NYdb::NTable;
-
-namespace {
-
-const char* FormatPragma(bool disableOpt) {
- if (disableOpt) {
- return "PRAGMA Kikimr.OptDisableSqlInToJoin = 'True';";
- }
- return "";
-}
-
-const bool DisableOpt = true;
-const bool EnableOpt = false;
-
-} // namespace
-
-
-Y_UNIT_TEST_SUITE(KqpSqlIn) {
-
- Y_UNIT_TEST_NEW_ENGINE(TableSource) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
+
+namespace NKikimr {
+namespace NKqp {
+
+using namespace NYdb;
+using namespace NYdb::NTable;
+
+namespace {
+
+const char* FormatPragma(bool disableOpt) {
+ if (disableOpt) {
+ return "PRAGMA Kikimr.OptDisableSqlInToJoin = 'True';";
+ }
+ return "";
+}
+
+const bool DisableOpt = true;
+const bool EnableOpt = false;
+
+} // namespace
+
+
+Y_UNIT_TEST_SUITE(KqpSqlIn) {
+
+ Y_UNIT_TEST_NEW_ENGINE(TableSource) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
CreateSampleTablesWithIndex(session);
-
- auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) {
- TString query = TStringBuilder()
- << "--!syntax_v1" << Endl
- << FormatPragma(disableOpt)
- << Q_(R"(
- DECLARE $in AS List<Struct<k: Uint64>>;
- SELECT Key, Value
- FROM `/Root/KeyValue` WHERE Key IN (SELECT k FROM AS_TABLE($in))
- )");
-
- auto params = TParamsBuilder().AddParam("$in").BeginList()
- .AddListItem().BeginStruct().AddMember("k").Uint64(1).EndStruct()
- .AddListItem().BeginStruct().AddMember("k").Uint64(2).EndStruct()
- .AddListItem().BeginStruct().AddMember("k").Uint64(5).EndStruct()
- .AddListItem().BeginStruct().AddMember("k").Uint64(10).EndStruct()
- .EndList().Build().Build();
-
- auto result = ExecQueryAndTestResult(session, query, params,
- R"([[[1u];["One"]];
- [[2u];["Two"]];
- [[10u];["Ten"]]])");
- assertFn(result);
- };
-
- test(DisableOpt, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/KeyValue", 3);
- });
-
- test(EnableOpt, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/KeyValue", 3);
- });
- }
-
- Y_UNIT_TEST_NEW_ENGINE(CantRewrite) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
+
+ auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) {
+ TString query = TStringBuilder()
+ << "--!syntax_v1" << Endl
+ << FormatPragma(disableOpt)
+ << Q_(R"(
+ DECLARE $in AS List<Struct<k: Uint64>>;
+ SELECT Key, Value
+ FROM `/Root/KeyValue` WHERE Key IN (SELECT k FROM AS_TABLE($in))
+ )");
+
+ auto params = TParamsBuilder().AddParam("$in").BeginList()
+ .AddListItem().BeginStruct().AddMember("k").Uint64(1).EndStruct()
+ .AddListItem().BeginStruct().AddMember("k").Uint64(2).EndStruct()
+ .AddListItem().BeginStruct().AddMember("k").Uint64(5).EndStruct()
+ .AddListItem().BeginStruct().AddMember("k").Uint64(10).EndStruct()
+ .EndList().Build().Build();
+
+ auto result = ExecQueryAndTestResult(session, query, params,
+ R"([[[1u];["One"]];
+ [[2u];["Two"]];
+ [[10u];["Ten"]]])");
+ assertFn(result);
+ };
+
+ test(DisableOpt, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/KeyValue", 3);
+ });
+
+ test(EnableOpt, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/KeyValue", 3);
+ });
+ }
+
+ Y_UNIT_TEST_NEW_ENGINE(CantRewrite) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
CreateSampleTablesWithIndex(session);
-
- auto query = TStringBuilder()
- << "--!syntax_v1" << Endl
- << Q_(R"(
- DECLARE $in AS List<Uint64>;
- SELECT Key, Value FROM `/Root/KeyValue`
- WHERE Value = "One" AND Key IN $in
- )");
-
- // empty parameters
- {
- TMap<TString, TType> paramsType;
- paramsType.emplace("$in", TTypeBuilder().BeginList().Primitive(EPrimitiveType::Uint64).EndList().Build());
- auto params = TParamsBuilder(paramsType);
-
- auto result = ExecQueryAndTestResult(session, query, params.Build(), R"([])");
- AssertTableReads(result, "/Root/KeyValue", 6);
- }
-
- // not empty parameters
- {
- auto params = TParamsBuilder();
- auto& pl = params.AddParam("$in").BeginList();
- for (auto v : {1, 2, 3, 42, 50, 100}) {
- pl.AddListItem().Uint64(v);
- }
- pl.EndList().Build();
-
- auto result = ExecQueryAndTestResult(session, query, params.Build(), R"([[[1u];["One"]]])");
- AssertTableReads(result, "/Root/KeyValue", 6);
- }
- }
-
- Y_UNIT_TEST_NEW_ENGINE(SimpleKey) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
+
+ auto query = TStringBuilder()
+ << "--!syntax_v1" << Endl
+ << Q_(R"(
+ DECLARE $in AS List<Uint64>;
+ SELECT Key, Value FROM `/Root/KeyValue`
+ WHERE Value = "One" AND Key IN $in
+ )");
+
+ // empty parameters
+ {
+ TMap<TString, TType> paramsType;
+ paramsType.emplace("$in", TTypeBuilder().BeginList().Primitive(EPrimitiveType::Uint64).EndList().Build());
+ auto params = TParamsBuilder(paramsType);
+
+ auto result = ExecQueryAndTestResult(session, query, params.Build(), R"([])");
+ AssertTableReads(result, "/Root/KeyValue", 6);
+ }
+
+ // not empty parameters
+ {
+ auto params = TParamsBuilder();
+ auto& pl = params.AddParam("$in").BeginList();
+ for (auto v : {1, 2, 3, 42, 50, 100}) {
+ pl.AddListItem().Uint64(v);
+ }
+ pl.EndList().Build();
+
+ auto result = ExecQueryAndTestResult(session, query, params.Build(), R"([[[1u];["One"]]])");
+ AssertTableReads(result, "/Root/KeyValue", 6);
+ }
+ }
+
+ Y_UNIT_TEST_NEW_ENGINE(SimpleKey) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
CreateSampleTablesWithIndex(session);
-
- auto test = [&](bool disableOpt, bool optionalParam, std::function<void(const TDataQueryResult&)> assertFn) {
- auto query = TStringBuilder()
- << "--!syntax_v1" << Endl
- << FormatPragma(disableOpt) << Endl
- << Q_(Sprintf(R"(
- DECLARE $in AS List<Uint64%s>;
- SELECT Key, Value FROM `/Root/KeyValue` WHERE Key IN $in
- ORDER BY Key
- )", (optionalParam ? "?" : "")));
-
- auto params = TParamsBuilder();
- auto& pl = params.AddParam("$in").BeginList();
- for (auto v : {1, 2, 3, 42, 50, 100}) {
- if (optionalParam) {
- pl.AddListItem().OptionalUint64(v);
- } else {
- pl.AddListItem().Uint64(v);
- }
- }
- if (optionalParam) {
- pl.AddListItem().OptionalUint64(Nothing());
- }
- pl.EndList().Build();
-
- auto result = ExecQueryAndTestResult(session, query, params.Build(),
- R"([[[1u];["One"]];
- [[2u];["Two"]];
- [[3u];["Three"]]])");
- assertFn(result);
- };
-
- test(DisableOpt, true /* optionalParams */, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/KeyValue", 6);
- });
- test(DisableOpt, false /* optionalParams */, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/KeyValue", 6);
- });
-
- test(EnableOpt, true /* optionalParams */, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/KeyValue", 3);
- });
- test(EnableOpt, false /* optionalParams */, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/KeyValue", 3);
- });
- }
-
- Y_UNIT_TEST_NEW_ENGINE(SimpleKey_Negated) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
+
+ auto test = [&](bool disableOpt, bool optionalParam, std::function<void(const TDataQueryResult&)> assertFn) {
+ auto query = TStringBuilder()
+ << "--!syntax_v1" << Endl
+ << FormatPragma(disableOpt) << Endl
+ << Q_(Sprintf(R"(
+ DECLARE $in AS List<Uint64%s>;
+ SELECT Key, Value FROM `/Root/KeyValue` WHERE Key IN $in
+ ORDER BY Key
+ )", (optionalParam ? "?" : "")));
+
+ auto params = TParamsBuilder();
+ auto& pl = params.AddParam("$in").BeginList();
+ for (auto v : {1, 2, 3, 42, 50, 100}) {
+ if (optionalParam) {
+ pl.AddListItem().OptionalUint64(v);
+ } else {
+ pl.AddListItem().Uint64(v);
+ }
+ }
+ if (optionalParam) {
+ pl.AddListItem().OptionalUint64(Nothing());
+ }
+ pl.EndList().Build();
+
+ auto result = ExecQueryAndTestResult(session, query, params.Build(),
+ R"([[[1u];["One"]];
+ [[2u];["Two"]];
+ [[3u];["Three"]]])");
+ assertFn(result);
+ };
+
+ test(DisableOpt, true /* optionalParams */, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/KeyValue", 6);
+ });
+ test(DisableOpt, false /* optionalParams */, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/KeyValue", 6);
+ });
+
+ test(EnableOpt, true /* optionalParams */, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/KeyValue", 3);
+ });
+ test(EnableOpt, false /* optionalParams */, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/KeyValue", 3);
+ });
+ }
+
+ Y_UNIT_TEST_NEW_ENGINE(SimpleKey_Negated) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
CreateSampleTablesWithIndex(session);
-
- auto query = TStringBuilder()
- << "--!syntax_v1" << Endl
- << Q_(R"(
- DECLARE $in AS List<Uint64?>;
- SELECT Key, Value FROM `/Root/KeyValue` WHERE Key NOT IN $in
- )");
- auto params = TParamsBuilder().AddParam("$in").BeginList()
- .AddListItem().OptionalUint64(1)
- .AddListItem().OptionalUint64(2)
- .AddListItem().OptionalUint64(42)
- .AddListItem().OptionalUint64(Nothing())
- .EndList().Build().Build();
- auto result = ExecQueryAndTestResult(session, query, params,
- R"([[[3u];["Three"]];
- [[4u];["Four"]];
- [[10u];["Ten"]]])");
- AssertTableReads(result, "/Root/KeyValue", 6); // not optimized
- }
-
- Y_UNIT_TEST_NEW_ENGINE(KeySuffix) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
+
+ auto query = TStringBuilder()
+ << "--!syntax_v1" << Endl
+ << Q_(R"(
+ DECLARE $in AS List<Uint64?>;
+ SELECT Key, Value FROM `/Root/KeyValue` WHERE Key NOT IN $in
+ )");
+ auto params = TParamsBuilder().AddParam("$in").BeginList()
+ .AddListItem().OptionalUint64(1)
+ .AddListItem().OptionalUint64(2)
+ .AddListItem().OptionalUint64(42)
+ .AddListItem().OptionalUint64(Nothing())
+ .EndList().Build().Build();
+ auto result = ExecQueryAndTestResult(session, query, params,
+ R"([[[3u];["Three"]];
+ [[4u];["Four"]];
+ [[10u];["Ten"]]])");
+ AssertTableReads(result, "/Root/KeyValue", 6); // not optimized
+ }
+
+ Y_UNIT_TEST_NEW_ENGINE(KeySuffix) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
CreateSampleTablesWithIndex(session);
-
- auto test = [&](bool disableOpt, bool optionalParam, std::function<void(const TDataQueryResult&)> assertFn) {
- auto query = TStringBuilder()
- << "--!syntax_v1" << Endl
- << Q_(Sprintf(R"(
- %s
- DECLARE $in AS List<String%s>;
- SELECT Group, Name, Amount, Comment
- FROM `/Root/Test`
- WHERE Group = 1 AND Name IN $in
- )", FormatPragma(disableOpt), (optionalParam ? "?" : "")));
-
- auto params = TParamsBuilder();
- auto& pl = params.AddParam("$in").BeginList();
- for (auto& v : {"Anna", "Tony", "Bob", "Muhammad", "Jack"}) {
- if (optionalParam) {
- pl.AddListItem().OptionalString(v);
- } else {
- pl.AddListItem().String(v);
- }
- }
- if (optionalParam) {
- pl.AddListItem().OptionalString(Nothing());
- }
- pl.EndList().Build();
-
- auto result = ExecQueryAndTestResult(session, query, params.Build(),
- R"([[[1u];["Anna"];[3500u];["None"]];
- [[1u];["Jack"];[100500u];["Just Jack"]]])");
- assertFn(result);
- };
-
- test(DisableOpt, true /* optionalParam */, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/Test", 3); // 3 rows have Group == 1
- });
- test(DisableOpt, false /* optionalParam */, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/Test", 3);
- });
-
- test(EnableOpt, true /* optionalParam */, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/Test", 2);
- });
- test(EnableOpt, false /* optionalParam */, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/Test", 2);
- });
- }
-
- Y_UNIT_TEST_NEW_ENGINE(KeySuffix_OnlyTail) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
+
+ auto test = [&](bool disableOpt, bool optionalParam, std::function<void(const TDataQueryResult&)> assertFn) {
+ auto query = TStringBuilder()
+ << "--!syntax_v1" << Endl
+ << Q_(Sprintf(R"(
+ %s
+ DECLARE $in AS List<String%s>;
+ SELECT Group, Name, Amount, Comment
+ FROM `/Root/Test`
+ WHERE Group = 1 AND Name IN $in
+ )", FormatPragma(disableOpt), (optionalParam ? "?" : "")));
+
+ auto params = TParamsBuilder();
+ auto& pl = params.AddParam("$in").BeginList();
+ for (auto& v : {"Anna", "Tony", "Bob", "Muhammad", "Jack"}) {
+ if (optionalParam) {
+ pl.AddListItem().OptionalString(v);
+ } else {
+ pl.AddListItem().String(v);
+ }
+ }
+ if (optionalParam) {
+ pl.AddListItem().OptionalString(Nothing());
+ }
+ pl.EndList().Build();
+
+ auto result = ExecQueryAndTestResult(session, query, params.Build(),
+ R"([[[1u];["Anna"];[3500u];["None"]];
+ [[1u];["Jack"];[100500u];["Just Jack"]]])");
+ assertFn(result);
+ };
+
+ test(DisableOpt, true /* optionalParam */, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/Test", 3); // 3 rows have Group == 1
+ });
+ test(DisableOpt, false /* optionalParam */, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/Test", 3);
+ });
+
+ test(EnableOpt, true /* optionalParam */, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/Test", 2);
+ });
+ test(EnableOpt, false /* optionalParam */, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/Test", 2);
+ });
+ }
+
+ Y_UNIT_TEST_NEW_ENGINE(KeySuffix_OnlyTail) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
CreateSampleTablesWithIndex(session);
-
- auto query = TStringBuilder()
- << "--!syntax_v1" << Endl
- << Q_(R"(
- DECLARE $in AS List<String?>;
- SELECT Group, Name, Amount, Comment
- FROM `/Root/Test`
- WHERE Name IN $in
- ORDER BY Group, Name
- )");
- auto params = TParamsBuilder().AddParam("$in").BeginList()
- .AddListItem().OptionalString("Anna")
- .AddListItem().OptionalString("Tony")
- .AddListItem().OptionalString("Hugo")
- .AddListItem().OptionalString("Logan")
- .AddListItem().OptionalString(Nothing())
- .EndList().Build().Build();
- auto result = ExecQueryAndTestResult(session, query, params,
- R"([[[1u];["Anna"];[3500u];["None"]];
- [[2u];["Tony"];[7200u];["None"]];
- [[4u];["Hugo"];[77u];["Boss"]]])");
- AssertTableReads(result, "/Root/Test", 8); // not optimized
- }
-
- Y_UNIT_TEST_NEW_ENGINE(KeySuffix_NotPointPrefix) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
+
+ auto query = TStringBuilder()
+ << "--!syntax_v1" << Endl
+ << Q_(R"(
+ DECLARE $in AS List<String?>;
+ SELECT Group, Name, Amount, Comment
+ FROM `/Root/Test`
+ WHERE Name IN $in
+ ORDER BY Group, Name
+ )");
+ auto params = TParamsBuilder().AddParam("$in").BeginList()
+ .AddListItem().OptionalString("Anna")
+ .AddListItem().OptionalString("Tony")
+ .AddListItem().OptionalString("Hugo")
+ .AddListItem().OptionalString("Logan")
+ .AddListItem().OptionalString(Nothing())
+ .EndList().Build().Build();
+ auto result = ExecQueryAndTestResult(session, query, params,
+ R"([[[1u];["Anna"];[3500u];["None"]];
+ [[2u];["Tony"];[7200u];["None"]];
+ [[4u];["Hugo"];[77u];["Boss"]]])");
+ AssertTableReads(result, "/Root/Test", 8); // not optimized
+ }
+
+ Y_UNIT_TEST_NEW_ENGINE(KeySuffix_NotPointPrefix) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
CreateSampleTablesWithIndex(session);
-
- auto query = TStringBuilder()
- << "--!syntax_v1" << Endl
- << Q_(R"(
- DECLARE $in AS List<String?>;
- SELECT Group, Name, Amount, Comment
- FROM `/Root/Test`
- WHERE (Group > 1 AND Group < 4) AND Name IN $in
- )");
- auto params = TParamsBuilder().AddParam("$in").BeginList()
- .AddListItem().OptionalString("Anna")
- .AddListItem().OptionalString("Tony")
- .AddListItem().OptionalString("Harry")
- .AddListItem().OptionalString("Hugo")
- .AddListItem().OptionalString(Nothing())
- .EndList().Build().Build();
- auto result = ExecQueryAndTestResult(session, query, params,
- R"([[[2u];["Tony"];[7200u];["None"]];
- [[3u];["Harry"];[5600u];["Not Potter"]]])");
- AssertTableReads(result, "/Root/Test", 4); // not optimized
- }
-
- Y_UNIT_TEST_NEW_ENGINE(ComplexKey) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
+
+ auto query = TStringBuilder()
+ << "--!syntax_v1" << Endl
+ << Q_(R"(
+ DECLARE $in AS List<String?>;
+ SELECT Group, Name, Amount, Comment
+ FROM `/Root/Test`
+ WHERE (Group > 1 AND Group < 4) AND Name IN $in
+ )");
+ auto params = TParamsBuilder().AddParam("$in").BeginList()
+ .AddListItem().OptionalString("Anna")
+ .AddListItem().OptionalString("Tony")
+ .AddListItem().OptionalString("Harry")
+ .AddListItem().OptionalString("Hugo")
+ .AddListItem().OptionalString(Nothing())
+ .EndList().Build().Build();
+ auto result = ExecQueryAndTestResult(session, query, params,
+ R"([[[2u];["Tony"];[7200u];["None"]];
+ [[3u];["Harry"];[5600u];["Not Potter"]]])");
+ AssertTableReads(result, "/Root/Test", 4); // not optimized
+ }
+
+ Y_UNIT_TEST_NEW_ENGINE(ComplexKey) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ CreateSampleTablesWithIndex(session);
+
+ auto query = TStringBuilder()
+ << "--!syntax_v1" << Endl
+ << Q_(R"(
+ DECLARE $groups AS List<Uint32?>;
+ DECLARE $names AS List<String?>;
+ SELECT Group, Name, Amount, Comment
+ FROM `/Root/Test`
+ WHERE Group IN $groups AND Name IN $names;
+ )");
+
+ auto params = TParamsBuilder()
+ .AddParam("$groups").BeginList()
+ .AddListItem().OptionalUint32(1)
+ .AddListItem().OptionalUint32(3)
+ .AddListItem().OptionalUint32(50)
+ .EndList().Build()
+ .AddParam("$names").BeginList()
+ .AddListItem().OptionalString("Anna")
+ .AddListItem().OptionalString("Tony")
+ .AddListItem().OptionalString("Jack")
+ .AddListItem().OptionalString("Hugo")
+ .AddListItem().OptionalString(Nothing())
+ .EndList().Build()
+ .Build();
+
+ auto result = ExecQueryAndTestResult(session, query, params,
+ R"([[[1u];["Anna"];[3500u];["None"]];
+ [[1u];["Jack"];[100500u];["Just Jack"]]])");
+ AssertTableReads(result, "/Root/Test", 6);
+ }
+
+ Y_UNIT_TEST_NEW_ENGINE(KeyTypeMissmatch_Int) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
CreateSampleTablesWithIndex(session);
-
- auto query = TStringBuilder()
- << "--!syntax_v1" << Endl
- << Q_(R"(
- DECLARE $groups AS List<Uint32?>;
- DECLARE $names AS List<String?>;
- SELECT Group, Name, Amount, Comment
- FROM `/Root/Test`
- WHERE Group IN $groups AND Name IN $names;
- )");
-
- auto params = TParamsBuilder()
- .AddParam("$groups").BeginList()
- .AddListItem().OptionalUint32(1)
- .AddListItem().OptionalUint32(3)
- .AddListItem().OptionalUint32(50)
- .EndList().Build()
- .AddParam("$names").BeginList()
- .AddListItem().OptionalString("Anna")
- .AddListItem().OptionalString("Tony")
- .AddListItem().OptionalString("Jack")
- .AddListItem().OptionalString("Hugo")
- .AddListItem().OptionalString(Nothing())
- .EndList().Build()
- .Build();
-
- auto result = ExecQueryAndTestResult(session, query, params,
- R"([[[1u];["Anna"];[3500u];["None"]];
- [[1u];["Jack"];[100500u];["Just Jack"]]])");
- AssertTableReads(result, "/Root/Test", 6);
- }
-
- Y_UNIT_TEST_NEW_ENGINE(KeyTypeMissmatch_Int) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- CreateSampleTablesWithIndex(session);
-
- auto query = TStringBuilder()
- << "--!syntax_v1" << Endl
- << Q_(R"(
- DECLARE $groups AS List<Int32>; -- not Uint32
- SELECT Group, Name, Amount, Comment
- FROM `/Root/Test`
- WHERE Group IN $groups;
- )");
-
- auto params = TParamsBuilder()
- .AddParam("$groups").BeginList()
- .AddListItem().Int32(1)
- .AddListItem().Int32(-1)
- .EndList().Build()
- .Build();
-
- auto result = ExecQueryAndTestResult(session, query, params,
- R"([[[1u];["Anna"];[3500u];["None"]];
- [[1u];["Jack"];[100500u];["Just Jack"]];
- [[1u];["Paul"];[300u];["None"]]])");
- AssertTableReads(result, "/Root/Test", UseNewEngine ? 3 : 8);
- }
-
- Y_UNIT_TEST_NEW_ENGINE(KeyTypeMissmatch_Str) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto query = TStringBuilder()
- << "--!syntax_v1" << Endl
- << Q_(R"(
- DECLARE $keys AS List<Utf8>; -- not String
- SELECT Key, Value
- FROM `/Root/KeyValue2`
- WHERE Key IN $keys;
- )");
-
- auto params = TParamsBuilder()
- .AddParam("$keys").BeginList()
- .AddListItem().Utf8("1")
+
+ auto query = TStringBuilder()
+ << "--!syntax_v1" << Endl
+ << Q_(R"(
+ DECLARE $groups AS List<Int32>; -- not Uint32
+ SELECT Group, Name, Amount, Comment
+ FROM `/Root/Test`
+ WHERE Group IN $groups;
+ )");
+
+ auto params = TParamsBuilder()
+ .AddParam("$groups").BeginList()
+ .AddListItem().Int32(1)
+ .AddListItem().Int32(-1)
+ .EndList().Build()
+ .Build();
+
+ auto result = ExecQueryAndTestResult(session, query, params,
+ R"([[[1u];["Anna"];[3500u];["None"]];
+ [[1u];["Jack"];[100500u];["Just Jack"]];
+ [[1u];["Paul"];[300u];["None"]]])");
+ AssertTableReads(result, "/Root/Test", UseNewEngine ? 3 : 8);
+ }
+
+ Y_UNIT_TEST_NEW_ENGINE(KeyTypeMissmatch_Str) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto query = TStringBuilder()
+ << "--!syntax_v1" << Endl
+ << Q_(R"(
+ DECLARE $keys AS List<Utf8>; -- not String
+ SELECT Key, Value
+ FROM `/Root/KeyValue2`
+ WHERE Key IN $keys;
+ )");
+
+ auto params = TParamsBuilder()
+ .AddParam("$keys").BeginList()
+ .AddListItem().Utf8("1")
.AddListItem().Utf8("¿cómo estás?")
- .EndList().Build()
- .Build();
-
- auto result = ExecQueryAndTestResult(session, query, params, R"([[["1"];["One"]]])");
- AssertTableReads(result, "/Root/KeyValue2", 1);
- }
-
- Y_UNIT_TEST_NEW_ENGINE(Dict) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
+ .EndList().Build()
+ .Build();
+
+ auto result = ExecQueryAndTestResult(session, query, params, R"([[["1"];["One"]]])");
+ AssertTableReads(result, "/Root/KeyValue2", 1);
+ }
+
+ Y_UNIT_TEST_NEW_ENGINE(Dict) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
CreateSampleTablesWithIndex(session);
-
- auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) {
- auto query = TStringBuilder()
- << "--!syntax_v1" << Endl
- << Q_(Sprintf(R"(
- %s
- DECLARE $in AS Dict<Uint64, String>;
- SELECT Key, Value FROM `/Root/KeyValue` WHERE Key IN $in
- ORDER BY Key
- )", FormatPragma(disableOpt)));
-
- auto params = TParamsBuilder().AddParam("$in").BeginDict()
- .AddDictItem().DictKey().Uint64(1).DictPayload().String("any-1")
- .AddDictItem().DictKey().Uint64(2).DictPayload().String("any-2")
- .AddDictItem().DictKey().Uint64(7).DictPayload().String("any-7")
- .EndDict().Build().Build();
-
- auto result = ExecQueryAndTestResult(session, query, params,
- R"([[[1u];["One"]];
- [[2u];["Two"]]])");
- assertFn(result);
- };
-
- test(DisableOpt, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/KeyValue", 6);
- });
-
- test(EnableOpt, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/KeyValue", 2);
- });
- }
-
- Y_UNIT_TEST_NEW_ENGINE(TupleParameter) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
+
+ auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) {
+ auto query = TStringBuilder()
+ << "--!syntax_v1" << Endl
+ << Q_(Sprintf(R"(
+ %s
+ DECLARE $in AS Dict<Uint64, String>;
+ SELECT Key, Value FROM `/Root/KeyValue` WHERE Key IN $in
+ ORDER BY Key
+ )", FormatPragma(disableOpt)));
+
+ auto params = TParamsBuilder().AddParam("$in").BeginDict()
+ .AddDictItem().DictKey().Uint64(1).DictPayload().String("any-1")
+ .AddDictItem().DictKey().Uint64(2).DictPayload().String("any-2")
+ .AddDictItem().DictKey().Uint64(7).DictPayload().String("any-7")
+ .EndDict().Build().Build();
+
+ auto result = ExecQueryAndTestResult(session, query, params,
+ R"([[[1u];["One"]];
+ [[2u];["Two"]]])");
+ assertFn(result);
+ };
+
+ test(DisableOpt, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/KeyValue", 6);
+ });
+
+ test(EnableOpt, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/KeyValue", 2);
+ });
+ }
+
+ Y_UNIT_TEST_NEW_ENGINE(TupleParameter) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
CreateSampleTablesWithIndex(session);
-
- auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) {
- auto query = TStringBuilder()
- << "--!syntax_v1" << Endl
- << Q_(Sprintf(R"(
- %s
- DECLARE $in AS List<Tuple<String, Uint32>>;
- SELECT Group AS g, Amount AS a, Comment
- FROM `/Root/Test`
- WHERE (Name, Group) IN $in -- reverse key prefix
- ORDER BY g, a
- )", FormatPragma(disableOpt)));
-
- auto params = TParamsBuilder().AddParam("$in").BeginList()
- .AddListItem().BeginTuple().AddElement().String("Anna").AddElement().Uint32(1).EndTuple()
- .AddListItem().BeginTuple().AddElement().String("Jack").AddElement().Uint32(1).EndTuple()
- .AddListItem().BeginTuple().AddElement().String("Hugo").AddElement().Uint32(4).EndTuple()
- .AddListItem().BeginTuple().AddElement().String("Harry").AddElement().Uint32(9).EndTuple()
- .EndList().Build().Build();
-
- auto result = ExecQueryAndTestResult(session, query, params,
- R"([[[1u];[3500u];["None"]];
- [[1u];[100500u];["Just Jack"]];
- [[4u];[77u];["Boss"]]])");
- assertFn(result);
- };
-
- test(DisableOpt, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/Test", 8);
- });
-
- test(EnableOpt, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/Test", 3);
- });
- }
-
- Y_UNIT_TEST_NEW_ENGINE(TupleLiteral) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
+
+ auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) {
+ auto query = TStringBuilder()
+ << "--!syntax_v1" << Endl
+ << Q_(Sprintf(R"(
+ %s
+ DECLARE $in AS List<Tuple<String, Uint32>>;
+ SELECT Group AS g, Amount AS a, Comment
+ FROM `/Root/Test`
+ WHERE (Name, Group) IN $in -- reverse key prefix
+ ORDER BY g, a
+ )", FormatPragma(disableOpt)));
+
+ auto params = TParamsBuilder().AddParam("$in").BeginList()
+ .AddListItem().BeginTuple().AddElement().String("Anna").AddElement().Uint32(1).EndTuple()
+ .AddListItem().BeginTuple().AddElement().String("Jack").AddElement().Uint32(1).EndTuple()
+ .AddListItem().BeginTuple().AddElement().String("Hugo").AddElement().Uint32(4).EndTuple()
+ .AddListItem().BeginTuple().AddElement().String("Harry").AddElement().Uint32(9).EndTuple()
+ .EndList().Build().Build();
+
+ auto result = ExecQueryAndTestResult(session, query, params,
+ R"([[[1u];[3500u];["None"]];
+ [[1u];[100500u];["Just Jack"]];
+ [[4u];[77u];["Boss"]]])");
+ assertFn(result);
+ };
+
+ test(DisableOpt, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/Test", 8);
+ });
+
+ test(EnableOpt, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/Test", 3);
+ });
+ }
+
+ Y_UNIT_TEST_NEW_ENGINE(TupleLiteral) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
CreateSampleTablesWithIndex(session);
-
- auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) {
- auto query = TStringBuilder()
- << "--!syntax_v1" << Endl
- << Q_(Sprintf(R"(
- %s
- SELECT Group AS g, Amount AS a, Comment
- FROM `/Root/Test`
- WHERE (Group, Name) IN AsList(AsTuple(1u, "Anna"), AsTuple(1u, "Jack"),
- AsTuple(4u, "Hugo"), AsTuple(9u, "Harry"))
- ORDER BY g, a
- )", FormatPragma(disableOpt)));
-
- auto result = ExecQueryAndTestResult(session, query, TParamsBuilder().Build(),
- R"([[[1u];[3500u];["None"]];
- [[1u];[100500u];["Just Jack"]];
- [[4u];[77u];["Boss"]]])");
- assertFn(result);
- };
-
- test(DisableOpt, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/Test", 8);
- });
-
- test(EnableOpt, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/Test", 3);
- });
- }
-
- Y_UNIT_TEST_NEW_ENGINE(TupleSelect) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
+
+ auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) {
+ auto query = TStringBuilder()
+ << "--!syntax_v1" << Endl
+ << Q_(Sprintf(R"(
+ %s
+ SELECT Group AS g, Amount AS a, Comment
+ FROM `/Root/Test`
+ WHERE (Group, Name) IN AsList(AsTuple(1u, "Anna"), AsTuple(1u, "Jack"),
+ AsTuple(4u, "Hugo"), AsTuple(9u, "Harry"))
+ ORDER BY g, a
+ )", FormatPragma(disableOpt)));
+
+ auto result = ExecQueryAndTestResult(session, query, TParamsBuilder().Build(),
+ R"([[[1u];[3500u];["None"]];
+ [[1u];[100500u];["Just Jack"]];
+ [[4u];[77u];["Boss"]]])");
+ assertFn(result);
+ };
+
+ test(DisableOpt, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/Test", 8);
+ });
+
+ test(EnableOpt, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/Test", 3);
+ });
+ }
+
+ Y_UNIT_TEST_NEW_ENGINE(TupleSelect) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
CreateSampleTablesWithIndex(session);
-
- auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) {
- auto query = TStringBuilder()
+
+ auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) {
+ auto query = TStringBuilder()
<< Q1_(Sprintf(R"(
- PRAGMA Kikimr.OptDisableSqlInToJoin = "True";
+ PRAGMA Kikimr.OptDisableSqlInToJoin = "True";
PRAGMA Kikimr.OptDisableJoinReverseTableLookupLeftSemi = "%s"; -- not depends on `Kikimr.OptDisableSqlInToJoin` pragma
- DECLARE $in AS List<Struct<k: Uint32, v: String>>;
- SELECT Group AS g, Amount AS a, Comment
- FROM `/Root/Test`
- WHERE (Group, Name) IN (SELECT (k, v) FROM AS_TABLE($in)) -- table source
- ORDER BY g, a
- )", disableOpt ? "True" : "False"));
-
- auto params = TParamsBuilder().AddParam("$in").BeginList()
- .AddListItem().BeginStruct().AddMember("k").Uint32(1).AddMember("v").String("Anna").EndStruct()
- .AddListItem().BeginStruct().AddMember("k").Uint32(1).AddMember("v").String("Jack").EndStruct()
- .AddListItem().BeginStruct().AddMember("k").Uint32(4).AddMember("v").String("Hugo").EndStruct()
- .AddListItem().BeginStruct().AddMember("k").Uint32(9).AddMember("v").String("Harry").EndStruct()
- .EndList().Build().Build();
-
- auto result = ExecQueryAndTestResult(session, query, params,
- R"([[[1u];[3500u];["None"]];
- [[1u];[100500u];["Just Jack"]];
- [[4u];[77u];["Boss"]]])");
- assertFn(result);
- };
-
- test(DisableOpt, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/Test", 8);
- });
-
- test(EnableOpt, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/Test", 3);
- });
- }
-
- Y_UNIT_TEST_NEW_ENGINE(SelectNotAllElements) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
+ DECLARE $in AS List<Struct<k: Uint32, v: String>>;
+ SELECT Group AS g, Amount AS a, Comment
+ FROM `/Root/Test`
+ WHERE (Group, Name) IN (SELECT (k, v) FROM AS_TABLE($in)) -- table source
+ ORDER BY g, a
+ )", disableOpt ? "True" : "False"));
+
+ auto params = TParamsBuilder().AddParam("$in").BeginList()
+ .AddListItem().BeginStruct().AddMember("k").Uint32(1).AddMember("v").String("Anna").EndStruct()
+ .AddListItem().BeginStruct().AddMember("k").Uint32(1).AddMember("v").String("Jack").EndStruct()
+ .AddListItem().BeginStruct().AddMember("k").Uint32(4).AddMember("v").String("Hugo").EndStruct()
+ .AddListItem().BeginStruct().AddMember("k").Uint32(9).AddMember("v").String("Harry").EndStruct()
+ .EndList().Build().Build();
+
+ auto result = ExecQueryAndTestResult(session, query, params,
+ R"([[[1u];[3500u];["None"]];
+ [[1u];[100500u];["Just Jack"]];
+ [[4u];[77u];["Boss"]]])");
+ assertFn(result);
+ };
+
+ test(DisableOpt, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/Test", 8);
+ });
+
+ test(EnableOpt, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/Test", 3);
+ });
+ }
+
+ Y_UNIT_TEST_NEW_ENGINE(SelectNotAllElements) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
CreateSampleTablesWithIndex(session);
-
- const TString query = Q1_(R"(
- DECLARE $in AS List<Int32?>;
- DECLARE $in2 AS List<String?>;
- SELECT Value AS V FROM `/Root/SecondaryKeys`
- WHERE Key IN $in AND Value IN $in2
- ORDER BY V
- )");
-
- auto params = TParamsBuilder()
- .AddParam("$in").BeginList()
- .AddListItem().OptionalInt32(1)
- .AddListItem().OptionalInt32(2)
- .AddListItem().OptionalInt32(42)
- .AddListItem().OptionalInt32(Nothing())
- .EndList().Build()
- .AddParam("$in2").BeginList()
- .AddListItem().OptionalString("Payload1")
- .AddListItem().OptionalString("Payload2")
- .AddListItem().OptionalString("Payload0")
- .AddListItem().OptionalString(Nothing())
- .EndList().Build()
- .Build();
-
- auto result = ExecQueryAndTestResult(session, query, params, R"([[["Payload1"]];[["Payload2"]]])");
- AssertTableReads(result, "/Root/SecondaryKeys", 2);
- }
-
+
+ const TString query = Q1_(R"(
+ DECLARE $in AS List<Int32?>;
+ DECLARE $in2 AS List<String?>;
+ SELECT Value AS V FROM `/Root/SecondaryKeys`
+ WHERE Key IN $in AND Value IN $in2
+ ORDER BY V
+ )");
+
+ auto params = TParamsBuilder()
+ .AddParam("$in").BeginList()
+ .AddListItem().OptionalInt32(1)
+ .AddListItem().OptionalInt32(2)
+ .AddListItem().OptionalInt32(42)
+ .AddListItem().OptionalInt32(Nothing())
+ .EndList().Build()
+ .AddParam("$in2").BeginList()
+ .AddListItem().OptionalString("Payload1")
+ .AddListItem().OptionalString("Payload2")
+ .AddListItem().OptionalString("Payload0")
+ .AddListItem().OptionalString(Nothing())
+ .EndList().Build()
+ .Build();
+
+ auto result = ExecQueryAndTestResult(session, query, params, R"([[["Payload1"]];[["Payload2"]]])");
+ AssertTableReads(result, "/Root/SecondaryKeys", 2);
+ }
+
Y_UNIT_TEST_NEW_ENGINE(SecondaryIndex_SimpleKey) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
CreateSampleTablesWithIndex(session);
-
- auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) {
+
+ auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) {
const TString query = Q1_(Sprintf(R"(
- %s
- DECLARE $in AS List<Int32?>;
- SELECT Value FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk IN $in
- ORDER BY Value
+ %s
+ DECLARE $in AS List<Int32?>;
+ SELECT Value FROM `/Root/SecondaryKeys` VIEW Index WHERE Fk IN $in
+ ORDER BY Value
)", FormatPragma(disableOpt)));
-
- auto params = TParamsBuilder().AddParam("$in").BeginList()
- .AddListItem().OptionalInt32(1)
- .AddListItem().OptionalInt32(2)
- .AddListItem().OptionalInt32(42)
- .AddListItem().OptionalInt32(Nothing())
- .EndList().Build().Build();
-
- auto result = ExecQueryAndTestResult(session, query, params, R"([[["Payload1"]];[["Payload2"]]])");
- assertFn(result);
- };
-
- test(DisableOpt, [](const TDataQueryResult& result) {
+
+ auto params = TParamsBuilder().AddParam("$in").BeginList()
+ .AddListItem().OptionalInt32(1)
+ .AddListItem().OptionalInt32(2)
+ .AddListItem().OptionalInt32(42)
+ .AddListItem().OptionalInt32(Nothing())
+ .EndList().Build().Build();
+
+ auto result = ExecQueryAndTestResult(session, query, params, R"([[["Payload1"]];[["Payload2"]]])");
+ assertFn(result);
+ };
+
+ test(DisableOpt, [](const TDataQueryResult& result) {
AssertTableReads(result, "/Root/SecondaryKeys/Index/indexImplTable", 5);
- AssertTableReads(result, "/Root/SecondaryKeys", 5);
- });
-
- test(EnableOpt, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/SecondaryKeys/Index/indexImplTable", 2);
- AssertTableReads(result, "/Root/SecondaryKeys", 2);
- });
- }
-
+ AssertTableReads(result, "/Root/SecondaryKeys", 5);
+ });
+
+ test(EnableOpt, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/SecondaryKeys/Index/indexImplTable", 2);
+ AssertTableReads(result, "/Root/SecondaryKeys", 2);
+ });
+ }
+
Y_UNIT_TEST_NEW_ENGINE(SecondaryIndex_SimpleKey_In_And) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
@@ -788,196 +788,196 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) {
}
Y_UNIT_TEST_NEW_ENGINE(SecondaryIndex_TupleParameter) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
CreateSampleTablesWithIndex(session);
-
- auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) {
+
+ auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) {
const TString query = Q1_(Sprintf(R"(
- %s
- DECLARE $in AS List<Tuple<Int32?, String>>;
- SELECT Value
- FROM `/Root/SecondaryComplexKeys` VIEW Index
- WHERE (Fk1, Fk2) IN $in
- ORDER BY Value
+ %s
+ DECLARE $in AS List<Tuple<Int32?, String>>;
+ SELECT Value
+ FROM `/Root/SecondaryComplexKeys` VIEW Index
+ WHERE (Fk1, Fk2) IN $in
+ ORDER BY Value
)", FormatPragma(disableOpt)));
-
- auto params = TParamsBuilder().AddParam("$in").BeginList()
- .AddListItem().BeginTuple().AddElement().OptionalInt32(1).AddElement().String("Fk1").EndTuple()
- .AddListItem().BeginTuple().AddElement().OptionalInt32(2).AddElement().String("Fk2").EndTuple()
- .AddListItem().BeginTuple().AddElement().OptionalInt32(42).AddElement().String("Fk5").EndTuple()
- .AddListItem().BeginTuple().AddElement().OptionalInt32(Nothing()).AddElement().String("FkNull").EndTuple()
- .EndList().Build().Build();
-
- auto result = ExecQueryAndTestResult(session, query, params, R"([[["Payload1"]];[["Payload2"]]])");
- assertFn(result);
- };
-
- test(DisableOpt, [](const TDataQueryResult& result) {
+
+ auto params = TParamsBuilder().AddParam("$in").BeginList()
+ .AddListItem().BeginTuple().AddElement().OptionalInt32(1).AddElement().String("Fk1").EndTuple()
+ .AddListItem().BeginTuple().AddElement().OptionalInt32(2).AddElement().String("Fk2").EndTuple()
+ .AddListItem().BeginTuple().AddElement().OptionalInt32(42).AddElement().String("Fk5").EndTuple()
+ .AddListItem().BeginTuple().AddElement().OptionalInt32(Nothing()).AddElement().String("FkNull").EndTuple()
+ .EndList().Build().Build();
+
+ auto result = ExecQueryAndTestResult(session, query, params, R"([[["Payload1"]];[["Payload2"]]])");
+ assertFn(result);
+ };
+
+ test(DisableOpt, [](const TDataQueryResult& result) {
AssertTableReads(result, "/Root/SecondaryComplexKeys/Index/indexImplTable", 5);
- AssertTableReads(result, "/Root/SecondaryComplexKeys", 5);
- });
-
- test(EnableOpt, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/SecondaryComplexKeys/Index/indexImplTable", 2);
- AssertTableReads(result, "/Root/SecondaryComplexKeys", 2);
- });
- }
-
+ AssertTableReads(result, "/Root/SecondaryComplexKeys", 5);
+ });
+
+ test(EnableOpt, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/SecondaryComplexKeys/Index/indexImplTable", 2);
+ AssertTableReads(result, "/Root/SecondaryComplexKeys", 2);
+ });
+ }
+
Y_UNIT_TEST_NEW_ENGINE(SecondaryIndex_TupleLiteral) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
CreateSampleTablesWithIndex(session);
-
- auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) {
+
+ auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) {
const TString query = Q1_(Sprintf(R"(
- %s
- SELECT Value
- FROM `/Root/SecondaryComplexKeys` VIEW Index
- WHERE (Fk1, Fk2) IN AsList((1, "Fk1"), (2, "Fk2"), (42, "Fk5"), (Null, "FkNull"))
- ORDER BY Value
+ %s
+ SELECT Value
+ FROM `/Root/SecondaryComplexKeys` VIEW Index
+ WHERE (Fk1, Fk2) IN AsList((1, "Fk1"), (2, "Fk2"), (42, "Fk5"), (Null, "FkNull"))
+ ORDER BY Value
)", FormatPragma(disableOpt)));
-
- auto result = ExecQueryAndTestResult(session, query, R"([[["Payload1"]];[["Payload2"]]])");
- assertFn(result);
- };
-
- test(DisableOpt, [](const TDataQueryResult& result) {
+
+ auto result = ExecQueryAndTestResult(session, query, R"([[["Payload1"]];[["Payload2"]]])");
+ assertFn(result);
+ };
+
+ test(DisableOpt, [](const TDataQueryResult& result) {
AssertTableReads(result, "/Root/SecondaryComplexKeys/Index/indexImplTable", 5);
- AssertTableReads(result, "/Root/SecondaryComplexKeys", 5);
- });
-
- test(EnableOpt, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/SecondaryComplexKeys/Index/indexImplTable", 2);
- AssertTableReads(result, "/Root/SecondaryComplexKeys", 2);
- });
- }
-
+ AssertTableReads(result, "/Root/SecondaryComplexKeys", 5);
+ });
+
+ test(EnableOpt, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/SecondaryComplexKeys/Index/indexImplTable", 2);
+ AssertTableReads(result, "/Root/SecondaryComplexKeys", 2);
+ });
+ }
+
Y_UNIT_TEST_NEW_ENGINE(SecondaryIndex_TupleSelect) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
CreateSampleTablesWithIndex(session);
-
- auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) {
+
+ auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) {
const TString query = Q1_(Sprintf(R"(
- PRAGMA Kikimr.OptDisableSqlInToJoin = "True";
+ PRAGMA Kikimr.OptDisableSqlInToJoin = "True";
PRAGMA Kikimr.OptDisableJoinReverseTableLookupLeftSemi = "%s";
- DECLARE $in AS List<Struct<k: Int32?, v: String>>;
- SELECT Value
- FROM `/Root/SecondaryComplexKeys` VIEW Index
- WHERE (Fk1, Fk2) IN (SELECT (k, v) FROM AS_TABLE($in))
- ORDER BY Value
+ DECLARE $in AS List<Struct<k: Int32?, v: String>>;
+ SELECT Value
+ FROM `/Root/SecondaryComplexKeys` VIEW Index
+ WHERE (Fk1, Fk2) IN (SELECT (k, v) FROM AS_TABLE($in))
+ ORDER BY Value
)", disableOpt ? "True" : "False"));
-
- auto params = TParamsBuilder().AddParam("$in").BeginList()
- .AddListItem().BeginStruct().AddMember("k").OptionalInt32(1).AddMember("v").String("Fk1").EndStruct()
- .AddListItem().BeginStruct().AddMember("k").OptionalInt32(2).AddMember("v").String("Fk2").EndStruct()
- .AddListItem().BeginStruct().AddMember("k").OptionalInt32(42).AddMember("v").String("Fk5").EndStruct()
- .AddListItem().BeginStruct().AddMember("k").OptionalInt32(Nothing()).AddMember("v").String("FkNull").EndStruct()
- .EndList().Build().Build();
-
- auto result = ExecQueryAndTestResult(session, query, params, R"([[["Payload1"]];[["Payload2"]]])");
- assertFn(result);
- };
-
- test(DisableOpt, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/SecondaryComplexKeys/Index/indexImplTable", 5);
- AssertTableReads(result, "/Root/SecondaryComplexKeys", 5);
- });
-
- test(EnableOpt, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/SecondaryComplexKeys/Index/indexImplTable", 2);
- AssertTableReads(result, "/Root/SecondaryComplexKeys", 2);
- });
- }
-
- Y_UNIT_TEST_NEW_ENGINE(TupleNotOnlyOfKeys) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
+
+ auto params = TParamsBuilder().AddParam("$in").BeginList()
+ .AddListItem().BeginStruct().AddMember("k").OptionalInt32(1).AddMember("v").String("Fk1").EndStruct()
+ .AddListItem().BeginStruct().AddMember("k").OptionalInt32(2).AddMember("v").String("Fk2").EndStruct()
+ .AddListItem().BeginStruct().AddMember("k").OptionalInt32(42).AddMember("v").String("Fk5").EndStruct()
+ .AddListItem().BeginStruct().AddMember("k").OptionalInt32(Nothing()).AddMember("v").String("FkNull").EndStruct()
+ .EndList().Build().Build();
+
+ auto result = ExecQueryAndTestResult(session, query, params, R"([[["Payload1"]];[["Payload2"]]])");
+ assertFn(result);
+ };
+
+ test(DisableOpt, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/SecondaryComplexKeys/Index/indexImplTable", 5);
+ AssertTableReads(result, "/Root/SecondaryComplexKeys", 5);
+ });
+
+ test(EnableOpt, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/SecondaryComplexKeys/Index/indexImplTable", 2);
+ AssertTableReads(result, "/Root/SecondaryComplexKeys", 2);
+ });
+ }
+
+ Y_UNIT_TEST_NEW_ENGINE(TupleNotOnlyOfKeys) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
CreateSampleTablesWithIndex(session);
-
- auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) {
- auto query = TStringBuilder()
+
+ auto test = [&](bool disableOpt, std::function<void(const TDataQueryResult&)> assertFn) {
+ auto query = TStringBuilder()
<< Q1_(Sprintf(R"(
- %s
- DECLARE $in AS List<Tuple<String, Uint32>>;
- SELECT Group AS g, Name as n, Amount AS a
- FROM `/Root/Test`
- WHERE (Comment, Group) IN $in -- (not key, key prefix)
- ORDER BY g, n, a
- )", FormatPragma(disableOpt)));
-
- auto params = TParamsBuilder().AddParam("$in").BeginList()
- .AddListItem().BeginTuple()
- .AddElement().String("None")
- .AddElement().Uint32(1).EndTuple()
- .AddListItem().BeginTuple()
- .AddElement().String("Just Jack")
- .AddElement().Uint32(1).EndTuple()
- .AddListItem().BeginTuple()
- .AddElement().String("Boss")
- .AddElement().Uint32(4).EndTuple()
- .AddListItem().BeginTuple()
- .AddElement().String("Not Potter")
- .AddElement().Uint32(9).EndTuple()
- .EndList().Build().Build();
-
- auto result = ExecQueryAndTestResult(session, query, params,
- R"([[[1u];["Anna"];[3500u]];
- [[1u];["Jack"];[100500u]];
- [[1u];["Paul"];[300u]];
- [[4u];["Hugo"];[77u]]])");
- assertFn(result);
- };
-
- test(DisableOpt, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/Test", 8);
- });
-
- test(EnableOpt, [](const TDataQueryResult& result) {
- AssertTableReads(result, "/Root/Test", 8); // todo: should be 4
- });
- }
-
- Y_UNIT_TEST_NEW_ENGINE(Delete) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
+ %s
+ DECLARE $in AS List<Tuple<String, Uint32>>;
+ SELECT Group AS g, Name as n, Amount AS a
+ FROM `/Root/Test`
+ WHERE (Comment, Group) IN $in -- (not key, key prefix)
+ ORDER BY g, n, a
+ )", FormatPragma(disableOpt)));
+
+ auto params = TParamsBuilder().AddParam("$in").BeginList()
+ .AddListItem().BeginTuple()
+ .AddElement().String("None")
+ .AddElement().Uint32(1).EndTuple()
+ .AddListItem().BeginTuple()
+ .AddElement().String("Just Jack")
+ .AddElement().Uint32(1).EndTuple()
+ .AddListItem().BeginTuple()
+ .AddElement().String("Boss")
+ .AddElement().Uint32(4).EndTuple()
+ .AddListItem().BeginTuple()
+ .AddElement().String("Not Potter")
+ .AddElement().Uint32(9).EndTuple()
+ .EndList().Build().Build();
+
+ auto result = ExecQueryAndTestResult(session, query, params,
+ R"([[[1u];["Anna"];[3500u]];
+ [[1u];["Jack"];[100500u]];
+ [[1u];["Paul"];[300u]];
+ [[4u];["Hugo"];[77u]]])");
+ assertFn(result);
+ };
+
+ test(DisableOpt, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/Test", 8);
+ });
+
+ test(EnableOpt, [](const TDataQueryResult& result) {
+ AssertTableReads(result, "/Root/Test", 8); // todo: should be 4
+ });
+ }
+
+ Y_UNIT_TEST_NEW_ENGINE(Delete) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
CreateSampleTablesWithIndex(session);
-
- auto query = TStringBuilder()
+
+ auto query = TStringBuilder()
<< Q1_(R"(
- DECLARE $in AS List<Uint64>;
- DELETE FROM `/Root/KeyValue` WHERE Key IN $in
- )");
-
- auto params = TParamsBuilder().AddParam("$in").BeginList()
- .AddListItem().Uint64(1)
- .AddListItem().Uint64(2)
- .AddListItem().Uint64(42)
- .EndList().Build().Build();
-
- NYdb::NTable::TExecDataQuerySettings settings;
- settings.CollectQueryStats(ECollectQueryStatsMode::Basic);
-
- auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), params, settings)
- .ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
+ DECLARE $in AS List<Uint64>;
+ DELETE FROM `/Root/KeyValue` WHERE Key IN $in
+ )");
+
+ auto params = TParamsBuilder().AddParam("$in").BeginList()
+ .AddListItem().Uint64(1)
+ .AddListItem().Uint64(2)
+ .AddListItem().Uint64(42)
+ .EndList().Build().Build();
+
+ NYdb::NTable::TExecDataQuerySettings settings;
+ settings.CollectQueryStats(ECollectQueryStatsMode::Basic);
+
+ auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), params, settings)
+ .ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
AssertTableStats(result, "/Root/KeyValue", {
.ExpectedReads = UseNewEngine ? 0 : 2,
.ExpectedDeletes = UseNewEngine ? 3 : 2,
});
- }
+ }
Y_UNIT_TEST_NEW_ENGINE(InWithCast) {
TKikimrRunner kikimr;
@@ -1009,7 +1009,7 @@ Y_UNIT_TEST_SUITE(KqpSqlIn) {
.ExpectedReads = UseNewEngine ? 1 : 6,
});
}
-}
-
-} // namespace NKqp
-} // namespace NKikimr
+}
+
+} // namespace NKqp
+} // namespace NKikimr
diff --git a/ydb/core/kqp/ut/kqp_stats_ut.cpp b/ydb/core/kqp/ut/kqp_stats_ut.cpp
index c1bd9cb645..f7815c016e 100644
--- a/ydb/core/kqp/ut/kqp_stats_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_stats_ut.cpp
@@ -75,12 +75,12 @@ Y_UNIT_TEST(JoinStatsBasic) {
UNIT_ASSERT_VALUES_EQUAL(res.QueryStats->query_phases(0).table_access(0).partitions_count(), 1);
UNIT_ASSERT_VALUES_EQUAL(res.QueryStats->query_phases(0).table_access(1).name(), "/Root/EightShard");
UNIT_ASSERT_VALUES_EQUAL(res.QueryStats->query_phases(0).table_access(1).partitions_count(), 8);
- } else {
+ } else {
UNIT_ASSERT_VALUES_EQUAL(res.QueryStats->query_phases(0).table_access(0).name(), "/Root/EightShard");
UNIT_ASSERT_VALUES_EQUAL(res.QueryStats->query_phases(0).table_access(0).partitions_count(), 8);
UNIT_ASSERT_VALUES_EQUAL(res.QueryStats->query_phases(0).table_access(1).name(), "/Root/KeyValue");
UNIT_ASSERT_VALUES_EQUAL(res.QueryStats->query_phases(0).table_access(1).partitions_count(), 1);
- }
+ }
UNIT_ASSERT(!res.PlanJson);
}
@@ -127,8 +127,8 @@ Y_UNIT_TEST(DeferredEffects) {
auto result = session.ExecuteDataQuery(R"(
PRAGMA kikimr.UseNewEngine = "true";
- UPSERT INTO `/Root/TwoShard`
- SELECT Key + 100u AS Key, Value1 FROM `/Root/TwoShard` WHERE Key in (1,2,3,4,5);
+ UPSERT INTO `/Root/TwoShard`
+ SELECT Key + 100u AS Key, Value1 FROM `/Root/TwoShard` WHERE Key in (1,2,3,4,5);
)", TTxControl::BeginTx(), settings).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
@@ -156,7 +156,7 @@ Y_UNIT_TEST(DeferredEffects) {
DECLARE $key AS Uint32;
DECLARE $value AS String;
- UPSERT INTO `/Root/TwoShard` (Key, Value1) VALUES
+ UPSERT INTO `/Root/TwoShard` (Key, Value1) VALUES
($key, $value);
)", TTxControl::Tx(*tx).CommitTx(), std::move(params), settings).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
@@ -166,8 +166,8 @@ Y_UNIT_TEST(DeferredEffects) {
result = session.ExecuteDataQuery(R"(
PRAGMA kikimr.UseNewEngine = "true";
- SELECT * FROM `/Root/TwoShard`;
- UPDATE `/Root/TwoShard` SET Value1 = "XXX" WHERE Key in (3,600);
+ SELECT * FROM `/Root/TwoShard`;
+ UPDATE `/Root/TwoShard` SET Value1 = "XXX" WHERE Key in (3,600);
)", TTxControl::BeginTx().CommitTx(), settings).ExtractValueSync();
result.GetIssues().PrintTo(Cerr);
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
@@ -192,8 +192,8 @@ Y_UNIT_TEST(DataQueryWithEffects) {
auto result = session.ExecuteDataQuery(R"(
PRAGMA kikimr.UseNewEngine = "true";
- UPSERT INTO `/Root/TwoShard`
- SELECT Key + 1u AS Key, Value1 FROM `/Root/TwoShard`;
+ UPSERT INTO `/Root/TwoShard`
+ SELECT Key + 1u AS Key, Value1 FROM `/Root/TwoShard`;
)", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), settings).ExtractValueSync();
result.GetIssues().PrintTo(Cerr);
AssertSuccessResult(result);
@@ -216,8 +216,8 @@ Y_UNIT_TEST(DataQueryOldEngine) {
auto result = session.ExecuteDataQuery(R"(
PRAGMA kikimr.UseNewEngine = "false";
- UPSERT INTO `/Root/TwoShard`
- SELECT Key + 1u AS Key, Value1 FROM `/Root/TwoShard`;
+ UPSERT INTO `/Root/TwoShard`
+ SELECT Key + 1u AS Key, Value1 FROM `/Root/TwoShard`;
)", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), settings).ExtractValueSync();
result.GetIssues().PrintTo(Cerr);
AssertSuccessResult(result);
@@ -247,14 +247,14 @@ Y_UNIT_TEST(DataQueryMulti) {
UNIT_ASSERT_EQUAL_C(plan.GetMapSafe().at("Plan").GetMapSafe().at("Plans").GetArraySafe().size(), 0, result.GetQueryPlan());
}
-Y_UNIT_TEST_NEW_ENGINE(RequestUnitForBadRequestExecute) {
+Y_UNIT_TEST_NEW_ENGINE(RequestUnitForBadRequestExecute) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
INCORRECT_STMT
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), TExecDataQuerySettings().ReportCostInfo(true))
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), TExecDataQuerySettings().ReportCostInfo(true))
.ExtractValueSync();
result.GetIssues().PrintTo(Cerr);
@@ -265,14 +265,14 @@ Y_UNIT_TEST_NEW_ENGINE(RequestUnitForBadRequestExecute) {
UNIT_ASSERT(result.GetConsumedRu() > 0);
}
-Y_UNIT_TEST_NEW_ENGINE(RequestUnitForBadRequestExplicitPrepare) {
+Y_UNIT_TEST_NEW_ENGINE(RequestUnitForBadRequestExplicitPrepare) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- auto result = session.PrepareDataQuery(Q_(R"(
+ auto result = session.PrepareDataQuery(Q_(R"(
INCORRECT_STMT
- )"), TPrepareDataQuerySettings().ReportCostInfo(true)).ExtractValueSync();
+ )"), TPrepareDataQuerySettings().ReportCostInfo(true)).ExtractValueSync();
result.GetIssues().PrintTo(Cerr);
auto ru = result.GetResponseMetadata().find(NYdb::YDB_CONSUMED_UNITS_HEADER);
@@ -282,15 +282,15 @@ Y_UNIT_TEST_NEW_ENGINE(RequestUnitForBadRequestExplicitPrepare) {
UNIT_ASSERT(result.GetConsumedRu() > 0);
}
-Y_UNIT_TEST_NEW_ENGINE(RequestUnitForSuccessExplicitPrepare) {
+Y_UNIT_TEST_NEW_ENGINE(RequestUnitForSuccessExplicitPrepare) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- auto result = session.PrepareDataQuery(Q_(R"(
+ auto result = session.PrepareDataQuery(Q_(R"(
SELECT 0; SELECT 1; SELECT 2; SELECT 3; SELECT 4;
SELECT 5; SELECT 6; SELECT 7; SELECT 8; SELECT 9;
- )"), TPrepareDataQuerySettings().ReportCostInfo(true)).ExtractValueSync();
+ )"), TPrepareDataQuerySettings().ReportCostInfo(true)).ExtractValueSync();
result.GetIssues().PrintTo(Cerr);
auto ru = result.GetResponseMetadata().find(NYdb::YDB_CONSUMED_UNITS_HEADER);
diff --git a/ydb/core/kqp/ut/kqp_sys_col_ut.cpp b/ydb/core/kqp/ut/kqp_sys_col_ut.cpp
index 97c6abee5c..13fb8fa8ec 100644
--- a/ydb/core/kqp/ut/kqp_sys_col_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_sys_col_ut.cpp
@@ -33,12 +33,12 @@ TStreamPartIterator ExecuteStreamQuery(TKikimrRunner& kikimr, const TString& que
return it;
}
-void SelectRowAsteriskCommon(bool useNewEngine) {
+void SelectRowAsteriskCommon(bool useNewEngine) {
TStringBuilder query;
query << R"(
- PRAGMA kikimr.UseNewEngine = ")" << (useNewEngine ? "true" : "false") << R"(";
+ PRAGMA kikimr.UseNewEngine = ")" << (useNewEngine ? "true" : "false") << R"(";
PRAGMA kikimr.EnableSystemColumns = "true";
- SELECT * FROM `/Root/TwoShard` WHERE Key = 1;
+ SELECT * FROM `/Root/TwoShard` WHERE Key = 1;
)";
auto result = ExecuteDataQuery(query);
UNIT_ASSERT(result.GetResultSets().size());
@@ -46,12 +46,12 @@ void SelectRowAsteriskCommon(bool useNewEngine) {
FormatResultSetYson(result.GetResultSet(0)));
}
-void SelectRowByIdCommon(bool useNewEngine) {
+void SelectRowByIdCommon(bool useNewEngine) {
TStringBuilder query;
query << R"(
- PRAGMA kikimr.UseNewEngine = ")" << (useNewEngine ? "true" : "false") << R"(";
+ PRAGMA kikimr.UseNewEngine = ")" << (useNewEngine ? "true" : "false") << R"(";
PRAGMA kikimr.EnableSystemColumns = "true";
- SELECT * FROM `/Root/TwoShard` WHERE _yql_partition_id = 72075186224037888ul;
+ SELECT * FROM `/Root/TwoShard` WHERE _yql_partition_id = 72075186224037888ul;
)";
auto result = ExecuteDataQuery(query);
UNIT_ASSERT(result.GetResultSets().size());
@@ -59,12 +59,12 @@ void SelectRowByIdCommon(bool useNewEngine) {
FormatResultSetYson(result.GetResultSet(0)));
}
-void SelectRangeCommon(bool useNewEngine) {
+void SelectRangeCommon(bool useNewEngine) {
TStringBuilder query;
query << R"(
- PRAGMA kikimr.UseNewEngine = ")" << (useNewEngine ? "true" : "false") << R"(";
+ PRAGMA kikimr.UseNewEngine = ")" << (useNewEngine ? "true" : "false") << R"(";
PRAGMA kikimr.EnableSystemColumns = "true";
- SELECT _yql_partition_id FROM `/Root/TwoShard` WHERE Key < 3;
+ SELECT _yql_partition_id FROM `/Root/TwoShard` WHERE Key < 3;
)";
auto result = ExecuteDataQuery(query);
UNIT_ASSERT(result.GetResultSets().size());
@@ -74,27 +74,27 @@ void SelectRangeCommon(bool useNewEngine) {
Y_UNIT_TEST_SUITE(KqpSysColV0) {
Y_UNIT_TEST(SelectRowAsterisk) {
- SelectRowAsteriskCommon(false);
+ SelectRowAsteriskCommon(false);
}
Y_UNIT_TEST(SelectRowAsteriskNewEngine) {
- SelectRowAsteriskCommon(true);
+ SelectRowAsteriskCommon(true);
}
Y_UNIT_TEST(SelectRowById) {
- SelectRowByIdCommon(false);
+ SelectRowByIdCommon(false);
}
Y_UNIT_TEST(SelectRowByIdNewEngine) {
- SelectRowByIdCommon(true);
+ SelectRowByIdCommon(true);
}
Y_UNIT_TEST(SelectRange) {
- SelectRangeCommon(false);
+ SelectRangeCommon(false);
}
Y_UNIT_TEST(SelectRangeNewEngine) {
- SelectRangeCommon(true);
+ SelectRangeCommon(true);
}
Y_UNIT_TEST(UpdateAndDelete) {
@@ -202,93 +202,93 @@ Y_UNIT_TEST_SUITE(KqpSysColV0) {
}
Y_UNIT_TEST_SUITE(KqpSysColV1) {
- Y_UNIT_TEST_NEW_ENGINE(SelectRowAsterisk) {
- auto query = Q_(R"(
+ Y_UNIT_TEST_NEW_ENGINE(SelectRowAsterisk) {
+ auto query = Q_(R"(
PRAGMA kikimr.EnableSystemColumns = "true";
SELECT * FROM `/Root/TwoShard` WHERE Key = 1;
- )");
+ )");
auto result = ExecuteDataQuery(query);
UNIT_ASSERT(result.GetResultSets().size());
CompareYson(R"([[[1u];["One"];[-1]]])",
FormatResultSetYson(result.GetResultSet(0)));
}
- Y_UNIT_TEST_NEW_ENGINE(SelectRowById) {
- auto query = Q_(R"(
+ Y_UNIT_TEST_NEW_ENGINE(SelectRowById) {
+ auto query = Q_(R"(
PRAGMA kikimr.EnableSystemColumns = "true";
SELECT * FROM `/Root/TwoShard` WHERE _yql_partition_id = 72075186224037888ul;
- )");
+ )");
auto result = ExecuteDataQuery(query);
UNIT_ASSERT(result.GetResultSets().size());
CompareYson(R"([[[1u];["One"];[-1]];[[2u];["Two"];[0]];[[3u];["Three"];[1]]])",
FormatResultSetYson(result.GetResultSet(0)));
}
- Y_UNIT_TEST_NEW_ENGINE(SelectRange) {
- auto query = Q_(R"(
+ Y_UNIT_TEST_NEW_ENGINE(SelectRange) {
+ auto query = Q_(R"(
PRAGMA kikimr.EnableSystemColumns = "true";
SELECT _yql_partition_id FROM `/Root/TwoShard` WHERE Key < 3;
- )");
+ )");
auto result = ExecuteDataQuery(query);
UNIT_ASSERT(result.GetResultSets().size());
CompareYson(R"([[[72075186224037888u]];[[72075186224037888u]]])",
FormatResultSetYson(result.GetResultSet(0)));
}
- Y_UNIT_TEST_NEW_ENGINE(UpdateAndDelete) {
+ Y_UNIT_TEST_NEW_ENGINE(UpdateAndDelete) {
TKikimrRunner kikimr;
{
- auto query = Q_(R"(
+ auto query = Q_(R"(
PRAGMA kikimr.EnableSystemColumns = "true";
REPLACE INTO `/Root/TwoShard` (Key, Value1, Value2)
VALUES (4u, "Four", -4);
- )");
+ )");
auto result = ExecuteDataQuery(kikimr, query);
}
{
- auto query = Q_(R"(
+ auto query = Q_(R"(
PRAGMA kikimr.EnableSystemColumns = "true";
SELECT COUNT(*) FROM `/Root/TwoShard`
WHERE _yql_partition_id = 72075186224037888ul AND Value2 = -4;
- )");
+ )");
auto result = ExecuteDataQuery(kikimr, query);
UNIT_ASSERT(result.GetResultSets().size());
CompareYson(R"([[1u]])",
FormatResultSetYson(result.GetResultSet(0)));
}
{
- auto query = Q_(R"(
+ auto query = Q_(R"(
PRAGMA kikimr.EnableSystemColumns = "true";
UPDATE `/Root/TwoShard` SET Value2 = -44
WHERE _yql_partition_id = 72075186224037888ul AND Value2 = -4;
- )");
+ )");
ExecuteDataQuery(kikimr, query);
}
{
- auto query = Q_(R"(
+ auto query = Q_(R"(
PRAGMA kikimr.EnableSystemColumns = "true";
SELECT COUNT(*) FROM `/Root/TwoShard`
WHERE _yql_partition_id = 72075186224037888ul AND Value2 = -44;
- )");
+ )");
auto result = ExecuteDataQuery(kikimr, query);
UNIT_ASSERT(result.GetResultSets().size());
CompareYson(R"([[1u]])",
FormatResultSetYson(result.GetResultSet(0)));
}
{
- auto query = Q_(R"(
+ auto query = Q_(R"(
PRAGMA kikimr.EnableSystemColumns = "true";
DELETE FROM `/Root/TwoShard`
WHERE _yql_partition_id = 72075186224037888ul AND Value2 = -44;
- )");
+ )");
auto result = ExecuteDataQuery(kikimr, query);
}
{
- auto query = Q_(R"(
+ auto query = Q_(R"(
PRAGMA kikimr.EnableSystemColumns = "true";
SELECT COUNT(*) FROM `/Root/TwoShard`
WHERE _yql_partition_id = 72075186224037888ul AND Value2 = -44;
- )");
+ )");
auto result = ExecuteDataQuery(kikimr, query);
UNIT_ASSERT(result.GetResultSets().size());
CompareYson(R"([[0u]])",
@@ -296,42 +296,42 @@ Y_UNIT_TEST_SUITE(KqpSysColV1) {
}
}
- Y_UNIT_TEST_NEW_ENGINE(InnerJoinTables) {
- auto query = Q_(R"(
+ Y_UNIT_TEST_NEW_ENGINE(InnerJoinTables) {
+ auto query = Q_(R"(
PRAGMA kikimr.EnableSystemColumns = "true";
SELECT * FROM `/Root/Join1` AS t1
INNER JOIN `/Root/Join2` AS t2
ON t1.Fk21 == t2.Key1 AND t1.Fk22 == t2.Key2
WHERE t1.Value == "Value5" AND t2.Value2 == "Value31";
- )");
+ )");
auto result = ExecuteDataQuery(query);
UNIT_ASSERT(result.GetResultSets().size());
CompareYson(R"([[[108u];["One"];[8];["Value5"];[108u];["One"];#;["Value31"]]])",
FormatResultSetYson(result.GetResultSet(0)));
}
- Y_UNIT_TEST_NEW_ENGINE(InnerJoinSelect) {
- auto query = Q_(R"(
+ Y_UNIT_TEST_NEW_ENGINE(InnerJoinSelect) {
+ auto query = Q_(R"(
PRAGMA kikimr.EnableSystemColumns = "true";
SELECT * FROM `/Root/Join1` AS t1
INNER JOIN (SELECT Key1, Key2, Value2 FROM `/Root/Join2`) AS t2
ON t1.Fk21 == t2.Key1 AND t1.Fk22 == t2.Key2
WHERE t1.Value == "Value5" AND t2.Value2 == "Value31";
- )");
+ )");
auto result = ExecuteDataQuery(query);
UNIT_ASSERT(result.GetResultSets().size());
CompareYson(R"([[[108u];["One"];[8];["Value5"];[108u];["One"];["Value31"]]])",
FormatResultSetYson(result.GetResultSet(0)));
}
- Y_UNIT_TEST_NEW_ENGINE(InnerJoinSelectAsterisk) {
- auto query = Q_(R"(
+ Y_UNIT_TEST_NEW_ENGINE(InnerJoinSelectAsterisk) {
+ auto query = Q_(R"(
PRAGMA kikimr.EnableSystemColumns = "true";
SELECT * FROM `/Root/Join1` AS t1
INNER JOIN (SELECT * FROM `/Root/Join2`) AS t2
ON t1.Fk21 == t2.Key1 AND t1.Fk22 == t2.Key2
WHERE t1.Value == "Value5" AND t2.Value2 == "Value31";
- )");
+ )");
auto result = ExecuteDataQuery(query);
UNIT_ASSERT(result.GetResultSets().size());
CompareYson(R"([[[108u];["One"];[8];["Value5"];[108u];["One"];#;["Value31"]]])",
@@ -408,5 +408,5 @@ Y_UNIT_TEST_SUITE(KqpSysColV1) {
}
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/ut/kqp_sys_view_ut.cpp b/ydb/core/kqp/ut/kqp_sys_view_ut.cpp
index 65fb5138fa..1f8988517c 100644
--- a/ydb/core/kqp/ut/kqp_sys_view_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_sys_view_ut.cpp
@@ -11,45 +11,45 @@ using namespace NYdb::NScheme;
Y_UNIT_TEST_SUITE(KqpSystemView) {
- Y_UNIT_TEST(Join) {
- return; // nodes table is currently switched off
+ Y_UNIT_TEST(Join) {
+ return; // nodes table is currently switched off
- TKikimrRunner kikimr;
+ TKikimrRunner kikimr;
auto client = kikimr.GetTableClient();
-
- while (true) {
+
+ while (true) {
auto it = client.StreamExecuteScanQuery(
"select NodeId from `/Root/.sys/partition_stats` where Path = '/Root/KeyValue' limit 1"
- ).GetValueSync();
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- if (StreamResultToYson(it) == "[[#]]") {
- ::Sleep(TDuration::Seconds(1));
- Cerr << "waiting..." << Endl;
- Cerr.Flush();
- } else {
- break;
- }
- }
-
+ ).GetValueSync();
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ if (StreamResultToYson(it) == "[[#]]") {
+ ::Sleep(TDuration::Seconds(1));
+ Cerr << "waiting..." << Endl;
+ Cerr.Flush();
+ } else {
+ break;
+ }
+ }
+
auto it = client.StreamExecuteScanQuery(R"(
- --!syntax_v1
- select n.Host, ps.Path, ps.RowCount
- from `/Root/.sys/partition_stats` as ps
- join `/Root/.sys/nodes` as n
- on ps.NodeId = n.NodeId
- where ps.Path = '/Root/KeyValue'
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- CompareYson(R"([[["::1"];["/Root/KeyValue"];[2u]]])", StreamResultToYson(it));
- }
-
+ --!syntax_v1
+ select n.Host, ps.Path, ps.RowCount
+ from `/Root/.sys/partition_stats` as ps
+ join `/Root/.sys/nodes` as n
+ on ps.NodeId = n.NodeId
+ where ps.Path = '/Root/KeyValue'
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ CompareYson(R"([[["::1"];["/Root/KeyValue"];[2u]]])", StreamResultToYson(it));
+ }
+
Y_UNIT_TEST(PartitionStatsSimple) {
TKikimrRunner kikimr;
auto client = kikimr.GetTableClient();
auto it = client.StreamExecuteScanQuery(R"(
- SELECT OwnerId, PartIdx, Path, PathId
+ SELECT OwnerId, PartIdx, Path, PathId
FROM `/Root/.sys/partition_stats`
ORDER BY PathId, PartIdx;
)").GetValueSync();
@@ -165,11 +165,11 @@ Y_UNIT_TEST_SUITE(KqpSystemView) {
UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
CompareYson(R"([
- [[72057594046644480u];[5u];[2u];["/Root/BatchUpload"]];
- [[72057594046644480u];[5u];[3u];["/Root/BatchUpload"]];
- [[72057594046644480u];[5u];[4u];["/Root/BatchUpload"]];
- [[72057594046644480u];[5u];[5u];["/Root/BatchUpload"]];
- [[72057594046644480u];[5u];[6u];["/Root/BatchUpload"]];
+ [[72057594046644480u];[5u];[2u];["/Root/BatchUpload"]];
+ [[72057594046644480u];[5u];[3u];["/Root/BatchUpload"]];
+ [[72057594046644480u];[5u];[4u];["/Root/BatchUpload"]];
+ [[72057594046644480u];[5u];[5u];["/Root/BatchUpload"]];
+ [[72057594046644480u];[5u];[6u];["/Root/BatchUpload"]];
])", StreamResultToYson(it));
}
@@ -189,9 +189,9 @@ Y_UNIT_TEST_SUITE(KqpSystemView) {
UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
auto expected = Sprintf(R"([
- [[%du];["::1"]];
- [[%du];["::1"]];
- [[%du];["::1"]]
+ [[%du];["::1"]];
+ [[%du];["::1"]];
+ [[%du];["::1"]]
])", offset, offset + 1, offset + 2);
CompareYson(expected, StreamResultToYson(it));
@@ -215,9 +215,9 @@ Y_UNIT_TEST_SUITE(KqpSystemView) {
UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
auto expected = Sprintf(R"([
- [[%du];["::1"]];
- [[%du];["::1"]];
- [[%du];["::1"]]
+ [[%du];["::1"]];
+ [[%du];["::1"]];
+ [[%du];["::1"]]
])", offset + 1, offset + 2, offset + 3);
CompareYson(expected, StreamResultToYson(it));
@@ -241,13 +241,13 @@ Y_UNIT_TEST_SUITE(KqpSystemView) {
UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
auto expected = Sprintf(R"([
- [[%du];["::1"]];
+ [[%du];["::1"]];
])", offset + 2);
CompareYson(expected, StreamResultToYson(it));
}
- Y_UNIT_TEST_NEW_ENGINE(QueryStatsSimple) {
+ Y_UNIT_TEST_NEW_ENGINE(QueryStatsSimple) {
auto checkTable = [&] (const TStringBuf tableName) {
TKikimrRunner kikimr("", KikimrDefaultUtDomainRoot, 3);
@@ -259,15 +259,15 @@ Y_UNIT_TEST_SUITE(KqpSystemView) {
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
}
{
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT * FROM `/Root/TwoShard`
- )"), TTxControl::BeginTx().CommitTx()).GetValueSync();
+ )"), TTxControl::BeginTx().CommitTx()).GetValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
}
{
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT * FROM `/Root/EightShard`
- )"), TTxControl::BeginTx().CommitTx()).GetValueSync();
+ )"), TTxControl::BeginTx().CommitTx()).GetValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
}
@@ -349,7 +349,7 @@ Y_UNIT_TEST_SUITE(KqpSystemView) {
UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
auto streamPart = it.ReadNext().GetValueSync();
- UNIT_ASSERT_VALUES_EQUAL(streamPart.GetStatus(), EStatus::SCHEME_ERROR);
+ UNIT_ASSERT_VALUES_EQUAL(streamPart.GetStatus(), EStatus::SCHEME_ERROR);
driver.Stop(true);
}
diff --git a/ydb/core/kqp/ut/kqp_table_predicate_ut.cpp b/ydb/core/kqp/ut/kqp_table_predicate_ut.cpp
index 1adb8a8949..5ab5108968 100644
--- a/ydb/core/kqp/ut/kqp_table_predicate_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_table_predicate_ut.cpp
@@ -201,7 +201,7 @@ void ExecuteStreamQueryAndCheck(NExperimental::TStreamQueryClient& db, const TSt
CompareYson(expectedYson, resultYson);
- NYql::NDqProto::TDqExecutionStats stats;
+ NYql::NDqProto::TDqExecutionStats stats;
// First stage is computation, second scan read.
google::protobuf::TextFormat::ParseFromString(profiles[1], &stats);
@@ -343,67 +343,67 @@ void RunPredicateTest(const std::vector<TString>& predicates, bool withNulls) {
} // anonymous namespace end
Y_UNIT_TEST_SUITE(KqpTablePredicate) {
- Y_UNIT_TEST_NEW_ENGINE(IsNull) {
+ Y_UNIT_TEST_NEW_ENGINE(IsNull) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
CreateSampleTables(session);
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT Value FROM [/Root/TestNulls] WHERE
- Key1 IS NULL AND Key2 IS NULL
- )"),
- TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ Key1 IS NULL AND Key2 IS NULL
+ )"),
+ TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
CompareYson(R"([[["One"]]])", FormatResultSetYson(result.GetResultSet(0)));
}
- Y_UNIT_TEST_NEW_ENGINE(IsNullPartial) {
+ Y_UNIT_TEST_NEW_ENGINE(IsNullPartial) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
CreateSampleTables(session);
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT * FROM [/Root/Test]
- WHERE Group == 1 AND Name IS NULL
- )"),
- TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ WHERE Group == 1 AND Name IS NULL
+ )"),
+ TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
CompareYson(R"([])", FormatResultSetYson(result.GetResultSet(0)));
}
- Y_UNIT_TEST_NEW_ENGINE(NullInKey) {
+ Y_UNIT_TEST_NEW_ENGINE(NullInKey) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
CreateSampleTables(session);
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT Value FROM [/Root/TestNulls] WHERE
- Key1 <= 1
- ORDER BY Value
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ Key1 <= 1
+ ORDER BY Value
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
- CompareYson(R"([[["Five"]];[["Four"]];[["Six"]]])", FormatResultSetYson(result.GetResultSet(0)));
+ CompareYson(R"([[["Five"]];[["Four"]];[["Six"]]])", FormatResultSetYson(result.GetResultSet(0)));
}
- Y_UNIT_TEST_NEW_ENGINE(NullInKeySuffix) {
+ Y_UNIT_TEST_NEW_ENGINE(NullInKeySuffix) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
CreateSampleTables(session);
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT Value FROM [/Root/TestNulls] WHERE Key1 > 1
- )"),
- TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ )"),
+ TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
CompareYson(R"([
@@ -411,7 +411,7 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
])", FormatResultSetYson(result.GetResultSet(0)));
}
- Y_UNIT_TEST_NEW_ENGINE(NullInPredicate) {
+ Y_UNIT_TEST_NEW_ENGINE(NullInPredicate) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -424,19 +424,19 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
.Build()
.Build();
- auto result = session.ExecuteDataQuery(Q_(R"(
- DECLARE $key1 AS 'Uint32?';
- DECLARE $key2 AS 'Uint32?';
+ auto result = session.ExecuteDataQuery(Q_(R"(
+ DECLARE $key1 AS 'Uint32?';
+ DECLARE $key2 AS 'Uint32?';
SELECT Value FROM [/Root/TestNulls] WHERE
- Key1 = $key1 AND Key2 >= $key2
- )"),
- TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), std::move(params)).ExtractValueSync();
+ Key1 = $key1 AND Key2 >= $key2
+ )"),
+ TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), std::move(params)).ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
CompareYson(R"([])", FormatResultSetYson(result.GetResultSet(0)));
}
- Y_UNIT_TEST_NEW_ENGINE(NullInPredicateRow) {
+ Y_UNIT_TEST_NEW_ENGINE(NullInPredicateRow) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -449,19 +449,19 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
.Build()
.Build();
- auto result = session.ExecuteDataQuery(Q_(R"(
- DECLARE $key1 AS 'Uint32?';
- DECLARE $key2 AS 'Uint32?';
+ auto result = session.ExecuteDataQuery(Q_(R"(
+ DECLARE $key1 AS 'Uint32?';
+ DECLARE $key2 AS 'Uint32?';
SELECT Value FROM [/Root/TestNulls] WHERE
- Key1 = $key1 AND Key2 == $key2
- )"),
- TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), std::move(params)).ExtractValueSync();
+ Key1 = $key1 AND Key2 == $key2
+ )"),
+ TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), std::move(params)).ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
CompareYson(R"([])", FormatResultSetYson(result.GetResultSet(0)));
}
- Y_UNIT_TEST(AllowNullCompareInIndex) {
+ Y_UNIT_TEST(AllowNullCompareInIndex) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -474,23 +474,23 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
.Build()
.Build();
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA Kikimr.UseNewEngine = 'false';
- PRAGMA kikimr.AllowNullCompareInIndex = "true";
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA Kikimr.UseNewEngine = 'false';
+ PRAGMA kikimr.AllowNullCompareInIndex = "true";
- DECLARE $key1 AS 'Uint32?';
- DECLARE $key2 AS 'Uint32?';
+ DECLARE $key1 AS 'Uint32?';
+ DECLARE $key2 AS 'Uint32?';
SELECT Value FROM [/Root/TestNulls] WHERE
- Key1 = $key1 AND Key2 <= $key2
- )",
- TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), std::move(params)).ExtractValueSync();
+ Key1 = $key1 AND Key2 <= $key2
+ )",
+ TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), std::move(params)).ExtractValueSync();
result.GetIssues().PrintTo(Cerr);
UNIT_ASSERT(result.IsSuccess());
CompareYson(R"([[["One"]];[["Two"]]])", FormatResultSetYson(result.GetResultSet(0)));
}
- Y_UNIT_TEST_NEW_ENGINE(WhereInSubquery) {
+ Y_UNIT_TEST_NEW_ENGINE(WhereInSubquery) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -501,50 +501,50 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
.Build()
.Build();
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
DECLARE $name AS String;
$groups = (SELECT Group FROM `/Root/Test` WHERE Name = $name);
SELECT * FROM `/Root/TwoShard` WHERE Key in $groups;
- )"), TTxControl::BeginTx().CommitTx(), std::move(params)).ExtractValueSync();
+ )"), TTxControl::BeginTx().CommitTx(), std::move(params)).ExtractValueSync();
result.GetIssues().PrintTo(Cerr);
UNIT_ASSERT(result.IsSuccess());
CompareYson(R"([[[1u];["One"];[-1]]])", FormatResultSetYson(result.GetResultSet(0)));
}
- Y_UNIT_TEST_NEW_ENGINE(UpdateMulti) {
+ Y_UNIT_TEST_NEW_ENGINE(UpdateMulti) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- {
+ {
auto query = Q_(R"(UPDATE `/Root/KeyValue` SET Value = Value || "_updated" WHERE Key IN (1, 2, 3, 4))");
-
- auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
- }
-
- {
+
+ auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ }
+
+ {
auto query = Q_(R"(SELECT Key, Value FROM `/Root/KeyValue` ORDER BY Key)");
-
- auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- CompareYson(R"([[[1u];["One_updated"]];
- [[2u];["Two_updated"]]])", FormatResultSetYson(result.GetResultSet(0)));
- }
- }
-
- Y_UNIT_TEST_NEW_ENGINE(UpdateWhereInNoFullScan) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
+
+ auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ CompareYson(R"([[[1u];["One_updated"]];
+ [[2u];["Two_updated"]]])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+ }
+
+ Y_UNIT_TEST_NEW_ENGINE(UpdateWhereInNoFullScan) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
CreateSampleTables(session);
const TString query = Q_("UPDATE [/Root/MultiShardTable] SET Value = 'aaaaa' WHERE Key IN (1, 500)");
- if (!UseNewEngine) {
+ if (!UseNewEngine) {
auto result = session.ExplainDataQuery(query).GetValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
@@ -568,23 +568,23 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
UNIT_ASSERT(result.IsSuccess());
auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2);
-
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).affected_shards(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/MultiShardTable");
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).partitions_count(), 1);
-
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).affected_shards(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/MultiShardTable");
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).updates().rows(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).partitions_count(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2);
+
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).affected_shards(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/MultiShardTable");
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).partitions_count(), 1);
+
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).affected_shards(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/MultiShardTable");
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).updates().rows(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).partitions_count(), 1);
}
}
- Y_UNIT_TEST_NEW_ENGINE(UpdateWhereInWithNull) {
+ Y_UNIT_TEST_NEW_ENGINE(UpdateWhereInWithNull) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -593,9 +593,9 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
CreateSampleTables(session);
- const TString query = Q_(R"(
+ const TString query = Q_(R"(
UPDATE [/Root/MultiShardTable] SET ValueInt = ValueInt + 1
- WHERE Key IN (1,2,3,4,NULL))");
+ WHERE Key IN (1,2,3,4,NULL))");
{
auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).GetValueSync();
Cerr << result.GetIssues().ToString() << Endl;
@@ -603,10 +603,10 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
}
{
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT Key, ValueInt FROM [/Root/MultiShardTable] ORDER BY Key;
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
CompareYson(R"([[[1u];#];[[2u];#];[[3u];#];[[4u];#];[[10u];[10]];[[4294967295u];#]])",
@@ -614,26 +614,26 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
}
}
- Y_UNIT_TEST_NEW_ENGINE(UpdateWhereInBigLiteralList) {
+ Y_UNIT_TEST_NEW_ENGINE(UpdateWhereInBigLiteralList) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
CreateSampleTables(session);
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
UPDATE `/Root/MultiShardTable` SET ValueInt = ValueInt + 1 WHERE Key IN
(1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,
31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,
61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,
91,92,93,94,95,96,97,98,99,100)
- )"), TTxControl::BeginTx().CommitTx()).GetValueSync();
+ )"), TTxControl::BeginTx().CommitTx()).GetValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
{
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT Key, ValueInt FROM `/Root/MultiShardTable` ORDER BY Key
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
CompareYson(R"([[[1u];#];[[2u];#];[[3u];#];[[4u];#];[[10u];[11]];[[4294967295u];#]])",
@@ -641,27 +641,27 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
}
}
- Y_UNIT_TEST_NEW_ENGINE(UpdateWhereInBigLiteralListPrefix) {
+ Y_UNIT_TEST_NEW_ENGINE(UpdateWhereInBigLiteralListPrefix) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
CreateSampleTables(session);
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
UPDATE `/Root/MultiShardTableCk` SET ValueInt = ValueInt + 1 WHERE Key1 IN
(1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,
31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,
61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,
91,92,93,94,95,96,97,98,99,100)
- )"), TTxControl::BeginTx().CommitTx()).GetValueSync();
+ )"), TTxControl::BeginTx().CommitTx()).GetValueSync();
result.GetIssues().PrintTo(Cerr);
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
{
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT Key1, ValueInt FROM `/Root/MultiShardTableCk` ORDER BY Key1;
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
CompareYson(R"([[[1u];#];[[2u];#];[[3u];#];[[4u];#];[[10u];[11]];[[4294967295u];#]])",
@@ -669,7 +669,7 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
}
}
- Y_UNIT_TEST_NEW_ENGINE(UpdateWhereInMultipleUpdate) {
+ Y_UNIT_TEST_NEW_ENGINE(UpdateWhereInMultipleUpdate) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -678,30 +678,30 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
CreateSampleTables(session);
- const TString query = Q_(R"(
+ const TString query = Q_(R"(
UPDATE `/Root/MultiShardTable` SET ValueInt = ValueInt + 1 WHERE Key IN (1,2,10);
UPDATE `/Root/TestNulls` SET Value = 'qq' WHERE Key1 IN (1,2);
- )");
-
+ )");
+
{
auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).GetValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
}
{
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT Key, ValueInt FROM `/Root/MultiShardTable` ORDER BY Key;
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
CompareYson(R"([[[1u];#];[[2u];#];[[3u];#];[[4u];#];[[10u];[11]];[[4294967295u];#]])",
FormatResultSetYson(result.GetResultSet(0)));
}
-
+
{
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT Key1, Value FROM `/Root/TestNulls` ORDER BY Key1;
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
CompareYson(R"([
@@ -730,8 +730,8 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
CreateSampleTables(session);
- const TString query = "PRAGMA Kikimr.UseNewEngine = 'false'; "
- "SELECT Key, ValueInt FROM [/Root/MultiShardTable] WHERE ValueInt = 1 "
+ const TString query = "PRAGMA Kikimr.UseNewEngine = 'false'; "
+ "SELECT Key, ValueInt FROM [/Root/MultiShardTable] WHERE ValueInt = 1 "
"UNION ALL SELECT Key, ValueInt FROM [/Root/MultiShardTable] WHERE ValueInt = 1;";
auto result = session.ExplainDataQuery(query).ExtractValueSync();
@@ -756,7 +756,7 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
}
}
- Y_UNIT_TEST_NEW_ENGINE(UpdateWhereInFullScan) {
+ Y_UNIT_TEST_NEW_ENGINE(UpdateWhereInFullScan) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -764,8 +764,8 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
CreateSampleTables(session);
const TString query = Q_("UPDATE `/Root/MultiShardTable` SET Value = 'aaaaa' WHERE Value IN ('One', 'www')");
-
- if (!UseNewEngine) {
+
+ if (!UseNewEngine) {
auto result = session.ExplainDataQuery(query).GetValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
@@ -789,27 +789,27 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
UNIT_ASSERT(result.IsSuccess());
auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2);
-
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).affected_shards(), 5);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/MultiShardTable");
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 6);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).partitions_count(), 5);
-
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).affected_shards(), 5);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/MultiShardTable");
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).updates().rows(), 1);
- if (UseNewEngine) {
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).partitions_count(), 1);
- } else {
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).partitions_count(), 5);
- }
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2);
+
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).affected_shards(), 5);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/MultiShardTable");
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 6);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).partitions_count(), 5);
+
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).affected_shards(), 5);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).name(), "/Root/MultiShardTable");
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).updates().rows(), 1);
+ if (UseNewEngine) {
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).partitions_count(), 1);
+ } else {
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).partitions_count(), 5);
+ }
}
}
- Y_UNIT_TEST_NEW_ENGINE(DateKeyPredicate) {
+ Y_UNIT_TEST_NEW_ENGINE(DateKeyPredicate) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -818,45 +818,45 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
NYdb::NTable::TExecDataQuerySettings execSettings;
execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT Value FROM [/Root/TestDate]
WHERE Key = Date("2019-07-01")
- )"), TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync();
+ )"), TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync();
result.GetIssues().PrintTo(Cerr);
UNIT_ASSERT(result.IsSuccess());
CompareYson(R"([[["Three"]]])", FormatResultSetYson(result.GetResultSet(0)));
- auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).affected_shards(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).partitions_count(), 1);
+ auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).affected_shards(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).partitions_count(), 1);
}
- Y_UNIT_TEST_NEW_ENGINE(DuplicateKeyPredicateLiteral) {
+ Y_UNIT_TEST_NEW_ENGINE(DuplicateKeyPredicateLiteral) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT Key FROM [/Root/EightShard] WHERE
Key > 200 AND Key >= 301 AND
Key < 600 AND Key <= 501
ORDER BY Key;
- )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
CompareYson(R"([[[301u]];[[302u]];[[303u]];[[401u]];[[402u]];[[403u]];[[501u]]])",
- FormatResultSetYson(result.GetResultSet(0)));
- }
-
- Y_UNIT_TEST_NEW_ENGINE(DuplicateKeyPredicateParam) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
+ FormatResultSetYson(result.GetResultSet(0)));
+ }
+
+ Y_UNIT_TEST_NEW_ENGINE(DuplicateKeyPredicateParam) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
auto params = db.GetParamsBuilder()
.AddParam("$key_from_1").Uint64(200).Build()
.AddParam("$key_from_2").Uint64(301).Build()
@@ -864,7 +864,7 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
.AddParam("$key_to_2").Uint64(501).Build()
.Build();
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
DECLARE $key_from_1 AS Uint64;
DECLARE $key_from_2 AS Uint64;
DECLARE $key_to_1 AS Uint64;
@@ -874,25 +874,25 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
Key > $key_from_1 AND Key >= $key_from_2 AND
Key < $key_to_1 AND Key <= $key_to_2
ORDER BY Key;
- )"), TTxControl::BeginTx().CommitTx(), params).ExtractValueSync();
+ )"), TTxControl::BeginTx().CommitTx(), params).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
CompareYson(R"([[[301u]];[[302u]];[[303u]];[[401u]];[[402u]];[[403u]];[[501u]]])",
FormatResultSetYson(result.GetResultSet(0)));
- }
-
- Y_UNIT_TEST_NEW_ENGINE(DuplicateKeyPredicateMixed) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto params = db.GetParamsBuilder()
+ }
+
+ Y_UNIT_TEST_NEW_ENGINE(DuplicateKeyPredicateMixed) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto params = db.GetParamsBuilder()
.AddParam("$key_from_1").Uint64(200).Build()
.AddParam("$key_to_1").Uint64(600).Build()
.Build();
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
DECLARE $key_from_1 AS Uint64;
DECLARE $key_to_1 AS Uint64;
@@ -900,26 +900,26 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
Key > $key_from_1 AND Key >= 301 AND
Key < $key_to_1 AND Key <= 501
ORDER BY Key;
- )"), TTxControl::BeginTx().CommitTx(), params).ExtractValueSync();
+ )"), TTxControl::BeginTx().CommitTx(), params).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
CompareYson(R"([[[301u]];[[302u]];[[303u]];[[401u]];[[402u]];[[403u]];[[501u]]])",
FormatResultSetYson(result.GetResultSet(0)));
}
- Y_UNIT_TEST_NEW_ENGINE(DuplicateCompositeKeyPredicate) {
+ Y_UNIT_TEST_NEW_ENGINE(DuplicateCompositeKeyPredicate) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
CreateSampleTables(session);
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT Key2 FROM [/Root/TestNulls] WHERE Key1 = 3 AND
Key2 >= 100 AND Key2 > 200 AND
Key2 <= 600 AND Key2 < 500
ORDER BY Key2;
- )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
CompareYson(R"([[[300u]];[[400u]]])",
@@ -932,7 +932,7 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
.AddParam("$key_to_2").OptionalUint32(500).Build()
.Build();
- result = session.ExecuteDataQuery(Q_(R"(
+ result = session.ExecuteDataQuery(Q_(R"(
DECLARE $key_from_1 AS 'Uint32?';
DECLARE $key_from_2 AS 'Uint32?';
DECLARE $key_to_1 AS 'Uint32?';
@@ -942,7 +942,7 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
Key2 >= $key_from_1 AND Key2 > $key_from_2 AND
Key2 <= $key_to_1 AND Key2 < $key_to_2
ORDER BY Key2;
- )"), TTxControl::BeginTx().CommitTx(), params).ExtractValueSync();
+ )"), TTxControl::BeginTx().CommitTx(), params).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
CompareYson(R"([[[300u]];[[400u]]])",
@@ -953,7 +953,7 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
.AddParam("$key_to_2").OptionalUint32(500).Build()
.Build();
- result = session.ExecuteDataQuery(Q_(R"(
+ result = session.ExecuteDataQuery(Q_(R"(
DECLARE $key_from_2 AS 'Uint32?';
DECLARE $key_to_2 AS 'Uint32?';
@@ -961,7 +961,7 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
Key2 >= 100 AND Key2 > $key_from_2 AND
Key2 <= 600 AND Key2 < $key_to_2
ORDER BY Key2;
- )"), TTxControl::BeginTx().CommitTx(), params).ExtractValueSync();
+ )"), TTxControl::BeginTx().CommitTx(), params).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
CompareYson(R"([[[300u]];[[400u]]])",
@@ -974,7 +974,7 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
auto session = db.CreateSession().GetValueSync().GetSession();
auto query = R"(
- PRAGMA Kikimr.UseNewEngine = 'false';
+ PRAGMA Kikimr.UseNewEngine = 'false';
DECLARE $key1_from AS Uint32;
DECLARE $name AS String;
SELECT * FROM [/Root/Join2] WHERE Key1 > $key1_from AND Name = $name;
@@ -994,53 +994,53 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
UNIT_ASSERT_VALUES_EQUAL(read["scan_by"].GetArray().size(), 2);
}
- Y_UNIT_TEST_NEW_ENGINE(DeleteNotFullScan) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto query = Q_(R"(
+ Y_UNIT_TEST_NEW_ENGINE(DeleteNotFullScan) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto query = Q_(R"(
DELETE FROM `/Root/Join2`
- WHERE Key1 = 102 AND Key2 = "One" OR
- Key1 = 101 AND Key2 = "Two" OR
- Key1 = 101 AND Key2 = "Three"
- )");
-
- NYdb::NTable::TExecDataQuerySettings settings;
- settings.CollectQueryStats(ECollectQueryStatsMode::Basic);
-
- auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), settings).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
- if (UseNewEngine) {
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2);
-
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).affected_shards(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 3);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).updates().rows(), 0);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).deletes().rows(), 0);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).partitions_count(), 1);
-
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).affected_shards(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 0);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).updates().rows(), 0);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).deletes().rows(), 3);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).partitions_count(), 1);
- } else {
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).affected_shards(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 0);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).updates().rows(), 0);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).deletes().rows(), 3);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).partitions_count(), 1);
- }
- }
-
- Y_UNIT_TEST_NEW_ENGINE(LiteralOr) {
+ WHERE Key1 = 102 AND Key2 = "One" OR
+ Key1 = 101 AND Key2 = "Two" OR
+ Key1 = 101 AND Key2 = "Three"
+ )");
+
+ NYdb::NTable::TExecDataQuerySettings settings;
+ settings.CollectQueryStats(ECollectQueryStatsMode::Basic);
+
+ auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), settings).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
+ if (UseNewEngine) {
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2);
+
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).affected_shards(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 3);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).updates().rows(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).deletes().rows(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).partitions_count(), 1);
+
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).affected_shards(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).updates().rows(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).deletes().rows(), 3);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).partitions_count(), 1);
+ } else {
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).affected_shards(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).updates().rows(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).deletes().rows(), 3);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).partitions_count(), 1);
+ }
+ }
+
+ Y_UNIT_TEST_NEW_ENGINE(LiteralOr) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -1049,11 +1049,11 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
NYdb::NTable::TExecDataQuerySettings execSettings;
execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT * FROM `/Root/EightShard`
WHERE Key = 101 OR Key = 302 OR Key = 403 OR Key = 705
ORDER BY Key;
- )"), TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync();
+ )"), TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
CompareYson(R"([
@@ -1062,14 +1062,14 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
[[2];[403u];["Value3"]]])", FormatResultSetYson(result.GetResultSet(0)));
auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).affected_shards(), 4);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 3);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).partitions_count(), 4);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).affected_shards(), 4);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 3);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).partitions_count(), 4);
}
- Y_UNIT_TEST_NEW_ENGINE(LiteralOrCompisite) {
+ Y_UNIT_TEST_NEW_ENGINE(LiteralOrCompisite) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -1078,7 +1078,7 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
NYdb::NTable::TExecDataQuerySettings execSettings;
execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT * FROM `/Root/Logs`
WHERE
App = "apache" AND Ts = 0 OR
@@ -1086,7 +1086,7 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
App = "kikimr-db" AND Ts = 4 OR
App = "ydb" AND Ts = 5
ORDER BY App, Ts;
- )"), TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync();
+ )"), TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
CompareYson(R"([
@@ -1096,21 +1096,21 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
])", FormatResultSetYson(result.GetResultSet(0)));
auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
- if (UseNewEngine) {
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2);
-
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 0);
-
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 3);
- } else {
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 3);
- }
+ if (UseNewEngine) {
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2);
+
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 0);
+
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 3);
+ } else {
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 3);
+ }
}
- Y_UNIT_TEST_NEW_ENGINE(LiteralOrCompisiteCollision) {
+ Y_UNIT_TEST_NEW_ENGINE(LiteralOrCompisiteCollision) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -1119,7 +1119,7 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
NYdb::NTable::TExecDataQuerySettings execSettings;
execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT * FROM `/Root/Logs`
WHERE
App = "apache" AND Ts = 0 OR
@@ -1127,7 +1127,7 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
App = "nginx" AND Ts = 3 OR
App = "ydb" AND Ts = 5
ORDER BY App, Ts;
- )"), TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync();
+ )"), TTxControl::BeginTx().CommitTx(), execSettings).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
CompareYson(R"([
@@ -1137,18 +1137,18 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
])", FormatResultSetYson(result.GetResultSet(0)));
auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
- if (UseNewEngine) {
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2);
-
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 0);
-
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 3);
- } else {
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 3);
- }
+ if (UseNewEngine) {
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 2);
+
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 0);
+
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 3);
+ } else {
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).reads().rows(), 3);
+ }
}
Y_UNIT_TEST(NoFullScanAtScanQuery) {
@@ -1506,5 +1506,5 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) {
}
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/ut/kqp_tx_ut.cpp b/ydb/core/kqp/ut/kqp_tx_ut.cpp
index c529835a35..5837331604 100644
--- a/ydb/core/kqp/ut/kqp_tx_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_tx_ut.cpp
@@ -46,7 +46,7 @@ Y_UNIT_TEST_SUITE(KqpTx) {
])", FormatResultSetYson(result.GetResultSet(0)));
}
- Y_UNIT_TEST_NEW_ENGINE(ExplicitTcl) {
+ Y_UNIT_TEST_NEW_ENGINE(ExplicitTcl) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -56,23 +56,23 @@ Y_UNIT_TEST_SUITE(KqpTx) {
.GetTransaction();
UNIT_ASSERT(tx.IsActive());
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
UPSERT INTO [/Root/KeyValue] (Key, Value) VALUES (10u, "New");
- )"), TTxControl::Tx(tx)).ExtractValueSync();
+ )"), TTxControl::Tx(tx)).ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
- result = session.ExecuteDataQuery(Q_(R"(
+ result = session.ExecuteDataQuery(Q_(R"(
SELECT * FROM [/Root/KeyValue] WHERE Value = "New";
- )"), TTxControl::BeginTx(TTxSettings::OnlineRO()).CommitTx()).ExtractValueSync();
+ )"), TTxControl::BeginTx(TTxSettings::OnlineRO()).CommitTx()).ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
CompareYson(R"([])", FormatResultSetYson(result.GetResultSet(0)));
auto commitResult = tx.Commit().ExtractValueSync();
- UNIT_ASSERT_C(commitResult.IsSuccess(), commitResult.GetIssues().ToString());
+ UNIT_ASSERT_C(commitResult.IsSuccess(), commitResult.GetIssues().ToString());
- result = session.ExecuteDataQuery(Q_(R"(
+ result = session.ExecuteDataQuery(Q_(R"(
SELECT * FROM [/Root/KeyValue] WHERE Value = "New";
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
CompareYson(R"([[[10u];["New"]]])", FormatResultSetYson(result.GetResultSet(0)));
@@ -129,31 +129,31 @@ Y_UNIT_TEST_SUITE(KqpTx) {
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
}
- Y_UNIT_TEST_NEW_ENGINE(CommitRequired) {
+ Y_UNIT_TEST_NEW_ENGINE(CommitRequired) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT * FROM [/Root/KeyValue] WHERE Value = "New";
- )"), TTxControl::BeginTx(TTxSettings::OnlineRO())).ExtractValueSync();
+ )"), TTxControl::BeginTx(TTxSettings::OnlineRO())).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
- result = session.ExecuteDataQuery(Q_(R"(
+ result = session.ExecuteDataQuery(Q_(R"(
SELECT * FROM [/Root/KeyValue] WHERE Value = "New";
- )"), TTxControl::BeginTx(TTxSettings::StaleRO())).ExtractValueSync();
+ )"), TTxControl::BeginTx(TTxSettings::StaleRO())).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
}
- Y_UNIT_TEST_NEW_ENGINE(RollbackTx) {
+ Y_UNIT_TEST_NEW_ENGINE(RollbackTx) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- // with effects, without locks
- auto result = session.ExecuteDataQuery(Q_(R"(
+ // with effects, without locks
+ auto result = session.ExecuteDataQuery(Q_(R"(
UPSERT INTO [/Root/KeyValue] (Key, Value) VALUES (10u, "New");
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW())).ExtractValueSync();
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW())).ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
auto tx = result.GetTransaction();
@@ -163,9 +163,9 @@ Y_UNIT_TEST_SUITE(KqpTx) {
auto rollbackResult = tx->Rollback().ExtractValueSync();
UNIT_ASSERT(rollbackResult.IsSuccess());
- result = session.ExecuteDataQuery(Q_(R"(
+ result = session.ExecuteDataQuery(Q_(R"(
SELECT * FROM [/Root/KeyValue] WHERE Value = "New";
- )"), TTxControl::BeginTx(TTxSettings::OnlineRO()).CommitTx()).ExtractValueSync();
+ )"), TTxControl::BeginTx(TTxSettings::OnlineRO()).CommitTx()).ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
CompareYson(R"([])", FormatResultSetYson(result.GetResultSet(0)));
@@ -174,113 +174,113 @@ Y_UNIT_TEST_SUITE(KqpTx) {
UNIT_ASSERT(HasIssue(rollbackResult.GetIssues(), NYql::TIssuesIds::KIKIMR_TRANSACTION_NOT_FOUND));
}
- Y_UNIT_TEST_NEW_ENGINE(RollbackTx2) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- // with effects, with locks
- auto result = session.ExecuteDataQuery(Q_(R"(
+ Y_UNIT_TEST_NEW_ENGINE(RollbackTx2) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ // with effects, with locks
+ auto result = session.ExecuteDataQuery(Q_(R"(
UPDATE [/Root/KeyValue] SET Value = "New" WHERE Key = 1;
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW())).ExtractValueSync();
- UNIT_ASSERT(result.IsSuccess());
-
- auto tx = result.GetTransaction();
- UNIT_ASSERT(tx);
- UNIT_ASSERT(tx->IsActive());
-
- auto rollbackResult = tx->Rollback().ExtractValueSync();
- UNIT_ASSERT(rollbackResult.IsSuccess());
-
- result = session.ExecuteDataQuery(Q_(R"(
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW())).ExtractValueSync();
+ UNIT_ASSERT(result.IsSuccess());
+
+ auto tx = result.GetTransaction();
+ UNIT_ASSERT(tx);
+ UNIT_ASSERT(tx->IsActive());
+
+ auto rollbackResult = tx->Rollback().ExtractValueSync();
+ UNIT_ASSERT(rollbackResult.IsSuccess());
+
+ result = session.ExecuteDataQuery(Q_(R"(
SELECT * FROM [/Root/KeyValue] WHERE Value = "New";
- )"), TTxControl::BeginTx(TTxSettings::OnlineRO()).CommitTx()).ExtractValueSync();
- UNIT_ASSERT(result.IsSuccess());
- CompareYson(R"([])", FormatResultSetYson(result.GetResultSet(0)));
-
- rollbackResult = tx->Rollback().ExtractValueSync();
- UNIT_ASSERT(!rollbackResult.IsSuccess());
- UNIT_ASSERT(HasIssue(rollbackResult.GetIssues(), NYql::TIssuesIds::KIKIMR_TRANSACTION_NOT_FOUND));
- }
-
- Y_UNIT_TEST_NEW_ENGINE(RollbackRoTx) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExecuteDataQuery(Q_(R"(
- SELECT * FROM `/Root/KeyValue` WHERE Key = 1
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW())).ExtractValueSync();
- UNIT_ASSERT(result.IsSuccess());
-
- auto tx = result.GetTransaction();
- UNIT_ASSERT(tx);
- UNIT_ASSERT(tx->IsActive());
-
- auto rollbackResult = tx->Rollback().ExtractValueSync();
- UNIT_ASSERT(rollbackResult.IsSuccess());
-
- rollbackResult = tx->Rollback().ExtractValueSync();
- UNIT_ASSERT(!rollbackResult.IsSuccess());
- UNIT_ASSERT(HasIssue(rollbackResult.GetIssues(), NYql::TIssuesIds::KIKIMR_TRANSACTION_NOT_FOUND));
- }
-
- Y_UNIT_TEST_NEW_ENGINE(CommitRoTx) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExecuteDataQuery(Q_(R"(
- SELECT * FROM `/Root/KeyValue` WHERE Key = 1
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW())).ExtractValueSync();
- UNIT_ASSERT(result.IsSuccess());
-
- auto tx = result.GetTransaction();
- UNIT_ASSERT(tx);
- UNIT_ASSERT(tx->IsActive());
-
- auto commitResult = tx->Commit().ExtractValueSync();
- UNIT_ASSERT(commitResult.IsSuccess());
- }
-
- Y_UNIT_TEST_NEW_ENGINE(CommitRoTx_TLI) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExecuteDataQuery(Q_(R"(
- SELECT * FROM `/Root/KeyValue` WHERE Key = 1
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW())).ExtractValueSync();
- UNIT_ASSERT(result.IsSuccess());
-
- auto tx = result.GetTransaction();
- UNIT_ASSERT(tx);
- UNIT_ASSERT(tx->IsActive());
-
- {
- result = session.ExecuteDataQuery(Q_(R"(
+ )"), TTxControl::BeginTx(TTxSettings::OnlineRO()).CommitTx()).ExtractValueSync();
+ UNIT_ASSERT(result.IsSuccess());
+ CompareYson(R"([])", FormatResultSetYson(result.GetResultSet(0)));
+
+ rollbackResult = tx->Rollback().ExtractValueSync();
+ UNIT_ASSERT(!rollbackResult.IsSuccess());
+ UNIT_ASSERT(HasIssue(rollbackResult.GetIssues(), NYql::TIssuesIds::KIKIMR_TRANSACTION_NOT_FOUND));
+ }
+
+ Y_UNIT_TEST_NEW_ENGINE(RollbackRoTx) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExecuteDataQuery(Q_(R"(
+ SELECT * FROM `/Root/KeyValue` WHERE Key = 1
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW())).ExtractValueSync();
+ UNIT_ASSERT(result.IsSuccess());
+
+ auto tx = result.GetTransaction();
+ UNIT_ASSERT(tx);
+ UNIT_ASSERT(tx->IsActive());
+
+ auto rollbackResult = tx->Rollback().ExtractValueSync();
+ UNIT_ASSERT(rollbackResult.IsSuccess());
+
+ rollbackResult = tx->Rollback().ExtractValueSync();
+ UNIT_ASSERT(!rollbackResult.IsSuccess());
+ UNIT_ASSERT(HasIssue(rollbackResult.GetIssues(), NYql::TIssuesIds::KIKIMR_TRANSACTION_NOT_FOUND));
+ }
+
+ Y_UNIT_TEST_NEW_ENGINE(CommitRoTx) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExecuteDataQuery(Q_(R"(
+ SELECT * FROM `/Root/KeyValue` WHERE Key = 1
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW())).ExtractValueSync();
+ UNIT_ASSERT(result.IsSuccess());
+
+ auto tx = result.GetTransaction();
+ UNIT_ASSERT(tx);
+ UNIT_ASSERT(tx->IsActive());
+
+ auto commitResult = tx->Commit().ExtractValueSync();
+ UNIT_ASSERT(commitResult.IsSuccess());
+ }
+
+ Y_UNIT_TEST_NEW_ENGINE(CommitRoTx_TLI) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExecuteDataQuery(Q_(R"(
+ SELECT * FROM `/Root/KeyValue` WHERE Key = 1
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW())).ExtractValueSync();
+ UNIT_ASSERT(result.IsSuccess());
+
+ auto tx = result.GetTransaction();
+ UNIT_ASSERT(tx);
+ UNIT_ASSERT(tx->IsActive());
+
+ {
+ result = session.ExecuteDataQuery(Q_(R"(
UPDATE `/Root/KeyValue` SET Value = "New" WHERE Key = 1;
- )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
- }
-
- auto commitResult = tx->Commit().ExtractValueSync();
+ )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ }
+
+ auto commitResult = tx->Commit().ExtractValueSync();
if (kikimr.IsUsingSnapshotReads()) {
UNIT_ASSERT_VALUES_EQUAL_C(commitResult.GetStatus(), EStatus::SUCCESS, commitResult.GetIssues().ToString());
} else {
UNIT_ASSERT_VALUES_EQUAL_C(commitResult.GetStatus(), EStatus::ABORTED, commitResult.GetIssues().ToString());
UNIT_ASSERT(HasIssue(commitResult.GetIssues(), NYql::TIssuesIds::KIKIMR_LOCKS_INVALIDATED));
}
- }
-
- Y_UNIT_TEST_NEW_ENGINE(EmptyTxOnCommit) {
+ }
+
+ Y_UNIT_TEST_NEW_ENGINE(EmptyTxOnCommit) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
UPSERT INTO [/Root/KeyValue] (Key, Value) VALUES (10u, "New");
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
auto tx = result.GetTransaction();
@@ -345,21 +345,21 @@ Y_UNIT_TEST_SUITE(KqpTx) {
UNIT_ASSERT(commitResult.IsSuccess());
}
- Y_UNIT_TEST_NEW_ENGINE(RollbackInvalidated) {
+ Y_UNIT_TEST_NEW_ENGINE(RollbackInvalidated) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
UPSERT INTO [/Root/KeyValue] (Key, Value) VALUES (10u, "New");
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW())).ExtractValueSync();
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW())).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
auto tx = result.GetTransaction();
- result = session.ExecuteDataQuery(Q_(R"(
+ result = session.ExecuteDataQuery(Q_(R"(
SELECT * FROM [BadTable];
- )"), TTxControl::Tx(*tx)).ExtractValueSync();
+ )"), TTxControl::Tx(*tx)).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SCHEME_ERROR, result.GetIssues().ToString());
UNIT_ASSERT(result.GetTransaction());
@@ -370,14 +370,14 @@ Y_UNIT_TEST_SUITE(KqpTx) {
UNIT_ASSERT(HasIssue(commitResult.GetIssues(), NYql::TIssuesIds::KIKIMR_TRANSACTION_NOT_FOUND));
}
- Y_UNIT_TEST_NEW_ENGINE(CommitPrepared) {
+ Y_UNIT_TEST_NEW_ENGINE(CommitPrepared) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
- auto query = session.PrepareDataQuery(Q_(R"(
+ auto query = session.PrepareDataQuery(Q_(R"(
UPSERT INTO [/Root/KeyValue] (Key, Value) VALUES (10u, "New");
- )")).ExtractValueSync().GetQuery();
+ )")).ExtractValueSync().GetQuery();
auto result = query.Execute(TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
@@ -385,14 +385,14 @@ Y_UNIT_TEST_SUITE(KqpTx) {
auto tx = result.GetTransaction();
UNIT_ASSERT(!tx->IsActive());
- result = session.ExecuteDataQuery(Q_(R"(
+ result = session.ExecuteDataQuery(Q_(R"(
SELECT * FROM [/Root/KeyValue] WHERE Value = "New";
- )"), TTxControl::BeginTx(TTxSettings::OnlineRO()).CommitTx()).ExtractValueSync();
+ )"), TTxControl::BeginTx(TTxSettings::OnlineRO()).CommitTx()).ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
CompareYson(R"([[[10u];["New"]]])", FormatResultSetYson(result.GetResultSet(0)));
}
- Y_UNIT_TEST_NEW_ENGINE(InvalidateOnError) {
+ Y_UNIT_TEST_NEW_ENGINE(InvalidateOnError) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -402,20 +402,20 @@ Y_UNIT_TEST_SUITE(KqpTx) {
.GetTransaction();
UNIT_ASSERT(tx.IsActive());
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
INSERT INTO [/Root/KeyValue] (Key, Value) VALUES (1u, "New");
- )"), TTxControl::Tx(tx)).ExtractValueSync();
- // result.GetIssues().PrintTo(Cerr);
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::PRECONDITION_FAILED, result.GetIssues().ToString());
+ )"), TTxControl::Tx(tx)).ExtractValueSync();
+ // result.GetIssues().PrintTo(Cerr);
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::PRECONDITION_FAILED, result.GetIssues().ToString());
- result = session.ExecuteDataQuery(Q_(R"(
+ result = session.ExecuteDataQuery(Q_(R"(
UPSERT INTO [/Root/KeyValue] (Key, Value) VALUES (1u, "New");
- )"), TTxControl::Tx(tx)).ExtractValueSync();
- // result.GetIssues().PrintTo(Cerr);
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::NOT_FOUND, result.GetIssues().ToString());
+ )"), TTxControl::Tx(tx)).ExtractValueSync();
+ // result.GetIssues().PrintTo(Cerr);
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::NOT_FOUND, result.GetIssues().ToString());
}
- Y_UNIT_TEST_NEW_ENGINE(CommitStats) {
+ Y_UNIT_TEST_NEW_ENGINE(CommitStats) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -425,9 +425,9 @@ Y_UNIT_TEST_SUITE(KqpTx) {
.GetTransaction();
UNIT_ASSERT(tx.IsActive());
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
UPSERT INTO [/Root/KeyValue] (Key, Value) VALUES (10u, "New");
- )"), TTxControl::Tx(tx)).ExtractValueSync();
+ )"), TTxControl::Tx(tx)).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
NYdb::NTable::TCommitTxSettings commitSettings;
@@ -443,38 +443,38 @@ Y_UNIT_TEST_SUITE(KqpTx) {
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).name(), "/Root/KeyValue");
UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access(0).updates().rows(), 1);
}
-
- Y_UNIT_TEST(MixEnginesOldNew) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto tx = session.BeginTransaction(TTxSettings::SerializableRW())
- .ExtractValueSync()
- .GetTransaction();
- UNIT_ASSERT(tx.IsActive());
-
- auto result = session.ExecuteDataQuery(R"(
- SELECT * FROM `/Root/KeyValue`
- )", TTxControl::Tx(tx)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
-
- result = session.ExecuteDataQuery(R"(
- PRAGMA Kikimr.UseNewEngine = "true";
- UPSERT INTO `/Root/KeyValue` (Key, Value) VALUES (1u, "New");
- )", TTxControl::Tx(tx).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
-
- result = session.ExecuteDataQuery(R"(
+
+ Y_UNIT_TEST(MixEnginesOldNew) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto tx = session.BeginTransaction(TTxSettings::SerializableRW())
+ .ExtractValueSync()
+ .GetTransaction();
+ UNIT_ASSERT(tx.IsActive());
+
+ auto result = session.ExecuteDataQuery(R"(
+ SELECT * FROM `/Root/KeyValue`
+ )", TTxControl::Tx(tx)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+
+ result = session.ExecuteDataQuery(R"(
+ PRAGMA Kikimr.UseNewEngine = "true";
+ UPSERT INTO `/Root/KeyValue` (Key, Value) VALUES (1u, "New");
+ )", TTxControl::Tx(tx).CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+
+ result = session.ExecuteDataQuery(R"(
SELECT * FROM `/Root/KeyValue`
- )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- CompareYson(R"([
- [[1u];["New"]];
- [[2u];["Two"]]
- ])", FormatResultSetYson(result.GetResultSet(0)));
- }
+ )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+ CompareYson(R"([
+ [[1u];["New"]];
+ [[2u];["Two"]]
+ ])", FormatResultSetYson(result.GetResultSet(0)));
+ }
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/ut/kqp_write_ut.cpp b/ydb/core/kqp/ut/kqp_write_ut.cpp
index a42f4b9ce2..d260211f58 100644
--- a/ydb/core/kqp/ut/kqp_write_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_write_ut.cpp
@@ -9,7 +9,7 @@ using namespace NYdb;
using namespace NYdb::NTable;
Y_UNIT_TEST_SUITE(KqpWrite) {
- Y_UNIT_TEST_NEW_ENGINE(UpsertNullKey) {
+ Y_UNIT_TEST_NEW_ENGINE(UpsertNullKey) {
auto setting = NKikimrKqp::TKqpSetting();
setting.SetName("_KqpYqlSyntaxVersion");
setting.SetValue("1");
@@ -19,7 +19,7 @@ Y_UNIT_TEST_SUITE(KqpWrite) {
auto session = db.CreateSession().GetValueSync().GetSession();
{
- const TString query = Q_(R"(
+ const TString query = Q_(R"(
UPSERT INTO `/Root/KeyValue` (Key, Value) VALUES
(Null, "Value1");
)");
@@ -31,9 +31,9 @@ Y_UNIT_TEST_SUITE(KqpWrite) {
{
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT * FROM `/Root/KeyValue`;
- )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
result.GetIssues().PrintTo(Cerr);
UNIT_ASSERT(result.IsSuccess());
UNIT_ASSERT_VALUES_EQUAL("[[#;[\"Value1\"]];[[1u];[\"One\"]];[[2u];[\"Two\"]]]",
@@ -41,7 +41,7 @@ Y_UNIT_TEST_SUITE(KqpWrite) {
}
{
- const TString query = Q_(R"(
+ const TString query = Q_(R"(
UPSERT INTO `/Root/KeyValue` (Key, Value) VALUES
(Null, "Value2");
)");
@@ -53,9 +53,9 @@ Y_UNIT_TEST_SUITE(KqpWrite) {
{
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT * FROM `/Root/KeyValue`;
- )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
result.GetIssues().PrintTo(Cerr);
UNIT_ASSERT(result.IsSuccess());
UNIT_ASSERT_VALUES_EQUAL("[[#;[\"Value2\"]];[[1u];[\"One\"]];[[2u];[\"Two\"]]]",
@@ -63,7 +63,7 @@ Y_UNIT_TEST_SUITE(KqpWrite) {
}
{
- const TString query = Q_(R"(
+ const TString query = Q_(R"(
UPSERT INTO `/Root/KeyValue` (Key) VALUES
(Null);
)");
@@ -75,9 +75,9 @@ Y_UNIT_TEST_SUITE(KqpWrite) {
{
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT * FROM `/Root/KeyValue`;
- )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
result.GetIssues().PrintTo(Cerr);
UNIT_ASSERT(result.IsSuccess());
UNIT_ASSERT_VALUES_EQUAL("[[#;[\"Value2\"]];[[1u];[\"One\"]];[[2u];[\"Two\"]]]",
@@ -98,7 +98,7 @@ Y_UNIT_TEST_SUITE(KqpWrite) {
.Build();
auto result = session.ExecuteDataQuery(R"(
- PRAGMA Kikimr.UseNewEngine = 'false';
+ PRAGMA Kikimr.UseNewEngine = 'false';
PRAGMA kikimr.CommitSafety = "Moderate";
DECLARE $key AS Uint64;
@@ -128,7 +128,7 @@ Y_UNIT_TEST_SUITE(KqpWrite) {
auto session = db.CreateSession().GetValueSync().GetSession();
auto queryPrefix = R"(
- PRAGMA Kikimr.UseNewEngine = 'false';
+ PRAGMA Kikimr.UseNewEngine = 'false';
PRAGMA kikimr.CommitSafety = "Moderate";
)";
@@ -210,7 +210,7 @@ Y_UNIT_TEST_SUITE(KqpWrite) {
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
TString query(R"(
- PRAGMA Kikimr.UseNewEngine = 'false';
+ PRAGMA Kikimr.UseNewEngine = 'false';
PRAGMA kikimr.CommitSafety = "%s";
DECLARE $Key AS Uint32;
@@ -266,7 +266,7 @@ Y_UNIT_TEST_SUITE(KqpWrite) {
.Build();
auto result = session.ExecuteDataQuery(R"(
- PRAGMA Kikimr.UseNewEngine = 'false';
+ PRAGMA Kikimr.UseNewEngine = 'false';
PRAGMA kikimr.CommitSafety = "Moderate";
DECLARE $key AS Uint64;
@@ -284,7 +284,7 @@ Y_UNIT_TEST_SUITE(KqpWrite) {
UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_OPERATION));
}
- Y_UNIT_TEST_NEW_ENGINE(Insert) {
+ Y_UNIT_TEST_NEW_ENGINE(Insert) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -296,15 +296,15 @@ Y_UNIT_TEST_SUITE(KqpWrite) {
);
)").GetValueSync().IsSuccess());
- auto insertQuery = Q1_(R"(
- DECLARE $rows AS List<Struct<Key: Uint64, Value: String>>;
+ auto insertQuery = Q1_(R"(
+ DECLARE $rows AS List<Struct<Key: Uint64, Value: String>>;
DECLARE $id AS Uint32;
- INSERT INTO `/Root/KeyValue`
+ INSERT INTO `/Root/KeyValue`
SELECT * FROM AS_TABLE($rows);
- UPSERT INTO `/Root/TxCheck` (Id) VALUES ($id);
- )");
+ UPSERT INTO `/Root/TxCheck` (Id) VALUES ($id);
+ )");
auto params = db.GetParamsBuilder()
.AddParam("$rows")
@@ -329,8 +329,8 @@ Y_UNIT_TEST_SUITE(KqpWrite) {
auto result = session.ExecuteDataQuery(insertQuery,
TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params).ExtractValueSync();
result.GetIssues().PrintTo(Cerr);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::PRECONDITION_FAILED);
- UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION));
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::PRECONDITION_FAILED);
+ UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION));
params = db.GetParamsBuilder()
.AddParam("$rows")
@@ -360,8 +360,8 @@ Y_UNIT_TEST_SUITE(KqpWrite) {
result = session.ExecuteDataQuery(insertQuery,
TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params).ExtractValueSync();
result.GetIssues().PrintTo(Cerr);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::PRECONDITION_FAILED);
- UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION));
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::PRECONDITION_FAILED);
+ UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION));
params = db.GetParamsBuilder()
.AddParam("$rows")
@@ -387,10 +387,10 @@ Y_UNIT_TEST_SUITE(KqpWrite) {
TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- result = session.ExecuteDataQuery(Q_(R"(
- SELECT * FROM `/Root/KeyValue`;
- SELECT * FROM `/Root/TxCheck`;
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ result = session.ExecuteDataQuery(Q_(R"(
+ SELECT * FROM `/Root/KeyValue`;
+ SELECT * FROM `/Root/TxCheck`;
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
CompareYson(R"([
[[1u];["One"]];
@@ -401,27 +401,27 @@ Y_UNIT_TEST_SUITE(KqpWrite) {
CompareYson(R"([[[3u]]])", FormatResultSetYson(result.GetResultSet(1)));
}
- Y_UNIT_TEST_NEW_ENGINE(InsertRevert) {
+ Y_UNIT_TEST_NEW_ENGINE(InsertRevert) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
UNIT_ASSERT(session.ExecuteSchemeQuery(R"(
- CREATE TABLE `/Root/TxCheck` (
+ CREATE TABLE `/Root/TxCheck` (
Id Uint32,
PRIMARY KEY (Id)
);
)").GetValueSync().IsSuccess());
- auto insertQuery = Q1_(R"(
- DECLARE $rows AS List<Struct<Key: Uint64, Value: String>>;
+ auto insertQuery = Q1_(R"(
+ DECLARE $rows AS List<Struct<Key: Uint64, Value: String>>;
DECLARE $id AS Uint32;
- INSERT OR REVERT INTO `/Root/KeyValue`
+ INSERT OR REVERT INTO `/Root/KeyValue`
SELECT * FROM AS_TABLE($rows);
- UPSERT INTO `/Root/TxCheck` (Id) VALUES ($id);
- )");
+ UPSERT INTO `/Root/TxCheck` (Id) VALUES ($id);
+ )");
auto params = db.GetParamsBuilder()
.AddParam("$rows")
@@ -447,12 +447,12 @@ Y_UNIT_TEST_SUITE(KqpWrite) {
TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params).ExtractValueSync();
result.GetIssues().PrintTo(Cerr);
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- if constexpr (UseNewEngine) {
- // ¯\_(ツ)_/¯
- UNIT_ASSERT_C(result.GetIssues().Size() == 0, result.GetIssues().ToString());
- } else {
- UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_OPERATION_REVERTED));
- }
+ if constexpr (UseNewEngine) {
+ // ¯\_(ツ)_/¯
+ UNIT_ASSERT_C(result.GetIssues().Size() == 0, result.GetIssues().ToString());
+ } else {
+ UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_OPERATION_REVERTED));
+ }
params = db.GetParamsBuilder()
.AddParam("$rows")
@@ -483,12 +483,12 @@ Y_UNIT_TEST_SUITE(KqpWrite) {
TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params).ExtractValueSync();
result.GetIssues().PrintTo(Cerr);
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- if constexpr (UseNewEngine) {
- // ¯\_(ツ)_/¯
- UNIT_ASSERT_C(result.GetIssues().Size() == 0, result.GetIssues().ToString());
- } else {
- UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_OPERATION_REVERTED));
- }
+ if constexpr (UseNewEngine) {
+ // ¯\_(ツ)_/¯
+ UNIT_ASSERT_C(result.GetIssues().Size() == 0, result.GetIssues().ToString());
+ } else {
+ UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_OPERATION_REVERTED));
+ }
params = db.GetParamsBuilder()
.AddParam("$rows")
@@ -514,10 +514,10 @@ Y_UNIT_TEST_SUITE(KqpWrite) {
TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- result = session.ExecuteDataQuery(Q_(R"(
- SELECT * FROM `/Root/KeyValue`;
- SELECT * FROM `/Root/TxCheck`;
- )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ result = session.ExecuteDataQuery(Q_(R"(
+ SELECT * FROM `/Root/KeyValue`;
+ SELECT * FROM `/Root/TxCheck`;
+ )"), TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
CompareYson(R"([
[[1u];["One"]];
@@ -528,7 +528,7 @@ Y_UNIT_TEST_SUITE(KqpWrite) {
CompareYson(R"([[[1u]];[[2u]];[[3u]]])", FormatResultSetYson(result.GetResultSet(1)));
}
- Y_UNIT_TEST_NEW_ENGINE(ProjectReplace) {
+ Y_UNIT_TEST_NEW_ENGINE(ProjectReplace) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -555,7 +555,7 @@ Y_UNIT_TEST_SUITE(KqpWrite) {
.Build()
.Build();
- auto result = session.ExecuteDataQuery(Q1_(R"(
+ auto result = session.ExecuteDataQuery(Q1_(R"(
DECLARE $rows AS List<Struct<
In_Group : Uint32?,
In_Name : String?,
@@ -568,12 +568,12 @@ Y_UNIT_TEST_SUITE(KqpWrite) {
In_Name AS Name,
In_Amount AS Amount
FROM AS_TABLE($rows);
- )"), TTxControl::BeginTx().CommitTx(), params, execSettings).ExtractValueSync();
+ )"), TTxControl::BeginTx().CommitTx(), params, execSettings).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
- UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), UseNewEngine ? 2 : 1);
+ UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), UseNewEngine ? 2 : 1);
}
Y_UNIT_TEST(Uint8Key) {
@@ -591,14 +591,14 @@ Y_UNIT_TEST_SUITE(KqpWrite) {
auto result = session.ExecuteDataQuery(R"(
--!syntax_v1
- PRAGMA Kikimr.UseNewEngine = 'false';
+ PRAGMA Kikimr.UseNewEngine = 'false';
UPSERT INTO `/Root/Temp` (Key) VALUES (127);
)", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
result = session.ExecuteDataQuery(R"(
--!syntax_v1
- PRAGMA Kikimr.UseNewEngine = 'false';
+ PRAGMA Kikimr.UseNewEngine = 'false';
UPSERT INTO `/Root/Temp` (Key) VALUES (128);
)", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
result.GetIssues().PrintTo(Cerr);
@@ -606,7 +606,7 @@ Y_UNIT_TEST_SUITE(KqpWrite) {
result = session.ExecuteDataQuery(R"(
--!syntax_v1
- PRAGMA Kikimr.UseNewEngine = 'false';
+ PRAGMA Kikimr.UseNewEngine = 'false';
DELETE FROM `/Root/Temp` ON (Key) VALUES (140);
)", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
result.GetIssues().PrintTo(Cerr);
@@ -614,7 +614,7 @@ Y_UNIT_TEST_SUITE(KqpWrite) {
result = session.ExecuteDataQuery(R"(
--!syntax_v1
- PRAGMA Kikimr.UseNewEngine = 'false';
+ PRAGMA Kikimr.UseNewEngine = 'false';
SELECT * FROM `/Root/Temp`;
)", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
@@ -693,5 +693,5 @@ Y_UNIT_TEST_SUITE(KqpWrite) {
}
}
-} // namespace NKqp
+} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/ut/ya.make b/ydb/core/kqp/ut/ya.make
index 04f29db39f..609ec3237f 100644
--- a/ydb/core/kqp/ut/ya.make
+++ b/ydb/core/kqp/ut/ya.make
@@ -24,8 +24,8 @@ SRCS(
kqp_document_api_ut.cpp
kqp_effects_perf_ut.cpp
kqp_explain_ut.cpp
- kqp_flip_join_ut.cpp
- kqp_index_lookup_join_ut.cpp
+ kqp_flip_join_ut.cpp
+ kqp_index_lookup_join_ut.cpp
kqp_indexes_ut.cpp
kqp_indexes_multishard_ut.cpp
kqp_join_ut.cpp
@@ -33,8 +33,8 @@ SRCS(
kqp_locks_ut.cpp
kqp_merge_connection_ut.cpp
kqp_mvcc_ut.cpp
- kqp_newengine_effects_ut.cpp
- kqp_newengine_flowcontrol_ut.cpp
+ kqp_newengine_effects_ut.cpp
+ kqp_newengine_flowcontrol_ut.cpp
kqp_newengine_ut.cpp
kqp_not_null_columns_ut.cpp
kqp_olap_ut.cpp
@@ -48,7 +48,7 @@ SRCS(
kqp_service_ut.cpp
kqp_sort_ut.cpp
kqp_stats_ut.cpp
- kqp_sqlin_ut.cpp
+ kqp_sqlin_ut.cpp
kqp_sys_view_ut.cpp
kqp_sys_col_ut.cpp
kqp_table_predicate_ut.cpp
@@ -76,9 +76,9 @@ REQUIREMENTS(ram:12)
END()
-RECURSE(
- fat
- ../rm/ut
+RECURSE(
+ fat
+ ../rm/ut
../proxy/ut
../runtime/ut
-)
+)
diff --git a/ydb/core/persqueue/cluster_tracker.cpp b/ydb/core/persqueue/cluster_tracker.cpp
index 5a63b24a4e..59fdea5c2b 100644
--- a/ydb/core/persqueue/cluster_tracker.cpp
+++ b/ydb/core/persqueue/cluster_tracker.cpp
@@ -57,7 +57,7 @@ private:
void HandleWhileWaiting(TEvClusterTracker::TEvSubscribe::TPtr& ev) {
AddSubscriber(ev->Sender);
- Become(&TThis::Working);
+ Become(&TThis::Working);
Send(Ctx().SelfID, new TEvents::TEvWakeup);
}
@@ -116,15 +116,15 @@ private:
void HandleWhileWorking(TEvents::TEvWakeup::TPtr&) {
auto req = MakeHolder<NKqp::TEvKqp::TEvQueryRequest>();
- req->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE);
- req->Record.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_SQL_DML);
+ req->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE);
+ req->Record.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_SQL_DML);
req->Record.MutableRequest()->SetKeepSession(false);
- req->Record.MutableRequest()->SetQuery(MakeListClustersQuery());
+ req->Record.MutableRequest()->SetQuery(MakeListClustersQuery());
req->Record.MutableRequest()->SetDatabase(GetDatabase());
- // useless without explicit session
- // req->Record.MutableRequest()->MutableQueryCachePolicy()->set_keep_in_cache(true);
- req->Record.MutableRequest()->MutableTxControl()->mutable_begin_tx()->mutable_serializable_read_write();
- req->Record.MutableRequest()->MutableTxControl()->set_commit_tx(true);
+ // useless without explicit session
+ // req->Record.MutableRequest()->MutableQueryCachePolicy()->set_keep_in_cache(true);
+ req->Record.MutableRequest()->MutableTxControl()->mutable_begin_tx()->mutable_serializable_read_write();
+ req->Record.MutableRequest()->MutableTxControl()->set_commit_tx(true);
Send(NKqp::MakeKqpProxyID(Ctx().SelfID.NodeId()), req.Release());
}
@@ -144,9 +144,9 @@ private:
} else {
LOG_ERROR_S(Ctx(), NKikimrServices::PERSQUEUE_CLUSTER_TRACKER, "failed to list clusters: " << record);
- ClustersList = nullptr;
-
- Schedule(TDuration::Seconds(Cfg().GetClustersUpdateTimeoutOnErrorSec()), new TEvents::TEvWakeup);
+ ClustersList = nullptr;
+
+ Schedule(TDuration::Seconds(Cfg().GetClustersUpdateTimeoutOnErrorSec()), new TEvents::TEvWakeup);
}
}
@@ -154,9 +154,9 @@ private:
const auto& record = ev->Get()->Record;
LOG_ERROR_S(Ctx(), NKikimrServices::PERSQUEUE_CLUSTER_TRACKER, "failed to list clusters: " << record);
- ClustersList = nullptr;
-
- Schedule(TDuration::Seconds(Cfg().GetClustersUpdateTimeoutOnErrorSec()), new TEvents::TEvWakeup);
+ ClustersList = nullptr;
+
+ Schedule(TDuration::Seconds(Cfg().GetClustersUpdateTimeoutOnErrorSec()), new TEvents::TEvWakeup);
}
template<typename TProtoRecord>
diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto
index 30d2bff2ba..d64169d4fc 100644
--- a/ydb/core/protos/config.proto
+++ b/ydb/core/protos/config.proto
@@ -640,12 +640,12 @@ message TFeatureFlags {
optional bool EnablePersistentQueryStats = 25 [default = false];
optional bool DisableDataShardBarrier = 26 [default = false];
optional bool EnablePutBatchingForBlobStorage = 27 [default = true];
- optional bool EnableKqpWideFlow = 28 [default = true]; // deprecated: always true
+ optional bool EnableKqpWideFlow = 28 [default = true]; // deprecated: always true
optional bool EnableKqpScanQueries = 29 [default = true]; // deprecated: always true
optional bool EnablePersistentPartitionStats = 30 [default = false];
optional bool EnableTtlOnIndexedTables = 31 [default = true]; // deprecated: always true
optional bool AllowUpdateChannelsBindingOfSolomonPartitions = 32 [default = false];
- optional bool DisableLegacyYql = 33 [default = true]; // deprecated: always true
+ optional bool DisableLegacyYql = 33 [default = true]; // deprecated: always true
optional bool EnableDataColumnForIndexTable = 34 [default = true];
optional bool AllowServerlessStorageBillingForSchemeShard = 35 [default = false];
optional bool EnableGracefulShutdown = 36 [default = true];
@@ -655,7 +655,7 @@ message TFeatureFlags {
optional bool EnableClockGettimeForUserCpuAccounting = 40 [default = false];
optional bool EnableAsyncIndexes = 41 [default = true];
optional bool AllowStreamExecuteYqlScript = 42 [default = true];
- optional bool EnableKqpScanOverPersistentSnapshot = 43 [default = true]; // deprecated: always true
+ optional bool EnableKqpScanOverPersistentSnapshot = 43 [default = true]; // deprecated: always true
optional bool EnableOlapSchemaOperations = 44 [default = false];
optional bool EnableVPatch = 45 [default = false];
optional bool EnableMvccSnapshotReads = 46 [default = false];
@@ -947,41 +947,41 @@ message TTableProfilesConfig {
message TTableServiceConfig {
message TResourceManager {
optional uint32 ComputeActorsCount = 1 [default = 1000];
- optional uint64 ChannelBufferSize = 2 [default = 4194304]; // 4 MB
- optional uint64 ScanBufferSize = 3 [default = 8388608]; // 8 MB
- optional uint64 MkqlLightProgramMemoryLimit = 4 [default = 31457280]; // 30 MB
- optional uint64 MkqlHeavyProgramMemoryLimit = 5 [default = 31457280]; // 30 MB
- optional uint64 QueryMemoryLimit = 6 [default = 32212254720]; // 30 GB
+ optional uint64 ChannelBufferSize = 2 [default = 4194304]; // 4 MB
+ optional uint64 ScanBufferSize = 3 [default = 8388608]; // 8 MB
+ optional uint64 MkqlLightProgramMemoryLimit = 4 [default = 31457280]; // 30 MB
+ optional uint64 MkqlHeavyProgramMemoryLimit = 5 [default = 31457280]; // 30 MB
+ optional uint64 QueryMemoryLimit = 6 [default = 32212254720]; // 30 GB
optional uint32 PublishStatisticsIntervalSec = 7 [default = 2];
- optional bool EnableInstantMkqlMemoryAlloc = 8 [default = true];
- reserved 9; // optional uint32 InstantMkqlMemoryAllocWaitTimeMs = 9 [default = 20];
- reserved 10; // optional uint32 InstantMkqlMemoryAllocStepTimeMs = 10 [default = 1];
-
- optional uint64 MaxTotalChannelBuffersSize = 11 [default = 2147483648]; // 2 GB
- optional uint64 MinChannelBufferSize = 12 [default = 2097152]; // 2 MB
-
- optional uint64 MaxTotalScanBuffersSize = 13 [default = 4294967296]; // 4 GB
- optional uint64 MinScanBufferSize = 14 [default = 2097152]; // 2 MB
+ optional bool EnableInstantMkqlMemoryAlloc = 8 [default = true];
+ reserved 9; // optional uint32 InstantMkqlMemoryAllocWaitTimeMs = 9 [default = 20];
+ reserved 10; // optional uint32 InstantMkqlMemoryAllocStepTimeMs = 10 [default = 1];
+
+ optional uint64 MaxTotalChannelBuffersSize = 11 [default = 2147483648]; // 2 GB
+ optional uint64 MinChannelBufferSize = 12 [default = 2097152]; // 2 MB
+
+ optional uint64 MaxTotalScanBuffersSize = 13 [default = 4294967296]; // 4 GB
+ optional uint64 MinScanBufferSize = 14 [default = 2097152]; // 2 MB
+ }
+
+ message TSpillingServiceConfig {
+ message TLocalFileConfig {
+ optional bool Enable = 1 [default = false];
+ optional string Root = 2 [default = "/tmp/kikimr_spilling/"];
+ optional uint64 MaxTotalSize = 3 [default = 21474836480]; // 20 GiB
+ optional uint64 MaxFileSize = 4 [default = 5368709120]; // 5 GiB
+ optional uint64 MaxFilePartSize = 5 [default = 104857600]; // 100 MB
+
+ message TIoThreadPoolConfig {
+ optional uint32 WorkersCount = 1 [default = 2];
+ optional uint32 QueueSize = 2 [default = 1000];
+ };
+ optional TIoThreadPoolConfig IoThreadPool = 6;
+ }
+
+ optional TLocalFileConfig LocalFileConfig = 1;
}
- message TSpillingServiceConfig {
- message TLocalFileConfig {
- optional bool Enable = 1 [default = false];
- optional string Root = 2 [default = "/tmp/kikimr_spilling/"];
- optional uint64 MaxTotalSize = 3 [default = 21474836480]; // 20 GiB
- optional uint64 MaxFileSize = 4 [default = 5368709120]; // 5 GiB
- optional uint64 MaxFilePartSize = 5 [default = 104857600]; // 100 MB
-
- message TIoThreadPoolConfig {
- optional uint32 WorkersCount = 1 [default = 2];
- optional uint32 QueueSize = 2 [default = 1000];
- };
- optional TIoThreadPoolConfig IoThreadPool = 6;
- }
-
- optional TLocalFileConfig LocalFileConfig = 1;
- }
-
message TQueryPhaseLimits {
optional uint32 AffectedShardsLimit = 1;
optional uint32 ReadsetCountLimit = 2;
@@ -1027,7 +1027,7 @@ message TTableServiceConfig {
// After SoftSessionShutdownTimeoutMs milliseconds
// session will not accept new query requests.
optional uint32 SoftSessionShutdownTimeoutMs = 6 [default = 1000];
- // Minimum number of sessions on node to start balancing. If number of sessions on node
+ // Minimum number of sessions on node to start balancing. If number of sessions on node
// less than MinNodeSessions, balancer will not be able to kick session from node.
optional uint32 MinNodeSessions = 7 [default = 10];
// If LocalDatacenterPolicy is true, balancer will count only local datacenter
@@ -1058,7 +1058,7 @@ message TTableServiceConfig {
optional uint32 QueryLimitBytes = 1;
optional uint32 ParametersLimitBytes = 2;
optional uint32 SessionsLimitPerNode = 3;
- reserved 4; // obsolete: optional uint32 SessionQueryCacheSize = 4 [default = 1000];
+ reserved 4; // obsolete: optional uint32 SessionQueryCacheSize = 4 [default = 1000];
reserved 5; // UseCompileService (always true)
optional uint32 CompileQueryCacheSize = 6 [default = 1000];
optional uint32 CompileMaxActiveRequests = 7 [default = 4];
@@ -1070,16 +1070,16 @@ message TTableServiceConfig {
optional TResourceManager ResourceManager = 13;
optional TQueryLimits QueryLimits = 14;
optional uint32 SqlVersion = 15;
- optional TSpillingServiceConfig SpillingServiceConfig = 16;
+ optional TSpillingServiceConfig SpillingServiceConfig = 16;
optional TShutdownSettings ShutdownSettings = 17;
optional TSessionBalancerSettings SessionBalancerSettings = 18;
- optional uint32 ForceNewEnginePercent = 19 [default = 0];
- // level:
- // 0 - Non interactive, read-only queries without joins, sql in and secondary indexes. With fallback to old engine.
- // 1 - Non interactive, read-only queries. With fallback to old engine.
- // 2 - All read-only queries. Without fallback to new engine (in case of rw tx, read queries will be executed with new engine, write queries - with old one).
- // 3 - New engine for all queries.
- optional uint32 ForceNewEngineLevel = 22 [default = 0];
+ optional uint32 ForceNewEnginePercent = 19 [default = 0];
+ // level:
+ // 0 - Non interactive, read-only queries without joins, sql in and secondary indexes. With fallback to old engine.
+ // 1 - Non interactive, read-only queries. With fallback to old engine.
+ // 2 - All read-only queries. Without fallback to new engine (in case of rw tx, read queries will be executed with new engine, write queries - with old one).
+ // 3 - New engine for all queries.
+ optional uint32 ForceNewEngineLevel = 22 [default = 0];
optional uint32 CompileQueryCacheTTLSec = 20 [default = 0];
optional TQueryReplayConfig QueryReplayConfig = 21;
};
diff --git a/ydb/core/protos/counters_datashard.proto b/ydb/core/protos/counters_datashard.proto
index 503c030424..956a136937 100644
--- a/ydb/core/protos/counters_datashard.proto
+++ b/ydb/core/protos/counters_datashard.proto
@@ -99,10 +99,10 @@ enum ECumulativeCounters {
COUNTER_ERASE_ROWS = 73 [(CounterOpts) = {Name: "EraseRows"}];
COUNTER_BUILD_DEPENDENCIES_USEC = 74 [(CounterOpts) = {Name: "BuildDependenciesMicrosec"}];
COUNTER_APPLY_LOCKS_USEC = 75 [(CounterOpts) = {Name: "ApplyLocksMicrosec"}];
- COUNTER_TX_NOT_ENOUGH_MEMORY_1 = 76 [(CounterOpts) = {Name: "TxNotEnoughMemory1"}];
- COUNTER_TX_NOT_ENOUGH_MEMORY_2 = 77 [(CounterOpts) = {Name: "TxNotEnoughMemory2"}];
- COUNTER_TX_NOT_ENOUGH_MEMORY_3 = 78 [(CounterOpts) = {Name: "TxNotEnoughMemory3"}];
- COUNTER_TX_NOT_ENOUGH_MEMORY_4 = 79 [(CounterOpts) = {Name: "TxNotEnoughMemory4"}];
+ COUNTER_TX_NOT_ENOUGH_MEMORY_1 = 76 [(CounterOpts) = {Name: "TxNotEnoughMemory1"}];
+ COUNTER_TX_NOT_ENOUGH_MEMORY_2 = 77 [(CounterOpts) = {Name: "TxNotEnoughMemory2"}];
+ COUNTER_TX_NOT_ENOUGH_MEMORY_3 = 78 [(CounterOpts) = {Name: "TxNotEnoughMemory3"}];
+ COUNTER_TX_NOT_ENOUGH_MEMORY_4 = 79 [(CounterOpts) = {Name: "TxNotEnoughMemory4"}];
COUNTER_TX_TABLET_NOT_READY = 80 [(CounterOpts) = {Name: "TxTabletNotReady"}];
COUNTER_TX_BACKGROUND_COMPACTION = 81 [(CounterOpts) = {Name: "TxCompactTable"}];
COUNTER_TX_BACKGROUND_COMPACTION_NOT_NEEDED = 82 [(CounterOpts) = {Name: "TxCompactTableNotNeeded"}];
diff --git a/ydb/core/protos/kqp.proto b/ydb/core/protos/kqp.proto
index 2462e06c88..9b46b720e1 100644
--- a/ydb/core/protos/kqp.proto
+++ b/ydb/core/protos/kqp.proto
@@ -79,7 +79,7 @@ message TQueryRequest {
optional bool KeepSession = 5;
optional bytes Cluster = 6;
optional NKikimrMiniKQL.TParams Parameters = 7;
- reserved 8; // (deprecated) SqlAutoCommit
+ reserved 8; // (deprecated) SqlAutoCommit
optional EQueryAction Action = 9;
optional bool Profile = 10; // DEPRECATED, use StatsMode instead
optional bytes PreparedQuery = 11;
@@ -91,7 +91,7 @@ message TQueryRequest {
optional uint64 TimeoutMs = 17;
optional Ydb.YqlInternal.ExecQuerySettings QuerySettings = 18;
reserved 19; // (deprecated) StatsMode
- optional NYql.NDqProto.EDqStatsMode StatsMode = 20;
+ optional NYql.NDqProto.EDqStatsMode StatsMode = 20;
}
message TKqpPathIdProto {
@@ -258,7 +258,7 @@ message TEvQueryResponse {
reserved 2; // (deprecated) KqpError
optional TQueryResponse Response = 3;
optional Ydb.StatusIds.StatusCode YdbStatus = 4;
- reserved 5; // (deprecated) ForcedNewEngine
+ reserved 5; // (deprecated) ForcedNewEngine
optional uint64 ConsumedRu = 6;
optional bool WorkerIsClosing = 7 [default = false];
}
@@ -365,7 +365,7 @@ message TExecuterTxResult {
repeated NKikimrMiniKQL.TResult Results = 3;
optional NKikimrMiniKQL.TResult Locks = 4;
reserved 5; // (deprecated) Stats
- optional NYql.NDqProto.TDqExecutionStats Stats = 6;
+ optional NYql.NDqProto.TDqExecutionStats Stats = 6;
};
message TExecuterTxResponse {
@@ -380,24 +380,24 @@ message TEvExecuterTxResponse {
message TEvExecuterStreamData {
optional Ydb.ResultSet ResultSet = 1;
- optional uint64 SeqNo = 2;
+ optional uint64 SeqNo = 2;
};
-message TEvExecuterStreamDataAck {
- optional uint64 SeqNo = 1;
- optional int64 FreeSpace = 2;
+message TEvExecuterStreamDataAck {
+ optional uint64 SeqNo = 1;
+ optional int64 FreeSpace = 2;
optional bool Enough = 3 [default = false];
-};
-
-message TEvExecuterStreamProfile {
+};
+
+message TEvExecuterStreamProfile {
reserved 1; // (deprecated) Profile
- optional NYql.NDqProto.TDqExecutionStats Profile = 2;
-};
-
-message TEvExecuterProgress {
+ optional NYql.NDqProto.TDqExecutionStats Profile = 2;
+};
+
+message TEvExecuterProgress {
optional NActorsProto.TActorId ExecuterActorId = 1;
-};
-
+};
+
message TKqpProxyNodeResources {
optional uint32 NodeId = 1;
optional uint64 DataCenterNumId = 2;
@@ -407,141 +407,141 @@ message TKqpProxyNodeResources {
optional uint32 Threads = 6;
}
-message TKqpSnapshot {
- optional uint64 Step = 1;
- optional uint64 TxId = 2;
-}
+message TKqpSnapshot {
+ optional uint64 Step = 1;
+ optional uint64 TxId = 2;
+}
-/// Legacy ResourceManager events //////////////////////////////////////////////////////////////////////////////////////
-message TEvStartComputeTasks {
- optional uint64 TxId = 1;
- repeated NYql.NDqProto.TDqTask Tasks = 2;
+/// Legacy ResourceManager events //////////////////////////////////////////////////////////////////////////////////////
+message TEvStartComputeTasks {
+ optional uint64 TxId = 1;
+ repeated NYql.NDqProto.TDqTask Tasks = 2;
reserved 3; // (deprecated) WaitResourcesTimeoutMs
- optional NYql.NDqProto.TComputeRuntimeSettings RuntimeSettings = 4;
- optional NActorsProto.TActorId ExecuterActorId = 5; // overwrite for ev->Sender
-
- optional TKqpSnapshot Snapshot = 8;
-}
-
-message TEvStartComputeTasksFailure {
- optional uint64 TxId = 1;
-
- enum EReason {
- MEMORY_LIMIT_EXCEEDED = 1; // total memory limit per query and per node
- NOT_ENOUGH_MEMORY = 2; // not enough memory at moment
- COMPUTE_ACTORS_LIMIT_EXCEEDED = 3; // -//-
- NOT_ENOUGH_COMPUTE_ACTORS = 4; // -//-
- REQUEST_ALREADY_EXISTS = 5;
- NOT_READY = 6; // ResourceManager not ready yet
- }
- optional EReason Reason = 2;
- optional string Message = 3;
-}
-
-message TEvStartedComputeTasks {
- optional uint64 TxId = 1;
-
- message TStartedTask {
- optional uint64 TaskId = 1;
- optional NActorsProto.TActorId ActorId = 2;
- }
- repeated TStartedTask StartedTasks = 2;
-}
-
-message TEvCancelComputeTasks {
- optional uint64 TxId = 1;
- repeated Ydb.Issue.IssueMessage Issues = 2;
-}
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-/// KQP Node Events ////////////////////////////////////////////////////////////////////////////////////////////////////
-message TEvStartKqpTasksRequest {
- optional uint64 TxId = 1;
- repeated NYql.NDqProto.TDqTask Tasks = 2;
- optional NYql.NDqProto.TComputeRuntimeSettings RuntimeSettings = 3;
- optional NActorsProto.TActorId ExecuterActorId = 4;
- optional TKqpSnapshot Snapshot = 5;
- optional bool StartAllOrFail = 6 [default = true];
-}
-
-message TEvStartKqpTasksResponse {
- message TStartedTask {
- optional uint64 TaskId = 1;
- optional NActorsProto.TActorId ActorId = 2;
- }
-
- enum ENotStartedTaskReason {
- // node limits
- NOT_ENOUGH_MEMORY = 1;
- NOT_ENOUGH_EXECUTION_UNITS = 2;
- // query limits
- QUERY_MEMORY_LIMIT_EXCEEDED = 3;
- QUERY_EXECUTION_UNITS_LIMIT_EXCEEDED = 4;
- // any other error
- INTERNAL_ERROR = 5;
- }
-
- message TNotStartedTask {
- optional uint64 TaskId = 1;
- optional ENotStartedTaskReason Reason = 2;
- optional string Message = 3;
- }
-
- optional uint64 TxId = 1;
- repeated TStartedTask StartedTasks = 2;
- repeated TNotStartedTask NotStartedTasks = 3;
-}
-
-message TEvCancelKqpTasksRequest {
- optional uint64 TxId = 1;
- optional string Reason = 2;
-}
-
-message TEvCancelKqpTasksResponse {
-}
-
-message TKqpNodeResources {
- optional NActorsProto.TActorId ResourceManagerActorId = 2; // legacy
- optional uint32 AvailableComputeActors = 4; // legacy
- optional uint64 UsedMemory = 5; // legacy
- optional uint64 TotalMemory = 7; // legacy
- message TTxResources {
- optional uint64 TxId = 1;
- optional uint32 ComputeActors = 2;
- optional uint64 Memory = 3;
- optional uint32 StartTimestamp = 4;
- }
- repeated TTxResources Transactions = 6; // legacy
-
- ////////////////////////////////////////////////////
-
- optional uint32 NodeId = 1;
- optional uint32 Timestamp = 3;
-
- message TMemory {
- optional uint32 Pool = 1;
- optional uint64 Available = 2;
- }
- repeated TMemory Memory = 8;
- optional uint32 ExecutionUnits = 9;
-}
-
-/// Scans
+ optional NYql.NDqProto.TComputeRuntimeSettings RuntimeSettings = 4;
+ optional NActorsProto.TActorId ExecuterActorId = 5; // overwrite for ev->Sender
+
+ optional TKqpSnapshot Snapshot = 8;
+}
+
+message TEvStartComputeTasksFailure {
+ optional uint64 TxId = 1;
+
+ enum EReason {
+ MEMORY_LIMIT_EXCEEDED = 1; // total memory limit per query and per node
+ NOT_ENOUGH_MEMORY = 2; // not enough memory at moment
+ COMPUTE_ACTORS_LIMIT_EXCEEDED = 3; // -//-
+ NOT_ENOUGH_COMPUTE_ACTORS = 4; // -//-
+ REQUEST_ALREADY_EXISTS = 5;
+ NOT_READY = 6; // ResourceManager not ready yet
+ }
+ optional EReason Reason = 2;
+ optional string Message = 3;
+}
+
+message TEvStartedComputeTasks {
+ optional uint64 TxId = 1;
+
+ message TStartedTask {
+ optional uint64 TaskId = 1;
+ optional NActorsProto.TActorId ActorId = 2;
+ }
+ repeated TStartedTask StartedTasks = 2;
+}
+
+message TEvCancelComputeTasks {
+ optional uint64 TxId = 1;
+ repeated Ydb.Issue.IssueMessage Issues = 2;
+}
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/// KQP Node Events ////////////////////////////////////////////////////////////////////////////////////////////////////
+message TEvStartKqpTasksRequest {
+ optional uint64 TxId = 1;
+ repeated NYql.NDqProto.TDqTask Tasks = 2;
+ optional NYql.NDqProto.TComputeRuntimeSettings RuntimeSettings = 3;
+ optional NActorsProto.TActorId ExecuterActorId = 4;
+ optional TKqpSnapshot Snapshot = 5;
+ optional bool StartAllOrFail = 6 [default = true];
+}
+
+message TEvStartKqpTasksResponse {
+ message TStartedTask {
+ optional uint64 TaskId = 1;
+ optional NActorsProto.TActorId ActorId = 2;
+ }
+
+ enum ENotStartedTaskReason {
+ // node limits
+ NOT_ENOUGH_MEMORY = 1;
+ NOT_ENOUGH_EXECUTION_UNITS = 2;
+ // query limits
+ QUERY_MEMORY_LIMIT_EXCEEDED = 3;
+ QUERY_EXECUTION_UNITS_LIMIT_EXCEEDED = 4;
+ // any other error
+ INTERNAL_ERROR = 5;
+ }
+
+ message TNotStartedTask {
+ optional uint64 TaskId = 1;
+ optional ENotStartedTaskReason Reason = 2;
+ optional string Message = 3;
+ }
+
+ optional uint64 TxId = 1;
+ repeated TStartedTask StartedTasks = 2;
+ repeated TNotStartedTask NotStartedTasks = 3;
+}
+
+message TEvCancelKqpTasksRequest {
+ optional uint64 TxId = 1;
+ optional string Reason = 2;
+}
+
+message TEvCancelKqpTasksResponse {
+}
+
+message TKqpNodeResources {
+ optional NActorsProto.TActorId ResourceManagerActorId = 2; // legacy
+ optional uint32 AvailableComputeActors = 4; // legacy
+ optional uint64 UsedMemory = 5; // legacy
+ optional uint64 TotalMemory = 7; // legacy
+ message TTxResources {
+ optional uint64 TxId = 1;
+ optional uint32 ComputeActors = 2;
+ optional uint64 Memory = 3;
+ optional uint32 StartTimestamp = 4;
+ }
+ repeated TTxResources Transactions = 6; // legacy
+
+ ////////////////////////////////////////////////////
+
+ optional uint32 NodeId = 1;
+ optional uint32 Timestamp = 3;
+
+ message TMemory {
+ optional uint32 Pool = 1;
+ optional uint64 Available = 2;
+ }
+ repeated TMemory Memory = 8;
+ optional uint32 ExecutionUnits = 9;
+}
+
+/// Scans
message TEvScanInitActor {
optional uint64 ScanId = 1;
optional NActorsProto.TActorId ScanActorId = 2;
- optional uint32 Generation = 3;
+ optional uint32 Generation = 3;
};
message TEvScanError {
optional Ydb.StatusIds.StatusCode Status = 1;
repeated Ydb.Issue.IssueMessage Issues = 2;
- optional uint32 Generation = 3;
+ optional uint32 Generation = 3;
}
message TEvRemoteScanData {
optional uint32 ScanId = 1;
- optional uint64 CpuTimeUs = 2;
+ optional uint64 CpuTimeUs = 2;
optional uint64 WaitTimeMs = 3;
optional uint32 PageFaults = 4;
optional bool PageFault = 5;
@@ -563,8 +563,8 @@ message TEvRemoteScanData {
message TEvRemoteScanDataAck {
optional uint64 FreeSpace = 1;
- optional uint32 Generation = 2;
+ optional uint32 Generation = 2;
+}
+
+message TEvKillScanTablet {
}
-
-message TEvKillScanTablet {
-}
diff --git a/ydb/core/protos/kqp_physical.proto b/ydb/core/protos/kqp_physical.proto
index d45857dc87..44d8f58716 100644
--- a/ydb/core/protos/kqp_physical.proto
+++ b/ydb/core/protos/kqp_physical.proto
@@ -68,32 +68,32 @@ message TKqpPhyParamElementValue {
uint32 ElementIndex = 2;
}
-message TKqpPhyLiteralValue {
- NKikimrMiniKQL.TType Type = 1;
- NKikimrMiniKQL.TValue Value = 2;
-}
-
-message TKqpPhyRowsList {
- message TValue {
- oneof Kind {
- TKqpPhyParamValue ParamValue = 1;
- TKqpPhyLiteralValue LiteralValue = 2;
- }
- }
-
- message TRow {
- map<string, TValue> Columns = 1;
- }
-
- repeated TRow Rows = 1;
-}
-
+message TKqpPhyLiteralValue {
+ NKikimrMiniKQL.TType Type = 1;
+ NKikimrMiniKQL.TValue Value = 2;
+}
+
+message TKqpPhyRowsList {
+ message TValue {
+ oneof Kind {
+ TKqpPhyParamValue ParamValue = 1;
+ TKqpPhyLiteralValue LiteralValue = 2;
+ }
+ }
+
+ message TRow {
+ map<string, TValue> Columns = 1;
+ }
+
+ repeated TRow Rows = 1;
+}
+
message TKqpPhyValue {
oneof Kind {
- TKqpPhyParamValue ParamValue = 1; // any type here (value, list of structs, ...)
- TKqpPhyParamElementValue ParamElementValue = 2; // any type here (value, list of structs, ...)
- TKqpPhyRowsList RowsList = 3; // list of structs
- TKqpPhyLiteralValue LiteralValue = 4; // value
+ TKqpPhyParamValue ParamValue = 1; // any type here (value, list of structs, ...)
+ TKqpPhyParamElementValue ParamElementValue = 2; // any type here (value, list of structs, ...)
+ TKqpPhyRowsList RowsList = 3; // list of structs
+ TKqpPhyLiteralValue LiteralValue = 4; // value
}
}
@@ -109,9 +109,9 @@ message TKqpPhyKeyRange {
message TKqpPhyOpReadRange {
TKqpPhyKeyRange KeyRange = 1;
- TKqpPhyParamValue ItemsLimit = 2;
- bool Reverse = 3;
- repeated bool SkipNullKeys = 4;
+ TKqpPhyParamValue ItemsLimit = 2;
+ bool Reverse = 3;
+ repeated bool SkipNullKeys = 4;
}
message TKqpPhyOpUpsertRows {
@@ -177,7 +177,7 @@ message TKqpPhyCnUnionAll {
message TKqpPhyCnMap {
}
-message TKqpPhyCnHashShuffle {
+message TKqpPhyCnHashShuffle {
repeated string KeyColumns = 1;
}
@@ -212,7 +212,7 @@ message TKqpPhyConnection {
oneof Type {
TKqpPhyCnUnionAll UnionAll = 3;
TKqpPhyCnMap Map = 4;
- TKqpPhyCnHashShuffle HashShuffle = 5;
+ TKqpPhyCnHashShuffle HashShuffle = 5;
TKqpPhyCnBroadcast Broadcast = 6;
TKqpPhyCnMapShard MapShard = 7;
TKqpPhyCnShuffleShard ShuffleShard = 8;
@@ -237,7 +237,7 @@ message TKqpPhyResult {
TKqpPhyConnection Connection = 1;
NKikimrMiniKQL.TType ItemType = 2;
bool IsStream = 3;
- repeated string ColumnHints = 4;
+ repeated string ColumnHints = 4;
}
message TKqpPhyTx {
@@ -253,7 +253,7 @@ message TKqpPhyTx {
repeated TKqpPhyResult Results = 3;
repeated TKqpPhyParamBinding ParamBindings = 4;
string Plan = 5;
- bool HasEffects = 6; // at least one stage has flag TKqpPhyStage::IsEffectStage set
+ bool HasEffects = 6; // at least one stage has flag TKqpPhyStage::IsEffectStage set
}
message TKqpTableInfo {
diff --git a/ydb/core/protos/kqp_query_settings.proto b/ydb/core/protos/kqp_query_settings.proto
index 444543a636..2cf97fbe9a 100644
--- a/ydb/core/protos/kqp_query_settings.proto
+++ b/ydb/core/protos/kqp_query_settings.proto
@@ -10,5 +10,5 @@ import "ydb/public/api/protos/ydb_common.proto";
message ExecQuerySettings {
Ydb.FeatureFlag.Status strict_dml = 1;
- reserved 2; // Ydb.FeatureFlag.Status use_new_engine = 2;
+ reserved 2; // Ydb.FeatureFlag.Status use_new_engine = 2;
}
diff --git a/ydb/core/protos/kqp_stats.proto b/ydb/core/protos/kqp_stats.proto
index 5c225df89d..c2579c24ff 100644
--- a/ydb/core/protos/kqp_stats.proto
+++ b/ydb/core/protos/kqp_stats.proto
@@ -12,63 +12,63 @@ message TKqpStatsCompile {
uint64 CpuTimeUs = 3;
}
-// Per datashard table extra stats
-message TKqpShardTableExtraStats {
- uint64 ShardId = 1;
- uint64 ShardCpuTimeUs = 2;
-}
-
-// aggregated shard extra stats for table
-message TKqpShardTableAggrExtraStats {
- uint32 AffectedShards = 1;
- NYql.NDqProto.TDqStatsAggr ShardCpuTimeUs = 2;
-}
-
-message TKqpScanTableExtraStats {
- // IScan stats
- uint64 IScanStartTimeMs = 1; // start IScan timestamp
- uint64 IScanFinishTimeMs = 2; // finish IScan timestamp
- uint64 IScanCpuTimeUs = 3; // IScan::Feed time
- uint64 IScanWaitTimeUs = 4; // wait for row inside IScan object
- uint64 IScanPageFaults = 5; // IScan page faults count
-
- // Network stats
- uint64 Messages = 6;
- uint64 MessagesByPageFault = 7;
-}
-
-// extra for NYql.NDqProto.TDqComputeActorStats
-message TKqpComputeActorExtraStats {
-}
-
-// extra for NYql.NDqProto.TDqStageStats
-message TKqpStageExtraStats {
- repeated NYql.NDqProto.TDqTaskStats DatashardTasks = 1;
-};
-
-// extra for NYql.NDqProto.TDqExecutionStats
-message TKqpExecutionExtraStats {
- // Basic stats
- uint32 AffectedShards = 1;
- NYql.NDqProto.TDqStatsAggr ComputeCpuTimeUs = 2; // compute actors time
- NYql.NDqProto.TDqStatsAggr ShardsCpuTimeUs = 3; // datashards time
-
- // Profile stats
- uint64 ResolveCpuTimeUs = 100;
- uint64 ResolveWallTimeUs = 101;
-}
-
+// Per datashard table extra stats
+message TKqpShardTableExtraStats {
+ uint64 ShardId = 1;
+ uint64 ShardCpuTimeUs = 2;
+}
+
+// aggregated shard extra stats for table
+message TKqpShardTableAggrExtraStats {
+ uint32 AffectedShards = 1;
+ NYql.NDqProto.TDqStatsAggr ShardCpuTimeUs = 2;
+}
+
+message TKqpScanTableExtraStats {
+ // IScan stats
+ uint64 IScanStartTimeMs = 1; // start IScan timestamp
+ uint64 IScanFinishTimeMs = 2; // finish IScan timestamp
+ uint64 IScanCpuTimeUs = 3; // IScan::Feed time
+ uint64 IScanWaitTimeUs = 4; // wait for row inside IScan object
+ uint64 IScanPageFaults = 5; // IScan page faults count
+
+ // Network stats
+ uint64 Messages = 6;
+ uint64 MessagesByPageFault = 7;
+}
+
+// extra for NYql.NDqProto.TDqComputeActorStats
+message TKqpComputeActorExtraStats {
+}
+
+// extra for NYql.NDqProto.TDqStageStats
+message TKqpStageExtraStats {
+ repeated NYql.NDqProto.TDqTaskStats DatashardTasks = 1;
+};
+
+// extra for NYql.NDqProto.TDqExecutionStats
+message TKqpExecutionExtraStats {
+ // Basic stats
+ uint32 AffectedShards = 1;
+ NYql.NDqProto.TDqStatsAggr ComputeCpuTimeUs = 2; // compute actors time
+ NYql.NDqProto.TDqStatsAggr ShardsCpuTimeUs = 3; // datashards time
+
+ // Profile stats
+ uint64 ResolveCpuTimeUs = 100;
+ uint64 ResolveWallTimeUs = 101;
+}
+
message TKqpStatsQuery {
// Basic stats
uint64 DurationUs = 1;
TKqpStatsCompile Compilation = 2;
-
- reserved 3; // repeated TKqpStatsExecution Executions = 3;
-
+
+ reserved 3; // repeated TKqpStatsExecution Executions = 3;
+
uint64 WorkerCpuTimeUs = 4;
uint64 ReadSetsCount = 5;
uint64 MaxShardProgramSize = 6;
uint64 MaxShardReplySize = 7;
-
- repeated NYql.NDqProto.TDqExecutionStats Executions = 8;
+
+ repeated NYql.NDqProto.TDqExecutionStats Executions = 8;
}
diff --git a/ydb/core/protos/services.proto b/ydb/core/protos/services.proto
index c0bef298fc..c17c8a7dc3 100644
--- a/ydb/core/protos/services.proto
+++ b/ydb/core/protos/services.proto
@@ -148,12 +148,12 @@ enum EServiceKikimr {
LOGGER = 410;
MSGBUS_TRACER = 411;
- MSGBUS_REQUEST = 412; // deprecated, use RPC_REQUEST
+ MSGBUS_REQUEST = 412; // deprecated, use RPC_REQUEST
GRPC_SERVER = 413;
GRPC_PROXY = 415;
GRPC_PROXY_NO_CONNECT_ACCESS = 417;
- READ_TABLE_API = 414; // deprecated, use RPC_REQUEST
- RPC_REQUEST = 416;
+ READ_TABLE_API = 414; // deprecated, use RPC_REQUEST
+ RPC_REQUEST = 416;
// KEY VALUE section
KEYVALUE = 420;
@@ -213,10 +213,10 @@ enum EServiceKikimr {
KQP_SLOW_LOG = 536;
KQP_COMPILE_SERVICE = 537;
KQP_COMPILE_ACTOR = 538;
- KQP_RESOURCE_MANAGER = 539;
- KQP_BLOBS_STORAGE = 541;
+ KQP_RESOURCE_MANAGER = 539;
+ KQP_BLOBS_STORAGE = 541;
KQP_COMPILE_REQUEST = 542;
- KQP_NODE = 543;
+ KQP_NODE = 543;
KQP_LOAD_TEST = 544;
KQP_SESSION = 545;
@@ -669,7 +669,7 @@ message TActivity {
BSC_STAT_PROCESSOR = 354;
SQS_QUEUES_LIST_READER_ACTOR = 355;
TX_PROXY_EXPORT = 356;
- KQP_SCAN_COMPUTE_ACTOR = 357;
+ KQP_SCAN_COMPUTE_ACTOR = 357;
KQP_TABLE_SCAN = 358;
CONFIGURED_BOOTSTRAPPER = 359;
TX_ALLOCATOR_CLIENT_ACTOR = 360;
@@ -805,12 +805,12 @@ message TActivity {
KQP_COMPILE_REQUEST = 486;
BS_DEFRAG = 487;
BS_DEFRAG_QUANTUM = 488;
- KQP_LITERAL_EXECUTER_ACTOR = 489;
+ KQP_LITERAL_EXECUTER_ACTOR = 489;
FILESTORE_SERVICE_PROXY = 490;
TX_DATASHARD_SOURCE_OFFSETS_SERVER = 491;
KQP_OLAP_SCAN = 492;
- KQP_DATA_EXECUTER_ACTOR = 493;
- KQP_NODE_SERVICE = 494;
+ KQP_DATA_EXECUTER_ACTOR = 493;
+ KQP_NODE_SERVICE = 494;
TX_DATASHARD_SOURCE_OFFSETS_CLIENT = 500;
YQL_PROXY_ACTOR = 510;
TEST_SHARD_ACTOR = 511;
diff --git a/ydb/core/protos/sys_view.proto b/ydb/core/protos/sys_view.proto
index 54c24e9814..cfc0366272 100644
--- a/ydb/core/protos/sys_view.proto
+++ b/ydb/core/protos/sys_view.proto
@@ -81,13 +81,13 @@ message TQueryDataStats {
optional uint64 PartitionCount = 7;
}
-message TStatsAggr {
- optional uint64 Min = 1;
- optional uint64 Max = 2;
- optional uint64 Sum = 3;
- optional uint64 Cnt = 4;
-}
-
+message TStatsAggr {
+ optional uint64 Min = 1;
+ optional uint64 Max = 2;
+ optional uint64 Sum = 3;
+ optional uint64 Cnt = 4;
+}
+
message TQueryStats {
optional string QueryText = 1;
optional uint64 QueryTextHash = 2;
@@ -99,8 +99,8 @@ message TQueryStats {
optional uint64 CompileDurationMs = 8;
optional bool FromQueryCache = 9;
optional uint32 NodeId = 10;
- optional TStatsAggr ShardsCpuTimeUs = 11;
- optional TStatsAggr ComputeCpuTimeUs = 12;
+ optional TStatsAggr ShardsCpuTimeUs = 11;
+ optional TStatsAggr ComputeCpuTimeUs = 12;
optional uint64 CompileCpuTimeUs = 13;
optional uint64 ProcessCpuTimeUs = 14;
optional uint64 TotalCpuTimeUs = 15;
diff --git a/ydb/core/protos/tx_datashard.proto b/ydb/core/protos/tx_datashard.proto
index e84eb8f8ca..89863b2540 100644
--- a/ydb/core/protos/tx_datashard.proto
+++ b/ydb/core/protos/tx_datashard.proto
@@ -112,18 +112,18 @@ message TReadTableTransaction {
optional uint64 SnapshotTxId = 6;
}
-message TKqpLocks {
+message TKqpLocks {
repeated TLock Locks = 1;
- repeated uint64 SendingShards = 2; // empty at Erase
- repeated uint64 ReceivingShards = 3; // empty at Erase
-
- enum ELocksOp {
- Unspecified = 0;
- Validate = 1;
- ValidateAndErase = 2;
- Erase = 3;
- }
- optional ELocksOp Op = 4;
+ repeated uint64 SendingShards = 2; // empty at Erase
+ repeated uint64 ReceivingShards = 3; // empty at Erase
+
+ enum ELocksOp {
+ Unspecified = 0;
+ Validate = 1;
+ ValidateAndErase = 2;
+ Erase = 3;
+ }
+ optional ELocksOp Op = 4;
}
message TKqpValidateLocksResult {
@@ -144,83 +144,83 @@ enum EScanDataFormat {
}
message TKqpTransaction {
- message TColumnMeta {
- optional uint32 Id = 1;
- optional string Name = 2;
- optional uint32 Type = 3;
- }
-
- message TTableKeyRange {
- // ordered lists of non-intersecting ranges and points
- repeated NKikimrTx.TKeyRange KeyRanges = 1;
- repeated bytes KeyPoints = 2;
- }
-
- message TTableMeta {
- optional TTableId TableId = 1;
- optional string TablePath = 2;
- optional uint64 SchemaVersion = 3;
- // reserved 4
- optional string SysViewInfo = 5;
+ message TColumnMeta {
+ optional uint32 Id = 1;
+ optional string Name = 2;
+ optional uint32 Type = 3;
+ }
+
+ message TTableKeyRange {
+ // ordered lists of non-intersecting ranges and points
+ repeated NKikimrTx.TKeyRange KeyRanges = 1;
+ repeated bytes KeyPoints = 2;
+ }
+
+ message TTableMeta {
+ optional TTableId TableId = 1;
+ optional string TablePath = 2;
+ optional uint64 SchemaVersion = 3;
+ // reserved 4
+ optional string SysViewInfo = 5;
optional uint32 TableKind = 6; // NKikimr::NKqp::ETableKind
- }
-
- // Data-query task meta
- message TDataTaskMeta {
- // key range - can be either explicit full datashard range or ordered list of non-intersecting ranges
- message TKeyRange {
- oneof Kind {
- NKikimrTx.TKeyRange FullRange = 1;
- TTableKeyRange Ranges = 2;
- }
- }
-
- // single Read operation (== single ReadTable callable)
- message TReadOpMeta {
- repeated TColumnMeta Columns = 1;
- optional TKeyRange Range = 2;
- optional uint64 ItemsLimit = 3;
- optional bool Reverse = 4;
- }
-
- message TWriteOpMeta {
- optional TKeyRange Range = 1;
- }
-
- optional TTableMeta Table = 1;
- repeated TReadOpMeta Reads = 2; // if not set/empty -> no reads
- optional TWriteOpMeta Writes = 3; // if not set -> no writes
- };
-
- // Scan-query task meta
- message TScanTaskMeta {
- // represents single scan instance over the specified table, with given limit and ordering
- // can cover multiple datashards
- message TReadOpMeta {
- // shard id hint, may change (split, merge, kick, ...)
- optional uint64 ShardId = 1;
- // ordered list of non-intersecting ranges (not adjacent), with points (`To` is empty and `ToInclusive` is true)
- repeated NKikimrTx.TKeyRange KeyRanges = 2;
- }
-
- optional TTableMeta Table = 1;
- repeated TColumnMeta Columns = 2;
- repeated uint32 KeyColumnTypes = 3; // for debug logs only
- repeated bool SkipNullKeys = 4;
- repeated TReadOpMeta Reads = 5;
+ }
+
+ // Data-query task meta
+ message TDataTaskMeta {
+ // key range - can be either explicit full datashard range or ordered list of non-intersecting ranges
+ message TKeyRange {
+ oneof Kind {
+ NKikimrTx.TKeyRange FullRange = 1;
+ TTableKeyRange Ranges = 2;
+ }
+ }
+
+ // single Read operation (== single ReadTable callable)
+ message TReadOpMeta {
+ repeated TColumnMeta Columns = 1;
+ optional TKeyRange Range = 2;
+ optional uint64 ItemsLimit = 3;
+ optional bool Reverse = 4;
+ }
+
+ message TWriteOpMeta {
+ optional TKeyRange Range = 1;
+ }
+
+ optional TTableMeta Table = 1;
+ repeated TReadOpMeta Reads = 2; // if not set/empty -> no reads
+ optional TWriteOpMeta Writes = 3; // if not set -> no writes
+ };
+
+ // Scan-query task meta
+ message TScanTaskMeta {
+ // represents single scan instance over the specified table, with given limit and ordering
+ // can cover multiple datashards
+ message TReadOpMeta {
+ // shard id hint, may change (split, merge, kick, ...)
+ optional uint64 ShardId = 1;
+ // ordered list of non-intersecting ranges (not adjacent), with points (`To` is empty and `ToInclusive` is true)
+ repeated NKikimrTx.TKeyRange KeyRanges = 2;
+ }
+
+ optional TTableMeta Table = 1;
+ repeated TColumnMeta Columns = 2;
+ repeated uint32 KeyColumnTypes = 3; // for debug logs only
+ repeated bool SkipNullKeys = 4;
+ repeated TReadOpMeta Reads = 5;
optional uint64 ItemsLimit = 6;
optional bool Reverse = 7;
reserved 8; // optional bytes ProcessProgram = 8;
optional EScanDataFormat DataFormat = 9;
optional NKikimrSSA.TOlapProgram OlapProgram = 10; // Currently only for OLAP tables
- }
-
+ }
+
optional EKqpTransactionType Type = 1;
repeated NYql.NDqProto.TDqTask Tasks = 2;
- optional TKqpLocks Locks = 3;
-
- optional NYql.NDqProto.TComputeRuntimeSettings RuntimeSettings = 4;
- reserved 5;
+ optional TKqpLocks Locks = 3;
+
+ optional NYql.NDqProto.TComputeRuntimeSettings RuntimeSettings = 4;
+ reserved 5;
optional NKikimrKqp.TKqpSnapshot Snapshot = 6;
}
@@ -231,12 +231,12 @@ message TKqpTaskInfo {
}
message TKqpReply {
- repeated NYql.NDqProto.TChannelData Outputs = 1;
+ repeated NYql.NDqProto.TChannelData Outputs = 1;
repeated TKqpTaskInfo Tasks = 2;
}
message TKqpReadset {
- repeated NYql.NDqProto.TChannelData Outputs = 1;
+ repeated NYql.NDqProto.TChannelData Outputs = 1;
optional TKqpValidateLocksResult ValidateLocksResult = 2;
}
@@ -579,8 +579,8 @@ message TEvProposeTransactionResult {
optional NKikimrQueryStats.TTxStats TxStats = 23;
optional uint64 DataSeqNo = 24; // Response data seqno (1, 2, ...)
optional bytes DataLastKey = 25; // Response data last key (for retries)
- reserved 26; // optional NKqpProto.TKqpStatsRun KqpRunStats = 26;
- optional NYql.NDqProto.TDqComputeActorStats ComputeActorStats = 27; // overall time + per-task statistics
+ reserved 26; // optional NKqpProto.TKqpStatsRun KqpRunStats = 26;
+ optional NYql.NDqProto.TDqComputeActorStats ComputeActorStats = 27; // overall time + per-task statistics
}
message TEvProposeTransactionRestart {
@@ -1385,9 +1385,9 @@ message TEvKqpScan {
repeated bool SkipNullKeys = 8;
repeated NKikimrTx.TKeyRange Ranges = 9;
optional NKikimrKqp.TKqpSnapshot Snapshot = 10;
- reserved 11; // optional NKqpProto.EKqpStatsMode StatsMode = 11;
+ reserved 11; // optional NKqpProto.EKqpStatsMode StatsMode = 11;
optional uint64 TimeoutMs = 12;
- optional uint32 Generation = 13;
+ optional uint32 Generation = 13;
// Seralized OlapProgram in old format without parameters
reserved 14;
optional uint64 ItemsLimit = 15;
@@ -1577,7 +1577,7 @@ message TEvReadCancel {
message TEvApplyReplicationChanges {
message TUpdates {
// Column tags
- repeated uint32 Tags = 1;
+ repeated uint32 Tags = 1;
// Serialized values (TSerializedCellVec)
optional bytes Data = 2;
}
diff --git a/ydb/core/scheme/scheme_tablecell.h b/ydb/core/scheme/scheme_tablecell.h
index 19e6d5a3e4..bccfdca97d 100644
--- a/ydb/core/scheme/scheme_tablecell.h
+++ b/ydb/core/scheme/scheme_tablecell.h
@@ -255,14 +255,14 @@ inline ui64 GetValueHash(NScheme::TTypeId type, const TCell& cell) {
return THash<double>()(ReadUnaligned<double>((const double*)cell.Data()));
case NYql::NProto::TypeIds::Date:
- return THash<ui16>()(ReadUnaligned<ui16>((const ui16*)cell.Data()));
+ return THash<ui16>()(ReadUnaligned<ui16>((const ui16*)cell.Data()));
case NYql::NProto::TypeIds::Datetime:
return THash<ui32>()(ReadUnaligned<ui32>((const ui32*)cell.Data()));
case NYql::NProto::TypeIds::Timestamp:
return THash<ui32>()(ReadUnaligned<ui64>((const ui64*)cell.Data()));
case NYql::NProto::TypeIds::Interval:
return THash<ui32>()(ReadUnaligned<ui64>((const ui64*)cell.Data()));
-
+
case NYql::NProto::TypeIds::String:
case NYql::NProto::TypeIds::Utf8:
case NYql::NProto::TypeIds::Yson:
diff --git a/ydb/core/scheme/scheme_tabledefs.h b/ydb/core/scheme/scheme_tabledefs.h
index 1d96993204..190233fdee 100644
--- a/ydb/core/scheme/scheme_tabledefs.h
+++ b/ydb/core/scheme/scheme_tabledefs.h
@@ -167,12 +167,12 @@ public:
, To(toValues)
, InclusiveFrom(inclusiveFrom || point)
, InclusiveTo(inclusiveTo || point)
- , Point(point)
- {
- if (Point) {
- Y_VERIFY_DEBUG(toValues.empty() || fromValues.size() == toValues.size());
- }
- }
+ , Point(point)
+ {
+ if (Point) {
+ Y_VERIFY_DEBUG(toValues.empty() || fromValues.size() == toValues.size());
+ }
+ }
bool IsEmptyRange(TConstArrayRef<const NScheme::TTypeId> cellTypeIds) const;
};
@@ -183,8 +183,8 @@ public:
TSerializedCellVec To;
bool FromInclusive = false;
bool ToInclusive = false;
- bool Point = false;
-
+ bool Point = false;
+
TSerializedTableRange() {}
TSerializedTableRange(const TString& from, const TString& to, bool fromInclusive, bool toInclusive)
@@ -203,9 +203,9 @@ public:
explicit TSerializedTableRange(const TTableRange& range)
: TSerializedTableRange(range.From, range.InclusiveFrom, range.To, range.InclusiveTo)
- {
- Point = range.Point;
- }
+ {
+ Point = range.Point;
+ }
explicit TSerializedTableRange(const NKikimrTx::TKeyRange& range) {
Load(range);
@@ -226,20 +226,20 @@ public:
void Serialize(NKikimrTx::TKeyRange& range) const {
range.SetFrom(From.GetBuffer());
range.SetFromInclusive(FromInclusive);
- if (Point) {
- Y_VERIFY_DEBUG(FromInclusive);
- range.SetTo(From.GetBuffer());
- range.SetToInclusive(true);
- } else {
- range.SetTo(To.GetBuffer());
- range.SetToInclusive(ToInclusive);
- }
+ if (Point) {
+ Y_VERIFY_DEBUG(FromInclusive);
+ range.SetTo(From.GetBuffer());
+ range.SetToInclusive(true);
+ } else {
+ range.SetTo(To.GetBuffer());
+ range.SetToInclusive(ToInclusive);
+ }
}
bool IsEmpty(TConstArrayRef<NScheme::TTypeId> type) const;
TTableRange ToTableRange() const {
- return TTableRange(From.GetCells(), FromInclusive, To.GetCells(), ToInclusive, Point);
+ return TTableRange(From.GetCells(), FromInclusive, To.GetCells(), ToInclusive, Point);
}
};
@@ -678,8 +678,8 @@ public:
TVector<TPartitionInfo> Partitions;
TIntrusivePtr<TSecurityObject> SecurityObject;
- bool IsSystemView() const { return Partitions.empty(); }
-
+ bool IsSystemView() const { return Partitions.empty(); }
+
template<typename TKeyColumnTypes, typename TColumns>
TKeyDesc(const TTableId& tableId, const TTableRange& range, ERowOperation rowOperation,
const TKeyColumnTypes &keyColumnTypes, const TColumns &columns,
diff --git a/ydb/core/sys_view/common/scan_actor_base_impl.h b/ydb/core/sys_view/common/scan_actor_base_impl.h
index 04711e6eff..298a8efbc0 100644
--- a/ydb/core/sys_view/common/scan_actor_base_impl.h
+++ b/ydb/core/sys_view/common/scan_actor_base_impl.h
@@ -67,18 +67,18 @@ protected:
ReplyErrorAndDie(Ydb::StatusIds::TIMEOUT, "System view: timeout");
}
- void HandleAbortExecution(NKqp::TEvKqp::TEvAbortExecution::TPtr& ev) {
- LOG_ERROR_S(TlsActivationContext->AsActorContext(), NKikimrServices::SYSTEM_VIEWS,
- "Got abort execution event, actor: " << TBase::SelfId()
- << ", owner: " << OwnerActorId
- << ", scan id: " << ScanId
- << ", table id: " << TableId
- << ", code: " << Ydb::StatusIds::StatusCode_Name(ev->Get()->Record.GetStatusCode())
- << ", error: " << ev->Get()->Record.GetMessage());
-
- this->PassAway();
- }
-
+ void HandleAbortExecution(NKqp::TEvKqp::TEvAbortExecution::TPtr& ev) {
+ LOG_ERROR_S(TlsActivationContext->AsActorContext(), NKikimrServices::SYSTEM_VIEWS,
+ "Got abort execution event, actor: " << TBase::SelfId()
+ << ", owner: " << OwnerActorId
+ << ", scan id: " << ScanId
+ << ", table id: " << TableId
+ << ", code: " << Ydb::StatusIds::StatusCode_Name(ev->Get()->Record.GetStatusCode())
+ << ", error: " << ev->Get()->Record.GetMessage());
+
+ this->PassAway();
+ }
+
void ReplyErrorAndDie(Ydb::StatusIds::StatusCode status, const TString& message) {
LOG_ERROR_S(TlsActivationContext->AsActorContext(), NKikimrServices::SYSTEM_VIEWS,
"Scan error, actor: " << TBase::SelfId()
@@ -293,7 +293,7 @@ private:
switch (ev->GetTypeRewrite()) {
hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, HandleScanAck);
hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleNavigate);
- hFunc(NKqp::TEvKqp::TEvAbortExecution, HandleAbortExecution);
+ hFunc(NKqp::TEvKqp::TEvAbortExecution, HandleAbortExecution);
cFunc(TEvents::TEvWakeup::EventType, HandleTimeout);
cFunc(TEvents::TEvPoison::EventType, this->PassAway);
default:
@@ -306,7 +306,7 @@ private:
switch (ev->GetTypeRewrite()) {
hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, HandleScanAck);
hFunc(TEvTenantNodeEnumerator::TEvLookupResult, HandleLookup);
- hFunc(NKqp::TEvKqp::TEvAbortExecution, HandleAbortExecution);
+ hFunc(NKqp::TEvKqp::TEvAbortExecution, HandleAbortExecution);
cFunc(TEvents::TEvWakeup::EventType, HandleTimeout);
cFunc(TEvents::TEvPoison::EventType, this->PassAway);
default:
diff --git a/ydb/core/sys_view/nodes/nodes.cpp b/ydb/core/sys_view/nodes/nodes.cpp
index a03eb0c698..a2322de9f9 100644
--- a/ydb/core/sys_view/nodes/nodes.cpp
+++ b/ydb/core/sys_view/nodes/nodes.cpp
@@ -62,7 +62,7 @@ public:
hFunc(TEvents::TEvUndelivered, Undelivered);
hFunc(TEvInterconnect::TEvNodeConnected, Connected);
hFunc(TEvInterconnect::TEvNodeDisconnected, Disconnected);
- hFunc(NKqp::TEvKqp::TEvAbortExecution, HandleAbortExecution);
+ hFunc(NKqp::TEvKqp::TEvAbortExecution, HandleAbortExecution);
cFunc(TEvents::TEvWakeup::EventType, HandleTimeout);
cFunc(TEvents::TEvPoison::EventType, PassAway);
default:
diff --git a/ydb/core/sys_view/partition_stats/partition_stats.cpp b/ydb/core/sys_view/partition_stats/partition_stats.cpp
index c97160d291..d6ea9ff73b 100644
--- a/ydb/core/sys_view/partition_stats/partition_stats.cpp
+++ b/ydb/core/sys_view/partition_stats/partition_stats.cpp
@@ -354,7 +354,7 @@ public:
hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, Handle);
hFunc(TEvSysView::TEvGetPartitionStatsResult, Handle);
hFunc(TEvPipeCache::TEvDeliveryProblem, Handle);
- hFunc(NKqp::TEvKqp::TEvAbortExecution, HandleAbortExecution);
+ hFunc(NKqp::TEvKqp::TEvAbortExecution, HandleAbortExecution);
cFunc(TEvents::TEvWakeup::EventType, HandleTimeout);
cFunc(TEvents::TEvPoison::EventType, PassAway);
default:
@@ -461,7 +461,7 @@ private:
return s.GetStats().HasUpdateTime() ? TCell::Make<ui64>(s.GetStats().GetUpdateTime() * 1000) : TCell();
}});
insert({TSchema::InFlightTxCount::ColumnId, [] (const TPartitionStats& s) {
- return TCell::Make<ui32>(s.GetStats().GetInFlightTxCount());
+ return TCell::Make<ui32>(s.GetStats().GetInFlightTxCount());
}});
insert({TSchema::RowUpdates::ColumnId, [] (const TPartitionStats& s) {
return TCell::Make<ui64>(s.GetStats().GetRowUpdates());
diff --git a/ydb/core/sys_view/query_stats/query_stats.cpp b/ydb/core/sys_view/query_stats/query_stats.cpp
index cac0020abf..918c9b781e 100644
--- a/ydb/core/sys_view/query_stats/query_stats.cpp
+++ b/ydb/core/sys_view/query_stats/query_stats.cpp
@@ -103,7 +103,7 @@ public:
, StatsType(statsType)
, BucketRange(this->TableRange, bucketSize)
{
- auto now = TAppData::TimeProvider->Now();
+ auto now = TAppData::TimeProvider->Now();
History = MakeHolder<TScanQueryHistory<TGreater>>(bucketCount, bucketSize, now);
ConvertKeyRange<NKikimrSysView::TEvGetQueryMetricsRequest, ui64, ui32>(Request, this->TableRange);
@@ -120,7 +120,7 @@ public:
hFunc(TEvPrivate::TEvRetryNode, RetryNode);
hFunc(TEvSysView::TEvGetQueryStatsResponse, Handle);
hFunc(TEvPipeCache::TEvDeliveryProblem, Handle);
- hFunc(NKqp::TEvKqp::TEvAbortExecution, TBase::HandleAbortExecution);
+ hFunc(NKqp::TEvKqp::TEvAbortExecution, TBase::HandleAbortExecution);
cFunc(TEvents::TEvWakeup::EventType, TBase::HandleTimeout);
cFunc(TEvents::TEvPoison::EventType, PassAway);
default:
diff --git a/ydb/core/sys_view/service/sysview_service.cpp b/ydb/core/sys_view/service/sysview_service.cpp
index 6f82455d3c..1f56906599 100644
--- a/ydb/core/sys_view/service/sysview_service.cpp
+++ b/ydb/core/sys_view/service/sysview_service.cpp
@@ -83,22 +83,22 @@ void CollectQueryStats(const TActorContext& ctx, const NKqpProto::TKqpStatsQuery
shardsCpuTime.SetMin(Max<ui64>());
computeCpuTime.SetMin(Max<ui64>());
- auto aggregate = [] (NKikimrSysView::TStatsAggr& to, const NYql::NDqProto::TDqStatsAggr& from) {
+ auto aggregate = [] (NKikimrSysView::TStatsAggr& to, const NYql::NDqProto::TDqStatsAggr& from) {
to.SetMin(std::min(to.GetMin(), from.GetMin()));
to.SetMax(std::max(to.GetMax(), from.GetMax()));
to.SetSum(to.GetSum() + from.GetSum());
to.SetCnt(to.GetCnt() + from.GetCnt());
};
- for (const NYql::NDqProto::TDqExecutionStats& exec : queryStats->GetExecutions()) {
- NKqpProto::TKqpExecutionExtraStats execExtra;
- if (exec.HasExtra()) {
- bool ok = exec.GetExtra().UnpackTo(&execExtra);
- Y_UNUSED(ok);
- }
+ for (const NYql::NDqProto::TDqExecutionStats& exec : queryStats->GetExecutions()) {
+ NKqpProto::TKqpExecutionExtraStats execExtra;
+ if (exec.HasExtra()) {
+ bool ok = exec.GetExtra().UnpackTo(&execExtra);
+ Y_UNUSED(ok);
+ }
+
+ dataStats.SetPartitionCount(dataStats.GetPartitionCount() + execExtra.GetAffectedShards());
- dataStats.SetPartitionCount(dataStats.GetPartitionCount() + execExtra.GetAffectedShards());
-
for (auto& table : exec.GetTables()) {
dataStats.SetReadRows(dataStats.GetReadRows() + table.GetReadRows());
dataStats.SetReadBytes(dataStats.GetReadBytes() + table.GetReadBytes());
@@ -107,8 +107,8 @@ void CollectQueryStats(const TActorContext& ctx, const NKqpProto::TKqpStatsQuery
dataStats.SetDeleteRows(dataStats.GetDeleteRows() + table.GetEraseRows());
}
- aggregate(shardsCpuTime, execExtra.GetShardsCpuTimeUs());
- aggregate(computeCpuTime, execExtra.GetComputeCpuTimeUs());
+ aggregate(shardsCpuTime, execExtra.GetShardsCpuTimeUs());
+ aggregate(computeCpuTime, execExtra.GetComputeCpuTimeUs());
}
if (queryStats->HasCompilation()) {
diff --git a/ydb/core/sys_view/tablets/tablets.cpp b/ydb/core/sys_view/tablets/tablets.cpp
index e83ef54897..88627748d8 100644
--- a/ydb/core/sys_view/tablets/tablets.cpp
+++ b/ydb/core/sys_view/tablets/tablets.cpp
@@ -266,7 +266,7 @@ private:
if (!entry.HasGeneration()) {
return TCell();
}
- return TCell::Make<ui32>(entry.GetGeneration());
+ return TCell::Make<ui32>(entry.GetGeneration());
}});
insert({TSchema::NodeId::ColumnId, [] (const TEntry& entry) {
return TCell::Make<ui32>(entry.GetNodeId());
diff --git a/ydb/core/sys_view/ut_kqp.cpp b/ydb/core/sys_view/ut_kqp.cpp
index b9e5b90534..5e90eb5992 100644
--- a/ydb/core/sys_view/ut_kqp.cpp
+++ b/ydb/core/sys_view/ut_kqp.cpp
@@ -242,7 +242,7 @@ Y_UNIT_TEST_SUITE(SystemView) {
UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
NKqp::CompareYson(R"([
- [[4u];[0u];["/Root/Table0"]]
+ [[4u];[0u];["/Root/Table0"]]
])", NKqp::StreamResultToYson(it));
}
{
@@ -253,7 +253,7 @@ Y_UNIT_TEST_SUITE(SystemView) {
UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
NKqp::CompareYson(R"([
- [[5u];[0u];["/Root/Tenant1/Table1"]]
+ [[5u];[0u];["/Root/Tenant1/Table1"]]
])", NKqp::StreamResultToYson(it));
}
{
@@ -264,7 +264,7 @@ Y_UNIT_TEST_SUITE(SystemView) {
UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
NKqp::CompareYson(R"([
- [[6u];[0u];["/Root/Tenant2/Table2"]]
+ [[6u];[0u];["/Root/Tenant2/Table2"]]
])", NKqp::StreamResultToYson(it));
}
}
@@ -273,8 +273,8 @@ Y_UNIT_TEST_SUITE(SystemView) {
TTestEnv env;
CreateTenantsAndTables(env, false);
- env.GetServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_EXECUTER, NActors::NLog::PRI_DEBUG);
-
+ env.GetServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_EXECUTER, NActors::NLog::PRI_DEBUG);
+
TTableClient client(env.GetDriver());
auto session = client.CreateSession().GetValueSync().GetSession();
{
diff --git a/ydb/core/tablet/resource_broker.cpp b/ydb/core/tablet/resource_broker.cpp
index ac5ad3bc26..fff30b2601 100644
--- a/ydb/core/tablet/resource_broker.cpp
+++ b/ydb/core/tablet/resource_broker.cpp
@@ -374,19 +374,19 @@ TScheduler::~TScheduler()
entry.second->Queue->EraseTask(entry.second, false, Now);
}
-const TTask* TScheduler::FindTask(ui64 taskId, const TActorId &client) const {
- auto it = Tasks.find(std::make_pair(client, taskId));
- return it != Tasks.end() ? it->second.Get() : nullptr;
-}
-
+const TTask* TScheduler::FindTask(ui64 taskId, const TActorId &client) const {
+ auto it = Tasks.find(std::make_pair(client, taskId));
+ return it != Tasks.end() ? it->second.Get() : nullptr;
+}
+
bool TScheduler::SubmitTask(const TEvResourceBroker::TTask &task,
const TActorId &client,
- const TActorSystem &as)
+ const TActorSystem &as)
{
auto &config = TaskConfig(task.Type);
TTaskPtr newTask = new TTask(task, client, Now, config.Counters);
- LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
+ LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
"Submitted new %s task %s priority=%" PRIu64 " resources={%s}",
config.Name.data(), newTask->GetIdString().data(), task.Priority,
JoinSeq(", ", task.RequiredResources).data());
@@ -398,13 +398,13 @@ bool TScheduler::SubmitTask(const TEvResourceBroker::TTask &task,
} while (Tasks.contains(id));
newTask->TaskId = id.second;
- LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
+ LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
"Use ID %" PRIu64 " for submitted task",
id.second);
} else if (Tasks.contains(id)) {
- LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
- "SubmitTask failed for task %" PRIu64 " to %s: task with the same ID has been already submitted",
- task.TaskId, ToString(client).c_str());
+ LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
+ "SubmitTask failed for task %" PRIu64 " to %s: task with the same ID has been already submitted",
+ task.TaskId, ToString(client).c_str());
return false;
}
@@ -412,7 +412,7 @@ bool TScheduler::SubmitTask(const TEvResourceBroker::TTask &task,
MissingTaskTypeCounter->Inc();
Tasks.emplace(id, newTask);
- AssignTask(newTask, as);
+ AssignTask(newTask, as);
return true;
}
@@ -423,18 +423,18 @@ bool TScheduler::UpdateTask(ui64 taskId,
ui64 priority,
const TString &type,
bool resubmit,
- const TActorSystem &as)
+ const TActorSystem &as)
{
auto it = Tasks.find(std::make_pair(client, taskId));
if (it == Tasks.end()) {
- LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
- "UpdateTask failed for task %" PRIu64 " to %s: cannot update unknown task",
- taskId, ToString(client).c_str());
+ LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
+ "UpdateTask failed for task %" PRIu64 " to %s: cannot update unknown task",
+ taskId, ToString(client).c_str());
return false;
}
auto task = it->second;
- LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
+ LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
"Update task %s (priority=%" PRIu64 " type=%s resources={%s} resubmit=%" PRIu32 ")",
task->GetIdString().data(), priority, type.data(), JoinSeq(", ", requiredResources).data(),
(ui32)resubmit);
@@ -448,82 +448,82 @@ bool TScheduler::UpdateTask(ui64 taskId,
if (resubmit && task->InFly)
task->InFly = false;
- AssignTask(task, as);
+ AssignTask(task, as);
return true;
}
-bool TScheduler::UpdateTaskCookie(ui64 taskId,
+bool TScheduler::UpdateTaskCookie(ui64 taskId,
const TActorId &client,
TIntrusivePtr<TThrRefBase> cookie,
- const TActorSystem &as)
+ const TActorSystem &as)
{
auto it = Tasks.find(std::make_pair(client, taskId));
if (it == Tasks.end()) {
- LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
- "UpdateTaskCookie failed for task %" PRIu64 " to %s: cannot update unknown task's cookie",
- taskId, ToString(client).c_str());
- return false;
+ LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
+ "UpdateTaskCookie failed for task %" PRIu64 " to %s: cannot update unknown task's cookie",
+ taskId, ToString(client).c_str());
+ return false;
}
auto task = it->second;
- LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
+ LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
"Update cookie for task %s", task->GetIdString().data());
task->Cookie = cookie;
- return true;
+ return true;
}
-TScheduler::TTerminateTaskResult TScheduler::RemoveQueuedTask(ui64 taskId,
- const TActorId &client,
- const TActorSystem &as)
+TScheduler::TTerminateTaskResult TScheduler::RemoveQueuedTask(ui64 taskId,
+ const TActorId &client,
+ const TActorSystem &as)
{
auto it = Tasks.find(std::make_pair(client, taskId));
if (it == Tasks.end()) {
- LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
- "RemoveQueuedTask failed for task %" PRIu64 " to %s: cannot remove unknown task",
- taskId, ToString(client).c_str());
- return TTerminateTaskResult(false, nullptr);
+ LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
+ "RemoveQueuedTask failed for task %" PRIu64 " to %s: cannot remove unknown task",
+ taskId, ToString(client).c_str());
+ return TTerminateTaskResult(false, nullptr);
}
auto task = it->second;
if (task->InFly) {
- LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
- "RemoveQueuedTask failed for task %" PRIu64 " to %s: cannot remove in-fly task",
- taskId, ToString(client).c_str());
- return TTerminateTaskResult(false, task);
+ LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
+ "RemoveQueuedTask failed for task %" PRIu64 " to %s: cannot remove in-fly task",
+ taskId, ToString(client).c_str());
+ return TTerminateTaskResult(false, task);
}
- LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER, "Removing task %s",
+ LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER, "Removing task %s",
task->GetIdString().data());
- EraseTask(task, false, as);
-
- return TTerminateTaskResult(true, task);
+ EraseTask(task, false, as);
+
+ return TTerminateTaskResult(true, task);
}
-TScheduler::TTerminateTaskResult TScheduler::FinishTask(ui64 taskId,
- const TActorId &client,
- bool cancel,
- const TActorSystem &as)
+TScheduler::TTerminateTaskResult TScheduler::FinishTask(ui64 taskId,
+ const TActorId &client,
+ bool cancel,
+ const TActorSystem &as)
{
auto it = Tasks.find(std::make_pair(client, taskId));
if (it == Tasks.end()) {
- LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
- "FinishTask failed for task %" PRIu64 " to %s: cannot finish unknown task",
- taskId, ToString(client).c_str());
- return TTerminateTaskResult(false, nullptr);
+ LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
+ "FinishTask failed for task %" PRIu64 " to %s: cannot finish unknown task",
+ taskId, ToString(client).c_str());
+ return TTerminateTaskResult(false, nullptr);
}
auto task = it->second;
if (!task->InFly) {
- LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
- "FinishTask failed for task %" PRIu64 " to %s: cannot finish queued task",
- taskId, ToString(client).c_str());
- return TTerminateTaskResult(false, task);
+ LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
+ "FinishTask failed for task %" PRIu64 " to %s: cannot finish queued task",
+ taskId, ToString(client).c_str());
+ return TTerminateTaskResult(false, task);
}
- LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER, "Finish task %s (release resources {%s})",
+ LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER, "Finish task %s (release resources {%s})",
task->GetIdString().data(), JoinSeq(", ", task->RequiredResources).data());
// Add execution time to statistics but cancelled tasks
@@ -533,12 +533,12 @@ TScheduler::TTerminateTaskResult TScheduler::FinishTask(ui64 taskId,
if (!cancel)
config.ExecTime.Add(execTime);
- EraseTask(task, true, as);
+ EraseTask(task, true, as);
- return TTerminateTaskResult(true, task);
+ return TTerminateTaskResult(true, task);
}
-bool TScheduler::RemoveTasks(const TActorId &client, const TActorSystem &as)
+bool TScheduler::RemoveTasks(const TActorId &client, const TActorSystem &as)
{
bool res = false;
@@ -550,9 +550,9 @@ bool TScheduler::RemoveTasks(const TActorId &client, const TActorSystem &as)
++it;
if (inFly)
- FinishTask(id, client, true, as);
+ FinishTask(id, client, true, as);
else
- RemoveQueuedTask(id, client, as);
+ RemoveQueuedTask(id, client, as);
res = true;
} else
@@ -562,7 +562,7 @@ bool TScheduler::RemoveTasks(const TActorId &client, const TActorSystem &as)
return res;
}
-void TScheduler::EraseTask(TTaskPtr task, bool finished, const TActorSystem &as)
+void TScheduler::EraseTask(TTaskPtr task, bool finished, const TActorSystem &as)
{
auto queue = task->Queue;
auto oldp = queue->PlannedResourceUsage;
@@ -571,22 +571,22 @@ void TScheduler::EraseTask(TTaskPtr task, bool finished, const TActorSystem &as)
queue->EraseTask(task, finished, Now);
if (oldp != queue->PlannedResourceUsage)
- LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
+ LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
"Updated planned resource usage for queue %s from %f to %f (remove task %s)",
queue->Name.data(), oldp, queue->PlannedResourceUsage, task->GetIdString().data());
if (oldr != queue->RealResourceUsage)
- LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
+ LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
"Updated real resource usage for queue %s from %f to %f",
queue->Name.data(), oldr, queue->RealResourceUsage);
Tasks.erase(std::make_pair(task->Client, task->TaskId));
}
-void TScheduler::ScheduleTasks(const TActorSystem &as,
- std::function<void(const TTask &task)> &&onTaskSchedule)
+void TScheduler::ScheduleTasks(const TActorSystem &as,
+ std::function<void(const TTask &task)> &&onTaskSchedule)
{
- UpdateResourceUsage(as);
+ UpdateResourceUsage(as);
TSet<TTaskQueuePtr, TTaskQueueLess> pending;
for (auto &entry : Queues) {
@@ -613,7 +613,7 @@ void TScheduler::ScheduleTasks(const TActorSystem &as,
// Allow resource over-usage if no tasks are running.
if (!ResourceLimit->HasResources(task->RequiredResources)
&& *TotalCounters->InFlyTasks) {
- LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
+ LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
"Not enough resources to start task %s",
task->GetIdString().data());
blockedResources |= task->GetRequiredResourcesMask();
@@ -624,13 +624,13 @@ void TScheduler::ScheduleTasks(const TActorSystem &as,
// Allow resource over-usage if no tasks are running in this queue.
if (!queue->QueueLimit.HasResources(task->RequiredResources)
&& *queue->QueueCounters.InFlyTasks) {
- LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
+ LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
"Skip queue %s due to exceeded limits",
queue->Name.data());
continue;
}
- LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
+ LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
"Allocate resources {%s} for task %s from queue %s",
JoinSeq(", ", task->RequiredResources).data(),
task->GetIdString().data(), queue->Name.data());
@@ -639,16 +639,16 @@ void TScheduler::ScheduleTasks(const TActorSystem &as,
task->InFly = true;
task->StartTime = Now;
task->FinishTime = Now + EstimateTaskExecutionTime(task);
- AssignTask(task, as);
+ AssignTask(task, as);
- onTaskSchedule(*task);
+ onTaskSchedule(*task);
if (!queue->Empty())
pending.insert(queue);
}
}
-void TScheduler::UpdateResourceUsage(const TActorSystem &as)
+void TScheduler::UpdateResourceUsage(const TActorSystem &as)
{
for (auto &entry : Queues) {
auto &queue = entry.second;
@@ -656,24 +656,24 @@ void TScheduler::UpdateResourceUsage(const TActorSystem &as)
queue->UpdateRealResourceUsage(Now);
if (old != queue->RealResourceUsage)
- LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
+ LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
"Updated real resource usage for queue %s from %f to %f (in-fly consumption {%s})",
queue->Name.data(), old, queue->RealResourceUsage,
JoinSeq(", ", queue->QueueLimit.Used).data());
}
}
-void TScheduler::AssignTask(TTaskPtr &task, const TActorSystem &as)
+void TScheduler::AssignTask(TTaskPtr &task, const TActorSystem &as)
{
TString state = task->InFly ? "in-fly" : "waiting";
TTaskQueuePtr queue = TaskConfig(task->Type).Queue;
if (!TaskConfigs.contains(task->Type)) {
- LOG_ERROR(as, NKikimrServices::RESOURCE_BROKER,
+ LOG_ERROR(as, NKikimrServices::RESOURCE_BROKER,
"Assigning %s task '%s' of unknown type '%s' to default queue",
state.data(), task->GetIdString().data(), task->Type.data());
} else {
- LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
+ LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
"Assigning %s task %s to queue %s",
state.data(), task->GetIdString().data(), queue->Name.data());
}
@@ -700,12 +700,12 @@ void TScheduler::AssignTask(TTaskPtr &task, const TActorSystem &as)
queue->InsertTask(task, Now);
if (oldr != queue->RealResourceUsage)
- LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
+ LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
"Updated real resource usage for queue %s from %f to %f",
queue->Name.data(), oldr, queue->RealResourceUsage);
if (oldp != queue->PlannedResourceUsage)
- LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
+ LOG_DEBUG(as, NKikimrServices::RESOURCE_BROKER,
"Updated planned resource usage for queue %s from %f to %f (insert task %s)",
queue->Name.data(), oldp, queue->PlannedResourceUsage, task->GetIdString().data());
}
@@ -729,7 +729,7 @@ TDuration TScheduler::EstimateTaskExecutionTime(TTaskPtr task)
return TaskConfig(task->Type).ExecTime.GetAverage();
}
-void TScheduler::Configure(const TResourceBrokerConfig &config, const TActorSystem &as)
+void TScheduler::Configure(const TResourceBrokerConfig &config, const TActorSystem &as)
{
// Remove all tasks from queues.
for (auto &entry : Tasks)
@@ -764,7 +764,7 @@ void TScheduler::Configure(const TResourceBrokerConfig &config, const TActorSyst
// Move all tasks to queues.
for (auto &entry : Tasks)
- AssignTask(entry.second, as);
+ AssignTask(entry.second, as);
}
void TScheduler::UpdateTime(TInstant now)
@@ -800,312 +800,312 @@ void TScheduler::OutputState(IOutputStream &os) const
}
TResourceBroker::TResourceBroker(const TResourceBrokerConfig &config,
- const NMonitoring::TDynamicCounterPtr &counters,
- TActorSystem *actorSystem)
+ const NMonitoring::TDynamicCounterPtr &counters,
+ TActorSystem *actorSystem)
: Config(config)
, Scheduler(counters)
- , ActorSystem(actorSystem)
-{
- with_lock(Lock) {
- Scheduler.Configure(Config, *ActorSystem);
- }
-}
-
-void TResourceBroker::Configure(const TResourceBrokerConfig &config)
-{
- with_lock(Lock) {
- Config = config;
-
- Scheduler.UpdateTime(ActorSystem->Timestamp());
- Scheduler.Configure(Config, *ActorSystem);
- Scheduler.ScheduleTasks(*ActorSystem, [this](const TTask &task) {
- ActorSystem->Send(task.Client, new TEvResourceBroker::TEvResourceAllocated(task.TaskId, task.Cookie));
- });
- }
-}
-
-TResourceBroker::TOpError TResourceBroker::SubmitTask(const TEvResourceBroker::TEvSubmitTask &ev,
- const TActorId &sender)
-{
- with_lock(Lock) {
- Scheduler.UpdateTime(ActorSystem->Timestamp());
- if (Scheduler.SubmitTask(ev.Task, sender, *ActorSystem)) {
- Scheduler.ScheduleTasks(*ActorSystem, [this](const TTask &task) {
- ActorSystem->Send(task.Client, new TEvResourceBroker::TEvResourceAllocated(task.TaskId, task.Cookie));
- });
- return {};
- }
- }
-
- auto error = MakeHolder<TEvResourceBroker::TEvTaskOperationError>();
- error->TaskId = ev.Task.TaskId;
- error->Status.Code = TEvResourceBroker::TStatus::ALREADY_EXISTS;
- error->Status.Message = "task with the same ID has been already submitted";
- error->Cookie = ev.Task.Cookie;
- return error;
-}
-
-bool TResourceBroker::SubmitTaskInstant(const TEvResourceBroker::TEvSubmitTask& ev, const TActorId& sender)
-{
- with_lock(Lock) {
- bool success = false;
-
- Scheduler.UpdateTime(ActorSystem->Timestamp());
- if (Scheduler.SubmitTask(ev.Task, sender, *ActorSystem)) {
- Scheduler.ScheduleTasks(*ActorSystem, [this, taskId = ev.Task.TaskId, &success](const TTask &task) {
- if (task.TaskId == taskId) {
- success = true;
- } else {
- ActorSystem->Send(task.Client, new TEvResourceBroker::TEvResourceAllocated(task.TaskId, task.Cookie));
- }
- });
-
- if (!success) {
- auto removed = Scheduler.RemoveQueuedTask(ev.Task.TaskId, sender, *ActorSystem);
- Y_VERIFY_DEBUG(removed.Success);
- }
- }
-
- return success;
- }
-}
-
-TResourceBroker::TOpError TResourceBroker::UpdateTask(const TEvResourceBroker::TEvUpdateTask& ev,
- const TActorId& sender)
-{
- with_lock(Lock) {
- Scheduler.UpdateTime(ActorSystem->Timestamp());
- if (Scheduler.UpdateTask(ev.TaskId, sender, ev.RequiredResources, ev.Priority, ev.Type, ev.Resubmit, *ActorSystem)) {
- Scheduler.ScheduleTasks(*ActorSystem, [this](const TTask &task) {
- ActorSystem->Send(task.Client, new TEvResourceBroker::TEvResourceAllocated(task.TaskId, task.Cookie));
- });
- return {};
- }
- }
-
- auto error = MakeHolder<TEvResourceBroker::TEvTaskOperationError>();
- error->TaskId = ev.TaskId;
- error->Status.Code = TEvResourceBroker::TStatus::UNKNOWN_TASK;
- error->Status.Message = "cannot update unknown task";
- error->Cookie = nullptr;
- return error;
-}
-
-bool TResourceBroker::MergeTasksInstant(ui64 recipientTaskId, ui64 donorTaskId, const TActorId &sender) {
- with_lock(Lock) {
- auto recipientTask = Scheduler.FindTask(recipientTaskId, sender);
- if (!recipientTask || !recipientTask->InFly) {
- return false;
- }
-
- auto donorTask = Scheduler.FindTask(donorTaskId, sender);
- if (!donorTask || !donorTask->InFly) {
- return false;
- }
-
- if (recipientTask->Type != donorTask->Type) {
- return false;
- }
-
- TResourceValues mergedResources;
- for (ui64 i = 0; i < recipientTask->RequiredResources.size(); ++i) {
- mergedResources[i] = recipientTask->RequiredResources[i] + donorTask->RequiredResources[i];
- }
-
- Scheduler.UpdateTime(ActorSystem->Timestamp());
-
- bool updated = Scheduler.UpdateTask(recipientTaskId, sender, mergedResources, recipientTask->Priority,
- recipientTask->Type, /* resubmit */ false, *ActorSystem);
- Y_VERIFY_DEBUG(updated);
-
- auto finished = Scheduler.FinishTask(donorTaskId, sender, /* cancel */ false, *ActorSystem);
- Y_VERIFY_DEBUG(finished.Success);
-
- Scheduler.ScheduleTasks(*ActorSystem, [this, recipientTaskId](const TTask &task) {
- Y_VERIFY_DEBUG(task.TaskId != recipientTaskId);
- ActorSystem->Send(task.Client, new TEvResourceBroker::TEvResourceAllocated(task.TaskId, task.Cookie));
- });
-
- return true;
- }
-}
-
-bool TResourceBroker::ReduceTaskResourcesInstant(ui64 taskId, const TResourceValues& reduceBy, const TActorId& sender)
-{
- with_lock(Lock) {
- auto task = Scheduler.FindTask(taskId, sender);
- if (!task) {
- return false;
- }
-
- auto resources = task->RequiredResources;
- for (ui32 i = 0; i < resources.size(); ++i) {
- if (i < reduceBy.size()) {
- resources[i] = resources[i] > reduceBy[i] ? resources[i] - reduceBy[i] : 0;
- }
- }
-
- Scheduler.UpdateTime(ActorSystem->Timestamp());
-
- if (Scheduler.UpdateTask(taskId, sender, resources, task->Priority, task->Type, false, *ActorSystem)) {
- Scheduler.ScheduleTasks(*ActorSystem, [this, taskId](const TTask &task) {
- if (task.TaskId != taskId) {
- ActorSystem->Send(task.Client, new TEvResourceBroker::TEvResourceAllocated(task.TaskId, task.Cookie));
- }
- });
- return true;
- }
-
- LOG_ERROR(*ActorSystem, NKikimrServices::RESOURCE_BROKER,
- "ReduceTaskResourcesInstant failed for task %" PRIu64, taskId);
- return false;
- }
-}
-
-TResourceBroker::TOpError TResourceBroker::UpdateTaskCookie(const TEvResourceBroker::TEvUpdateTaskCookie &ev,
- const TActorId &sender)
-{
- with_lock(Lock) {
- Scheduler.UpdateTime(ActorSystem->Timestamp());
- if (Scheduler.UpdateTaskCookie(ev.TaskId, sender, ev.Cookie, *ActorSystem)) {
- return {};
- }
- }
-
- auto error = MakeHolder<TEvResourceBroker::TEvTaskOperationError>();
- error->TaskId = ev.TaskId;
- error->Status.Code = TEvResourceBroker::TStatus::UNKNOWN_TASK;
- error->Status.Message = "cannot update unknown task's cookie";
- error->Cookie = nullptr;
- return error;
-}
-
-TResourceBroker::TOpError TResourceBroker::RemoveTask(const TEvResourceBroker::TEvRemoveTask &ev,
- const TActorId &sender)
-{
- with_lock(Lock) {
- Scheduler.UpdateTime(ActorSystem->Timestamp());
-
- auto result = Scheduler.RemoveQueuedTask(ev.TaskId, sender, *ActorSystem);
-
- if (result.Success) {
- if (ev.ReplyOnSuccess) {
- auto resp = MakeHolder<TEvResourceBroker::TEvTaskRemoved>();
- resp->TaskId = result.Task->TaskId;
- resp->Cookie = result.Task->Cookie;
-
- ActorSystem->Send(sender, resp.Release());
- }
-
- Scheduler.ScheduleTasks(*ActorSystem, [this](const TTask &task) {
- ActorSystem->Send(task.Client, new TEvResourceBroker::TEvResourceAllocated(task.TaskId, task.Cookie));
- });
-
- return {};
- }
-
- auto error = MakeHolder<TEvResourceBroker::TEvTaskOperationError>();
- error->TaskId = ev.TaskId;
-
- if (result.Task) {
- error->Status.Code = TEvResourceBroker::TStatus::TASK_IN_FLY;
- error->Status.Message = "cannot remove in-fly task";
- error->Cookie = result.Task->Cookie;
- } else {
- error->Status.Code = TEvResourceBroker::TStatus::UNKNOWN_TASK;
- error->Status.Message = "cannot remove unknown task";
- error->Cookie = nullptr;
- }
-
- return error;
- }
-}
-
-TResourceBroker::TOpError TResourceBroker::FinishTask(const TEvResourceBroker::TEvFinishTask &ev,
- const TActorId &sender)
-{
- with_lock(Lock) {
- Scheduler.UpdateTime(ActorSystem->Timestamp());
-
- auto result = Scheduler.FinishTask(ev.TaskId, sender, ev.Cancel, *ActorSystem);
-
- if (result.Success) {
- Scheduler.ScheduleTasks(*ActorSystem, [this](const TTask &task) {
- ActorSystem->Send(task.Client, new TEvResourceBroker::TEvResourceAllocated(task.TaskId, task.Cookie));
- });
-
- return {};
- }
-
- auto error = MakeHolder<TEvResourceBroker::TEvTaskOperationError>();
- error->TaskId = ev.TaskId;
-
- if (result.Task) {
- error->Status.Code = TEvResourceBroker::TStatus::TASK_IN_QUEUE;
- error->Status.Message = "cannot finish queued task";
- error->Cookie = result.Task->Cookie;
- } else {
- error->Status.Code = TEvResourceBroker::TStatus::UNKNOWN_TASK;
- error->Status.Message = "cannot finish unknown task";
- error->Cookie = nullptr;
- }
-
- return error;
- }
-}
-
-bool TResourceBroker::FinishTaskInstant(const TEvResourceBroker::TEvFinishTask &ev,
- const TActorId &sender)
-{
- with_lock(Lock) {
- Scheduler.UpdateTime(ActorSystem->Timestamp());
-
- auto result = Scheduler.FinishTask(ev.TaskId, sender, ev.Cancel, *ActorSystem);
-
- if (result.Success) {
- Scheduler.ScheduleTasks(*ActorSystem, [this](const TTask &task) {
- ActorSystem->Send(task.Client, new TEvResourceBroker::TEvResourceAllocated(task.TaskId, task.Cookie));
- });
- } else {
- LOG_ERROR(*ActorSystem, NKikimrServices::RESOURCE_BROKER,
- "FinishTaskInstant failed for task %" PRIu64 ": %s",
- ev.TaskId, (result.Task ? "cannot finish queued task" : "cannot finish unknown task"));
- }
-
- return result.Success;
- }
-}
-
-void TResourceBroker::NotifyActorDied(const TEvResourceBroker::TEvNotifyActorDied &, const TActorId &sender)
-{
- with_lock(Lock) {
- Scheduler.UpdateTime(ActorSystem->Timestamp());
- if (Scheduler.RemoveTasks(sender, *ActorSystem)) {
- Scheduler.ScheduleTasks(*ActorSystem, [this](const TTask &task) {
- ActorSystem->Send(task.Client, new TEvResourceBroker::TEvResourceAllocated(task.TaskId, task.Cookie));
- });
- }
- }
-}
-
-void TResourceBroker::OutputState(TStringStream& str)
-{
- with_lock(Lock) {
- Scheduler.OutputState(str);
- }
-}
-
-TResourceBrokerActor::TResourceBrokerActor(const TResourceBrokerConfig &config,
- const NMonitoring::TDynamicCounterPtr &counters)
- : Config(config)
- , Counters(counters)
-{
-}
-
-void TResourceBrokerActor::Bootstrap(const TActorContext &ctx)
-{
- LOG_DEBUG(ctx, NKikimrServices::RESOURCE_BROKER, "TResourceBrokerActor bootstrap");
-
+ , ActorSystem(actorSystem)
+{
+ with_lock(Lock) {
+ Scheduler.Configure(Config, *ActorSystem);
+ }
+}
+
+void TResourceBroker::Configure(const TResourceBrokerConfig &config)
+{
+ with_lock(Lock) {
+ Config = config;
+
+ Scheduler.UpdateTime(ActorSystem->Timestamp());
+ Scheduler.Configure(Config, *ActorSystem);
+ Scheduler.ScheduleTasks(*ActorSystem, [this](const TTask &task) {
+ ActorSystem->Send(task.Client, new TEvResourceBroker::TEvResourceAllocated(task.TaskId, task.Cookie));
+ });
+ }
+}
+
+TResourceBroker::TOpError TResourceBroker::SubmitTask(const TEvResourceBroker::TEvSubmitTask &ev,
+ const TActorId &sender)
+{
+ with_lock(Lock) {
+ Scheduler.UpdateTime(ActorSystem->Timestamp());
+ if (Scheduler.SubmitTask(ev.Task, sender, *ActorSystem)) {
+ Scheduler.ScheduleTasks(*ActorSystem, [this](const TTask &task) {
+ ActorSystem->Send(task.Client, new TEvResourceBroker::TEvResourceAllocated(task.TaskId, task.Cookie));
+ });
+ return {};
+ }
+ }
+
+ auto error = MakeHolder<TEvResourceBroker::TEvTaskOperationError>();
+ error->TaskId = ev.Task.TaskId;
+ error->Status.Code = TEvResourceBroker::TStatus::ALREADY_EXISTS;
+ error->Status.Message = "task with the same ID has been already submitted";
+ error->Cookie = ev.Task.Cookie;
+ return error;
+}
+
+bool TResourceBroker::SubmitTaskInstant(const TEvResourceBroker::TEvSubmitTask& ev, const TActorId& sender)
+{
+ with_lock(Lock) {
+ bool success = false;
+
+ Scheduler.UpdateTime(ActorSystem->Timestamp());
+ if (Scheduler.SubmitTask(ev.Task, sender, *ActorSystem)) {
+ Scheduler.ScheduleTasks(*ActorSystem, [this, taskId = ev.Task.TaskId, &success](const TTask &task) {
+ if (task.TaskId == taskId) {
+ success = true;
+ } else {
+ ActorSystem->Send(task.Client, new TEvResourceBroker::TEvResourceAllocated(task.TaskId, task.Cookie));
+ }
+ });
+
+ if (!success) {
+ auto removed = Scheduler.RemoveQueuedTask(ev.Task.TaskId, sender, *ActorSystem);
+ Y_VERIFY_DEBUG(removed.Success);
+ }
+ }
+
+ return success;
+ }
+}
+
+TResourceBroker::TOpError TResourceBroker::UpdateTask(const TEvResourceBroker::TEvUpdateTask& ev,
+ const TActorId& sender)
+{
+ with_lock(Lock) {
+ Scheduler.UpdateTime(ActorSystem->Timestamp());
+ if (Scheduler.UpdateTask(ev.TaskId, sender, ev.RequiredResources, ev.Priority, ev.Type, ev.Resubmit, *ActorSystem)) {
+ Scheduler.ScheduleTasks(*ActorSystem, [this](const TTask &task) {
+ ActorSystem->Send(task.Client, new TEvResourceBroker::TEvResourceAllocated(task.TaskId, task.Cookie));
+ });
+ return {};
+ }
+ }
+
+ auto error = MakeHolder<TEvResourceBroker::TEvTaskOperationError>();
+ error->TaskId = ev.TaskId;
+ error->Status.Code = TEvResourceBroker::TStatus::UNKNOWN_TASK;
+ error->Status.Message = "cannot update unknown task";
+ error->Cookie = nullptr;
+ return error;
+}
+
+bool TResourceBroker::MergeTasksInstant(ui64 recipientTaskId, ui64 donorTaskId, const TActorId &sender) {
+ with_lock(Lock) {
+ auto recipientTask = Scheduler.FindTask(recipientTaskId, sender);
+ if (!recipientTask || !recipientTask->InFly) {
+ return false;
+ }
+
+ auto donorTask = Scheduler.FindTask(donorTaskId, sender);
+ if (!donorTask || !donorTask->InFly) {
+ return false;
+ }
+
+ if (recipientTask->Type != donorTask->Type) {
+ return false;
+ }
+
+ TResourceValues mergedResources;
+ for (ui64 i = 0; i < recipientTask->RequiredResources.size(); ++i) {
+ mergedResources[i] = recipientTask->RequiredResources[i] + donorTask->RequiredResources[i];
+ }
+
+ Scheduler.UpdateTime(ActorSystem->Timestamp());
+
+ bool updated = Scheduler.UpdateTask(recipientTaskId, sender, mergedResources, recipientTask->Priority,
+ recipientTask->Type, /* resubmit */ false, *ActorSystem);
+ Y_VERIFY_DEBUG(updated);
+
+ auto finished = Scheduler.FinishTask(donorTaskId, sender, /* cancel */ false, *ActorSystem);
+ Y_VERIFY_DEBUG(finished.Success);
+
+ Scheduler.ScheduleTasks(*ActorSystem, [this, recipientTaskId](const TTask &task) {
+ Y_VERIFY_DEBUG(task.TaskId != recipientTaskId);
+ ActorSystem->Send(task.Client, new TEvResourceBroker::TEvResourceAllocated(task.TaskId, task.Cookie));
+ });
+
+ return true;
+ }
+}
+
+bool TResourceBroker::ReduceTaskResourcesInstant(ui64 taskId, const TResourceValues& reduceBy, const TActorId& sender)
+{
+ with_lock(Lock) {
+ auto task = Scheduler.FindTask(taskId, sender);
+ if (!task) {
+ return false;
+ }
+
+ auto resources = task->RequiredResources;
+ for (ui32 i = 0; i < resources.size(); ++i) {
+ if (i < reduceBy.size()) {
+ resources[i] = resources[i] > reduceBy[i] ? resources[i] - reduceBy[i] : 0;
+ }
+ }
+
+ Scheduler.UpdateTime(ActorSystem->Timestamp());
+
+ if (Scheduler.UpdateTask(taskId, sender, resources, task->Priority, task->Type, false, *ActorSystem)) {
+ Scheduler.ScheduleTasks(*ActorSystem, [this, taskId](const TTask &task) {
+ if (task.TaskId != taskId) {
+ ActorSystem->Send(task.Client, new TEvResourceBroker::TEvResourceAllocated(task.TaskId, task.Cookie));
+ }
+ });
+ return true;
+ }
+
+ LOG_ERROR(*ActorSystem, NKikimrServices::RESOURCE_BROKER,
+ "ReduceTaskResourcesInstant failed for task %" PRIu64, taskId);
+ return false;
+ }
+}
+
+TResourceBroker::TOpError TResourceBroker::UpdateTaskCookie(const TEvResourceBroker::TEvUpdateTaskCookie &ev,
+ const TActorId &sender)
+{
+ with_lock(Lock) {
+ Scheduler.UpdateTime(ActorSystem->Timestamp());
+ if (Scheduler.UpdateTaskCookie(ev.TaskId, sender, ev.Cookie, *ActorSystem)) {
+ return {};
+ }
+ }
+
+ auto error = MakeHolder<TEvResourceBroker::TEvTaskOperationError>();
+ error->TaskId = ev.TaskId;
+ error->Status.Code = TEvResourceBroker::TStatus::UNKNOWN_TASK;
+ error->Status.Message = "cannot update unknown task's cookie";
+ error->Cookie = nullptr;
+ return error;
+}
+
+TResourceBroker::TOpError TResourceBroker::RemoveTask(const TEvResourceBroker::TEvRemoveTask &ev,
+ const TActorId &sender)
+{
+ with_lock(Lock) {
+ Scheduler.UpdateTime(ActorSystem->Timestamp());
+
+ auto result = Scheduler.RemoveQueuedTask(ev.TaskId, sender, *ActorSystem);
+
+ if (result.Success) {
+ if (ev.ReplyOnSuccess) {
+ auto resp = MakeHolder<TEvResourceBroker::TEvTaskRemoved>();
+ resp->TaskId = result.Task->TaskId;
+ resp->Cookie = result.Task->Cookie;
+
+ ActorSystem->Send(sender, resp.Release());
+ }
+
+ Scheduler.ScheduleTasks(*ActorSystem, [this](const TTask &task) {
+ ActorSystem->Send(task.Client, new TEvResourceBroker::TEvResourceAllocated(task.TaskId, task.Cookie));
+ });
+
+ return {};
+ }
+
+ auto error = MakeHolder<TEvResourceBroker::TEvTaskOperationError>();
+ error->TaskId = ev.TaskId;
+
+ if (result.Task) {
+ error->Status.Code = TEvResourceBroker::TStatus::TASK_IN_FLY;
+ error->Status.Message = "cannot remove in-fly task";
+ error->Cookie = result.Task->Cookie;
+ } else {
+ error->Status.Code = TEvResourceBroker::TStatus::UNKNOWN_TASK;
+ error->Status.Message = "cannot remove unknown task";
+ error->Cookie = nullptr;
+ }
+
+ return error;
+ }
+}
+
+TResourceBroker::TOpError TResourceBroker::FinishTask(const TEvResourceBroker::TEvFinishTask &ev,
+ const TActorId &sender)
+{
+ with_lock(Lock) {
+ Scheduler.UpdateTime(ActorSystem->Timestamp());
+
+ auto result = Scheduler.FinishTask(ev.TaskId, sender, ev.Cancel, *ActorSystem);
+
+ if (result.Success) {
+ Scheduler.ScheduleTasks(*ActorSystem, [this](const TTask &task) {
+ ActorSystem->Send(task.Client, new TEvResourceBroker::TEvResourceAllocated(task.TaskId, task.Cookie));
+ });
+
+ return {};
+ }
+
+ auto error = MakeHolder<TEvResourceBroker::TEvTaskOperationError>();
+ error->TaskId = ev.TaskId;
+
+ if (result.Task) {
+ error->Status.Code = TEvResourceBroker::TStatus::TASK_IN_QUEUE;
+ error->Status.Message = "cannot finish queued task";
+ error->Cookie = result.Task->Cookie;
+ } else {
+ error->Status.Code = TEvResourceBroker::TStatus::UNKNOWN_TASK;
+ error->Status.Message = "cannot finish unknown task";
+ error->Cookie = nullptr;
+ }
+
+ return error;
+ }
+}
+
+bool TResourceBroker::FinishTaskInstant(const TEvResourceBroker::TEvFinishTask &ev,
+ const TActorId &sender)
+{
+ with_lock(Lock) {
+ Scheduler.UpdateTime(ActorSystem->Timestamp());
+
+ auto result = Scheduler.FinishTask(ev.TaskId, sender, ev.Cancel, *ActorSystem);
+
+ if (result.Success) {
+ Scheduler.ScheduleTasks(*ActorSystem, [this](const TTask &task) {
+ ActorSystem->Send(task.Client, new TEvResourceBroker::TEvResourceAllocated(task.TaskId, task.Cookie));
+ });
+ } else {
+ LOG_ERROR(*ActorSystem, NKikimrServices::RESOURCE_BROKER,
+ "FinishTaskInstant failed for task %" PRIu64 ": %s",
+ ev.TaskId, (result.Task ? "cannot finish queued task" : "cannot finish unknown task"));
+ }
+
+ return result.Success;
+ }
+}
+
+void TResourceBroker::NotifyActorDied(const TEvResourceBroker::TEvNotifyActorDied &, const TActorId &sender)
+{
+ with_lock(Lock) {
+ Scheduler.UpdateTime(ActorSystem->Timestamp());
+ if (Scheduler.RemoveTasks(sender, *ActorSystem)) {
+ Scheduler.ScheduleTasks(*ActorSystem, [this](const TTask &task) {
+ ActorSystem->Send(task.Client, new TEvResourceBroker::TEvResourceAllocated(task.TaskId, task.Cookie));
+ });
+ }
+ }
+}
+
+void TResourceBroker::OutputState(TStringStream& str)
+{
+ with_lock(Lock) {
+ Scheduler.OutputState(str);
+ }
+}
+
+TResourceBrokerActor::TResourceBrokerActor(const TResourceBrokerConfig &config,
+ const NMonitoring::TDynamicCounterPtr &counters)
+ : Config(config)
+ , Counters(counters)
+{
+}
+
+void TResourceBrokerActor::Bootstrap(const TActorContext &ctx)
+{
+ LOG_DEBUG(ctx, NKikimrServices::RESOURCE_BROKER, "TResourceBrokerActor bootstrap");
+
NActors::TMon* mon = AppData(ctx)->Mon;
if (mon) {
NMonitoring::TIndexMonPage *actorsMonPage = mon->RegisterIndexPage("actors", "Actors");
@@ -1113,68 +1113,68 @@ void TResourceBrokerActor::Bootstrap(const TActorContext &ctx)
false, ctx.ExecutorThread.ActorSystem, ctx.SelfID);
}
- ResourceBroker = MakeIntrusive<TResourceBroker>(std::move(Config), std::move(Counters), ctx.ActorSystem());
+ ResourceBroker = MakeIntrusive<TResourceBroker>(std::move(Config), std::move(Counters), ctx.ActorSystem());
Become(&TThis::StateWork);
}
-void TResourceBrokerActor::Handle(TEvResourceBroker::TEvSubmitTask::TPtr &ev,
- const TActorContext &ctx)
+void TResourceBrokerActor::Handle(TEvResourceBroker::TEvSubmitTask::TPtr &ev,
+ const TActorContext &ctx)
{
- auto error = ResourceBroker->SubmitTask(*ev->Get(), ev->Sender);
-
- if (error) {
- ctx.Send(ev->Sender, error.Release());
- }
+ auto error = ResourceBroker->SubmitTask(*ev->Get(), ev->Sender);
+
+ if (error) {
+ ctx.Send(ev->Sender, error.Release());
+ }
}
-void TResourceBrokerActor::Handle(TEvResourceBroker::TEvUpdateTask::TPtr &ev,
- const TActorContext &ctx)
+void TResourceBrokerActor::Handle(TEvResourceBroker::TEvUpdateTask::TPtr &ev,
+ const TActorContext &ctx)
{
- auto error = ResourceBroker->UpdateTask(*ev->Get(), ev->Sender);
-
- if (error) {
- ctx.Send(ev->Sender, error.Release());
- }
+ auto error = ResourceBroker->UpdateTask(*ev->Get(), ev->Sender);
+
+ if (error) {
+ ctx.Send(ev->Sender, error.Release());
+ }
}
-void TResourceBrokerActor::Handle(TEvResourceBroker::TEvUpdateTaskCookie::TPtr &ev,
- const TActorContext &ctx)
+void TResourceBrokerActor::Handle(TEvResourceBroker::TEvUpdateTaskCookie::TPtr &ev,
+ const TActorContext &ctx)
{
- auto error = ResourceBroker->UpdateTaskCookie(*ev->Get(), ev->Sender);
-
- if (error) {
- ctx.Send(ev->Sender, error.Release());
- }
+ auto error = ResourceBroker->UpdateTaskCookie(*ev->Get(), ev->Sender);
+
+ if (error) {
+ ctx.Send(ev->Sender, error.Release());
+ }
}
-void TResourceBrokerActor::Handle(TEvResourceBroker::TEvRemoveTask::TPtr &ev,
- const TActorContext &ctx)
+void TResourceBrokerActor::Handle(TEvResourceBroker::TEvRemoveTask::TPtr &ev,
+ const TActorContext &ctx)
{
- auto error = ResourceBroker->RemoveTask(*ev->Get(), ev->Sender);
-
- if (error) {
- ctx.Send(ev->Sender, error.Release());
- }
+ auto error = ResourceBroker->RemoveTask(*ev->Get(), ev->Sender);
+
+ if (error) {
+ ctx.Send(ev->Sender, error.Release());
+ }
}
-void TResourceBrokerActor::Handle(TEvResourceBroker::TEvFinishTask::TPtr &ev,
- const TActorContext &ctx)
+void TResourceBrokerActor::Handle(TEvResourceBroker::TEvFinishTask::TPtr &ev,
+ const TActorContext &ctx)
{
- auto error = ResourceBroker->FinishTask(*ev->Get(), ev->Sender);
-
- if (error) {
- ctx.Send(ev->Sender, error.Release());
- }
+ auto error = ResourceBroker->FinishTask(*ev->Get(), ev->Sender);
+
+ if (error) {
+ ctx.Send(ev->Sender, error.Release());
+ }
}
-void TResourceBrokerActor::Handle(TEvResourceBroker::TEvNotifyActorDied::TPtr &ev,
- const TActorContext &)
+void TResourceBrokerActor::Handle(TEvResourceBroker::TEvNotifyActorDied::TPtr &ev,
+ const TActorContext &)
{
- ResourceBroker->NotifyActorDied(*ev->Get(), ev->Sender);
+ ResourceBroker->NotifyActorDied(*ev->Get(), ev->Sender);
}
-void TResourceBrokerActor::Handle(TEvResourceBroker::TEvConfigure::TPtr &ev,
- const TActorContext &ctx)
+void TResourceBrokerActor::Handle(TEvResourceBroker::TEvConfigure::TPtr &ev,
+ const TActorContext &ctx)
{
auto &rec = ev->Get()->Record;
TAutoPtr<TEvResourceBroker::TEvConfigureResult> response
@@ -1218,7 +1218,7 @@ void TResourceBrokerActor::Handle(TEvResourceBroker::TEvConfigure::TPtr &ev,
} else {
response->Record.SetSuccess(true);
- ResourceBroker->Configure(std::move(ev->Get()->Record));
+ ResourceBroker->Configure(std::move(ev->Get()->Record));
}
LOG_DEBUG(ctx, NKikimrServices::RESOURCE_BROKER, "Configure result: %s",
@@ -1227,34 +1227,34 @@ void TResourceBrokerActor::Handle(TEvResourceBroker::TEvConfigure::TPtr &ev,
ctx.Send(ev->Sender, response.Release());
}
-void TResourceBrokerActor::Handle(TEvResourceBroker::TEvConfigRequest::TPtr& ev, const TActorContext&)
-{
- auto resp = MakeHolder<TEvResourceBroker::TEvConfigResponse>();
- for (auto& queue : Config.GetQueues()) {
- if (queue.GetName() == ev->Get()->Queue) {
- resp->QueueConfig = queue;
- break;
- }
- }
- Send(ev->Sender, resp.Release());
-}
-
-void TResourceBrokerActor::Handle(TEvResourceBroker::TEvResourceBrokerRequest::TPtr &ev, const TActorContext &ctx)
-{
- auto resp = MakeHolder<TEvResourceBroker::TEvResourceBrokerResponse>();
- resp->ResourceBroker = ResourceBroker;
-
- ctx.Send(ev->Sender, resp.Release());
-}
-
-void TResourceBrokerActor::Handle(NMon::TEvHttpInfo::TPtr &ev, const TActorContext &ctx)
-{
+void TResourceBrokerActor::Handle(TEvResourceBroker::TEvConfigRequest::TPtr& ev, const TActorContext&)
+{
+ auto resp = MakeHolder<TEvResourceBroker::TEvConfigResponse>();
+ for (auto& queue : Config.GetQueues()) {
+ if (queue.GetName() == ev->Get()->Queue) {
+ resp->QueueConfig = queue;
+ break;
+ }
+ }
+ Send(ev->Sender, resp.Release());
+}
+
+void TResourceBrokerActor::Handle(TEvResourceBroker::TEvResourceBrokerRequest::TPtr &ev, const TActorContext &ctx)
+{
+ auto resp = MakeHolder<TEvResourceBroker::TEvResourceBrokerResponse>();
+ resp->ResourceBroker = ResourceBroker;
+
+ ctx.Send(ev->Sender, resp.Release());
+}
+
+void TResourceBrokerActor::Handle(NMon::TEvHttpInfo::TPtr &ev, const TActorContext &ctx)
+{
TStringStream str;
HTML(str) {
PRE() {
str << "Current config:" << Endl
<< Config.DebugString() << Endl;
- ResourceBroker->OutputState(str);
+ ResourceBroker->OutputState(str);
}
}
ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str()));
@@ -1264,16 +1264,16 @@ NKikimrResourceBroker::TResourceBrokerConfig MakeDefaultConfig()
{
NKikimrResourceBroker::TResourceBrokerConfig config;
- const ui64 DefaultQueueCPU = 2;
-
+ const ui64 DefaultQueueCPU = 2;
+
const ui64 KqpRmQueueCPU = 4;
- const ui64 KqpRmQueueMemory = 10ULL << 30;
-
+ const ui64 KqpRmQueueMemory = 10ULL << 30;
+
const ui64 TotalCPU = 20;
- const ui64 TotalMemory = 16ULL << 30;
-
- static_assert(KqpRmQueueMemory < TotalMemory);
-
+ const ui64 TotalMemory = 16ULL << 30;
+
+ static_assert(KqpRmQueueMemory < TotalMemory);
+
auto queue = config.AddQueues();
queue->SetName(NLocalDb::DefaultQueueName);
queue->SetWeight(30);
@@ -1319,7 +1319,7 @@ NKikimrResourceBroker::TResourceBrokerConfig MakeDefaultConfig()
queue->SetWeight(100);
queue->MutableLimit()->SetCpu(10);
- queue = config.AddQueues();
+ queue = config.AddQueues();
queue->SetName("queue_backup");
queue->SetWeight(100);
queue->MutableLimit()->SetCpu(2);
@@ -1330,11 +1330,11 @@ NKikimrResourceBroker::TResourceBrokerConfig MakeDefaultConfig()
queue->MutableLimit()->SetCpu(10);
queue = config.AddQueues();
- queue->SetName(NLocalDb::KqpResourceManagerQueue);
+ queue->SetName(NLocalDb::KqpResourceManagerQueue);
queue->SetWeight(30);
queue->MutableLimit()->SetCpu(KqpRmQueueCPU);
queue->MutableLimit()->SetMemory(KqpRmQueueMemory);
-
+
queue = config.AddQueues();
queue->SetName("queue_build_index");
queue->SetWeight(100);
@@ -1405,7 +1405,7 @@ NKikimrResourceBroker::TResourceBrokerConfig MakeDefaultConfig()
task->SetQueueName("queue_scan");
task->SetDefaultDuration(TDuration::Minutes(5).GetValue());
- task = config.AddTasks();
+ task = config.AddTasks();
task->SetName("backup");
task->SetQueueName("queue_backup");
task->SetDefaultDuration(TDuration::Minutes(5).GetValue());
@@ -1416,10 +1416,10 @@ NKikimrResourceBroker::TResourceBrokerConfig MakeDefaultConfig()
task->SetDefaultDuration(TDuration::Minutes(5).GetValue());
task = config.AddTasks();
- task->SetName(NLocalDb::KqpResourceManagerTaskName);
- task->SetQueueName(NLocalDb::KqpResourceManagerQueue);
- task->SetDefaultDuration(TDuration::Minutes(10).GetValue());
-
+ task->SetName(NLocalDb::KqpResourceManagerTaskName);
+ task->SetQueueName(NLocalDb::KqpResourceManagerQueue);
+ task->SetDefaultDuration(TDuration::Minutes(10).GetValue());
+
task = config.AddTasks();
task->SetName("build_index");
task->SetQueueName("queue_build_index");
@@ -1537,10 +1537,10 @@ void MergeConfigUpdates(
}
-IActor* CreateResourceBrokerActor(const NKikimrResourceBroker::TResourceBrokerConfig &config,
- const NMonitoring::TDynamicCounterPtr &counters)
+IActor* CreateResourceBrokerActor(const NKikimrResourceBroker::TResourceBrokerConfig &config,
+ const NMonitoring::TDynamicCounterPtr &counters)
{
- return new TResourceBrokerActor(config, counters);
+ return new TResourceBrokerActor(config, counters);
}
diff --git a/ydb/core/tablet/resource_broker.h b/ydb/core/tablet/resource_broker.h
index bc93ab7eff..095702cc19 100644
--- a/ydb/core/tablet/resource_broker.h
+++ b/ydb/core/tablet/resource_broker.h
@@ -24,8 +24,8 @@ inline TActorId MakeResourceBrokerID(ui32 node = 0) {
using TResourceValues = std::array<ui64, RESOURCE_COUNT>;
-class IResourceBroker;
-
+class IResourceBroker;
+
struct TEvResourceBroker {
enum EEv {
EvSubmitTask = EventSpaceBegin(TKikimrEvents::ES_RESOURCE_BROKER),
@@ -35,19 +35,19 @@ struct TEvResourceBroker {
EvFinishTask,
EvNotifyActorDied,
EvConfigure,
- EvConfigRequest,
- EvTaskRemoved,
- EvResourceBrokerRequest,
+ EvConfigRequest,
+ EvTaskRemoved,
+ EvResourceBrokerRequest,
EvTaskOperationError = EvSubmitTask + 512,
EvResourceAllocated,
- EvConfigureResult,
- EvConfigResponse,
- EvResourceBrokerResponse
+ EvConfigureResult,
+ EvConfigResponse,
+ EvResourceBrokerResponse
};
- static_assert(EvResourceBrokerRequest < EvTaskOperationError);
-
+ static_assert(EvResourceBrokerRequest < EvTaskOperationError);
+
struct TTask {
/**
* 0 is invalid task ID value and is used to choose task ID
@@ -147,21 +147,21 @@ struct TEvResourceBroker {
struct TEvRemoveTask : public TEventLocal<TEvRemoveTask, EvRemoveTask> {
ui64 TaskId;
- bool ReplyOnSuccess;
+ bool ReplyOnSuccess;
- TEvRemoveTask(ui64 taskId, bool replyOnSuccess = false)
+ TEvRemoveTask(ui64 taskId, bool replyOnSuccess = false)
: TaskId(taskId)
- , ReplyOnSuccess(replyOnSuccess)
+ , ReplyOnSuccess(replyOnSuccess)
{
Y_VERIFY(taskId);
}
};
- struct TEvTaskRemoved : public TEventLocal<TEvTaskRemoved, EvTaskRemoved> {
- ui64 TaskId;
- TIntrusivePtr<TThrRefBase> Cookie;
- };
-
+ struct TEvTaskRemoved : public TEventLocal<TEvTaskRemoved, EvTaskRemoved> {
+ ui64 TaskId;
+ TIntrusivePtr<TThrRefBase> Cookie;
+ };
+
struct TEvFinishTask : public TEventLocal<TEvFinishTask, EvFinishTask> {
ui64 TaskId;
bool Cancel;
@@ -197,44 +197,44 @@ struct TEvResourceBroker {
NKikimrResourceBroker::TResourceBrokerConfigResult,
EvConfigureResult> {
};
-
- struct TEvConfigRequest : public TEventLocal<TEvConfigRequest, EvConfigRequest> {
- TString Queue;
-
- TEvConfigRequest(const TString& queue)
- : Queue(queue) {}
- };
-
- struct TEvConfigResponse : public TEventLocal<TEvConfigResponse, EvConfigResponse> {
- TMaybe<NKikimrResourceBroker::TQueueConfig> QueueConfig;
- };
-
- struct TEvResourceBrokerRequest : public TEventLocal<TEvResourceBrokerRequest, EvResourceBrokerRequest> {};
-
- struct TEvResourceBrokerResponse : public TEventLocal<TEvResourceBrokerResponse, EvResourceBrokerResponse> {
- TIntrusivePtr<IResourceBroker> ResourceBroker;
- };
+
+ struct TEvConfigRequest : public TEventLocal<TEvConfigRequest, EvConfigRequest> {
+ TString Queue;
+
+ TEvConfigRequest(const TString& queue)
+ : Queue(queue) {}
+ };
+
+ struct TEvConfigResponse : public TEventLocal<TEvConfigResponse, EvConfigResponse> {
+ TMaybe<NKikimrResourceBroker::TQueueConfig> QueueConfig;
+ };
+
+ struct TEvResourceBrokerRequest : public TEventLocal<TEvResourceBrokerRequest, EvResourceBrokerRequest> {};
+
+ struct TEvResourceBrokerResponse : public TEventLocal<TEvResourceBrokerResponse, EvResourceBrokerResponse> {
+ TIntrusivePtr<IResourceBroker> ResourceBroker;
+ };
+};
+
+class IResourceBroker : public TThrRefBase {
+public:
+ virtual bool SubmitTaskInstant(const TEvResourceBroker::TEvSubmitTask &ev, const TActorId &sender) = 0;
+ virtual bool FinishTaskInstant(const TEvResourceBroker::TEvFinishTask &ev, const TActorId &sender) = 0;
+ /**
+ * Merges in-fly tasks of the same type. All donor's resources goes to recipient's ones.
+ * Donor task will be finished.
+ */
+ virtual bool MergeTasksInstant(ui64 recipientTaskId, ui64 donorTaskId, const TActorId &sender) = 0;
+ virtual bool ReduceTaskResourcesInstant(ui64 taskId, const TResourceValues& reduceBy, const TActorId& sender) = 0;
};
-class IResourceBroker : public TThrRefBase {
-public:
- virtual bool SubmitTaskInstant(const TEvResourceBroker::TEvSubmitTask &ev, const TActorId &sender) = 0;
- virtual bool FinishTaskInstant(const TEvResourceBroker::TEvFinishTask &ev, const TActorId &sender) = 0;
- /**
- * Merges in-fly tasks of the same type. All donor's resources goes to recipient's ones.
- * Donor task will be finished.
- */
- virtual bool MergeTasksInstant(ui64 recipientTaskId, ui64 donorTaskId, const TActorId &sender) = 0;
- virtual bool ReduceTaskResourcesInstant(ui64 taskId, const TResourceValues& reduceBy, const TActorId& sender) = 0;
-};
-
NKikimrResourceBroker::TResourceBrokerConfig MakeDefaultConfig();
-void MergeConfigUpdates(NKikimrResourceBroker::TResourceBrokerConfig &config,
- const NKikimrResourceBroker::TResourceBrokerConfig &updates);
+void MergeConfigUpdates(NKikimrResourceBroker::TResourceBrokerConfig &config,
+ const NKikimrResourceBroker::TResourceBrokerConfig &updates);
-IActor* CreateResourceBrokerActor(const NKikimrResourceBroker::TResourceBrokerConfig &config,
- const NMonitoring::TDynamicCounterPtr& counters);
+IActor* CreateResourceBrokerActor(const NKikimrResourceBroker::TResourceBrokerConfig &config,
+ const NMonitoring::TDynamicCounterPtr& counters);
} // NResourceBroker
} // NKikimr
diff --git a/ydb/core/tablet/resource_broker_impl.h b/ydb/core/tablet/resource_broker_impl.h
index 88e623eea0..2c781692f6 100644
--- a/ydb/core/tablet/resource_broker_impl.h
+++ b/ydb/core/tablet/resource_broker_impl.h
@@ -197,7 +197,7 @@ private:
*
* Real resource usage is the second parameter used to schedule resource
* allocation. It is computed before each queue resource consumption modification
- * and also may be additionally re-computed before choosing the next task by
+ * and also may be additionally re-computed before choosing the next task by
* scheduler.
*
* Real resource usage may exceed planned one. Therefore max of these two components
@@ -257,14 +257,14 @@ public:
/**
* Scheduler class manages tasks and queues. It directly communicates clients
- * in case of an error or successful resource allocation.
+ * in case of an error or successful resource allocation.
*
* Task submission/update/removal doesn't cause resource allocation. ScheduleTasks
- * method should be called to trigger resource allocation.
+ * method should be called to trigger resource allocation.
*
* Before next task allocation scheduler simply orders non-empty queues according
* to their resource usage and pick the top task of the first queue. If chosen
- * task cannot be run because of total resource limit then resource allocation
+ * task cannot be run because of total resource limit then resource allocation
* stops. If we have resources but queue's limit doesn't allow to chosen task
* then this queue is skipped.
*
@@ -305,50 +305,50 @@ public:
~TScheduler();
/**
- * Returns TTask raw pointer if it exists.
+ * Returns TTask raw pointer if it exists.
*/
- const TTask* FindTask(ui64 taskId, const TActorId &client) const;
+ const TTask* FindTask(ui64 taskId, const TActorId &client) const;
/**
- * Create new task and queue it. Return true on success, and false if task with the same id already exists.
+ * Create new task and queue it. Return true on success, and false if task with the same id already exists.
+ */
+ bool SubmitTask(const TEvResourceBroker::TTask &task, const TActorId &client, const TActorSystem &as);
+ /**
+ * Update queued task. Return true on success and false if task not found.
*/
- bool SubmitTask(const TEvResourceBroker::TTask &task, const TActorId &client, const TActorSystem &as);
- /**
- * Update queued task. Return true on success and false if task not found.
- */
bool UpdateTask(ui64 taskId, const TActorId &client, const TResourceValues &requiredResources,
- ui64 priority, const TString &type, bool resubmit, const TActorSystem &as);
+ ui64 priority, const TString &type, bool resubmit, const TActorSystem &as);
/**
- * Update cookie for submitted task. Return true on success and false if task not found.
+ * Update cookie for submitted task. Return true on success and false if task not found.
*/
- bool UpdateTaskCookie(ui64 taskId, const TActorId &client, TIntrusivePtr<TThrRefBase> cookie,
- const TActorSystem &as);
-
- struct TTerminateTaskResult {
- bool Success;
- TTaskPtr Task; // can be nullptr
-
- TTerminateTaskResult(bool success, TTaskPtr task)
- : Success(success), Task(task) {}
- };
-
+ bool UpdateTaskCookie(ui64 taskId, const TActorId &client, TIntrusivePtr<TThrRefBase> cookie,
+ const TActorSystem &as);
+
+ struct TTerminateTaskResult {
+ bool Success;
+ TTaskPtr Task; // can be nullptr
+
+ TTerminateTaskResult(bool success, TTaskPtr task)
+ : Success(success), Task(task) {}
+ };
+
/**
- * Remove queued task. Return <true, task> on success, and <false, task> otherwise.
+ * Remove queued task. Return <true, task> on success, and <false, task> otherwise.
*/
- TTerminateTaskResult RemoveQueuedTask(ui64 taskId, const TActorId &client, const TActorSystem &as);
+ TTerminateTaskResult RemoveQueuedTask(ui64 taskId, const TActorId &client, const TActorSystem &as);
/**
- * Finish/cancel task and release its resources. Return <true, task> on success, and <false, task> otherwise.
+ * Finish/cancel task and release its resources. Return <true, task> on success, and <false, task> otherwise.
*/
- TTerminateTaskResult FinishTask(ui64 taskId, const TActorId &client, bool cancel, const TActorSystem &as);
-
+ TTerminateTaskResult FinishTask(ui64 taskId, const TActorId &client, bool cancel, const TActorSystem &as);
+
/**
* Remove queued tasks and finish in-fly tasks submitted by specified client.
* Return true if any task was affected.
*/
- bool RemoveTasks(const TActorId &client, const TActorSystem &as);
+ bool RemoveTasks(const TActorId &client, const TActorSystem &as);
/**
* Try to allocate resources for next tasks until we are out of resources or tasks.
*/
- void ScheduleTasks(const TActorSystem &as, std::function<void(const TTask &task)> &&onTaskSchedule);
+ void ScheduleTasks(const TActorSystem &as, std::function<void(const TTask &task)> &&onTaskSchedule);
/**
* Build or reconfigure queues and limits according to new config. Config should
@@ -356,7 +356,7 @@ public:
* unknown task). On re-configuration tasks may migrate between queues according
* to new mapping.
*/
- void Configure(const NKikimrResourceBroker::TResourceBrokerConfig &config, const TActorSystem &as);
+ void Configure(const NKikimrResourceBroker::TResourceBrokerConfig &config, const TActorSystem &as);
/**
* Update current scheduler time which is used for all resource consumption
@@ -369,15 +369,15 @@ public:
private:
// Erase task from all structure.
- void EraseTask(TTaskPtr task, bool finished, const TActorSystem &as);
+ void EraseTask(TTaskPtr task, bool finished, const TActorSystem &as);
// Update resource usage for all queues.
- void UpdateResourceUsage(const TActorSystem &as);
+ void UpdateResourceUsage(const TActorSystem &as);
/**
* Insert task to queue according to its type. Also used to attach in-fly
* tasks to queues.
*/
- void AssignTask(TTaskPtr &task, const TActorSystem &as);
+ void AssignTask(TTaskPtr &task, const TActorSystem &as);
/**
* Get estimated execution time for task basing on default config and
* collected statistics.
@@ -399,49 +399,49 @@ private:
};
/**
- * TResourceBroker is a simple thread-safe wrapper for scheduler.
+ * TResourceBroker is a simple thread-safe wrapper for scheduler.
*/
-class TResourceBroker : public IResourceBroker {
+class TResourceBroker : public IResourceBroker {
+public:
+ TResourceBroker(const NKikimrResourceBroker::TResourceBrokerConfig &config,
+ const NMonitoring::TDynamicCounterPtr &counters,
+ TActorSystem *actorSystem);
+
+ bool SubmitTaskInstant(const TEvResourceBroker::TEvSubmitTask &ev, const TActorId &sender) override;
+ bool FinishTaskInstant(const TEvResourceBroker::TEvFinishTask &ev, const TActorId &sender) override;
+ bool MergeTasksInstant(ui64 recipientTaskId, ui64 donorTaskId, const TActorId &sender) override;
+ bool ReduceTaskResourcesInstant(ui64 taskId, const TResourceValues& reduceBy, const TActorId& sender) override;
+
+ void Configure(const NKikimrResourceBroker::TResourceBrokerConfig &config);
+
+ using TOpError = THolder<TEvResourceBroker::TEvTaskOperationError>;
+
+ TOpError SubmitTask(const TEvResourceBroker::TEvSubmitTask &ev, const TActorId &sender);
+ TOpError UpdateTask(const TEvResourceBroker::TEvUpdateTask &ev, const TActorId &sender);
+ TOpError UpdateTaskCookie(const TEvResourceBroker::TEvUpdateTaskCookie &ev, const TActorId &sender);
+ TOpError RemoveTask(const TEvResourceBroker::TEvRemoveTask &ev, const TActorId &sender);
+ TOpError FinishTask(const TEvResourceBroker::TEvFinishTask &ev, const TActorId &sender);
+
+ void NotifyActorDied(const TEvResourceBroker::TEvNotifyActorDied &ev, const TActorId &sender);
+
+ void OutputState(TStringStream &str);
+
+private:
+ NKikimrResourceBroker::TResourceBrokerConfig Config;
+ TScheduler Scheduler;
+ TMutex Lock;
+ TActorSystem* ActorSystem;
+};
+
+
+class TResourceBrokerActor : public TActorBootstrapped<TResourceBrokerActor> {
public:
- TResourceBroker(const NKikimrResourceBroker::TResourceBrokerConfig &config,
- const NMonitoring::TDynamicCounterPtr &counters,
- TActorSystem *actorSystem);
-
- bool SubmitTaskInstant(const TEvResourceBroker::TEvSubmitTask &ev, const TActorId &sender) override;
- bool FinishTaskInstant(const TEvResourceBroker::TEvFinishTask &ev, const TActorId &sender) override;
- bool MergeTasksInstant(ui64 recipientTaskId, ui64 donorTaskId, const TActorId &sender) override;
- bool ReduceTaskResourcesInstant(ui64 taskId, const TResourceValues& reduceBy, const TActorId& sender) override;
-
- void Configure(const NKikimrResourceBroker::TResourceBrokerConfig &config);
-
- using TOpError = THolder<TEvResourceBroker::TEvTaskOperationError>;
-
- TOpError SubmitTask(const TEvResourceBroker::TEvSubmitTask &ev, const TActorId &sender);
- TOpError UpdateTask(const TEvResourceBroker::TEvUpdateTask &ev, const TActorId &sender);
- TOpError UpdateTaskCookie(const TEvResourceBroker::TEvUpdateTaskCookie &ev, const TActorId &sender);
- TOpError RemoveTask(const TEvResourceBroker::TEvRemoveTask &ev, const TActorId &sender);
- TOpError FinishTask(const TEvResourceBroker::TEvFinishTask &ev, const TActorId &sender);
-
- void NotifyActorDied(const TEvResourceBroker::TEvNotifyActorDied &ev, const TActorId &sender);
-
- void OutputState(TStringStream &str);
-
-private:
- NKikimrResourceBroker::TResourceBrokerConfig Config;
- TScheduler Scheduler;
- TMutex Lock;
- TActorSystem* ActorSystem;
-};
-
-
-class TResourceBrokerActor : public TActorBootstrapped<TResourceBrokerActor> {
-public:
static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
return NKikimrServices::TActivity::TABLET_COMPACTION_BROKER;
}
- TResourceBrokerActor(const NKikimrResourceBroker::TResourceBrokerConfig &config,
- const NMonitoring::TDynamicCounterPtr &counters);
+ TResourceBrokerActor(const NKikimrResourceBroker::TResourceBrokerConfig &config,
+ const NMonitoring::TDynamicCounterPtr &counters);
void Bootstrap(const TActorContext &ctx);
@@ -454,8 +454,8 @@ public:
HFunc(TEvResourceBroker::TEvFinishTask, Handle);
HFunc(TEvResourceBroker::TEvNotifyActorDied, Handle);
HFunc(TEvResourceBroker::TEvConfigure, Handle);
- HFunc(TEvResourceBroker::TEvConfigRequest, Handle);
- HFunc(TEvResourceBroker::TEvResourceBrokerRequest, Handle);
+ HFunc(TEvResourceBroker::TEvConfigRequest, Handle);
+ HFunc(TEvResourceBroker::TEvResourceBrokerRequest, Handle);
HFunc(NMon::TEvHttpInfo, Handle);
default:
Y_FAIL("TResourceBroker::StateWork unexpected event type: %" PRIx32 " event: %s",
@@ -472,13 +472,13 @@ private:
void Handle(TEvResourceBroker::TEvFinishTask::TPtr &ev, const TActorContext &ctx);
void Handle(TEvResourceBroker::TEvNotifyActorDied::TPtr &ev, const TActorContext &ctx);
void Handle(TEvResourceBroker::TEvConfigure::TPtr &ev, const TActorContext &ctx);
- void Handle(TEvResourceBroker::TEvConfigRequest::TPtr &ev, const TActorContext &ctx);
- void Handle(TEvResourceBroker::TEvResourceBrokerRequest::TPtr &ev, const TActorContext &ctx);
+ void Handle(TEvResourceBroker::TEvConfigRequest::TPtr &ev, const TActorContext &ctx);
+ void Handle(TEvResourceBroker::TEvResourceBrokerRequest::TPtr &ev, const TActorContext &ctx);
void Handle(NMon::TEvHttpInfo::TPtr &ev, const TActorContext &ctx);
NKikimrResourceBroker::TResourceBrokerConfig Config;
- NMonitoring::TDynamicCounterPtr Counters;
- TIntrusivePtr<TResourceBroker> ResourceBroker;
+ NMonitoring::TDynamicCounterPtr Counters;
+ TIntrusivePtr<TResourceBroker> ResourceBroker;
};
} // NResourceBroker
diff --git a/ydb/core/tablet/resource_broker_ut.cpp b/ydb/core/tablet/resource_broker_ut.cpp
index 18406c22cd..5948509025 100644
--- a/ydb/core/tablet/resource_broker_ut.cpp
+++ b/ydb/core/tablet/resource_broker_ut.cpp
@@ -185,13 +185,13 @@ static void CheckConfigure(TTestActorRuntime &runtime, TActorId broker, TActorId
UNIT_ASSERT_VALUES_EQUAL((int)rec.GetSuccess(), (int)success);
}
-static
-TIntrusivePtr<IResourceBroker> GetInstantResourceBroker(TTestActorRuntime &runtime, TActorId broker, TActorId sender) {
- runtime.Send(new IEventHandle(broker, sender, new TEvResourceBroker::TEvResourceBrokerRequest));
- auto answer = runtime.GrabEdgeEvent<TEvResourceBroker::TEvResourceBrokerResponse>(sender);
- return answer->Get()->ResourceBroker;
-}
-
+static
+TIntrusivePtr<IResourceBroker> GetInstantResourceBroker(TTestActorRuntime &runtime, TActorId broker, TActorId sender) {
+ runtime.Send(new IEventHandle(broker, sender, new TEvResourceBroker::TEvResourceBrokerRequest));
+ auto answer = runtime.GrabEdgeEvent<TEvResourceBroker::TEvResourceBrokerResponse>(sender);
+ return answer->Get()->ResourceBroker;
+}
+
Y_UNIT_TEST_SUITE(TResourceBroker) {
Y_UNIT_TEST(TestErrors) {
TTestBasicRuntime runtime;
@@ -205,7 +205,7 @@ Y_UNIT_TEST_SUITE(TResourceBroker) {
NMonitoring::TDynamicCounterPtr counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
auto config = MakeTestConfig();
- auto broker = CreateResourceBrokerActor(config, counters);
+ auto broker = CreateResourceBrokerActor(config, counters);
auto brokerId = runtime.Register(broker);
WaitForBootstrap(runtime);
@@ -285,7 +285,7 @@ Y_UNIT_TEST_SUITE(TResourceBroker) {
NMonitoring::TDynamicCounterPtr counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
auto config = MakeTestConfig();
- auto broker = CreateResourceBrokerActor(config, counters);
+ auto broker = CreateResourceBrokerActor(config, counters);
auto brokerId = runtime.Register(broker);
WaitForBootstrap(runtime);
@@ -323,7 +323,7 @@ Y_UNIT_TEST_SUITE(TResourceBroker) {
NMonitoring::TDynamicCounterPtr counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
auto config = MakeTestConfig();
- auto broker = CreateResourceBrokerActor(config, counters);
+ auto broker = CreateResourceBrokerActor(config, counters);
auto brokerId = runtime.Register(broker);
WaitForBootstrap(runtime);
@@ -413,7 +413,7 @@ Y_UNIT_TEST_SUITE(TResourceBroker) {
NMonitoring::TDynamicCounterPtr counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
auto config = MakeTestConfig();
- auto broker = CreateResourceBrokerActor(config, counters);
+ auto broker = CreateResourceBrokerActor(config, counters);
auto brokerId = runtime.Register(broker);
WaitForBootstrap(runtime);
@@ -462,7 +462,7 @@ Y_UNIT_TEST_SUITE(TResourceBroker) {
NMonitoring::TDynamicCounterPtr counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
auto config = MakeTestConfig();
- auto broker = CreateResourceBrokerActor(config, counters);
+ auto broker = CreateResourceBrokerActor(config, counters);
auto brokerId = runtime.Register(broker);
WaitForBootstrap(runtime);
@@ -557,7 +557,7 @@ Y_UNIT_TEST_SUITE(TResourceBroker) {
NMonitoring::TDynamicCounterPtr counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
auto config = MakeTestConfig();
- auto broker = CreateResourceBrokerActor(config, counters);
+ auto broker = CreateResourceBrokerActor(config, counters);
auto brokerId = runtime.Register(broker);
WaitForBootstrap(runtime);
@@ -615,7 +615,7 @@ Y_UNIT_TEST_SUITE(TResourceBroker) {
NMonitoring::TDynamicCounterPtr counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
auto config = MakeTestConfig();
- auto broker = CreateResourceBrokerActor(config, counters);
+ auto broker = CreateResourceBrokerActor(config, counters);
auto brokerId = runtime.Register(broker);
WaitForBootstrap(runtime);
@@ -652,7 +652,7 @@ Y_UNIT_TEST_SUITE(TResourceBroker) {
NMonitoring::TDynamicCounterPtr counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
auto config = MakeTestConfig();
- auto broker = CreateResourceBrokerActor(config, counters);
+ auto broker = CreateResourceBrokerActor(config, counters);
auto brokerId = runtime.Register(broker);
WaitForBootstrap(runtime);
@@ -690,7 +690,7 @@ Y_UNIT_TEST_SUITE(TResourceBroker) {
NMonitoring::TDynamicCounterPtr counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
auto config = MakeTestConfig();
- auto broker = CreateResourceBrokerActor(config, counters);
+ auto broker = CreateResourceBrokerActor(config, counters);
auto brokerId = runtime.Register(broker);
WaitForBootstrap(runtime);
@@ -727,7 +727,7 @@ Y_UNIT_TEST_SUITE(TResourceBroker) {
NMonitoring::TDynamicCounterPtr counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
auto config = MakeTestConfig();
- auto broker = CreateResourceBrokerActor(config, counters);
+ auto broker = CreateResourceBrokerActor(config, counters);
auto brokerId = runtime.Register(broker);
WaitForBootstrap(runtime);
@@ -764,7 +764,7 @@ Y_UNIT_TEST_SUITE(TResourceBroker) {
NMonitoring::TDynamicCounterPtr counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
auto config = MakeTestConfig();
- auto broker = CreateResourceBrokerActor(config, counters);
+ auto broker = CreateResourceBrokerActor(config, counters);
auto brokerId = runtime.Register(broker);
WaitForBootstrap(runtime);
@@ -810,7 +810,7 @@ Y_UNIT_TEST_SUITE(TResourceBroker) {
NMonitoring::TDynamicCounterPtr counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
auto config = MakeTestConfig();
- auto broker = CreateResourceBrokerActor(config, counters);
+ auto broker = CreateResourceBrokerActor(config, counters);
auto brokerId = runtime.Register(broker);
WaitForBootstrap(runtime);
@@ -893,122 +893,122 @@ Y_UNIT_TEST_SUITE(TResourceBroker) {
}
};
-Y_UNIT_TEST_SUITE(TResourceBrokerInstant) {
- Y_UNIT_TEST(Test) {
- TTestBasicRuntime runtime;
- SetupTabletServices(runtime);
- SetupLogging(runtime);
-
- TActorId sender = runtime.AllocateEdgeActor();
-
- NMonitoring::TDynamicCounterPtr counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
- auto broker = runtime.Register(CreateResourceBrokerActor(MakeTestConfig(), counters));
- WaitForBootstrap(runtime);
-
- auto rb = GetInstantResourceBroker(runtime, broker, sender);
-
- bool ok = rb->SubmitTaskInstant({/* taskId */ 1, /* name */ "task-1", /* resources */ {100, 100},
- /* type */ "compaction0", /* priority */ 0, /* cookie */ nullptr}, sender);
- UNIT_ASSERT(ok);
- CheckCounters(counters, "queue", "total", 100, 100, 0, 0, 1);
-
- ok = rb->ReduceTaskResourcesInstant(/* taskId */ 1, /* reduceBy */ {20, 30}, sender);
- UNIT_ASSERT(ok);
- CheckCounters(counters, "queue", "total", 80, 70, 0, 0, 1);
-
- ok = rb->FinishTaskInstant({/* taskId */ 1, /* cancel */ false}, sender);
- UNIT_ASSERT(ok);
- CheckCounters(counters, "queue", "total", 0, 0, 1, 0, 0);
- }
-
- Y_UNIT_TEST(TestErrors) {
- TTestBasicRuntime runtime;
- SetupTabletServices(runtime);
- SetupLogging(runtime);
-
- TActorId sender = runtime.AllocateEdgeActor();
-
- NMonitoring::TDynamicCounterPtr counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
- auto broker = runtime.Register(CreateResourceBrokerActor(MakeTestConfig(), counters));
- WaitForBootstrap(runtime);
-
- auto rb = GetInstantResourceBroker(runtime, broker, sender);
-
- bool ok = rb->SubmitTaskInstant({/* taskId */ 1, /* name */ "task-1", /* resources */ {100, 100},
- /* type */ "compaction0", /* priority */ 0, /* cookie */ nullptr}, sender);
- UNIT_ASSERT(ok);
- CheckCounters(counters, "queue", "total", 100, 100, 0, 0, 1);
-
- ok = rb->SubmitTaskInstant({/* taskId */ 1, /* name */ "task-1", /* resources */ {100500, 100500},
- /* type */ "compaction0", /* priority */ 0, /* cookie */ nullptr}, sender);
- UNIT_ASSERT(!ok);
- CheckCounters(counters, "queue", "total", 100, 100, 0, 0, 1);
-
- ok = rb->ReduceTaskResourcesInstant(/* taskId */ 2, /* reduceBy */ {20, 30}, sender);
- UNIT_ASSERT(!ok);
- CheckCounters(counters, "queue", "total", 100, 100, 0, 0, 1);
-
- ok = rb->FinishTaskInstant({/* taskId */ 2, /* cancel */ false}, sender);
- UNIT_ASSERT(!ok);
- CheckCounters(counters, "queue", "total", 100, 100, 0, 0, 1);
- }
-
- Y_UNIT_TEST(TestMerge) {
- TTestBasicRuntime runtime;
- SetupTabletServices(runtime);
- SetupLogging(runtime);
-
- TActorId sender = runtime.AllocateEdgeActor();
-
- NMonitoring::TDynamicCounterPtr counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
- auto broker = runtime.Register(CreateResourceBrokerActor(MakeTestConfig(), counters));
- WaitForBootstrap(runtime);
-
- auto rb = GetInstantResourceBroker(runtime, broker, sender);
-
- // recipient task
- {
- bool ok = rb->SubmitTaskInstant({/* taskId */ 1, /* name */ "task-1", /* resources */ {100, 200},
- /* type */ "compaction0", /* priority */ 0, /* cookie */ nullptr}, sender);
- UNIT_ASSERT(ok);
- CheckCounters(counters, "queue", "total", 100, 200, 0, 0, 1);
- }
-
- // donor task
- {
- bool ok = rb->SubmitTaskInstant({/* taskId */ 2, /* name */ "task-2", /* resources */ {100, 100},
- /* type */ "compaction0", /* priority */ 0, /* cookie */ nullptr}, sender);
- UNIT_ASSERT(ok);
- CheckCounters(counters, "queue", "total", 200, 300, 0, 0, 2);
- }
-
- // merge tasks
- {
- bool ok = rb->MergeTasksInstant(/* recipientTaskId */ 1, /* donorTaskId */ 2, sender);
- UNIT_ASSERT(ok);
-
- CheckCounters(counters, "queue", "total", 200, 300, 1, 0, 1);
- }
-
- // donor task of different type
- {
- bool ok = rb->SubmitTaskInstant({/* taskId */ 3, /* name */ "task-3", /* resources */ {10, 20},
- /* type */ "compaction1", /* priority */ 0, /* cookie */ nullptr}, sender);
- UNIT_ASSERT(ok);
-
- CheckCounters(counters, "queue", "total", 210, 320, 1, 0, 2);
- }
-
- // merge tasks
- {
- bool ok = rb->MergeTasksInstant(/* recipientTaskId */ 1, /* donorTaskId */ 3, sender);
- UNIT_ASSERT(!ok);
-
- CheckCounters(counters, "queue", "total", 210, 320, 1, 0, 2);
- }
- }
-};
-
+Y_UNIT_TEST_SUITE(TResourceBrokerInstant) {
+ Y_UNIT_TEST(Test) {
+ TTestBasicRuntime runtime;
+ SetupTabletServices(runtime);
+ SetupLogging(runtime);
+
+ TActorId sender = runtime.AllocateEdgeActor();
+
+ NMonitoring::TDynamicCounterPtr counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
+ auto broker = runtime.Register(CreateResourceBrokerActor(MakeTestConfig(), counters));
+ WaitForBootstrap(runtime);
+
+ auto rb = GetInstantResourceBroker(runtime, broker, sender);
+
+ bool ok = rb->SubmitTaskInstant({/* taskId */ 1, /* name */ "task-1", /* resources */ {100, 100},
+ /* type */ "compaction0", /* priority */ 0, /* cookie */ nullptr}, sender);
+ UNIT_ASSERT(ok);
+ CheckCounters(counters, "queue", "total", 100, 100, 0, 0, 1);
+
+ ok = rb->ReduceTaskResourcesInstant(/* taskId */ 1, /* reduceBy */ {20, 30}, sender);
+ UNIT_ASSERT(ok);
+ CheckCounters(counters, "queue", "total", 80, 70, 0, 0, 1);
+
+ ok = rb->FinishTaskInstant({/* taskId */ 1, /* cancel */ false}, sender);
+ UNIT_ASSERT(ok);
+ CheckCounters(counters, "queue", "total", 0, 0, 1, 0, 0);
+ }
+
+ Y_UNIT_TEST(TestErrors) {
+ TTestBasicRuntime runtime;
+ SetupTabletServices(runtime);
+ SetupLogging(runtime);
+
+ TActorId sender = runtime.AllocateEdgeActor();
+
+ NMonitoring::TDynamicCounterPtr counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
+ auto broker = runtime.Register(CreateResourceBrokerActor(MakeTestConfig(), counters));
+ WaitForBootstrap(runtime);
+
+ auto rb = GetInstantResourceBroker(runtime, broker, sender);
+
+ bool ok = rb->SubmitTaskInstant({/* taskId */ 1, /* name */ "task-1", /* resources */ {100, 100},
+ /* type */ "compaction0", /* priority */ 0, /* cookie */ nullptr}, sender);
+ UNIT_ASSERT(ok);
+ CheckCounters(counters, "queue", "total", 100, 100, 0, 0, 1);
+
+ ok = rb->SubmitTaskInstant({/* taskId */ 1, /* name */ "task-1", /* resources */ {100500, 100500},
+ /* type */ "compaction0", /* priority */ 0, /* cookie */ nullptr}, sender);
+ UNIT_ASSERT(!ok);
+ CheckCounters(counters, "queue", "total", 100, 100, 0, 0, 1);
+
+ ok = rb->ReduceTaskResourcesInstant(/* taskId */ 2, /* reduceBy */ {20, 30}, sender);
+ UNIT_ASSERT(!ok);
+ CheckCounters(counters, "queue", "total", 100, 100, 0, 0, 1);
+
+ ok = rb->FinishTaskInstant({/* taskId */ 2, /* cancel */ false}, sender);
+ UNIT_ASSERT(!ok);
+ CheckCounters(counters, "queue", "total", 100, 100, 0, 0, 1);
+ }
+
+ Y_UNIT_TEST(TestMerge) {
+ TTestBasicRuntime runtime;
+ SetupTabletServices(runtime);
+ SetupLogging(runtime);
+
+ TActorId sender = runtime.AllocateEdgeActor();
+
+ NMonitoring::TDynamicCounterPtr counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
+ auto broker = runtime.Register(CreateResourceBrokerActor(MakeTestConfig(), counters));
+ WaitForBootstrap(runtime);
+
+ auto rb = GetInstantResourceBroker(runtime, broker, sender);
+
+ // recipient task
+ {
+ bool ok = rb->SubmitTaskInstant({/* taskId */ 1, /* name */ "task-1", /* resources */ {100, 200},
+ /* type */ "compaction0", /* priority */ 0, /* cookie */ nullptr}, sender);
+ UNIT_ASSERT(ok);
+ CheckCounters(counters, "queue", "total", 100, 200, 0, 0, 1);
+ }
+
+ // donor task
+ {
+ bool ok = rb->SubmitTaskInstant({/* taskId */ 2, /* name */ "task-2", /* resources */ {100, 100},
+ /* type */ "compaction0", /* priority */ 0, /* cookie */ nullptr}, sender);
+ UNIT_ASSERT(ok);
+ CheckCounters(counters, "queue", "total", 200, 300, 0, 0, 2);
+ }
+
+ // merge tasks
+ {
+ bool ok = rb->MergeTasksInstant(/* recipientTaskId */ 1, /* donorTaskId */ 2, sender);
+ UNIT_ASSERT(ok);
+
+ CheckCounters(counters, "queue", "total", 200, 300, 1, 0, 1);
+ }
+
+ // donor task of different type
+ {
+ bool ok = rb->SubmitTaskInstant({/* taskId */ 3, /* name */ "task-3", /* resources */ {10, 20},
+ /* type */ "compaction1", /* priority */ 0, /* cookie */ nullptr}, sender);
+ UNIT_ASSERT(ok);
+
+ CheckCounters(counters, "queue", "total", 210, 320, 1, 0, 2);
+ }
+
+ // merge tasks
+ {
+ bool ok = rb->MergeTasksInstant(/* recipientTaskId */ 1, /* donorTaskId */ 3, sender);
+ UNIT_ASSERT(!ok);
+
+ CheckCounters(counters, "queue", "total", 210, 320, 1, 0, 2);
+ }
+ }
+};
+
Y_UNIT_TEST_SUITE(TResourceBrokerConfig) {
Y_UNIT_TEST(UpdateQueues) {
diff --git a/ydb/core/tablet_flat/flat_exec_seat.h b/ydb/core/tablet_flat/flat_exec_seat.h
index 53f3b59b18..f37d3bfb0b 100644
--- a/ydb/core/tablet_flat/flat_exec_seat.h
+++ b/ydb/core/tablet_flat/flat_exec_seat.h
@@ -47,7 +47,7 @@ namespace NTabletFlatExecutor {
ui64 CurrentTxDataLimit = 0;
ui64 CurrentMemoryLimit = 0;
- ui32 NotEnoughMemoryCount = 0;
+ ui32 NotEnoughMemoryCount = 0;
ui64 TaskId = 0;
TAutoPtr<TMemoryToken> AttachedMemory;
diff --git a/ydb/core/tablet_flat/flat_executor.cpp b/ydb/core/tablet_flat/flat_executor.cpp
index c236b0f7f7..59afc46215 100644
--- a/ydb/core/tablet_flat/flat_executor.cpp
+++ b/ydb/core/tablet_flat/flat_executor.cpp
@@ -1466,7 +1466,7 @@ void TExecutor::ExecuteTransaction(TAutoPtr<TSeat> seat, const TActorContext &ct
TPageCollectionTxEnv env(*PrivatePageCache);
TTransactionContext txc(Owner->TabletID(), Generation(), Step(), *Database, env, seat->CurrentTxDataLimit, seat->TaskId);
- txc.NotEnoughMemory(seat->NotEnoughMemoryCount);
+ txc.NotEnoughMemory(seat->NotEnoughMemoryCount);
Database->Begin(Stamp(), env);
LWTRACK(TransactionExecuteBegin, seat->Self->Orbit, seat->UniqID);
@@ -1503,7 +1503,7 @@ void TExecutor::ExecuteTransaction(TAutoPtr<TSeat> seat, const TActorContext &ct
seat->AttachedMemory = txc.ExtractMemoryToken();
seat->RequestedMemory = txc.GetRequestedMemory();
seat->CapturedMemory = txc.GetMemoryGCToken();
- seat->NotEnoughMemoryCount = txc.GetNotEnoughMemoryCount();
+ seat->NotEnoughMemoryCount = txc.GetNotEnoughMemoryCount();
if (seat->AttachedMemory)
Counters->Cumulative()[TExecutorCounters::TX_MEM_ATTACHES].Increment(1);
diff --git a/ydb/core/tablet_flat/tablet_flat_executor.h b/ydb/core/tablet_flat/tablet_flat_executor.h
index 6b6d3d1c66..075ebce354 100644
--- a/ydb/core/tablet_flat/tablet_flat_executor.h
+++ b/ydb/core/tablet_flat/tablet_flat_executor.h
@@ -108,7 +108,7 @@ public:
: MemoryLimit(memoryLimit)
, TaskId(taskId)
, RequestedMemory(0)
- , NotEnoughMemoryCount(0)
+ , NotEnoughMemoryCount(0)
{}
~TTxMemoryProviderBase() {}
@@ -135,13 +135,13 @@ public:
ui64 GetRequestedMemory() const { return RequestedMemory; }
- void NotEnoughMemory(ui32 add = 1)
- {
- NotEnoughMemoryCount += add;
- }
-
- ui32 GetNotEnoughMemoryCount() const { return NotEnoughMemoryCount; }
-
+ void NotEnoughMemory(ui32 add = 1)
+ {
+ NotEnoughMemoryCount += add;
+ }
+
+ ui32 GetNotEnoughMemoryCount() const { return NotEnoughMemoryCount; }
+
/**
* Memory token should be captured only when tx is finished
* or within ReleaseTxData method.
@@ -177,7 +177,7 @@ private:
const ui64 MemoryLimit;
const ui64 TaskId;
ui64 RequestedMemory;
- ui32 NotEnoughMemoryCount;
+ ui32 NotEnoughMemoryCount;
TIntrusivePtr<TMemoryGCToken> MemoryGCToken;
TAutoPtr<TMemoryToken> MemoryToken;
};
diff --git a/ydb/core/tablet_flat/test/libs/exec/runner.h b/ydb/core/tablet_flat/test/libs/exec/runner.h
index 31ed857e2b..3b58ab814b 100644
--- a/ydb/core/tablet_flat/test/libs/exec/runner.h
+++ b/ydb/core/tablet_flat/test/libs/exec/runner.h
@@ -163,7 +163,7 @@ namespace NFake {
{ /*_ Resource broker service, used for generic scans */
using namespace NResourceBroker;
- auto *actor = CreateResourceBrokerActor(MakeDefaultConfig(), Env.GetDynamicCounters(0));
+ auto *actor = CreateResourceBrokerActor(MakeDefaultConfig(), Env.GetDynamicCounters(0));
AddService(MakeResourceBrokerID(), actor, EMail::Revolving);
}
diff --git a/ydb/core/testlib/actors/test_runtime.cpp b/ydb/core/testlib/actors/test_runtime.cpp
index e71e63f5fa..63f0436bff 100644
--- a/ydb/core/testlib/actors/test_runtime.cpp
+++ b/ydb/core/testlib/actors/test_runtime.cpp
@@ -128,7 +128,7 @@ namespace NActors {
nodeAppData->PQConfig = app0->PQConfig;
nodeAppData->NetClassifierConfig.CopyFrom(app0->NetClassifierConfig);
nodeAppData->StaticBlobStorageConfig->CopyFrom(*app0->StaticBlobStorageConfig);
- nodeAppData->EnableKqpSpilling = app0->EnableKqpSpilling;
+ nodeAppData->EnableKqpSpilling = app0->EnableKqpSpilling;
nodeAppData->FeatureFlags = app0->FeatureFlags;
nodeAppData->CompactionConfig = app0->CompactionConfig;
nodeAppData->HiveConfig = app0->HiveConfig;
diff --git a/ydb/core/testlib/basics/appdata.h b/ydb/core/testlib/basics/appdata.h
index b1e97e10c3..409869161f 100644
--- a/ydb/core/testlib/basics/appdata.h
+++ b/ydb/core/testlib/basics/appdata.h
@@ -73,7 +73,7 @@ namespace NKikimr {
NKikimrBlobStorage::TNodeWardenServiceSet BSConf;
TIntrusivePtr<TDomainsInfo> Domains;
TMap<ui32, NKikimrProto::TKeyConfig> Keys;
- bool EnableKqpSpilling = false;
+ bool EnableKqpSpilling = false;
NKikimrConfig::TCompactionConfig CompactionConfig;
TString NetDataSourceUrl;
NKikimrConfig::THiveConfig HiveConfig;
diff --git a/ydb/core/testlib/basics/services.cpp b/ydb/core/testlib/basics/services.cpp
index 7d8bdf9f8d..b4585ad96c 100644
--- a/ydb/core/testlib/basics/services.cpp
+++ b/ydb/core/testlib/basics/services.cpp
@@ -86,7 +86,7 @@ namespace NPDisk {
{
runtime.AddLocalService(NResourceBroker::MakeResourceBrokerID(),
TActorSetupCmd(
- NResourceBroker::CreateResourceBrokerActor(NResourceBroker::MakeDefaultConfig(), runtime.GetDynamicCounters(0)),
+ NResourceBroker::CreateResourceBrokerActor(NResourceBroker::MakeDefaultConfig(), runtime.GetDynamicCounters(0)),
TMailboxType::Revolving, 0),
nodeIndex);
}
diff --git a/ydb/core/testlib/minikql_compile.h b/ydb/core/testlib/minikql_compile.h
index d21b27dcb3..4d76ca8d73 100644
--- a/ydb/core/testlib/minikql_compile.h
+++ b/ydb/core/testlib/minikql_compile.h
@@ -7,23 +7,23 @@
#include <util/thread/pool.h>
#include <library/cpp/testing/unittest/registar.h>
-class TMockDbSchemeResolver : public NYql::IDbSchemeResolver {
+class TMockDbSchemeResolver : public NYql::IDbSchemeResolver {
public:
TMockDbSchemeResolver()
{
MtpQueue.Start(2);
}
- template <typename Func>
- NThreading::TFuture<NThreading::TFutureType<::TFunctionResult<Func>>> Async(Func&& func, IThreadPool& queue) {
- auto promise = NThreading::NewPromise<NThreading::TFutureType<::TFunctionResult<Func>>>();
- auto lambda = [promise, func = std::forward<Func>(func)]() mutable {
- NThreading::NImpl::SetValue(promise, func);
- };
- queue.SafeAddFunc(std::move(lambda));
- return promise.GetFuture();
- }
-
+ template <typename Func>
+ NThreading::TFuture<NThreading::TFutureType<::TFunctionResult<Func>>> Async(Func&& func, IThreadPool& queue) {
+ auto promise = NThreading::NewPromise<NThreading::TFutureType<::TFunctionResult<Func>>>();
+ auto lambda = [promise, func = std::forward<Func>(func)]() mutable {
+ NThreading::NImpl::SetValue(promise, func);
+ };
+ queue.SafeAddFunc(std::move(lambda));
+ return promise.GetFuture();
+ }
+
virtual NThreading::TFuture<TTableResults> ResolveTables(const TVector<TTable>& tables) override {
TTableResults results;
results.reserve(tables.size());
@@ -51,7 +51,7 @@ public:
results.push_back(result);
}
- return Async([results]() {
+ return Async([results]() {
return results;
}, MtpQueue);
}
@@ -74,8 +74,8 @@ private:
THashMap<TString, IDbSchemeResolver::TTableResult> Tables;
};
-namespace NYql {
-
+namespace NYql {
+
inline TExprContainer::TPtr ParseText(const TString& programText) {
TAstParseResult astRes = ParseAst(programText);
astRes.Issues.PrintTo(Cerr);
diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp
index f40af52021..d4907c26f1 100644
--- a/ydb/core/testlib/test_client.cpp
+++ b/ydb/core/testlib/test_client.cpp
@@ -159,7 +159,7 @@ namespace Tests {
TAppPrepare app; /* will cook TAppData */
app.SetNetDataSourceUrl(Settings->NetClassifierConfig.GetUpdaterConfig().GetNetDataSourceUrl());
- app.SetEnableKqpSpilling(Settings->EnableKqpSpilling);
+ app.SetEnableKqpSpilling(Settings->EnableKqpSpilling);
app.SetKeepSnapshotTimeout(Settings->KeepSnapshotTimeout);
app.SetChangesQueueItemsLimit(Settings->ChangesQueueItemsLimit);
app.SetChangesQueueBytesLimit(Settings->ChangesQueueBytesLimit);
@@ -249,8 +249,8 @@ namespace Tests {
auto grpcService = new NGRpcProxy::TGRpcService();
auto system(Runtime->GetAnyNodeActorSystem());
- auto grpcRequestProxy = NGRpcService::CreateGRpcRequestProxy(Settings->AppConfig);
- auto grpcRequestProxyId = system->Register(grpcRequestProxy, TMailboxType::ReadAsFilled);
+ auto grpcRequestProxy = NGRpcService::CreateGRpcRequestProxy(Settings->AppConfig);
+ auto grpcRequestProxyId = system->Register(grpcRequestProxy, TMailboxType::ReadAsFilled);
system->RegisterLocalService(NGRpcService::CreateGRpcRequestProxyId(), grpcRequestProxyId);
auto grpcMon = system->Register(NGRpcService::CreateGrpcMonService(), TMailboxType::ReadAsFilled);
system->RegisterLocalService(NGRpcService::GrpcMonServiceId(), grpcMon);
@@ -598,14 +598,14 @@ namespace Tests {
}
{
- IActor* kqpRmService = NKqp::CreateKqpResourceManagerActor(Settings->AppConfig.GetTableServiceConfig().GetResourceManager(), nullptr);
- TActorId kqpRmServiceId = Runtime->Register(kqpRmService, nodeIdx);
- Runtime->RegisterService(NKqp::MakeKqpRmServiceID(Runtime->GetNodeId(nodeIdx)), kqpRmServiceId, nodeIdx);
- }
-
- {
+ IActor* kqpRmService = NKqp::CreateKqpResourceManagerActor(Settings->AppConfig.GetTableServiceConfig().GetResourceManager(), nullptr);
+ TActorId kqpRmServiceId = Runtime->Register(kqpRmService, nodeIdx);
+ Runtime->RegisterService(NKqp::MakeKqpRmServiceID(Runtime->GetNodeId(nodeIdx)), kqpRmServiceId, nodeIdx);
+ }
+
+ {
IActor* kqpProxyService = NKqp::CreateKqpProxyService(Settings->AppConfig.GetLogConfig(),
- Settings->AppConfig.GetTableServiceConfig(),
+ Settings->AppConfig.GetTableServiceConfig(),
TVector<NKikimrKqp::TKqpSetting>(Settings->KqpSettings),
nullptr);
TActorId kqpProxyServiceId = Runtime->Register(kqpProxyService, nodeIdx);
diff --git a/ydb/core/testlib/test_client.h b/ydb/core/testlib/test_client.h
index d6503fb1c1..2064752ab4 100644
--- a/ydb/core/testlib/test_client.h
+++ b/ydb/core/testlib/test_client.h
@@ -111,7 +111,7 @@ namespace Tests {
bool EnableConsole = true;
bool EnableConfigsDispatcher = true;
bool UseRealThreads = true;
- bool EnableKqpSpilling = false;
+ bool EnableKqpSpilling = false;
bool EnableYq = false;
TDuration KeepSnapshotTimeout = TDuration::Zero();
ui64 ChangesQueueItemsLimit = 0;
@@ -149,7 +149,7 @@ namespace Tests {
TServerSettings& SetUseRealThreads(bool value) { UseRealThreads = value; return *this; }
TServerSettings& SetAppConfig(const NKikimrConfig::TAppConfig value) { AppConfig = value; return *this; }
TServerSettings& SetKeyFor(ui32 nodeId, TString keyValue) { NodeKeys[nodeId] = keyValue; return *this; }
- TServerSettings& SetEnableKqpSpilling(bool value) { EnableKqpSpilling = value; return *this; }
+ TServerSettings& SetEnableKqpSpilling(bool value) { EnableKqpSpilling = value; return *this; }
TServerSettings& SetDomainPlanResolution(ui64 resolution) { DomainPlanResolution = resolution; return *this; }
TServerSettings& SetFeatureFlags(const NKikimrConfig::TFeatureFlags& value) { FeatureFlags = value; return *this; }
TServerSettings& SetCompactionConfig(const NKikimrConfig::TCompactionConfig& value) { CompactionConfig = value; return *this; }
diff --git a/ydb/core/tx/columnshard/columnshard__scan.cpp b/ydb/core/tx/columnshard/columnshard__scan.cpp
index 16dfe45b1b..e2f7ba5a51 100644
--- a/ydb/core/tx/columnshard/columnshard__scan.cpp
+++ b/ydb/core/tx/columnshard/columnshard__scan.cpp
@@ -38,7 +38,7 @@ public:
, BlobCacheActorId(NBlobCache::MakeBlobCacheServiceId())
, ScanId(scanId)
, TxId(txId)
- , ScanGen(scanGen)
+ , ScanGen(scanGen)
, RequestCookie(requestCookie)
, DataFormat(dataFormat)
, TablePath(table)
@@ -102,7 +102,7 @@ private:
}
Y_VERIFY(ev->Get()->Generation == ScanGen);
-
+
PeerFreeSpace = ev->Get()->FreeSpace;
ContinueProcessing();
@@ -299,7 +299,7 @@ private:
private:
void MakeResult(size_t reserveRows = 0) {
if (!Finished && !Result) {
- Result = MakeHolder<TEvKqpCompute::TEvScanData>(ScanId, ScanGen);
+ Result = MakeHolder<TEvKqpCompute::TEvScanData>(ScanId, ScanGen);
if (reserveRows) {
Y_VERIFY(DataFormat != NKikimrTxDataShard::EScanDataFormat::ARROW);
Result->Rows.reserve(reserveRows);
@@ -363,8 +363,8 @@ private:
Result->PageFaults = PageFaults;
Result->Finished = lastBatch;
TDuration totalElapsedTime = TDuration::Seconds(GetElapsedTicksAsSeconds());
- // Result->TotalTime = totalElapsedTime - LastReportedElapsedTime;
- // TODO: Result->CpuTime = ...
+ // Result->TotalTime = totalElapsedTime - LastReportedElapsedTime;
+ // TODO: Result->CpuTime = ...
LastReportedElapsedTime = totalElapsedTime;
PageFaults = 0;
@@ -427,7 +427,7 @@ private:
TActorId BlobCacheActorId;
const ui32 ScanId;
const ui64 TxId;
- const ui32 ScanGen;
+ const ui32 ScanGen;
const ui64 RequestCookie;
const i64 MaxReadAheadBytes = DEFAULT_READ_AHEAD_BYTES;
const NKikimrTxDataShard::EScanDataFormat DataFormat;
@@ -667,7 +667,7 @@ void TTxScan::Complete(const TActorContext& ctx) {
const auto& snapshot = request.GetSnapshot();
const auto scanId = request.GetScanId();
const ui64 txId = request.GetTxId();
- const ui32 scanGen = request.GetGeneration();
+ const ui32 scanGen = request.GetGeneration();
TString table = request.GetTablePath();
auto dataFormat = request.GetDataFormat();
TDuration timeout = TDuration::MilliSeconds(request.GetTimeoutMs());
@@ -686,7 +686,7 @@ void TTxScan::Complete(const TActorContext& ctx) {
LOG_S_DEBUG("Starting scan"
<< ", txId: " << txId
<< ", scanId: " << scanId
- << ", gen: " << scanGen
+ << ", gen: " << scanGen
<< ", table: " << table
<< ", snapshot: " << snapshot
<< ", shard: " << Self->TabletID()
diff --git a/ydb/core/tx/datashard/build_data_tx_out_rs_unit.cpp b/ydb/core/tx/datashard/build_data_tx_out_rs_unit.cpp
index 3d6b806b85..cd8d3dab69 100644
--- a/ydb/core/tx/datashard/build_data_tx_out_rs_unit.cpp
+++ b/ydb/core/tx/datashard/build_data_tx_out_rs_unit.cpp
@@ -91,9 +91,9 @@ EExecutionStatus TBuildDataTxOutRSUnit::Execute(TOperation::TPtr op,
<< " and requests " << txc.GetMemoryLimit() * MEMORY_REQUEST_FACTOR
<< " more for the next try");
- txc.NotEnoughMemory();
- DataShard.IncCounter(DataShard.NotEnoughMemoryCounter(txc.GetNotEnoughMemoryCount()));
-
+ txc.NotEnoughMemory();
+ DataShard.IncCounter(DataShard.NotEnoughMemoryCounter(txc.GetNotEnoughMemoryCount()));
+
engine->ReleaseUnusedMemory();
txc.RequestMemory(txc.GetMemoryLimit() * MEMORY_REQUEST_FACTOR);
@@ -105,8 +105,8 @@ EExecutionStatus TBuildDataTxOutRSUnit::Execute(TOperation::TPtr op,
"Tablet " << DataShard.TabletID() << " is not ready for " << *op
<< " execution");
- DataShard.IncCounter(COUNTER_TX_TABLET_NOT_READY);
-
+ DataShard.IncCounter(COUNTER_TX_TABLET_NOT_READY);
+
tx->ReleaseTxData(txc, ctx);
return EExecutionStatus::Restart;
diff --git a/ydb/core/tx/datashard/build_kqp_data_tx_out_rs_unit.cpp b/ydb/core/tx/datashard/build_kqp_data_tx_out_rs_unit.cpp
index ecf8805b83..50a3c1f401 100644
--- a/ydb/core/tx/datashard/build_kqp_data_tx_out_rs_unit.cpp
+++ b/ydb/core/tx/datashard/build_kqp_data_tx_out_rs_unit.cpp
@@ -5,18 +5,18 @@
#include "setup_sys_locks.h"
#include <ydb/core/kqp/rm/kqp_rm.h>
-
+
namespace NKikimr {
namespace NDataShard {
using namespace NMiniKQL;
-#define LOG_T(stream) LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, stream)
-#define LOG_D(stream) LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, stream)
-#define LOG_E(stream) LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, stream)
-#define LOG_C(stream) LOG_CRIT_S(ctx, NKikimrServices::TX_DATASHARD, stream)
-#define LOG_W(stream) LOG_WARN_S(ctx, NKikimrServices::TX_DATASHARD, stream)
-
+#define LOG_T(stream) LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, stream)
+#define LOG_D(stream) LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, stream)
+#define LOG_E(stream) LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, stream)
+#define LOG_C(stream) LOG_CRIT_S(ctx, NKikimrServices::TX_DATASHARD, stream)
+#define LOG_W(stream) LOG_WARN_S(ctx, NKikimrServices::TX_DATASHARD, stream)
+
class TBuildKqpDataTxOutRSUnit : public TExecutionUnit {
public:
TBuildKqpDataTxOutRSUnit(TDataShard& dataShard, TPipeline& pipeline);
@@ -25,10 +25,10 @@ public:
bool IsReadyToExecute(TOperation::TPtr op) const override;
EExecutionStatus Execute(TOperation::TPtr op, TTransactionContext& txc, const TActorContext& ctx) override;
void Complete(TOperation::TPtr op, const TActorContext& ctx) override;
-
-private:
- EExecutionStatus OnTabletNotReady(TActiveTransaction& tx, TValidatedDataTx& dataTx, TTransactionContext& txc,
- const TActorContext& ctx);
+
+private:
+ EExecutionStatus OnTabletNotReady(TActiveTransaction& tx, TValidatedDataTx& dataTx, TTransactionContext& txc,
+ const TActorContext& ctx);
};
TBuildKqpDataTxOutRSUnit::TBuildKqpDataTxOutRSUnit(TDataShard& dataShard, TPipeline& pipeline)
@@ -60,76 +60,76 @@ EExecutionStatus TBuildKqpDataTxOutRSUnit::Execute(TOperation::TPtr op, TTransac
}
}
- const auto& dataTx = tx->GetDataTx();
- ui64 tabletId = DataShard.TabletID();
-
- if (tx->GetDataTx()->CheckCancelled()) {
- tx->ReleaseTxData(txc, ctx);
- BuildResult(op, NKikimrTxDataShard::TEvProposeTransactionResult::CANCELLED)
- ->AddError(NKikimrTxDataShard::TError::EXECUTION_CANCELLED, "Tx was cancelled");
-
- DataShard.IncCounter(op->IsImmediate() ? COUNTER_IMMEDIATE_TX_CANCELLED : COUNTER_PLANNED_TX_CANCELLED);
-
- return EExecutionStatus::Executed;
- }
-
+ const auto& dataTx = tx->GetDataTx();
+ ui64 tabletId = DataShard.TabletID();
+
+ if (tx->GetDataTx()->CheckCancelled()) {
+ tx->ReleaseTxData(txc, ctx);
+ BuildResult(op, NKikimrTxDataShard::TEvProposeTransactionResult::CANCELLED)
+ ->AddError(NKikimrTxDataShard::TError::EXECUTION_CANCELLED, "Tx was cancelled");
+
+ DataShard.IncCounter(op->IsImmediate() ? COUNTER_IMMEDIATE_TX_CANCELLED : COUNTER_PLANNED_TX_CANCELLED);
+
+ return EExecutionStatus::Executed;
+ }
+
try {
- const auto& kqpTx = dataTx->GetKqpTransaction();
- auto& tasksRunner = dataTx->GetKqpTasksRunner();
-
- auto allocGuard = tasksRunner.BindAllocator(txc.GetMemoryLimit() - dataTx->GetTxSize());
-
- NKqp::NRm::TKqpResourcesRequest req;
- req.MemoryPool = NKqp::NRm::EKqpMemoryPool::DataQuery;
- req.Memory = txc.GetMemoryLimit();
- ui64 taskId = kqpTx.GetTasks().empty() ? std::numeric_limits<ui64>::max() : kqpTx.GetTasks()[0].GetId();
- NKqp::GetKqpResourceManager()->NotifyExternalResourcesAllocated(tx->GetTxId(), taskId, req);
-
- Y_DEFER {
- NKqp::GetKqpResourceManager()->NotifyExternalResourcesFreed(tx->GetTxId(), taskId);
- };
-
- LOG_T("Operation " << *op << " (build_kqp_data_tx_out_rs) at " << tabletId
- << " set memory limit " << (txc.GetMemoryLimit() - dataTx->GetTxSize()));
-
+ const auto& kqpTx = dataTx->GetKqpTransaction();
+ auto& tasksRunner = dataTx->GetKqpTasksRunner();
+
+ auto allocGuard = tasksRunner.BindAllocator(txc.GetMemoryLimit() - dataTx->GetTxSize());
+
+ NKqp::NRm::TKqpResourcesRequest req;
+ req.MemoryPool = NKqp::NRm::EKqpMemoryPool::DataQuery;
+ req.Memory = txc.GetMemoryLimit();
+ ui64 taskId = kqpTx.GetTasks().empty() ? std::numeric_limits<ui64>::max() : kqpTx.GetTasks()[0].GetId();
+ NKqp::GetKqpResourceManager()->NotifyExternalResourcesAllocated(tx->GetTxId(), taskId, req);
+
+ Y_DEFER {
+ NKqp::GetKqpResourceManager()->NotifyExternalResourcesFreed(tx->GetTxId(), taskId);
+ };
+
+ LOG_T("Operation " << *op << " (build_kqp_data_tx_out_rs) at " << tabletId
+ << " set memory limit " << (txc.GetMemoryLimit() - dataTx->GetTxSize()));
+
dataTx->SetReadVersion(DataShard.GetReadWriteVersions(tx).ReadVersion);
-
- if (dataTx->GetKqpComputeCtx().HasPersistentChannels()) {
- auto result = KqpRunTransaction(ctx, op->GetTxId(), dataTx->GetKqpTasks(), tasksRunner);
-
- if (result == NYql::NDq::ERunStatus::PendingInput && dataTx->GetKqpComputeCtx().IsTabletNotReady()) {
- return OnTabletNotReady(*tx, *dataTx, txc, ctx);
- }
- }
-
- KqpFillOutReadSets(op->OutReadSets(), kqpTx, tasksRunner, DataShard.SysLocksTable(), tabletId);
- } catch (const TMemoryLimitExceededException&) {
- LOG_T("Operation " << *op << " at " << tabletId
- << " exceeded memory limit " << txc.GetMemoryLimit()
- << " and requests " << txc.GetMemoryLimit() * MEMORY_REQUEST_FACTOR
- << " more for the next try");
-
- txc.NotEnoughMemory();
- DataShard.IncCounter(DataShard.NotEnoughMemoryCounter(txc.GetNotEnoughMemoryCount()));
-
- txc.RequestMemory(txc.GetMemoryLimit() * MEMORY_REQUEST_FACTOR);
- tx->ReleaseTxData(txc, ctx);
-
- return EExecutionStatus::Restart;
- } catch (const TNotReadyTabletException&) {
- LOG_C("Unexpected TNotReadyTabletException exception at build out rs");
- return OnTabletNotReady(*tx, *dataTx, txc, ctx);
+
+ if (dataTx->GetKqpComputeCtx().HasPersistentChannels()) {
+ auto result = KqpRunTransaction(ctx, op->GetTxId(), dataTx->GetKqpTasks(), tasksRunner);
+
+ if (result == NYql::NDq::ERunStatus::PendingInput && dataTx->GetKqpComputeCtx().IsTabletNotReady()) {
+ return OnTabletNotReady(*tx, *dataTx, txc, ctx);
+ }
+ }
+
+ KqpFillOutReadSets(op->OutReadSets(), kqpTx, tasksRunner, DataShard.SysLocksTable(), tabletId);
+ } catch (const TMemoryLimitExceededException&) {
+ LOG_T("Operation " << *op << " at " << tabletId
+ << " exceeded memory limit " << txc.GetMemoryLimit()
+ << " and requests " << txc.GetMemoryLimit() * MEMORY_REQUEST_FACTOR
+ << " more for the next try");
+
+ txc.NotEnoughMemory();
+ DataShard.IncCounter(DataShard.NotEnoughMemoryCounter(txc.GetNotEnoughMemoryCount()));
+
+ txc.RequestMemory(txc.GetMemoryLimit() * MEMORY_REQUEST_FACTOR);
+ tx->ReleaseTxData(txc, ctx);
+
+ return EExecutionStatus::Restart;
+ } catch (const TNotReadyTabletException&) {
+ LOG_C("Unexpected TNotReadyTabletException exception at build out rs");
+ return OnTabletNotReady(*tx, *dataTx, txc, ctx);
} catch (const yexception& e) {
- LOG_C("Exception while preparing out-readsets for KQP transaction " << *op << " at " << DataShard.TabletID()
- << ": " << e.what());
- if (op->IsReadOnly() || op->IsImmediate()) {
- tx->ReleaseTxData(txc, ctx);
- BuildResult(op, NKikimrTxDataShard::TEvProposeTransactionResult::EXEC_ERROR)
- ->AddError(NKikimrTxDataShard::TError::PROGRAM_ERROR, TStringBuilder() << "Tx was terminated: " << e.what());
- return EExecutionStatus::Executed;
- } else {
- Y_FAIL_S("Unexpected exception in KQP out-readsets prepare: " << e.what());
- }
+ LOG_C("Exception while preparing out-readsets for KQP transaction " << *op << " at " << DataShard.TabletID()
+ << ": " << e.what());
+ if (op->IsReadOnly() || op->IsImmediate()) {
+ tx->ReleaseTxData(txc, ctx);
+ BuildResult(op, NKikimrTxDataShard::TEvProposeTransactionResult::EXEC_ERROR)
+ ->AddError(NKikimrTxDataShard::TError::PROGRAM_ERROR, TStringBuilder() << "Tx was terminated: " << e.what());
+ return EExecutionStatus::Executed;
+ } else {
+ Y_FAIL_S("Unexpected exception in KQP out-readsets prepare: " << e.what());
+ }
}
return EExecutionStatus::Executed;
@@ -137,20 +137,20 @@ EExecutionStatus TBuildKqpDataTxOutRSUnit::Execute(TOperation::TPtr op, TTransac
void TBuildKqpDataTxOutRSUnit::Complete(TOperation::TPtr, const TActorContext&) {}
-EExecutionStatus TBuildKqpDataTxOutRSUnit::OnTabletNotReady(TActiveTransaction& tx, TValidatedDataTx& dataTx,
- TTransactionContext& txc, const TActorContext& ctx)
-{
- LOG_T("Tablet " << DataShard.TabletID() << " is not ready for " << tx << " execution");
-
- DataShard.IncCounter(COUNTER_TX_TABLET_NOT_READY);
-
- ui64 pageFaultCount = tx.IncrementPageFaultCount();
- dataTx.GetKqpComputeCtx().PinPages(dataTx.TxInfo().Keys, pageFaultCount);
-
- tx.ReleaseTxData(txc, ctx);
- return EExecutionStatus::Restart;
-}
-
+EExecutionStatus TBuildKqpDataTxOutRSUnit::OnTabletNotReady(TActiveTransaction& tx, TValidatedDataTx& dataTx,
+ TTransactionContext& txc, const TActorContext& ctx)
+{
+ LOG_T("Tablet " << DataShard.TabletID() << " is not ready for " << tx << " execution");
+
+ DataShard.IncCounter(COUNTER_TX_TABLET_NOT_READY);
+
+ ui64 pageFaultCount = tx.IncrementPageFaultCount();
+ dataTx.GetKqpComputeCtx().PinPages(dataTx.TxInfo().Keys, pageFaultCount);
+
+ tx.ReleaseTxData(txc, ctx);
+ return EExecutionStatus::Restart;
+}
+
THolder<TExecutionUnit> CreateBuildKqpDataTxOutRSUnit(TDataShard& dataShard, TPipeline& pipeline) {
return THolder(new TBuildKqpDataTxOutRSUnit(dataShard, pipeline));
}
diff --git a/ydb/core/tx/datashard/datashard.cpp b/ydb/core/tx/datashard/datashard.cpp
index 687bf2036f..d86568b4a1 100644
--- a/ydb/core/tx/datashard/datashard.cpp
+++ b/ydb/core/tx/datashard/datashard.cpp
@@ -2545,7 +2545,7 @@ void TDataShard::ScanComplete(NTable::EAbort,
LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD,
"Scan complete at " << TabletID() << " for unknown tx " << cookie);
}
- }
+ }
}
// Continue current Tx
diff --git a/ydb/core/tx/datashard/datashard__engine_host.cpp b/ydb/core/tx/datashard/datashard__engine_host.cpp
index a639f96ceb..876c337093 100644
--- a/ydb/core/tx/datashard/datashard__engine_host.cpp
+++ b/ydb/core/tx/datashard/datashard__engine_host.cpp
@@ -541,7 +541,7 @@ TEngineBay::TEngineBay(TDataShard * self, TTransactionContext& txc, const TActor
<< ": " << message);
};
- ComputeCtx = MakeHolder<TKqpDatashardComputeContext>(self, EngineHostCounters, now);
+ ComputeCtx = MakeHolder<TKqpDatashardComputeContext>(self, EngineHostCounters, now);
ComputeCtx->Database = &txc.DB;
auto kqpApplyCtx = MakeHolder<TKqpDatashardApplyContext>();
@@ -549,64 +549,64 @@ TEngineBay::TEngineBay(TDataShard * self, TTransactionContext& txc, const TActor
KqpApplyCtx.Reset(kqpApplyCtx.Release());
- KqpAlloc = MakeHolder<TScopedAlloc>(TAlignedPagePoolCounters(), AppData(ctx)->FunctionRegistry->SupportsSizedAllocators());
- KqpTypeEnv = MakeHolder<TTypeEnvironment>(*KqpAlloc);
- KqpAlloc->Release();
-
+ KqpAlloc = MakeHolder<TScopedAlloc>(TAlignedPagePoolCounters(), AppData(ctx)->FunctionRegistry->SupportsSizedAllocators());
+ KqpTypeEnv = MakeHolder<TTypeEnvironment>(*KqpAlloc);
+ KqpAlloc->Release();
+
KqpExecCtx.FuncRegistry = AppData(ctx)->FunctionRegistry;
KqpExecCtx.ComputeCtx = ComputeCtx.Get();
KqpExecCtx.ComputationFactory = GetKqpDatashardComputeFactory(ComputeCtx.Get());
KqpExecCtx.RandomProvider = TAppData::RandomProvider.Get();
KqpExecCtx.TimeProvider = TAppData::TimeProvider.Get();
KqpExecCtx.ApplyCtx = KqpApplyCtx.Get();
- KqpExecCtx.Alloc = KqpAlloc.Get();
- KqpExecCtx.TypeEnv = KqpTypeEnv.Get();
+ KqpExecCtx.Alloc = KqpAlloc.Get();
+ KqpExecCtx.TypeEnv = KqpTypeEnv.Get();
+}
+
+TEngineBay::~TEngineBay() {
+ if (KqpTasksRunner) {
+ KqpTasksRunner.Reset();
+ auto guard = TGuard(*KqpAlloc);
+ KqpTypeEnv.Reset();
+ }
}
-TEngineBay::~TEngineBay() {
- if (KqpTasksRunner) {
- KqpTasksRunner.Reset();
- auto guard = TGuard(*KqpAlloc);
- KqpTypeEnv.Reset();
- }
-}
-
-void TEngineBay::AddReadRange(const TTableId& tableId, const TVector<ui32>& columns, const TTableRange& range,
- const TVector<NScheme::TTypeId>& keyTypes, ui64 itemsLimit, bool reverse)
+void TEngineBay::AddReadRange(const TTableId& tableId, const TVector<ui32>& columns, const TTableRange& range,
+ const TVector<NScheme::TTypeId>& keyTypes, ui64 itemsLimit, bool reverse)
{
- TVector<TKeyDesc::TColumnOp> columnOps;
- columnOps.reserve(columns.size());
- for (ui32 column : columns) {
- TKeyDesc::TColumnOp op;
- op.Column = column;
- op.Operation = TKeyDesc::EColumnOperation::Read;
- columnOps.emplace_back(std::move(op));
- }
-
- LOG_TRACE_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD,
- "-- AddReadRange: " << DebugPrintRange(keyTypes, range, *AppData()->TypeRegistry));
-
- auto desc = MakeHolder<TKeyDesc>(tableId, range, TKeyDesc::ERowOperation::Read, keyTypes, columnOps, itemsLimit,
- 0 /* bytesLimit */, reverse);
- Info.Keys.emplace_back(TValidatedKey(std::move(desc), /* isWrite */ false));
- // Info.Keys.back().IsResultPart = not a lock key? // TODO: KIKIMR-11134
+ TVector<TKeyDesc::TColumnOp> columnOps;
+ columnOps.reserve(columns.size());
+ for (ui32 column : columns) {
+ TKeyDesc::TColumnOp op;
+ op.Column = column;
+ op.Operation = TKeyDesc::EColumnOperation::Read;
+ columnOps.emplace_back(std::move(op));
+ }
+
+ LOG_TRACE_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD,
+ "-- AddReadRange: " << DebugPrintRange(keyTypes, range, *AppData()->TypeRegistry));
+
+ auto desc = MakeHolder<TKeyDesc>(tableId, range, TKeyDesc::ERowOperation::Read, keyTypes, columnOps, itemsLimit,
+ 0 /* bytesLimit */, reverse);
+ Info.Keys.emplace_back(TValidatedKey(std::move(desc), /* isWrite */ false));
+ // Info.Keys.back().IsResultPart = not a lock key? // TODO: KIKIMR-11134
++Info.ReadsCount;
Info.Loaded = true;
}
-void TEngineBay::AddWriteRange(const TTableId& tableId, const TTableRange& range,
- const TVector<NScheme::TTypeId>& keyTypes)
-{
- auto desc = MakeHolder<TKeyDesc>(tableId, range, TKeyDesc::ERowOperation::Update,
- keyTypes, TVector<TKeyDesc::TColumnOp>());
- Info.Keys.emplace_back(TValidatedKey(std::move(desc), /* isWrite */ true));
- ++Info.WritesCount;
- if (!range.Point) {
- ++Info.DynKeysCount;
- }
- Info.Loaded = true;
-}
-
+void TEngineBay::AddWriteRange(const TTableId& tableId, const TTableRange& range,
+ const TVector<NScheme::TTypeId>& keyTypes)
+{
+ auto desc = MakeHolder<TKeyDesc>(tableId, range, TKeyDesc::ERowOperation::Update,
+ keyTypes, TVector<TKeyDesc::TColumnOp>());
+ Info.Keys.emplace_back(TValidatedKey(std::move(desc), /* isWrite */ true));
+ ++Info.WritesCount;
+ if (!range.Point) {
+ ++Info.DynKeysCount;
+ }
+ Info.Loaded = true;
+}
+
TEngineBay::TSizes TEngineBay::CalcSizes(bool needsTotalKeysSize) const {
Y_VERIFY(EngineHost);
@@ -686,25 +686,25 @@ void TEngineBay::SetLockTxId(ui64 lockTxId) {
}
}
-NKqp::TKqpTasksRunner& TEngineBay::GetKqpTasksRunner(const NKikimrTxDataShard::TKqpTransaction& tx) {
+NKqp::TKqpTasksRunner& TEngineBay::GetKqpTasksRunner(const NKikimrTxDataShard::TKqpTransaction& tx) {
if (!KqpTasksRunner) {
- NYql::NDq::TDqTaskRunnerSettings settings;
-
- if (tx.HasRuntimeSettings() && tx.GetRuntimeSettings().HasStatsMode()) {
- auto statsMode = tx.GetRuntimeSettings().GetStatsMode();
- settings.CollectBasicStats = statsMode >= NYql::NDqProto::DQ_STATS_MODE_BASIC;
- settings.CollectProfileStats = statsMode >= NYql::NDqProto::DQ_STATS_MODE_PROFILE;
- } else {
- settings.CollectBasicStats = false;
- settings.CollectProfileStats = false;
- }
-
+ NYql::NDq::TDqTaskRunnerSettings settings;
+
+ if (tx.HasRuntimeSettings() && tx.GetRuntimeSettings().HasStatsMode()) {
+ auto statsMode = tx.GetRuntimeSettings().GetStatsMode();
+ settings.CollectBasicStats = statsMode >= NYql::NDqProto::DQ_STATS_MODE_BASIC;
+ settings.CollectProfileStats = statsMode >= NYql::NDqProto::DQ_STATS_MODE_PROFILE;
+ } else {
+ settings.CollectBasicStats = false;
+ settings.CollectProfileStats = false;
+ }
+
settings.OptLLVM = "OFF";
- settings.TerminateOnError = false;
- settings.AllowGeneratorsInUnboxedValues = false;
-
- KqpAlloc->SetLimit(10_MB);
- KqpTasksRunner = NKqp::CreateKqpTasksRunner(tx.GetTasks(), KqpExecCtx, settings, KqpLogFunc);
+ settings.TerminateOnError = false;
+ settings.AllowGeneratorsInUnboxedValues = false;
+
+ KqpAlloc->SetLimit(10_MB);
+ KqpTasksRunner = NKqp::CreateKqpTasksRunner(tx.GetTasks(), KqpExecCtx, settings, KqpLogFunc);
}
return *KqpTasksRunner;
diff --git a/ydb/core/tx/datashard/datashard__engine_host.h b/ydb/core/tx/datashard/datashard__engine_host.h
index 4690c66e54..8e3064c83d 100644
--- a/ydb/core/tx/datashard/datashard__engine_host.h
+++ b/ydb/core/tx/datashard/datashard__engine_host.h
@@ -40,8 +40,8 @@ public:
TEngineBay(TDataShard * self, TTransactionContext& txc, const TActorContext& ctx,
std::pair<ui64, ui64> stepTxId);
- virtual ~TEngineBay();
-
+ virtual ~TEngineBay();
+
const NMiniKQL::IEngineFlat * GetEngine() const { return Engine.Get(); }
NMiniKQL::IEngineFlat * GetEngine();
void SetLockTxId(ui64 lockTxId);
@@ -60,28 +60,28 @@ public:
return Engine->ValidateKeys(Info);
}
- void AddReadRange(const TTableId& tableId, const TVector<ui32>& columns, const TTableRange& range,
- const TVector<NScheme::TTypeId>& keyTypes, ui64 itemsLimit = 0, bool reverse = false);
+ void AddReadRange(const TTableId& tableId, const TVector<ui32>& columns, const TTableRange& range,
+ const TVector<NScheme::TTypeId>& keyTypes, ui64 itemsLimit = 0, bool reverse = false);
+
+ void AddWriteRange(const TTableId& tableId, const TTableRange& range, const TVector<NScheme::TTypeId>& keyTypes);
+
+ void MarkTxLoaded() {
+ Info.Loaded = true;
+ }
- void AddWriteRange(const TTableId& tableId, const TTableRange& range, const TVector<NScheme::TTypeId>& keyTypes);
-
- void MarkTxLoaded() {
- Info.Loaded = true;
- }
-
/// @note it expects TValidationInfo keys are materialized outsize of engine's allocs
void DestroyEngine() {
ComputeCtx->Clear();
- if (KqpTasksRunner) {
- KqpTasksRunner.Reset();
- {
- auto guard = TGuard(*KqpAlloc);
- KqpTypeEnv.Reset();
- }
- KqpAlloc.Reset();
- }
- KqpExecCtx = {};
-
+ if (KqpTasksRunner) {
+ KqpTasksRunner.Reset();
+ {
+ auto guard = TGuard(*KqpAlloc);
+ KqpTypeEnv.Reset();
+ }
+ KqpAlloc.Reset();
+ }
+ KqpExecCtx = {};
+
Engine.Reset();
EngineHost.Reset();
TraceMessage.clear();
@@ -99,8 +99,8 @@ public:
void ResetCounters() { EngineHostCounters = TEngineHostCounters(); }
const TEngineHostCounters& GetCounters() const { return EngineHostCounters; }
- NKqp::TKqpTasksRunner& GetKqpTasksRunner(const NKikimrTxDataShard::TKqpTransaction& tx);
- NMiniKQL::TKqpDatashardComputeContext& GetKqpComputeCtx();
+ NKqp::TKqpTasksRunner& GetKqpTasksRunner(const NKikimrTxDataShard::TKqpTransaction& tx);
+ NMiniKQL::TKqpDatashardComputeContext& GetKqpComputeCtx();
private:
std::pair<ui64, ui64> StepTxId;
@@ -114,10 +114,10 @@ private:
NYql::NDq::TLogFunc KqpLogFunc;
THolder<NUdf::IApplyContext> KqpApplyCtx;
THolder<NMiniKQL::TKqpDatashardComputeContext> ComputeCtx;
- THolder<NMiniKQL::TScopedAlloc> KqpAlloc;
- THolder<NMiniKQL::TTypeEnvironment> KqpTypeEnv;
- NYql::NDq::TDqTaskRunnerContext KqpExecCtx;
- TIntrusivePtr<NKqp::TKqpTasksRunner> KqpTasksRunner;
+ THolder<NMiniKQL::TScopedAlloc> KqpAlloc;
+ THolder<NMiniKQL::TTypeEnvironment> KqpTypeEnv;
+ NYql::NDq::TDqTaskRunnerContext KqpExecCtx;
+ TIntrusivePtr<NKqp::TKqpTasksRunner> KqpTasksRunner;
};
}}
diff --git a/ydb/core/tx/datashard/datashard__kqp_scan.cpp b/ydb/core/tx/datashard/datashard__kqp_scan.cpp
index 6d4971c1a9..b6b6b7e337 100644
--- a/ydb/core/tx/datashard/datashard__kqp_scan.cpp
+++ b/ydb/core/tx/datashard/datashard__kqp_scan.cpp
@@ -37,21 +37,21 @@ public:
TKqpScan(const TActorId& computeActorId, const TActorId& datashardActorId, ui32 scanId,
NDataShard::TUserTable::TCPtr tableInfo, const TSmallVec<TSerializedTableRange>&& tableRanges,
const TSmallVec<NTable::TTag>&& columnTags, const TSmallVec<bool>&& skipNullKeys,
- const NYql::NDqProto::EDqStatsMode& statsMode, ui64 timeoutMs, ui32 generation,
+ const NYql::NDqProto::EDqStatsMode& statsMode, ui64 timeoutMs, ui32 generation,
NKikimrTxDataShard::EScanDataFormat dataFormat)
: TActor(&TKqpScan::StateScan)
, ComputeActorId(computeActorId)
, DatashardActorId(datashardActorId)
, ScanId(scanId)
, TableInfo(tableInfo)
- , TablePath(TableInfo->Path)
+ , TablePath(TableInfo->Path)
, TableRanges(std::move(tableRanges))
, CurrentRange(0)
, Tags(std::move(columnTags))
, SkipNullKeys(std::move(skipNullKeys))
, StatsMode(statsMode)
, Deadline(TInstant::Now() + (timeoutMs ? TDuration::MilliSeconds(timeoutMs) + SCAN_HARD_TIMEOUT_GAP : SCAN_HARD_TIMEOUT))
- , Generation(generation)
+ , Generation(generation)
, DataFormat(dataFormat)
, PeerFreeSpace(0)
, Sleep(true)
@@ -73,18 +73,18 @@ public:
YQL_ENSURE(started, "Failed to start BatchBuilder");
}
}
-
- for (auto& range : TableRanges) {
- LOG_TRACE_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "--> Scan range: "
- << DebugPrintRange(TableInfo->KeyColumnTypes, range.ToTableRange(), *AppData()->TypeRegistry));
- }
+
+ for (auto& range : TableRanges) {
+ LOG_TRACE_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "--> Scan range: "
+ << DebugPrintRange(TableInfo->KeyColumnTypes, range.ToTableRange(), *AppData()->TypeRegistry));
+ }
}
private:
STATEFN(StateScan) {
switch (ev->GetTypeRewrite()) {
hFunc(TEvKqpCompute::TEvScanDataAck, HandleScan);
- hFunc(TEvKqpCompute::TEvKillScanTablet, HandleScan);
+ hFunc(TEvKqpCompute::TEvKillScanTablet, HandleScan);
hFunc(TEvKqp::TEvAbortExecution, HandleScan);
hFunc(TEvents::TEvUndelivered, HandleScan);
hFunc(TEvents::TEvWakeup, HandleScan);
@@ -101,12 +101,12 @@ private:
}
LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Got ScanDataAck"
- << ", at: " << ScanActorId << ", scanId: " << ScanId << ", table: " << TablePath
- << ", gen: " << ev->Get()->Generation << ", tablet: " << DatashardActorId
+ << ", at: " << ScanActorId << ", scanId: " << ScanId << ", table: " << TablePath
+ << ", gen: " << ev->Get()->Generation << ", tablet: " << DatashardActorId
<< ", freeSpace: " << ev->Get()->FreeSpace << ", prevFreeSpace: " << PeerFreeSpace);
- YQL_ENSURE(ev->Get()->Generation == Generation, "expected: " << Generation << ", got: " << ev->Get()->Generation);
-
+ YQL_ENSURE(ev->Get()->Generation == Generation, "expected: " << Generation << ", got: " << ev->Get()->Generation);
+
if (!ComputeActorId) {
ComputeActorId = ev->Sender;
}
@@ -114,7 +114,7 @@ private:
PeerFreeSpace = ev->Get()->FreeSpace;
if (PeerFreeSpace > 0) {
if (Y_UNLIKELY(IsProfile())) {
- StartWaitTime = TInstant::Now();
+ StartWaitTime = TInstant::Now();
}
if (Sleep) {
Sleep = false;
@@ -124,11 +124,11 @@ private:
}
}
- void HandleScan(TEvKqpCompute::TEvKillScanTablet::TPtr&) {
- LOG_CRIT_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Kill self tablet " << DatashardActorId);
- Send(DatashardActorId, new TEvents::TEvPoison);
- }
-
+ void HandleScan(TEvKqpCompute::TEvKillScanTablet::TPtr&) {
+ LOG_CRIT_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Kill self tablet " << DatashardActorId);
+ Send(DatashardActorId, new TEvents::TEvPoison);
+ }
+
void HandleScan(TEvKqp::TEvAbortExecution::TPtr& ev) {
if (!Driver) {
LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Got AbortExecution while driver not set");
@@ -140,12 +140,12 @@ private:
auto prio = msg.GetStatusCode() == Ydb::StatusIds::SUCCESS ? NActors::NLog::PRI_DEBUG : NActors::NLog::PRI_WARN;
LOG_LOG_S(*TlsActivationContext, prio, NKikimrServices::TX_DATASHARD, "Got AbortExecution"
- << ", at: " << ScanActorId << ", tablet: " << DatashardActorId
+ << ", at: " << ScanActorId << ", tablet: " << DatashardActorId
<< ", scanId: " << ScanId << ", table: " << TablePath
<< ", code: " << Ydb::StatusIds_StatusCode_Name(msg.GetStatusCode())
<< ", reason: " << msg.GetMessage());
- AbortEvent = ev->Release();
+ AbortEvent = ev->Release();
Driver->Touch(EScan::Final);
}
@@ -156,12 +156,12 @@ private:
}
LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Undelivered event: " << ev->GetTypeRewrite()
- << ", at: " << ScanActorId << ", tablet: " << DatashardActorId
+ << ", at: " << ScanActorId << ", tablet: " << DatashardActorId
<< ", scanId: " << ScanId << ", table: " << TablePath);
switch (ev->GetTypeRewrite()) {
- case TEvKqpCompute::TEvScanInitActor::EventType:
- case TEvKqpCompute::TEvScanData::EventType:
+ case TEvKqpCompute::TEvScanInitActor::EventType:
+ case TEvKqpCompute::TEvScanData::EventType:
Driver->Touch(EScan::Final);
}
}
@@ -188,8 +188,8 @@ private:
ScanActorId = TActivationContext::AsActorContext().RegisterWithSameMailbox(this);
// propagate self actor id
- Send(ComputeActorId, new TEvKqpCompute::TEvScanInitActor(ScanId, ScanActorId, Generation),
- IEventHandle::FlagTrackDelivery);
+ Send(ComputeActorId, new TEvKqpCompute::TEvScanInitActor(ScanId, ScanActorId, Generation),
+ IEventHandle::FlagTrackDelivery);
Sleep = true;
@@ -202,12 +202,12 @@ private:
new IEventHandle(SelfId(), SelfId(), new TEvents::TEvWakeup));
if (Y_UNLIKELY(IsProfile())) {
- StartWaitTime = TInstant::Now();
+ StartWaitTime = TInstant::Now();
}
- LOG_INFO_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Start scan"
- << ", at: " << ScanActorId << ", tablet: " << DatashardActorId
- << ", scanId: " << ScanId << ", table: " << TablePath << ", gen: " << Generation
+ LOG_INFO_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Start scan"
+ << ", at: " << ScanActorId << ", tablet: " << DatashardActorId
+ << ", scanId: " << ScanId << ", table: " << TablePath << ", gen: " << Generation
<< ", deadline: " << Deadline);
return startConfig;
@@ -230,17 +230,17 @@ private:
int cmpTo;
cmpFrom = CompareBorders<false, false>(
range.From.GetCells(),
- TableInfo->Range.From.GetCells(),
+ TableInfo->Range.From.GetCells(),
range.FromInclusive,
- TableInfo->Range.FromInclusive,
- TableInfo->KeyColumnTypes);
+ TableInfo->Range.FromInclusive,
+ TableInfo->KeyColumnTypes);
cmpTo = CompareBorders<true, true>(
range.To.GetCells(),
- TableInfo->Range.To.GetCells(),
+ TableInfo->Range.To.GetCells(),
range.ToInclusive,
- TableInfo->Range.ToInclusive,
- TableInfo->KeyColumnTypes);
+ TableInfo->Range.ToInclusive,
+ TableInfo->KeyColumnTypes);
if (cmpFrom > 0) {
auto seek = range.FromInclusive ? NTable::ESeek::Lower : NTable::ESeek::Upper;
@@ -251,8 +251,8 @@ private:
if (cmpTo < 0) {
lead.Until(range.To.GetCells(), range.ToInclusive);
- }
-
+ }
+
return EScan::Feed;
}
@@ -322,7 +322,7 @@ private:
}
if (sent && PeerFreeSpace > 0 && Y_UNLIKELY(IsProfile())) {
- StartWaitTime = TInstant::Now();
+ StartWaitTime = TInstant::Now();
}
}
return EScan::Feed;
@@ -333,31 +333,31 @@ private:
auto prio = abort == EAbort::None ? NActors::NLog::PRI_DEBUG : NActors::NLog::PRI_ERROR;
LOG_LOG_S(*TlsActivationContext, prio, NKikimrServices::TX_DATASHARD, "Finish scan"
<< ", at: " << ScanActorId << ", scanId: " << ScanId
- << ", table: " << TablePath << ", reason: " << (int) abort
- << ", abortEvent: " << (AbortEvent ? AbortEvent->Record.ShortDebugString() : TString("<none>")));
-
- if (abort != EAbort::None || AbortEvent) {
- auto ev = MakeHolder<TEvKqpCompute::TEvScanError>(Generation);
-
- if (AbortEvent) {
- ev->Record.SetStatus(AbortEvent->Record.GetStatusCode());
- auto issue = NYql::YqlIssue({}, NYql::TIssuesIds::KIKIMR_OPERATION_ABORTED, TStringBuilder()
- << "Table " << TablePath << " scan failed, reason: " << AbortEvent->Record.GetMessage());
- IssueToMessage(issue, ev->Record.MutableIssues()->Add());
- } else {
- ev->Record.SetStatus(Ydb::StatusIds::ABORTED);
- auto issue = NYql::YqlIssue({}, NYql::TIssuesIds::KIKIMR_OPERATION_ABORTED, TStringBuilder()
- << "Table " << TablePath << " scan failed, reason: " << ToString((int) abort));
- IssueToMessage(issue, ev->Record.MutableIssues()->Add());
- }
-
- Send(ComputeActorId, ev.Release());
+ << ", table: " << TablePath << ", reason: " << (int) abort
+ << ", abortEvent: " << (AbortEvent ? AbortEvent->Record.ShortDebugString() : TString("<none>")));
+
+ if (abort != EAbort::None || AbortEvent) {
+ auto ev = MakeHolder<TEvKqpCompute::TEvScanError>(Generation);
+
+ if (AbortEvent) {
+ ev->Record.SetStatus(AbortEvent->Record.GetStatusCode());
+ auto issue = NYql::YqlIssue({}, NYql::TIssuesIds::KIKIMR_OPERATION_ABORTED, TStringBuilder()
+ << "Table " << TablePath << " scan failed, reason: " << AbortEvent->Record.GetMessage());
+ IssueToMessage(issue, ev->Record.MutableIssues()->Add());
+ } else {
+ ev->Record.SetStatus(Ydb::StatusIds::ABORTED);
+ auto issue = NYql::YqlIssue({}, NYql::TIssuesIds::KIKIMR_OPERATION_ABORTED, TStringBuilder()
+ << "Table " << TablePath << " scan failed, reason: " << ToString((int) abort));
+ IssueToMessage(issue, ev->Record.MutableIssues()->Add());
+ }
+
+ Send(ComputeActorId, ev.Release());
} else {
if (Result) {
- // TODO:
- // Result->CpuTime = CpuTime;
+ // TODO:
+ // Result->CpuTime = CpuTime;
} else {
- Result = MakeHolder<TEvKqpCompute::TEvScanData>(ScanId, Generation);
+ Result = MakeHolder<TEvKqpCompute::TEvScanData>(ScanId, Generation);
}
auto send = SendResult(Result->PageFault, true);
Y_VERIFY_DEBUG(send);
@@ -452,25 +452,25 @@ private:
PeerFreeSpace -= sendBytes;
}
- if (sendBytes >= 48_MB) {
- LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Query size limit exceeded.");
- if (finish) {
- bool sent = Send(ComputeActorId, new TEvKqp::TEvAbortExecution(Ydb::StatusIds::PRECONDITION_FAILED,
- "Query size limit exceeded."));
- Y_VERIFY(sent);
-
- ReportDatashardStats();
- return true;
- } else {
- bool sent = Send(SelfId(), new TEvKqp::TEvAbortExecution(Ydb::StatusIds::PRECONDITION_FAILED,
- "Query size limit exceeded."));
- Y_VERIFY(sent);
-
- ReportDatashardStats();
- return false;
- }
- }
-
+ if (sendBytes >= 48_MB) {
+ LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Query size limit exceeded.");
+ if (finish) {
+ bool sent = Send(ComputeActorId, new TEvKqp::TEvAbortExecution(Ydb::StatusIds::PRECONDITION_FAILED,
+ "Query size limit exceeded."));
+ Y_VERIFY(sent);
+
+ ReportDatashardStats();
+ return true;
+ } else {
+ bool sent = Send(SelfId(), new TEvKqp::TEvAbortExecution(Ydb::StatusIds::PRECONDITION_FAILED,
+ "Query size limit exceeded."));
+ Y_VERIFY(sent);
+
+ ReportDatashardStats();
+ return false;
+ }
+ }
+
Send(ComputeActorId, Result.Release(), IEventHandle::FlagTrackDelivery);
ReportDatashardStats();
@@ -496,7 +496,7 @@ private:
}
bool IsProfile() const {
- return StatsMode >= NYql::NDqProto::DQ_STATS_MODE_PROFILE;
+ return StatsMode >= NYql::NDqProto::DQ_STATS_MODE_PROFILE;
}
private:
@@ -510,9 +510,9 @@ private:
const TSmallVec<NTable::TTag> Tags;
TSmallVec<NScheme::TTypeId> Types;
const TSmallVec<bool> SkipNullKeys;
- const NYql::NDqProto::EDqStatsMode StatsMode;
+ const NYql::NDqProto::EDqStatsMode StatsMode;
const TInstant Deadline;
- const ui32 Generation;
+ const ui32 Generation;
const NKikimrTxDataShard::EScanDataFormat DataFormat;
ui64 PeerFreeSpace = 0;
bool Sleep;
@@ -521,7 +521,7 @@ private:
IDriver* Driver = nullptr;
TActorId ScanActorId;
TActorId TimeoutActorId;
- TAutoPtr<TEvKqp::TEvAbortExecution> AbortEvent;
+ TAutoPtr<TEvKqp::TEvAbortExecution> AbortEvent;
THolder<NArrow::TArrowBatchBuilder> BatchBuilder;
THolder<TEvKqpCompute::TEvScanData> Result;
@@ -538,15 +538,15 @@ private:
void TDataShard::Handle(TEvDataShard::TEvKqpScan::TPtr& ev, const TActorContext&) {
auto& request = ev->Get()->Record;
auto scanComputeActor = ev->Sender;
- auto generation = request.GetGeneration();
+ auto generation = request.GetGeneration();
auto infoIt = TableInfos.find(request.GetLocalPathId());
- auto reportError = [this, scanComputeActor, generation] (const TString& table, const TString& detailedReason) {
- auto ev = MakeHolder<TEvKqpCompute::TEvScanError>(generation);
+ auto reportError = [this, scanComputeActor, generation] (const TString& table, const TString& detailedReason) {
+ auto ev = MakeHolder<TEvKqpCompute::TEvScanError>(generation);
ev->Record.SetStatus(Ydb::StatusIds::ABORTED);
- auto issue = NYql::YqlIssue({}, NYql::TIssuesIds::KIKIMR_SCHEME_MISMATCH, TStringBuilder() <<
- "Table '" << table << "' scheme changed.");
+ auto issue = NYql::YqlIssue({}, NYql::TIssuesIds::KIKIMR_SCHEME_MISMATCH, TStringBuilder() <<
+ "Table '" << table << "' scheme changed.");
IssueToMessage(issue, ev->Record.MutableIssues()->Add());
Send(scanComputeActor, ev.Release(), IEventHandle::FlagTrackDelivery);
LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, detailedReason);
@@ -559,12 +559,12 @@ void TDataShard::Handle(TEvDataShard::TEvKqpScan::TPtr& ev, const TActorContext&
return;
}
- auto tableInfo = infoIt->second; // copy table info ptr here
- auto& tableColumns = tableInfo->Columns;
+ auto tableInfo = infoIt->second; // copy table info ptr here
+ auto& tableColumns = tableInfo->Columns;
Y_VERIFY(request.GetColumnTags().size() == request.GetColumnTypes().size());
- if (tableInfo->GetTableSchemaVersion() != 0 &&
- request.GetSchemaVersion() != tableInfo->GetTableSchemaVersion())
+ if (tableInfo->GetTableSchemaVersion() != 0 &&
+ request.GetSchemaVersion() != tableInfo->GetTableSchemaVersion())
{
reportError(request.GetTablePath(), TStringBuilder() << "TxId: " << request.GetTxId() << "."
<< " Table '" << request.GetTablePath() << "'"
@@ -590,28 +590,28 @@ void TDataShard::Handle(TEvDataShard::TEvKqpScan::TPtr& ev, const TActorContext&
}
}
- if (request.HasOlapProgram()) {
- auto msg = TStringBuilder() << "TxId: " << request.GetTxId() << "."
- << " Unexpected process program in datashard scan at " << TabletID();
- LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, msg);
-
- auto ev = MakeHolder<TEvKqpCompute::TEvScanError>(generation);
- ev->Record.SetStatus(Ydb::StatusIds::INTERNAL_ERROR);
- auto issue = NYql::YqlIssue({}, NYql::TIssuesIds::DEFAULT_ERROR, msg);
- IssueToMessage(issue, ev->Record.MutableIssues()->Add());
- Send(scanComputeActor, ev.Release(), IEventHandle::FlagTrackDelivery);
- return;
+ if (request.HasOlapProgram()) {
+ auto msg = TStringBuilder() << "TxId: " << request.GetTxId() << "."
+ << " Unexpected process program in datashard scan at " << TabletID();
+ LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, msg);
+
+ auto ev = MakeHolder<TEvKqpCompute::TEvScanError>(generation);
+ ev->Record.SetStatus(Ydb::StatusIds::INTERNAL_ERROR);
+ auto issue = NYql::YqlIssue({}, NYql::TIssuesIds::DEFAULT_ERROR, msg);
+ IssueToMessage(issue, ev->Record.MutableIssues()->Add());
+ Send(scanComputeActor, ev.Release(), IEventHandle::FlagTrackDelivery);
+ return;
}
auto& snapshot = request.GetSnapshot();
- auto snapshotKey = TSnapshotKey(PathOwnerId, request.GetLocalPathId(), snapshot.GetStep(), snapshot.GetTxId());
- if (!SnapshotManager.FindAvailable(snapshotKey)) {
- reportError(request.GetTablePath(), TStringBuilder() << "TxId: " << request.GetTxId() << "."
- << " Snapshot is not valid, tabletId: " << TabletID() << ", step: " << snapshot.GetStep());
- return;
- }
-
+ auto snapshotKey = TSnapshotKey(PathOwnerId, request.GetLocalPathId(), snapshot.GetStep(), snapshot.GetTxId());
+ if (!SnapshotManager.FindAvailable(snapshotKey)) {
+ reportError(request.GetTablePath(), TStringBuilder() << "TxId: " << request.GetTxId() << "."
+ << " Snapshot is not valid, tabletId: " << TabletID() << ", step: " << snapshot.GetStep());
+ return;
+ }
+
Pipeline.StartStreamingTx(snapshot.GetTxId(), 1);
TSmallVec<TSerializedTableRange> ranges;
@@ -630,7 +630,7 @@ void TDataShard::Handle(TEvDataShard::TEvKqpScan::TPtr& ev, const TActorContext&
std::move(TSmallVec<NTable::TTag>(request.GetColumnTags().begin(), request.GetColumnTags().end())),
std::move(TSmallVec<bool>(request.GetSkipNullKeys().begin(), request.GetSkipNullKeys().end())),
request.GetStatsMode(),
- request.GetTimeoutMs(),
+ request.GetTimeoutMs(),
generation,
request.GetDataFormat()
);
@@ -641,7 +641,7 @@ void TDataShard::Handle(TEvDataShard::TEvKqpScan::TPtr& ev, const TActorContext&
.SetReadAhead(READAHEAD_LO, READAHEAD_HI)
.SetSnapshotRowVersion(TRowVersion(snapshot.GetStep(), snapshot.GetTxId()));
- Executor()->QueueScan(tableInfo->LocalTid, tableScan, snapshot.GetTxId(), scanOptions);
+ Executor()->QueueScan(tableInfo->LocalTid, tableScan, snapshot.GetTxId(), scanOptions);
}
}
diff --git a/ydb/core/tx/datashard/datashard_active_transaction.cpp b/ydb/core/tx/datashard/datashard_active_transaction.cpp
index 623987c219..0226bd42e2 100644
--- a/ydb/core/tx/datashard/datashard_active_transaction.cpp
+++ b/ydb/core/tx/datashard/datashard_active_transaction.cpp
@@ -55,103 +55,103 @@ TValidatedDataTx::TValidatedDataTx(TDataShard *self,
if (self->TableInfos.contains(tx.GetTableId().GetTableId())) {
auto* info = self->TableInfos[tx.GetTableId().GetTableId()].Get();
Y_VERIFY(info, "Unexpected missing table info");
- TSerializedTableRange range(tx.GetRange());
+ TSerializedTableRange range(tx.GetRange());
EngineBay.AddReadRange(TTableId(tx.GetTableId().GetOwnerId(),
tx.GetTableId().GetTableId()),
- {}, range.ToTableRange(), info->KeyColumnTypes);
+ {}, range.ToTableRange(), info->KeyColumnTypes);
} else {
ErrCode = NKikimrTxDataShard::TError::SCHEME_ERROR;
ErrStr = "Trying to read from table that doesn't exist";
}
} else if (IsKqpTx()) {
- if (Y_UNLIKELY(!IsKqpDataTx())) {
- LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, "Unexpected KQP transaction type, shard: " << TabletId()
- << ", txid: " << StepTxId_.TxId << ", tx: " << Tx.DebugString());
- ErrCode = NKikimrTxDataShard::TError::BAD_TX_KIND;
- ErrStr = TStringBuilder() << "Unexpected KQP transaction type: "
- << NKikimrTxDataShard::EKqpTransactionType_Name(Tx.GetKqpTransaction().GetType()) << ".";
- return;
- }
-
- auto& typeRegistry = *AppData()->TypeRegistry;
- auto& computeCtx = EngineBay.GetKqpComputeCtx();
-
- try {
- bool hasPersistentChannels = false;
- if (!KqpValidateTransaction(GetKqpTransaction(), Immediate(), StepTxId_.TxId, ctx, hasPersistentChannels)) {
- LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, "KQP transaction validation failed, datashard: "
- << TabletId() << ", txid: " << StepTxId_.TxId);
- ErrCode = NKikimrTxDataShard::TError::PROGRAM_ERROR;
- ErrStr = "Transaction validation failed.";
- return;
- }
- computeCtx.SetHasPersistentChannels(hasPersistentChannels);
-
- for (auto& task : GetKqpTransaction().GetTasks()) {
- NKikimrTxDataShard::TKqpTransaction::TDataTaskMeta meta;
- if (!task.GetMeta().UnpackTo(&meta)) {
+ if (Y_UNLIKELY(!IsKqpDataTx())) {
+ LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, "Unexpected KQP transaction type, shard: " << TabletId()
+ << ", txid: " << StepTxId_.TxId << ", tx: " << Tx.DebugString());
+ ErrCode = NKikimrTxDataShard::TError::BAD_TX_KIND;
+ ErrStr = TStringBuilder() << "Unexpected KQP transaction type: "
+ << NKikimrTxDataShard::EKqpTransactionType_Name(Tx.GetKqpTransaction().GetType()) << ".";
+ return;
+ }
+
+ auto& typeRegistry = *AppData()->TypeRegistry;
+ auto& computeCtx = EngineBay.GetKqpComputeCtx();
+
+ try {
+ bool hasPersistentChannels = false;
+ if (!KqpValidateTransaction(GetKqpTransaction(), Immediate(), StepTxId_.TxId, ctx, hasPersistentChannels)) {
+ LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, "KQP transaction validation failed, datashard: "
+ << TabletId() << ", txid: " << StepTxId_.TxId);
+ ErrCode = NKikimrTxDataShard::TError::PROGRAM_ERROR;
+ ErrStr = "Transaction validation failed.";
+ return;
+ }
+ computeCtx.SetHasPersistentChannels(hasPersistentChannels);
+
+ for (auto& task : GetKqpTransaction().GetTasks()) {
+ NKikimrTxDataShard::TKqpTransaction::TDataTaskMeta meta;
+ if (!task.GetMeta().UnpackTo(&meta)) {
LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, "KQP transaction validation failed"
<< ", datashard: " << TabletId()
- << ", txid: " << StepTxId_.TxId
- << ", failed to load task meta: " << task.GetMeta().value());
+ << ", txid: " << StepTxId_.TxId
+ << ", failed to load task meta: " << task.GetMeta().value());
ErrCode = NKikimrTxDataShard::TError::PROGRAM_ERROR;
- ErrStr = "Transaction validation failed: invalid task metadata.";
+ ErrStr = "Transaction validation failed: invalid task metadata.";
return;
}
- LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, "TxId: " << StepTxId_.TxId << ", shard " << TabletId()
- << ", task: " << task.GetId() << ", meta: " << meta.ShortDebugString());
-
- auto& tableMeta = meta.GetTable();
-
- auto tableInfoPtr = self->TableInfos.FindPtr(tableMeta.GetTableId().GetTableId());
- if (!tableInfoPtr) {
- ErrCode = NKikimrTxDataShard::TError::SCHEME_ERROR;
- ErrStr = TStringBuilder() << "Table '" << tableMeta.GetTablePath() << "' doesn't exist";
- return;
- }
- auto tableInfo = tableInfoPtr->Get();
- YQL_ENSURE(tableInfo);
-
- if (tableInfo->GetTableSchemaVersion() != 0 &&
- tableMeta.GetSchemaVersion() != tableInfo->GetTableSchemaVersion())
- {
- ErrCode = NKikimrTxDataShard::TError::SCHEME_CHANGED;
- ErrStr = TStringBuilder() << "Table '" << tableMeta.GetTablePath() << "' scheme changed.";
- return;
- }
-
- for (auto& read : meta.GetReads()) {
- for (auto& column : read.GetColumns()) {
- if (tableInfo->Columns.contains(column.GetId()) || IsSystemColumn(column.GetName())) {
- // ok
- } else {
- ErrCode = NKikimrTxDataShard::TError::SCHEME_CHANGED;
- ErrStr = TStringBuilder() << "Table '" << tableMeta.GetTablePath() << "' scheme changed:"
- << " column '" << column.GetName() << "' not found.";
- return;
- }
- }
- }
-
- KqpSetTxKeys(TabletId(), task.GetId(), tableInfo, meta, typeRegistry, ctx, EngineBay);
-
- for (auto& output : task.GetOutputs()) {
- for (auto& channel : output.GetChannels()) {
- computeCtx.SetTaskOutputChannel(task.GetId(), channel.GetId(),
- ActorIdFromProto(channel.GetDstEndpoint().GetActorId()));
- }
- }
- }
-
- if (Tx.HasPerShardKeysSizeLimitBytes()) {
- PerShardKeysSizeLimitBytes_ = Tx.GetPerShardKeysSizeLimitBytes();
- }
-
- IsReadOnly = IsReadOnly && Tx.GetReadOnly();
-
- KqpSetTxLocksKeys(GetKqpTransaction().GetLocks(), self->SysLocksTable(), EngineBay);
- EngineBay.MarkTxLoaded();
+ LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, "TxId: " << StepTxId_.TxId << ", shard " << TabletId()
+ << ", task: " << task.GetId() << ", meta: " << meta.ShortDebugString());
+
+ auto& tableMeta = meta.GetTable();
+
+ auto tableInfoPtr = self->TableInfos.FindPtr(tableMeta.GetTableId().GetTableId());
+ if (!tableInfoPtr) {
+ ErrCode = NKikimrTxDataShard::TError::SCHEME_ERROR;
+ ErrStr = TStringBuilder() << "Table '" << tableMeta.GetTablePath() << "' doesn't exist";
+ return;
+ }
+ auto tableInfo = tableInfoPtr->Get();
+ YQL_ENSURE(tableInfo);
+
+ if (tableInfo->GetTableSchemaVersion() != 0 &&
+ tableMeta.GetSchemaVersion() != tableInfo->GetTableSchemaVersion())
+ {
+ ErrCode = NKikimrTxDataShard::TError::SCHEME_CHANGED;
+ ErrStr = TStringBuilder() << "Table '" << tableMeta.GetTablePath() << "' scheme changed.";
+ return;
+ }
+
+ for (auto& read : meta.GetReads()) {
+ for (auto& column : read.GetColumns()) {
+ if (tableInfo->Columns.contains(column.GetId()) || IsSystemColumn(column.GetName())) {
+ // ok
+ } else {
+ ErrCode = NKikimrTxDataShard::TError::SCHEME_CHANGED;
+ ErrStr = TStringBuilder() << "Table '" << tableMeta.GetTablePath() << "' scheme changed:"
+ << " column '" << column.GetName() << "' not found.";
+ return;
+ }
+ }
+ }
+
+ KqpSetTxKeys(TabletId(), task.GetId(), tableInfo, meta, typeRegistry, ctx, EngineBay);
+
+ for (auto& output : task.GetOutputs()) {
+ for (auto& channel : output.GetChannels()) {
+ computeCtx.SetTaskOutputChannel(task.GetId(), channel.GetId(),
+ ActorIdFromProto(channel.GetDstEndpoint().GetActorId()));
+ }
+ }
+ }
+
+ if (Tx.HasPerShardKeysSizeLimitBytes()) {
+ PerShardKeysSizeLimitBytes_ = Tx.GetPerShardKeysSizeLimitBytes();
+ }
+
+ IsReadOnly = IsReadOnly && Tx.GetReadOnly();
+
+ KqpSetTxLocksKeys(GetKqpTransaction().GetLocks(), self->SysLocksTable(), EngineBay);
+ EngineBay.MarkTxLoaded();
auto& tasksRunner = GetKqpTasksRunner(); // prepare tasks runner, can throw TMemoryLimitExceededException
@@ -159,17 +159,17 @@ TValidatedDataTx::TValidatedDataTx(TDataShard *self,
auto execCtx = DefaultKqpExecutionContext();
tasksRunner.Prepare(DefaultKqpDataReqMemoryLimits(), *execCtx);
- } catch (const TMemoryLimitExceededException&) {
- LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, "Not enough memory to create tasks runner, datashard: "
- << TabletId() << ", txid: " << StepTxId_.TxId);
- ErrCode = NKikimrTxDataShard::TError::PROGRAM_ERROR;
- ErrStr = TStringBuilder() << "Transaction validation failed: not enough memory.";
- return;
- } catch (const yexception& e) {
- LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, "Exception while validating KQP transaction, datashard: "
- << TabletId() << ", txid: " << StepTxId_.TxId << ", error: " << e.what());
+ } catch (const TMemoryLimitExceededException&) {
+ LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, "Not enough memory to create tasks runner, datashard: "
+ << TabletId() << ", txid: " << StepTxId_.TxId);
+ ErrCode = NKikimrTxDataShard::TError::PROGRAM_ERROR;
+ ErrStr = TStringBuilder() << "Transaction validation failed: not enough memory.";
+ return;
+ } catch (const yexception& e) {
+ LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, "Exception while validating KQP transaction, datashard: "
+ << TabletId() << ", txid: " << StepTxId_.TxId << ", error: " << e.what());
ErrCode = NKikimrTxDataShard::TError::PROGRAM_ERROR;
- ErrStr = TStringBuilder() << "Transaction validation failed: " << e.what() << ".";
+ ErrStr = TStringBuilder() << "Transaction validation failed: " << e.what() << ".";
return;
}
} else {
@@ -199,9 +199,9 @@ TValidatedDataTx::~TValidatedDataTx() {
NActors::NMemory::TLabel<MemoryLabelValidatedDataTx>::Sub(TxSize);
}
-const google::protobuf::RepeatedPtrField<NYql::NDqProto::TDqTask>& TValidatedDataTx::GetKqpTasks() const {
+const google::protobuf::RepeatedPtrField<NYql::NDqProto::TDqTask>& TValidatedDataTx::GetKqpTasks() const {
Y_VERIFY(IsKqpTx());
- return Tx.GetKqpTransaction().GetTasks();
+ return Tx.GetKqpTransaction().GetTasks();
}
ui32 TValidatedDataTx::ExtractKeys(bool allowErrors)
@@ -660,7 +660,7 @@ void TActiveTransaction::FinalizeDataTxPlan()
{
Y_VERIFY(IsDataTx());
Y_VERIFY(!IsImmediate());
- Y_VERIFY(!IsKqpScanTransaction());
+ Y_VERIFY(!IsKqpScanTransaction());
TVector<EExecutionUnitKind> plan;
@@ -709,7 +709,7 @@ public:
// Restore transaction type flags
if (dataTx->IsKqpDataTx() && !tx->IsKqpDataTransaction())
tx->SetKqpDataTransactionFlag();
- Y_VERIFY_S(!dataTx->IsKqpScanTx(), "unexpected kqp scan tx");
+ Y_VERIFY_S(!dataTx->IsKqpScanTx(), "unexpected kqp scan tx");
}
tx->FinalizeDataTxPlan();
@@ -732,7 +732,7 @@ THolder<TExecutionUnit> CreateFinalizeDataTxPlanUnit(TDataShard &dataShard, TPip
void TActiveTransaction::BuildExecutionPlan(bool loaded)
{
Y_VERIFY(GetExecutionPlan().empty());
- Y_VERIFY(!IsKqpScanTransaction());
+ Y_VERIFY(!IsKqpScanTransaction());
TVector<EExecutionUnitKind> plan;
if (IsDataTx()) {
diff --git a/ydb/core/tx/datashard/datashard_active_transaction.h b/ydb/core/tx/datashard/datashard_active_transaction.h
index f11117dea7..43abd525e1 100644
--- a/ydb/core/tx/datashard/datashard_active_transaction.h
+++ b/ydb/core/tx/datashard/datashard_active_transaction.h
@@ -196,9 +196,9 @@ public:
}
const NKikimrTxDataShard::TKqpTransaction &GetKqpTransaction() const { return Tx.GetKqpTransaction(); }
- const google::protobuf::RepeatedPtrField<NYql::NDqProto::TDqTask>& GetKqpTasks() const;
- NKqp::TKqpTasksRunner& GetKqpTasksRunner() { Y_VERIFY(IsKqpDataTx()); return EngineBay.GetKqpTasksRunner(Tx.GetKqpTransaction()); }
- NMiniKQL::TKqpDatashardComputeContext& GetKqpComputeCtx() { Y_VERIFY(IsKqpDataTx()); return EngineBay.GetKqpComputeCtx(); }
+ const google::protobuf::RepeatedPtrField<NYql::NDqProto::TDqTask>& GetKqpTasks() const;
+ NKqp::TKqpTasksRunner& GetKqpTasksRunner() { Y_VERIFY(IsKqpDataTx()); return EngineBay.GetKqpTasksRunner(Tx.GetKqpTransaction()); }
+ NMiniKQL::TKqpDatashardComputeContext& GetKqpComputeCtx() { Y_VERIFY(IsKqpDataTx()); return EngineBay.GetKqpComputeCtx(); }
bool HasStreamResponse() const { return Tx.GetStreamResponse(); }
TActorId GetSink() const { return ActorIdFromProto(Tx.GetSink()); }
@@ -419,13 +419,13 @@ public:
// out-of-order stuff
ui32 ExtractKeys() {
- if (DataTx && (DataTx->ProgramSize() || DataTx->IsKqpDataTx()))
+ if (DataTx && (DataTx->ProgramSize() || DataTx->IsKqpDataTx()))
return DataTx->ExtractKeys(false);
return 0;
}
bool ReValidateKeys() {
- if (DataTx && (DataTx->ProgramSize() || DataTx->IsKqpDataTx()))
+ if (DataTx && (DataTx->ProgramSize() || DataTx->IsKqpDataTx()))
return DataTx->ReValidateKeys();
return true;
}
diff --git a/ydb/core/tx/datashard/datashard_impl.h b/ydb/core/tx/datashard/datashard_impl.h
index afb324a1d3..11313474f8 100644
--- a/ydb/core/tx/datashard/datashard_impl.h
+++ b/ydb/core/tx/datashard/datashard_impl.h
@@ -238,7 +238,7 @@ class TDataShard
friend class TActiveTransaction;
friend class TValidatedDataTx;
friend class TEngineBay;
- friend class NMiniKQL::TKqpScanComputeContext;
+ friend class NMiniKQL::TKqpScanComputeContext;
friend class TSnapshotManager;
friend class TSchemaSnapshotManager;
friend class TReplicationSourceOffsetsClient;
@@ -1102,15 +1102,15 @@ public:
}
static NDataShard::ECumulativeCounters NotEnoughMemoryCounter(ui64 count) {
- if (count == 1)
- return COUNTER_TX_NOT_ENOUGH_MEMORY_1;
- if (count == 2)
- return COUNTER_TX_NOT_ENOUGH_MEMORY_2;
- if (count == 3)
- return COUNTER_TX_NOT_ENOUGH_MEMORY_3;
- return COUNTER_TX_NOT_ENOUGH_MEMORY_4;
- }
-
+ if (count == 1)
+ return COUNTER_TX_NOT_ENOUGH_MEMORY_1;
+ if (count == 2)
+ return COUNTER_TX_NOT_ENOUGH_MEMORY_2;
+ if (count == 3)
+ return COUNTER_TX_NOT_ENOUGH_MEMORY_3;
+ return COUNTER_TX_NOT_ENOUGH_MEMORY_4;
+ }
+
bool IsStateActive() const {
return State == TShardState::Ready ||
State == TShardState::Readonly ||
diff --git a/ydb/core/tx/datashard/datashard_kqp.cpp b/ydb/core/tx/datashard/datashard_kqp.cpp
index 5a4c4c4a77..a262c98de5 100644
--- a/ydb/core/tx/datashard/datashard_kqp.cpp
+++ b/ydb/core/tx/datashard/datashard_kqp.cpp
@@ -1,5 +1,5 @@
#include "datashard_kqp.h"
-#include "datashard_impl.h"
+#include "datashard_impl.h"
#include <ydb/core/kqp/kqp.h>
#include <ydb/core/kqp/runtime/kqp_tasks_runner.h>
@@ -11,8 +11,8 @@
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h>
#include <ydb/library/yql/dq/runtime/dq_transport.h>
-#include <util/generic/size_literals.h>
-
+#include <util/generic/size_literals.h>
+
namespace NKikimr {
namespace NDataShard {
@@ -20,15 +20,15 @@ namespace {
const ui32 MaxDatashardReplySize = 48 * 1024 * 1024; // 48 MB
-using namespace NYql;
-
-bool KqpValidateTask(const NYql::NDqProto::TDqTask& task, bool isImmediate, ui64 txId, const TActorContext& ctx,
- bool& hasPersistentChannels)
+using namespace NYql;
+
+bool KqpValidateTask(const NYql::NDqProto::TDqTask& task, bool isImmediate, ui64 txId, const TActorContext& ctx,
+ bool& hasPersistentChannels)
{
for (auto& input : task.GetInputs()) {
for (auto& channel : input.GetChannels()) {
if (channel.GetIsPersistent()) {
- hasPersistentChannels = true;
+ hasPersistentChannels = true;
if (isImmediate) {
LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, "KQP validate, txId: " << txId
<< ", immediate KQP transaction cannot have persistent input channels"
@@ -39,7 +39,7 @@ bool KqpValidateTask(const NYql::NDqProto::TDqTask& task, bool isImmediate, ui64
if (!channel.GetSrcEndpoint().HasTabletId()) {
LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, "KQP validate, txId: " << txId
- << ", persistent input channel without src tablet id"
+ << ", persistent input channel without src tablet id"
<< ", task: " << task.GetId()
<< ", channelId: " << channel.GetId());
return false;
@@ -51,7 +51,7 @@ bool KqpValidateTask(const NYql::NDqProto::TDqTask& task, bool isImmediate, ui64
for (auto& output : task.GetOutputs()) {
for (auto& channel : output.GetChannels()) {
if (channel.GetIsPersistent()) {
- hasPersistentChannels = true;
+ hasPersistentChannels = true;
if (isImmediate) {
LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, "KQP validate, txId: " << txId
<< ", immediate KQP transaction cannot have persistent output channels"
@@ -62,7 +62,7 @@ bool KqpValidateTask(const NYql::NDqProto::TDqTask& task, bool isImmediate, ui64
if (!channel.GetDstEndpoint().HasTabletId()) {
LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, "KQP validate, txId: " << txId
- << ", persistent output channel without dst tablet id"
+ << ", persistent output channel without dst tablet id"
<< ", task: " << task.GetId()
<< ", channelId: " << channel.GetId());
return false;
@@ -74,36 +74,36 @@ bool KqpValidateTask(const NYql::NDqProto::TDqTask& task, bool isImmediate, ui64
return true;
}
-NUdf::EFetchStatus FetchAllOutput(NDq::IDqOutputChannel* channel, NDqProto::TData& buffer) {
- auto result = channel->PopAll(buffer);
- Y_UNUSED(result);
-
- if (channel->IsFinished()) {
- return NUdf::EFetchStatus::Finish;
+NUdf::EFetchStatus FetchAllOutput(NDq::IDqOutputChannel* channel, NDqProto::TData& buffer) {
+ auto result = channel->PopAll(buffer);
+ Y_UNUSED(result);
+
+ if (channel->IsFinished()) {
+ return NUdf::EFetchStatus::Finish;
}
- return NUdf::EFetchStatus::Yield;
+ return NUdf::EFetchStatus::Yield;
}
-NDq::ERunStatus RunKqpTransactionInternal(const TActorContext& ctx, ui64 txId,
- const TInputOpData::TInReadSets* inReadSets, const google::protobuf::RepeatedPtrField<NDqProto::TDqTask>& tasks,
- NKqp::TKqpTasksRunner& tasksRunner, bool applyEffects)
-{
- THashMap<ui64, std::pair<ui64, ui32>> inputChannelsMap; // channelId -> (taskId, input index)
+NDq::ERunStatus RunKqpTransactionInternal(const TActorContext& ctx, ui64 txId,
+ const TInputOpData::TInReadSets* inReadSets, const google::protobuf::RepeatedPtrField<NDqProto::TDqTask>& tasks,
+ NKqp::TKqpTasksRunner& tasksRunner, bool applyEffects)
+{
+ THashMap<ui64, std::pair<ui64, ui32>> inputChannelsMap; // channelId -> (taskId, input index)
for (auto& task : tasks) {
for (ui32 i = 0; i < task.InputsSize(); ++i) {
auto& input = task.GetInputs(i);
for (auto& channel : input.GetChannels()) {
auto channelInfo = std::make_pair(task.GetId(), i);
- auto result = inputChannelsMap.emplace(channel.GetId(), channelInfo);
+ auto result = inputChannelsMap.emplace(channel.GetId(), channelInfo);
MKQL_ENSURE_S(result.second);
}
}
}
if (inReadSets) {
- YQL_ENSURE(applyEffects);
-
+ YQL_ENSURE(applyEffects);
+
for (auto& readSet : *inReadSets) {
auto& key = readSet.first;
auto& dataList = readSet.second;
@@ -115,62 +115,62 @@ NDq::ERunStatus RunKqpTransactionInternal(const TActorContext& ctx, ui64 txId,
NKikimrTxDataShard::TKqpReadset kqpReadset;
Y_PROTOBUF_SUPPRESS_NODISCARD kqpReadset.ParseFromString(data.Body);
- for (int outputId = 0; outputId < kqpReadset.GetOutputs().size(); ++outputId) {
- auto* channelData = kqpReadset.MutableOutputs()->Mutable(outputId);
- auto channelId = channelData->GetChannelId();
+ for (int outputId = 0; outputId < kqpReadset.GetOutputs().size(); ++outputId) {
+ auto* channelData = kqpReadset.MutableOutputs()->Mutable(outputId);
+ auto channelId = channelData->GetChannelId();
auto inputInfo = inputChannelsMap.FindPtr(channelId);
MKQL_ENSURE_S(inputInfo);
auto taskId = inputInfo->first;
LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, "Added KQP readset"
- << ", source: " << source << ", target: " << target << ", origin: " << data.Origin
- << ", TxId: " << txId << ", task: " << taskId << ", channelId: " << channelId);
+ << ", source: " << source << ", target: " << target << ", origin: " << data.Origin
+ << ", TxId: " << txId << ", task: " << taskId << ", channelId: " << channelId);
- auto channel = tasksRunner.GetInputChannel(taskId, channelId);
- channel->Push(std::move(*(channelData->MutableData())));
+ auto channel = tasksRunner.GetInputChannel(taskId, channelId);
+ channel->Push(std::move(*(channelData->MutableData())));
- MKQL_ENSURE_S(channelData->GetFinished());
- channel->Finish();
+ MKQL_ENSURE_S(channelData->GetFinished());
+ channel->Finish();
}
}
}
}
- auto runStatus = NDq::ERunStatus::PendingInput;
+ auto runStatus = NDq::ERunStatus::PendingInput;
bool hasInputChanges = true;
- while (runStatus == NDq::ERunStatus::PendingInput && hasInputChanges) {
+ while (runStatus == NDq::ERunStatus::PendingInput && hasInputChanges) {
runStatus = tasksRunner.Run(applyEffects);
- if (runStatus == NDq::ERunStatus::Finished) {
+ if (runStatus == NDq::ERunStatus::Finished) {
break;
}
- // we must set output buffers big enough to avoid PendingOutput state here
- MKQL_ENSURE_S(runStatus == NDq::ERunStatus::PendingInput);
+ // we must set output buffers big enough to avoid PendingOutput state here
+ MKQL_ENSURE_S(runStatus == NDq::ERunStatus::PendingInput);
hasInputChanges = false;
for (auto& task : tasks) {
for (ui32 i = 0; i < task.OutputsSize(); ++i) {
for (auto& channel : task.GetOutputs(i).GetChannels()) {
- if (auto* inputInfo = inputChannelsMap.FindPtr(channel.GetId())) {
- auto transferState = tasksRunner.TransferData(task.GetId(), channel.GetId(),
- inputInfo->first, channel.GetId());
-
- if (transferState.first) {
- hasInputChanges = true;
- LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, "Forwarded KQP channel data"
- << ", TxId: " << txId
- << ", srcTask: " << task.GetId() << ", dstTask: " << inputInfo->first
- << ", channelId: " << channel.GetId());
- }
-
- if (transferState.second) {
- hasInputChanges = true;
- LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, "Finished input channel"
- << ", TxId: " << txId
- << ", srcTask: " << task.GetId() << ", dstTask: " << inputInfo->first
- << ", channelId: " << channel.GetId());
+ if (auto* inputInfo = inputChannelsMap.FindPtr(channel.GetId())) {
+ auto transferState = tasksRunner.TransferData(task.GetId(), channel.GetId(),
+ inputInfo->first, channel.GetId());
+
+ if (transferState.first) {
+ hasInputChanges = true;
+ LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, "Forwarded KQP channel data"
+ << ", TxId: " << txId
+ << ", srcTask: " << task.GetId() << ", dstTask: " << inputInfo->first
+ << ", channelId: " << channel.GetId());
+ }
+
+ if (transferState.second) {
+ hasInputChanges = true;
+ LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, "Finished input channel"
+ << ", TxId: " << txId
+ << ", srcTask: " << task.GetId() << ", dstTask: " << inputInfo->first
+ << ", channelId: " << channel.GetId());
}
}
}
@@ -181,95 +181,95 @@ NDq::ERunStatus RunKqpTransactionInternal(const TActorContext& ctx, ui64 txId,
return runStatus;
}
-bool NeedValidateLocks(NKikimrTxDataShard::TKqpLocks_ELocksOp op) {
- switch (op) {
- case NKikimrTxDataShard::TKqpLocks::ValidateAndErase:
- case NKikimrTxDataShard::TKqpLocks::Validate:
- return true;
-
- case NKikimrTxDataShard::TKqpLocks::Erase:
- case NKikimrTxDataShard::TKqpLocks::Unspecified:
- return false;
- }
-}
-
-bool NeedEraseLocks(NKikimrTxDataShard::TKqpLocks_ELocksOp op) {
- switch (op) {
- case NKikimrTxDataShard::TKqpLocks::ValidateAndErase:
- case NKikimrTxDataShard::TKqpLocks::Erase:
- return true;
-
- case NKikimrTxDataShard::TKqpLocks::Validate:
- case NKikimrTxDataShard::TKqpLocks::Unspecified:
- return false;
- }
-}
-
-TVector<TCell> MakeLockKey(const NKikimrTxDataShard::TLock& lockProto) {
- auto lockId = lockProto.GetLockId();
- auto lockDatashard = lockProto.GetDataShard();
- auto lockSchemeShard = lockProto.GetSchemeShard();
- auto lockPathId = lockProto.GetPathId();
-
- Y_ASSERT(TCell::CanInline(sizeof(lockId)));
- Y_ASSERT(TCell::CanInline(sizeof(lockDatashard)));
- Y_ASSERT(TCell::CanInline(sizeof(lockSchemeShard)));
- Y_ASSERT(TCell::CanInline(sizeof(lockPathId)));
-
- TVector<TCell> lockKey{
- TCell(reinterpret_cast<const char*>(&lockId), sizeof(lockId)),
- TCell(reinterpret_cast<const char*>(&lockDatashard), sizeof(lockDatashard)),
- TCell(reinterpret_cast<const char*>(&lockSchemeShard), sizeof(lockSchemeShard)),
- TCell(reinterpret_cast<const char*>(&lockPathId), sizeof(lockPathId))};
-
- return lockKey;
-}
-
-// returns list of broken locks
-TVector<NKikimrTxDataShard::TLock> ValidateLocks(const NKikimrTxDataShard::TKqpLocks& txLocks, TSysLocks& sysLocks,
- ui64 tabletId)
+bool NeedValidateLocks(NKikimrTxDataShard::TKqpLocks_ELocksOp op) {
+ switch (op) {
+ case NKikimrTxDataShard::TKqpLocks::ValidateAndErase:
+ case NKikimrTxDataShard::TKqpLocks::Validate:
+ return true;
+
+ case NKikimrTxDataShard::TKqpLocks::Erase:
+ case NKikimrTxDataShard::TKqpLocks::Unspecified:
+ return false;
+ }
+}
+
+bool NeedEraseLocks(NKikimrTxDataShard::TKqpLocks_ELocksOp op) {
+ switch (op) {
+ case NKikimrTxDataShard::TKqpLocks::ValidateAndErase:
+ case NKikimrTxDataShard::TKqpLocks::Erase:
+ return true;
+
+ case NKikimrTxDataShard::TKqpLocks::Validate:
+ case NKikimrTxDataShard::TKqpLocks::Unspecified:
+ return false;
+ }
+}
+
+TVector<TCell> MakeLockKey(const NKikimrTxDataShard::TLock& lockProto) {
+ auto lockId = lockProto.GetLockId();
+ auto lockDatashard = lockProto.GetDataShard();
+ auto lockSchemeShard = lockProto.GetSchemeShard();
+ auto lockPathId = lockProto.GetPathId();
+
+ Y_ASSERT(TCell::CanInline(sizeof(lockId)));
+ Y_ASSERT(TCell::CanInline(sizeof(lockDatashard)));
+ Y_ASSERT(TCell::CanInline(sizeof(lockSchemeShard)));
+ Y_ASSERT(TCell::CanInline(sizeof(lockPathId)));
+
+ TVector<TCell> lockKey{
+ TCell(reinterpret_cast<const char*>(&lockId), sizeof(lockId)),
+ TCell(reinterpret_cast<const char*>(&lockDatashard), sizeof(lockDatashard)),
+ TCell(reinterpret_cast<const char*>(&lockSchemeShard), sizeof(lockSchemeShard)),
+ TCell(reinterpret_cast<const char*>(&lockPathId), sizeof(lockPathId))};
+
+ return lockKey;
+}
+
+// returns list of broken locks
+TVector<NKikimrTxDataShard::TLock> ValidateLocks(const NKikimrTxDataShard::TKqpLocks& txLocks, TSysLocks& sysLocks,
+ ui64 tabletId)
{
- TVector<NKikimrTxDataShard::TLock> brokenLocks;
-
- if (!NeedValidateLocks(txLocks.GetOp())) {
- return {};
- }
-
+ TVector<NKikimrTxDataShard::TLock> brokenLocks;
+
+ if (!NeedValidateLocks(txLocks.GetOp())) {
+ return {};
+ }
+
for (auto& lockProto : txLocks.GetLocks()) {
if (lockProto.GetDataShard() != tabletId) {
continue;
}
- auto lockKey = MakeLockKey(lockProto);
+ auto lockKey = MakeLockKey(lockProto);
auto lock = sysLocks.GetLock(lockKey);
if (lock.Generation != lockProto.GetGeneration() || lock.Counter != lockProto.GetCounter()) {
- brokenLocks.push_back(lockProto);
+ brokenLocks.push_back(lockProto);
}
}
- return brokenLocks;
+ return brokenLocks;
}
-bool SendLocks(const NKikimrTxDataShard::TKqpLocks& locks, ui64 shardId) {
- auto& sendingShards = locks.GetSendingShards();
+bool SendLocks(const NKikimrTxDataShard::TKqpLocks& locks, ui64 shardId) {
+ auto& sendingShards = locks.GetSendingShards();
auto it = std::find(sendingShards.begin(), sendingShards.end(), shardId);
return it != sendingShards.end();
}
-bool ReceiveLocks(const NKikimrTxDataShard::TKqpLocks& locks, ui64 shardId) {
- auto& receivingShards = locks.GetReceivingShards();
+bool ReceiveLocks(const NKikimrTxDataShard::TKqpLocks& locks, ui64 shardId) {
+ auto& receivingShards = locks.GetReceivingShards();
auto it = std::find(receivingShards.begin(), receivingShards.end(), shardId);
return it != receivingShards.end();
}
} // namespace
-bool KqpValidateTransaction(const NKikimrTxDataShard::TKqpTransaction& tx, bool isImmediate, ui64 txId,
- const TActorContext& ctx, bool& hasPersistentChannels)
+bool KqpValidateTransaction(const NKikimrTxDataShard::TKqpTransaction& tx, bool isImmediate, ui64 txId,
+ const TActorContext& ctx, bool& hasPersistentChannels)
{
- for (const auto& task : tx.GetTasks()) {
- if (!KqpValidateTask(task, isImmediate, txId, ctx, hasPersistentChannels)) {
+ for (const auto& task : tx.GetTasks()) {
+ if (!KqpValidateTask(task, isImmediate, txId, ctx, hasPersistentChannels)) {
return false;
}
}
@@ -277,172 +277,172 @@ bool KqpValidateTransaction(const NKikimrTxDataShard::TKqpTransaction& tx, bool
return true;
}
-namespace {
-
-template <bool Read>
-void KqpSetTxKeysImpl(ui64 tabletId, ui64 taskId, const TTableId& tableId, const TUserTable* tableInfo,
- const NKikimrTxDataShard::TKqpTransaction_TDataTaskMeta_TKeyRange& rangeKind, const TVector<ui32>& columns,
- ui64 itemsLimit, bool reverse, const NScheme::TTypeRegistry& typeRegistry, const TActorContext& ctx,
- TEngineBay& engineBay)
-{
- switch (rangeKind.Kind_case()) {
- case NKikimrTxDataShard::TKqpTransaction_TDataTaskMeta_TKeyRange::kRanges: {
- auto& ranges = rangeKind.GetRanges();
- Y_VERIFY_DEBUG(ranges.GetKeyRanges().size() + ranges.GetKeyPoints().size() > 0);
-
- for (auto& range : ranges.GetKeyRanges()) {
- TSerializedTableRange tableRange;
- tableRange.Load(range);
-
- LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, "Table " << tableInfo->Path
- << ", shard: " << tabletId << ", task: " << taskId << ", " << (Read ? "read range " : "write range ")
- << DebugPrintRange(tableInfo->KeyColumnTypes, tableRange.ToTableRange(), typeRegistry));
-
- Y_VERIFY_DEBUG(!(tableRange.To.GetCells().empty() && tableRange.ToInclusive));
-
- if constexpr (Read) {
- engineBay.AddReadRange(tableId, columns, tableRange.ToTableRange(), tableInfo->KeyColumnTypes, itemsLimit, reverse);
- } else {
- engineBay.AddWriteRange(tableId, tableRange.ToTableRange(), tableInfo->KeyColumnTypes);
- }
- }
-
- for (auto& point : ranges.GetKeyPoints()) {
- TSerializedTableRange tablePoint(point, point, true, true);
- tablePoint.Point = true;
-
- LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, "Table " << tableInfo->Path
- << ", shard: " << tabletId << ", task: " << taskId << ", " << (Read ? "read point " : "write point ")
- << DebugPrintPoint(tableInfo->KeyColumnTypes, tablePoint.From.GetCells(), typeRegistry));
-
- if constexpr (Read) {
- engineBay.AddReadRange(tableId, columns, tablePoint.ToTableRange(), tableInfo->KeyColumnTypes, itemsLimit, reverse);
- } else {
- engineBay.AddWriteRange(tableId, tablePoint.ToTableRange(), tableInfo->KeyColumnTypes);
- }
- }
-
- break;
- }
-
- case NKikimrTxDataShard::TKqpTransaction_TDataTaskMeta_TKeyRange::kFullRange: {
- TSerializedTableRange tableRange;
- tableRange.Load(rangeKind.GetFullRange());
-
- LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, "Table " << tableInfo->Path
- << ", shard: " << tabletId << ", task: " << taskId << ", " << (Read ? "read range: FULL " : "write range: FULL ")
- << DebugPrintRange(tableInfo->KeyColumnTypes, tableRange.ToTableRange(), typeRegistry));
-
- if constexpr (Read) {
- engineBay.AddReadRange(tableId, columns, tableRange.ToTableRange(), tableInfo->KeyColumnTypes, itemsLimit, reverse);
- } else {
- engineBay.AddWriteRange(tableId, tableRange.ToTableRange(), tableInfo->KeyColumnTypes);
- }
-
- break;
- }
-
- case NKikimrTxDataShard::TKqpTransaction_TDataTaskMeta_TKeyRange::KIND_NOT_SET: {
- LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, "Table " << tableInfo->Path
- << ", shard: " << tabletId << ", task: " << taskId << ", " << (Read ? "read range: UNSPECIFIED" : "write range: UNSPECIFIED"));
-
- if constexpr (Read) {
- engineBay.AddReadRange(tableId, columns, tableInfo->Range.ToTableRange(), tableInfo->KeyColumnTypes, itemsLimit, reverse);
- } else {
- engineBay.AddWriteRange(tableId, tableInfo->Range.ToTableRange(), tableInfo->KeyColumnTypes);
- }
-
- break;
- }
- }
-}
-
-} // anonymous namespace
-
-void KqpSetTxKeys(ui64 tabletId, ui64 taskId, const TUserTable* tableInfo,
- const NKikimrTxDataShard::TKqpTransaction_TDataTaskMeta& meta, const NScheme::TTypeRegistry& typeRegistry,
- const TActorContext& ctx, TEngineBay& engineBay)
-{
- auto& tableMeta = meta.GetTable();
- auto tableId = TTableId(tableMeta.GetTableId().GetOwnerId(), tableMeta.GetTableId().GetTableId(),
- tableMeta.GetSchemaVersion());
-
- for (auto& read : meta.GetReads()) {
- TVector<ui32> columns;
- columns.reserve(read.GetColumns().size());
- for (auto& c : read.GetColumns()) {
- columns.push_back(c.GetId());
- }
-
- KqpSetTxKeysImpl<true>(tabletId, taskId, tableId, tableInfo, read.GetRange(), columns, read.GetItemsLimit(),
- read.GetReverse(), typeRegistry, ctx, engineBay);
- }
-
- if (meta.HasWrites()) {
- KqpSetTxKeysImpl<false>(tabletId, taskId, tableId, tableInfo, meta.GetWrites().GetRange(), /* columns */ {}, 0,
- /* reverse */ false, typeRegistry, ctx, engineBay);
- }
-}
-
-void KqpSetTxLocksKeys(const NKikimrTxDataShard::TKqpLocks& locks, const TSysLocks& sysLocks, TEngineBay& engineBay) {
- if (locks.LocksSize() == 0) {
- return;
- }
-
- static TTableId sysLocksTableId = TTableId(TSysTables::SysSchemeShard, TSysTables::SysTableLocks2);
- static TVector<NScheme::TTypeId> lockRowType = {
- NScheme::TUint64::TypeId, NScheme::TUint64::TypeId,
- NScheme::TUint64::TypeId, NScheme::TUint64::TypeId,
- };
-
- for (auto& lock : locks.GetLocks()) {
- auto lockKey = MakeLockKey(lock);
- if (sysLocks.IsMyKey(lockKey)) {
- auto point = TTableRange(lockKey, true, {}, true, /* point */ true);
- engineBay.AddReadRange(sysLocksTableId, {}, point, lockRowType);
- }
- }
-}
-
+namespace {
+
+template <bool Read>
+void KqpSetTxKeysImpl(ui64 tabletId, ui64 taskId, const TTableId& tableId, const TUserTable* tableInfo,
+ const NKikimrTxDataShard::TKqpTransaction_TDataTaskMeta_TKeyRange& rangeKind, const TVector<ui32>& columns,
+ ui64 itemsLimit, bool reverse, const NScheme::TTypeRegistry& typeRegistry, const TActorContext& ctx,
+ TEngineBay& engineBay)
+{
+ switch (rangeKind.Kind_case()) {
+ case NKikimrTxDataShard::TKqpTransaction_TDataTaskMeta_TKeyRange::kRanges: {
+ auto& ranges = rangeKind.GetRanges();
+ Y_VERIFY_DEBUG(ranges.GetKeyRanges().size() + ranges.GetKeyPoints().size() > 0);
+
+ for (auto& range : ranges.GetKeyRanges()) {
+ TSerializedTableRange tableRange;
+ tableRange.Load(range);
+
+ LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, "Table " << tableInfo->Path
+ << ", shard: " << tabletId << ", task: " << taskId << ", " << (Read ? "read range " : "write range ")
+ << DebugPrintRange(tableInfo->KeyColumnTypes, tableRange.ToTableRange(), typeRegistry));
+
+ Y_VERIFY_DEBUG(!(tableRange.To.GetCells().empty() && tableRange.ToInclusive));
+
+ if constexpr (Read) {
+ engineBay.AddReadRange(tableId, columns, tableRange.ToTableRange(), tableInfo->KeyColumnTypes, itemsLimit, reverse);
+ } else {
+ engineBay.AddWriteRange(tableId, tableRange.ToTableRange(), tableInfo->KeyColumnTypes);
+ }
+ }
+
+ for (auto& point : ranges.GetKeyPoints()) {
+ TSerializedTableRange tablePoint(point, point, true, true);
+ tablePoint.Point = true;
+
+ LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, "Table " << tableInfo->Path
+ << ", shard: " << tabletId << ", task: " << taskId << ", " << (Read ? "read point " : "write point ")
+ << DebugPrintPoint(tableInfo->KeyColumnTypes, tablePoint.From.GetCells(), typeRegistry));
+
+ if constexpr (Read) {
+ engineBay.AddReadRange(tableId, columns, tablePoint.ToTableRange(), tableInfo->KeyColumnTypes, itemsLimit, reverse);
+ } else {
+ engineBay.AddWriteRange(tableId, tablePoint.ToTableRange(), tableInfo->KeyColumnTypes);
+ }
+ }
+
+ break;
+ }
+
+ case NKikimrTxDataShard::TKqpTransaction_TDataTaskMeta_TKeyRange::kFullRange: {
+ TSerializedTableRange tableRange;
+ tableRange.Load(rangeKind.GetFullRange());
+
+ LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, "Table " << tableInfo->Path
+ << ", shard: " << tabletId << ", task: " << taskId << ", " << (Read ? "read range: FULL " : "write range: FULL ")
+ << DebugPrintRange(tableInfo->KeyColumnTypes, tableRange.ToTableRange(), typeRegistry));
+
+ if constexpr (Read) {
+ engineBay.AddReadRange(tableId, columns, tableRange.ToTableRange(), tableInfo->KeyColumnTypes, itemsLimit, reverse);
+ } else {
+ engineBay.AddWriteRange(tableId, tableRange.ToTableRange(), tableInfo->KeyColumnTypes);
+ }
+
+ break;
+ }
+
+ case NKikimrTxDataShard::TKqpTransaction_TDataTaskMeta_TKeyRange::KIND_NOT_SET: {
+ LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, "Table " << tableInfo->Path
+ << ", shard: " << tabletId << ", task: " << taskId << ", " << (Read ? "read range: UNSPECIFIED" : "write range: UNSPECIFIED"));
+
+ if constexpr (Read) {
+ engineBay.AddReadRange(tableId, columns, tableInfo->Range.ToTableRange(), tableInfo->KeyColumnTypes, itemsLimit, reverse);
+ } else {
+ engineBay.AddWriteRange(tableId, tableInfo->Range.ToTableRange(), tableInfo->KeyColumnTypes);
+ }
+
+ break;
+ }
+ }
+}
+
+} // anonymous namespace
+
+void KqpSetTxKeys(ui64 tabletId, ui64 taskId, const TUserTable* tableInfo,
+ const NKikimrTxDataShard::TKqpTransaction_TDataTaskMeta& meta, const NScheme::TTypeRegistry& typeRegistry,
+ const TActorContext& ctx, TEngineBay& engineBay)
+{
+ auto& tableMeta = meta.GetTable();
+ auto tableId = TTableId(tableMeta.GetTableId().GetOwnerId(), tableMeta.GetTableId().GetTableId(),
+ tableMeta.GetSchemaVersion());
+
+ for (auto& read : meta.GetReads()) {
+ TVector<ui32> columns;
+ columns.reserve(read.GetColumns().size());
+ for (auto& c : read.GetColumns()) {
+ columns.push_back(c.GetId());
+ }
+
+ KqpSetTxKeysImpl<true>(tabletId, taskId, tableId, tableInfo, read.GetRange(), columns, read.GetItemsLimit(),
+ read.GetReverse(), typeRegistry, ctx, engineBay);
+ }
+
+ if (meta.HasWrites()) {
+ KqpSetTxKeysImpl<false>(tabletId, taskId, tableId, tableInfo, meta.GetWrites().GetRange(), /* columns */ {}, 0,
+ /* reverse */ false, typeRegistry, ctx, engineBay);
+ }
+}
+
+void KqpSetTxLocksKeys(const NKikimrTxDataShard::TKqpLocks& locks, const TSysLocks& sysLocks, TEngineBay& engineBay) {
+ if (locks.LocksSize() == 0) {
+ return;
+ }
+
+ static TTableId sysLocksTableId = TTableId(TSysTables::SysSchemeShard, TSysTables::SysTableLocks2);
+ static TVector<NScheme::TTypeId> lockRowType = {
+ NScheme::TUint64::TypeId, NScheme::TUint64::TypeId,
+ NScheme::TUint64::TypeId, NScheme::TUint64::TypeId,
+ };
+
+ for (auto& lock : locks.GetLocks()) {
+ auto lockKey = MakeLockKey(lock);
+ if (sysLocks.IsMyKey(lockKey)) {
+ auto point = TTableRange(lockKey, true, {}, true, /* point */ true);
+ engineBay.AddReadRange(sysLocksTableId, {}, point, lockRowType);
+ }
+ }
+}
+
NYql::NDq::ERunStatus KqpRunTransaction(const TActorContext& ctx, ui64 txId,
- const google::protobuf::RepeatedPtrField<NYql::NDqProto::TDqTask>& tasks, NKqp::TKqpTasksRunner& tasksRunner)
+ const google::protobuf::RepeatedPtrField<NYql::NDqProto::TDqTask>& tasks, NKqp::TKqpTasksRunner& tasksRunner)
{
- return RunKqpTransactionInternal(ctx, txId, /* inReadSets */ nullptr, tasks, tasksRunner, /* applyEffects */ false);
+ return RunKqpTransactionInternal(ctx, txId, /* inReadSets */ nullptr, tasks, tasksRunner, /* applyEffects */ false);
}
THolder<TEvDataShard::TEvProposeTransactionResult> KqpCompleteTransaction(const TActorContext& ctx,
ui64 origin, ui64 txId, const TInputOpData::TInReadSets* inReadSets,
- const google::protobuf::RepeatedPtrField<NDqProto::TDqTask>& tasks, NKqp::TKqpTasksRunner& tasksRunner,
- const NMiniKQL::TKqpDatashardComputeContext& computeCtx)
+ const google::protobuf::RepeatedPtrField<NDqProto::TDqTask>& tasks, NKqp::TKqpTasksRunner& tasksRunner,
+ const NMiniKQL::TKqpDatashardComputeContext& computeCtx)
{
- auto runStatus = RunKqpTransactionInternal(ctx, txId, inReadSets, tasks, tasksRunner, /* applyEffects */ true);
+ auto runStatus = RunKqpTransactionInternal(ctx, txId, inReadSets, tasks, tasksRunner, /* applyEffects */ true);
+
+ if (runStatus == NYql::NDq::ERunStatus::PendingInput && computeCtx.IsTabletNotReady()) {
+ return nullptr;
+ }
- if (runStatus == NYql::NDq::ERunStatus::PendingInput && computeCtx.IsTabletNotReady()) {
- return nullptr;
- }
-
MKQL_ENSURE_S(runStatus == NYql::NDq::ERunStatus::Finished);
auto result = MakeHolder<TEvDataShard::TEvProposeTransactionResult>(NKikimrTxDataShard::TX_KIND_DATA,
origin, txId, NKikimrTxDataShard::TEvProposeTransactionResult::COMPLETE);
for (auto& task : tasks) {
- auto& taskRunner = tasksRunner.GetTaskRunner(task.GetId());
+ auto& taskRunner = tasksRunner.GetTaskRunner(task.GetId());
for (ui32 i = 0; i < task.OutputsSize(); ++i) {
for (auto& channel : task.GetOutputs(i).GetChannels()) {
- auto computeActor = computeCtx.GetTaskOutputChannel(task.GetId(), channel.GetId());
- if (computeActor) {
- auto dataEv = MakeHolder<NYql::NDq::TEvDqCompute::TEvChannelData>();
- dataEv->Record.SetSeqNo(1);
- dataEv->Record.MutableChannelData()->SetChannelId(channel.GetId());
- dataEv->Record.MutableChannelData()->SetFinished(true);
- dataEv->Record.SetNoAck(true);
- auto outputData = dataEv->Record.MutableChannelData()->MutableData();
-
- auto fetchStatus = FetchAllOutput(taskRunner.GetOutputChannel(channel.GetId()).Get(), *outputData);
- MKQL_ENSURE_S(fetchStatus == NUdf::EFetchStatus::Finish);
-
+ auto computeActor = computeCtx.GetTaskOutputChannel(task.GetId(), channel.GetId());
+ if (computeActor) {
+ auto dataEv = MakeHolder<NYql::NDq::TEvDqCompute::TEvChannelData>();
+ dataEv->Record.SetSeqNo(1);
+ dataEv->Record.MutableChannelData()->SetChannelId(channel.GetId());
+ dataEv->Record.MutableChannelData()->SetFinished(true);
+ dataEv->Record.SetNoAck(true);
+ auto outputData = dataEv->Record.MutableChannelData()->MutableData();
+
+ auto fetchStatus = FetchAllOutput(taskRunner.GetOutputChannel(channel.GetId()).Get(), *outputData);
+ MKQL_ENSURE_S(fetchStatus == NUdf::EFetchStatus::Finish);
+
if (outputData->GetRaw().size() > MaxDatashardReplySize) {
auto message = TStringBuilder() << "Datashard " << origin
<< ": reply size limit exceeded (" << outputData->GetRaw().size() << " > "
@@ -453,31 +453,31 @@ THolder<TEvDataShard::TEvProposeTransactionResult> KqpCompleteTransaction(const
} else {
ctx.Send(computeActor, dataEv.Release());
}
- } else {
- NDqProto::TData outputData;
- auto fetchStatus = FetchAllOutput(taskRunner.GetOutputChannel(channel.GetId()).Get(), outputData);
- MKQL_ENSURE_S(fetchStatus == NUdf::EFetchStatus::Finish);
- MKQL_ENSURE_S(outputData.GetRows() == 0);
- }
+ } else {
+ NDqProto::TData outputData;
+ auto fetchStatus = FetchAllOutput(taskRunner.GetOutputChannel(channel.GetId()).Get(), outputData);
+ MKQL_ENSURE_S(fetchStatus == NUdf::EFetchStatus::Finish);
+ MKQL_ENSURE_S(outputData.GetRows() == 0);
+ }
}
}
}
TString replyStr;
- NKikimrTxDataShard::TKqpReply reply;
+ NKikimrTxDataShard::TKqpReply reply;
Y_PROTOBUF_SUPPRESS_NODISCARD reply.SerializeToString(&replyStr);
result->SetTxResult(replyStr);
return result;
}
-void KqpFillOutReadSets(TOutputOpData::TOutReadSets& outReadSets, const NKikimrTxDataShard::TKqpTransaction& kqpTx,
- NKqp::TKqpTasksRunner& tasksRunner, TSysLocks& sysLocks, ui64 tabletId)
+void KqpFillOutReadSets(TOutputOpData::TOutReadSets& outReadSets, const NKikimrTxDataShard::TKqpTransaction& kqpTx,
+ NKqp::TKqpTasksRunner& tasksRunner, TSysLocks& sysLocks, ui64 tabletId)
{
TMap<std::pair<ui64, ui64>, NKikimrTxDataShard::TKqpReadset> readsetData;
for (auto& task : kqpTx.GetTasks()) {
- auto& taskRunner = tasksRunner.GetTaskRunner(task.GetId());
+ auto& taskRunner = tasksRunner.GetTaskRunner(task.GetId());
for (ui32 i = 0; i < task.OutputsSize(); ++i) {
for (auto& channel : task.GetOutputs(i).GetChannels()) {
@@ -485,8 +485,8 @@ void KqpFillOutReadSets(TOutputOpData::TOutReadSets& outReadSets, const NKikimrT
MKQL_ENSURE_S(channel.GetSrcEndpoint().HasTabletId());
MKQL_ENSURE_S(channel.GetDstEndpoint().HasTabletId());
- NDqProto::TData outputData;
- auto fetchStatus = FetchAllOutput(taskRunner.GetOutputChannel(channel.GetId()).Get(), outputData);
+ NDqProto::TData outputData;
+ auto fetchStatus = FetchAllOutput(taskRunner.GetOutputChannel(channel.GetId()).Get(), outputData);
MKQL_ENSURE_S(fetchStatus == NUdf::EFetchStatus::Finish);
auto key = std::make_pair(channel.GetSrcEndpoint().GetTabletId(),
@@ -495,43 +495,43 @@ void KqpFillOutReadSets(TOutputOpData::TOutReadSets& outReadSets, const NKikimrT
channelData.SetChannelId(channel.GetId());
channelData.SetFinished(true);
- channelData.MutableData()->Swap(&outputData);
+ channelData.MutableData()->Swap(&outputData);
}
}
}
}
- if (kqpTx.HasLocks() && NeedValidateLocks(kqpTx.GetLocks().GetOp())) {
- bool sendLocks = SendLocks(kqpTx.GetLocks(), tabletId);
- YQL_ENSURE(sendLocks == !kqpTx.GetLocks().GetLocks().empty());
+ if (kqpTx.HasLocks() && NeedValidateLocks(kqpTx.GetLocks().GetOp())) {
+ bool sendLocks = SendLocks(kqpTx.GetLocks(), tabletId);
+ YQL_ENSURE(sendLocks == !kqpTx.GetLocks().GetLocks().empty());
- if (sendLocks && !kqpTx.GetLocks().GetReceivingShards().empty()) {
- auto brokenLocks = ValidateLocks(kqpTx.GetLocks(), sysLocks, tabletId);
+ if (sendLocks && !kqpTx.GetLocks().GetReceivingShards().empty()) {
+ auto brokenLocks = ValidateLocks(kqpTx.GetLocks(), sysLocks, tabletId);
NKikimrTxDataShard::TKqpValidateLocksResult validateLocksResult;
- validateLocksResult.SetSuccess(brokenLocks.empty());
-
- for (auto& lock : brokenLocks) {
- LOG_TRACE_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD,
- "Found broken lock: " << lock.ShortDebugString());
- validateLocksResult.AddBrokenLocks()->Swap(&lock);
+ validateLocksResult.SetSuccess(brokenLocks.empty());
+
+ for (auto& lock : brokenLocks) {
+ LOG_TRACE_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD,
+ "Found broken lock: " << lock.ShortDebugString());
+ validateLocksResult.AddBrokenLocks()->Swap(&lock);
}
- for (auto& dstTabletId : kqpTx.GetLocks().GetReceivingShards()) {
- if (tabletId == dstTabletId) {
- continue;
- }
-
- LOG_TRACE_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Send locks from "
- << tabletId << " to " << dstTabletId << ", locks: " << validateLocksResult.ShortDebugString());
-
- auto key = std::make_pair(tabletId, dstTabletId);
+ for (auto& dstTabletId : kqpTx.GetLocks().GetReceivingShards()) {
+ if (tabletId == dstTabletId) {
+ continue;
+ }
+
+ LOG_TRACE_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Send locks from "
+ << tabletId << " to " << dstTabletId << ", locks: " << validateLocksResult.ShortDebugString());
+
+ auto key = std::make_pair(tabletId, dstTabletId);
readsetData[key].MutableValidateLocksResult()->CopyFrom(validateLocksResult);
}
}
}
- for (auto& [key, data] : readsetData) {
+ for (auto& [key, data] : readsetData) {
TString bodyStr;
Y_PROTOBUF_SUPPRESS_NODISCARD data.SerializeToString(&bodyStr);
@@ -539,36 +539,36 @@ void KqpFillOutReadSets(TOutputOpData::TOutReadSets& outReadSets, const NKikimrT
}
}
-bool KqpValidateLocks(ui64 origin, TActiveTransaction* tx, TSysLocks& sysLocks) {
- auto& kqpTx = tx->GetDataTx()->GetKqpTransaction();
-
- if (kqpTx.HasLocks() && !NeedValidateLocks(kqpTx.GetLocks().GetOp())) {
- return true;
- }
-
- bool sendLocks = SendLocks(kqpTx.GetLocks(), origin);
- YQL_ENSURE(sendLocks == !kqpTx.GetLocks().GetLocks().empty());
-
- if (sendLocks) {
- auto brokenLocks = ValidateLocks(kqpTx.GetLocks(), sysLocks, origin);
-
- if (!brokenLocks.empty()) {
- tx->Result() = MakeHolder<TEvDataShard::TEvProposeTransactionResult>(
- NKikimrTxDataShard::TX_KIND_DATA,
- origin,
- tx->GetTxId(),
- NKikimrTxDataShard::TEvProposeTransactionResult::LOCKS_BROKEN);
-
- auto* protoLocks = tx->Result()->Record.MutableTxLocks();
- for (auto& brokenLock : brokenLocks) {
- protoLocks->Add()->Swap(&brokenLock);
- }
-
- return false;
- }
- }
-
- for (auto& readSet : tx->InReadSets()) {
+bool KqpValidateLocks(ui64 origin, TActiveTransaction* tx, TSysLocks& sysLocks) {
+ auto& kqpTx = tx->GetDataTx()->GetKqpTransaction();
+
+ if (kqpTx.HasLocks() && !NeedValidateLocks(kqpTx.GetLocks().GetOp())) {
+ return true;
+ }
+
+ bool sendLocks = SendLocks(kqpTx.GetLocks(), origin);
+ YQL_ENSURE(sendLocks == !kqpTx.GetLocks().GetLocks().empty());
+
+ if (sendLocks) {
+ auto brokenLocks = ValidateLocks(kqpTx.GetLocks(), sysLocks, origin);
+
+ if (!brokenLocks.empty()) {
+ tx->Result() = MakeHolder<TEvDataShard::TEvProposeTransactionResult>(
+ NKikimrTxDataShard::TX_KIND_DATA,
+ origin,
+ tx->GetTxId(),
+ NKikimrTxDataShard::TEvProposeTransactionResult::LOCKS_BROKEN);
+
+ auto* protoLocks = tx->Result()->Record.MutableTxLocks();
+ for (auto& brokenLock : brokenLocks) {
+ protoLocks->Add()->Swap(&brokenLock);
+ }
+
+ return false;
+ }
+ }
+
+ for (auto& readSet : tx->InReadSets()) {
for (auto& data : readSet.second) {
NKikimrTxDataShard::TKqpReadset kqpReadset;
Y_PROTOBUF_SUPPRESS_NODISCARD kqpReadset.ParseFromString(data.Body);
@@ -576,13 +576,13 @@ bool KqpValidateLocks(ui64 origin, TActiveTransaction* tx, TSysLocks& sysLocks)
if (kqpReadset.HasValidateLocksResult()) {
auto& validateResult = kqpReadset.GetValidateLocksResult();
if (!validateResult.GetSuccess()) {
- tx->Result() = MakeHolder<TEvDataShard::TEvProposeTransactionResult>(
+ tx->Result() = MakeHolder<TEvDataShard::TEvProposeTransactionResult>(
NKikimrTxDataShard::TX_KIND_DATA,
origin,
- tx->GetTxId(),
+ tx->GetTxId(),
NKikimrTxDataShard::TEvProposeTransactionResult::LOCKS_BROKEN);
- tx->Result()->Record.MutableTxLocks()->CopyFrom(validateResult.GetBrokenLocks());
+ tx->Result()->Record.MutableTxLocks()->CopyFrom(validateResult.GetBrokenLocks());
return false;
}
@@ -593,25 +593,25 @@ bool KqpValidateLocks(ui64 origin, TActiveTransaction* tx, TSysLocks& sysLocks)
return true;
}
-void KqpEraseLocks(ui64 origin, TActiveTransaction* tx, TSysLocks& sysLocks) {
- auto& kqpTx = tx->GetDataTx()->GetKqpTransaction();
-
- if (!kqpTx.HasLocks() || !NeedEraseLocks(kqpTx.GetLocks().GetOp())) {
- return;
- }
-
- for (auto& lockProto : kqpTx.GetLocks().GetLocks()) {
- if (lockProto.GetDataShard() != origin) {
- continue;
- }
-
- LOG_TRACE_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "KqpEraseLock " << lockProto.ShortDebugString());
-
- auto lockKey = MakeLockKey(lockProto);
- sysLocks.EraseLock(lockKey);
- }
-}
-
+void KqpEraseLocks(ui64 origin, TActiveTransaction* tx, TSysLocks& sysLocks) {
+ auto& kqpTx = tx->GetDataTx()->GetKqpTransaction();
+
+ if (!kqpTx.HasLocks() || !NeedEraseLocks(kqpTx.GetLocks().GetOp())) {
+ return;
+ }
+
+ for (auto& lockProto : kqpTx.GetLocks().GetLocks()) {
+ if (lockProto.GetDataShard() != origin) {
+ continue;
+ }
+
+ LOG_TRACE_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "KqpEraseLock " << lockProto.ShortDebugString());
+
+ auto lockKey = MakeLockKey(lockProto);
+ sysLocks.EraseLock(lockKey);
+ }
+}
+
void KqpPrepareInReadsets(TInputOpData::TInReadSets& inReadSets,
const NKikimrTxDataShard::TKqpTransaction& kqpTx, ui64 tabletId)
{
@@ -624,161 +624,161 @@ void KqpPrepareInReadsets(TInputOpData::TInReadSets& inReadSets,
auto key = std::make_pair(channel.GetSrcEndpoint().GetTabletId(),
channel.GetDstEndpoint().GetTabletId());
-
- inReadSets.emplace(key, TVector<TRSData>());
+
+ inReadSets.emplace(key, TVector<TRSData>());
}
}
}
}
- if (ReceiveLocks(kqpTx.GetLocks(), tabletId)) {
- for (ui64 shardId : kqpTx.GetLocks().GetSendingShards()) {
- if (shardId == tabletId) {
- continue;
- }
-
- LOG_TRACE_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Prepare InReadsets from " << shardId
- << " to " << tabletId);
-
+ if (ReceiveLocks(kqpTx.GetLocks(), tabletId)) {
+ for (ui64 shardId : kqpTx.GetLocks().GetSendingShards()) {
+ if (shardId == tabletId) {
+ continue;
+ }
+
+ LOG_TRACE_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Prepare InReadsets from " << shardId
+ << " to " << tabletId);
+
auto key = std::make_pair(shardId, tabletId);
- inReadSets.emplace(key, TVector<TRSData>());
+ inReadSets.emplace(key, TVector<TRSData>());
}
}
}
void KqpUpdateDataShardStatCounters(TDataShard& dataShard, const NMiniKQL::TEngineHostCounters& counters) {
- dataShard.IncCounter(COUNTER_ENGINE_HOST_SELECT_ROW, counters.NSelectRow);
- dataShard.IncCounter(COUNTER_ENGINE_HOST_SELECT_RANGE, counters.NSelectRange);
- dataShard.IncCounter(COUNTER_ENGINE_HOST_UPDATE_ROW, counters.NUpdateRow);
- dataShard.IncCounter(COUNTER_ENGINE_HOST_ERASE_ROW, counters.NEraseRow);
- dataShard.IncCounter(COUNTER_ENGINE_HOST_SELECT_ROW_BYTES, counters.SelectRowBytes);
- dataShard.IncCounter(COUNTER_ENGINE_HOST_SELECT_RANGE_ROWS, counters.SelectRangeRows);
- dataShard.IncCounter(COUNTER_ENGINE_HOST_SELECT_RANGE_BYTES, counters.SelectRangeBytes);
- dataShard.IncCounter(COUNTER_ENGINE_HOST_SELECT_RANGE_ROW_SKIPS, counters.SelectRangeDeletedRowSkips);
- dataShard.IncCounter(COUNTER_ENGINE_HOST_UPDATE_ROW_BYTES, counters.UpdateRowBytes);
- dataShard.IncCounter(COUNTER_ENGINE_HOST_ERASE_ROW_BYTES, counters.EraseRowBytes);
- if (counters.NSelectRow > 0) {
- dataShard.IncCounter(COUNTER_SELECT_ROWS_PER_REQUEST, counters.NSelectRow);
- }
- if (counters.NSelectRange > 0) {
- dataShard.IncCounter(COUNTER_RANGE_READ_ROWS_PER_REQUEST, counters.SelectRangeRows);
- }
-}
-
+ dataShard.IncCounter(COUNTER_ENGINE_HOST_SELECT_ROW, counters.NSelectRow);
+ dataShard.IncCounter(COUNTER_ENGINE_HOST_SELECT_RANGE, counters.NSelectRange);
+ dataShard.IncCounter(COUNTER_ENGINE_HOST_UPDATE_ROW, counters.NUpdateRow);
+ dataShard.IncCounter(COUNTER_ENGINE_HOST_ERASE_ROW, counters.NEraseRow);
+ dataShard.IncCounter(COUNTER_ENGINE_HOST_SELECT_ROW_BYTES, counters.SelectRowBytes);
+ dataShard.IncCounter(COUNTER_ENGINE_HOST_SELECT_RANGE_ROWS, counters.SelectRangeRows);
+ dataShard.IncCounter(COUNTER_ENGINE_HOST_SELECT_RANGE_BYTES, counters.SelectRangeBytes);
+ dataShard.IncCounter(COUNTER_ENGINE_HOST_SELECT_RANGE_ROW_SKIPS, counters.SelectRangeDeletedRowSkips);
+ dataShard.IncCounter(COUNTER_ENGINE_HOST_UPDATE_ROW_BYTES, counters.UpdateRowBytes);
+ dataShard.IncCounter(COUNTER_ENGINE_HOST_ERASE_ROW_BYTES, counters.EraseRowBytes);
+ if (counters.NSelectRow > 0) {
+ dataShard.IncCounter(COUNTER_SELECT_ROWS_PER_REQUEST, counters.NSelectRow);
+ }
+ if (counters.NSelectRange > 0) {
+ dataShard.IncCounter(COUNTER_RANGE_READ_ROWS_PER_REQUEST, counters.SelectRangeRows);
+ }
+}
+
void KqpFillTxStats(TDataShard& dataShard, const NMiniKQL::TEngineHostCounters& counters,
- TEvDataShard::TEvProposeTransactionResult& result)
+ TEvDataShard::TEvProposeTransactionResult& result)
{
- auto& stats = *result.Record.MutableTxStats();
- auto& perTable = *stats.AddTableAccessStats();
- perTable.MutableTableInfo()->SetSchemeshardId(dataShard.GetPathOwnerId());
- Y_VERIFY(dataShard.GetUserTables().size() == 1, "TODO: Fix handling of collocated tables");
- auto tableInfo = dataShard.GetUserTables().begin();
- perTable.MutableTableInfo()->SetPathId(tableInfo->first);
- perTable.MutableTableInfo()->SetName(tableInfo->second->Path);
- if (counters.NSelectRow) {
- perTable.MutableSelectRow()->SetCount(counters.NSelectRow);
- perTable.MutableSelectRow()->SetRows(counters.SelectRowRows);
- perTable.MutableSelectRow()->SetBytes(counters.SelectRowBytes);
- }
- if (counters.NSelectRange) {
- perTable.MutableSelectRange()->SetCount(counters.NSelectRange);
- perTable.MutableSelectRange()->SetRows(counters.SelectRangeRows);
- perTable.MutableSelectRange()->SetBytes(counters.SelectRangeBytes);
- }
- if (counters.NUpdateRow) {
- perTable.MutableUpdateRow()->SetCount(counters.NUpdateRow);
- perTable.MutableUpdateRow()->SetRows(counters.NUpdateRow);
- perTable.MutableUpdateRow()->SetBytes(counters.UpdateRowBytes);
- }
- if (counters.NEraseRow) {
- perTable.MutableEraseRow()->SetCount(counters.NEraseRow);
- perTable.MutableEraseRow()->SetRows(counters.NEraseRow);
- perTable.MutableEraseRow()->SetBytes(counters.EraseRowBytes);
- }
-}
+ auto& stats = *result.Record.MutableTxStats();
+ auto& perTable = *stats.AddTableAccessStats();
+ perTable.MutableTableInfo()->SetSchemeshardId(dataShard.GetPathOwnerId());
+ Y_VERIFY(dataShard.GetUserTables().size() == 1, "TODO: Fix handling of collocated tables");
+ auto tableInfo = dataShard.GetUserTables().begin();
+ perTable.MutableTableInfo()->SetPathId(tableInfo->first);
+ perTable.MutableTableInfo()->SetName(tableInfo->second->Path);
+ if (counters.NSelectRow) {
+ perTable.MutableSelectRow()->SetCount(counters.NSelectRow);
+ perTable.MutableSelectRow()->SetRows(counters.SelectRowRows);
+ perTable.MutableSelectRow()->SetBytes(counters.SelectRowBytes);
+ }
+ if (counters.NSelectRange) {
+ perTable.MutableSelectRange()->SetCount(counters.NSelectRange);
+ perTable.MutableSelectRange()->SetRows(counters.SelectRangeRows);
+ perTable.MutableSelectRange()->SetBytes(counters.SelectRangeBytes);
+ }
+ if (counters.NUpdateRow) {
+ perTable.MutableUpdateRow()->SetCount(counters.NUpdateRow);
+ perTable.MutableUpdateRow()->SetRows(counters.NUpdateRow);
+ perTable.MutableUpdateRow()->SetBytes(counters.UpdateRowBytes);
+ }
+ if (counters.NEraseRow) {
+ perTable.MutableEraseRow()->SetCount(counters.NEraseRow);
+ perTable.MutableEraseRow()->SetRows(counters.NEraseRow);
+ perTable.MutableEraseRow()->SetBytes(counters.EraseRowBytes);
+ }
+}
void KqpFillStats(TDataShard& dataShard, const NKqp::TKqpTasksRunner& tasksRunner,
- NMiniKQL::TKqpDatashardComputeContext& computeCtx, const NYql::NDqProto::EDqStatsMode& statsMode,
- TEvDataShard::TEvProposeTransactionResult& result)
-{
- Y_VERIFY(dataShard.GetUserTables().size() == 1, "TODO: Fix handling of collocated tables");
- auto tableInfo = dataShard.GetUserTables().begin();
-
- bool withProfileStats = statsMode >= NYql::NDqProto::DQ_STATS_MODE_PROFILE;
-
- auto& computeActorStats = *result.Record.MutableComputeActorStats();
-
- ui64 minFirstRowTimeMs = std::numeric_limits<ui64>::max();
- ui64 maxFinishTimeMs = 0;
-
- for (auto& [taskId, taskStats] : tasksRunner.GetTasksStats()) {
- auto* protoTask = computeActorStats.AddTasks();
- auto stageId = tasksRunner.GetTask(taskId).GetStageId();
- NYql::NDq::FillTaskRunnerStats(taskId, stageId, *taskStats, protoTask, withProfileStats);
-
- auto taskTableStats = computeCtx.GetTaskCounters(taskId);
-
- auto* protoTable = protoTask->AddTables();
- protoTable->SetTablePath(tableInfo->second->Path);
- protoTable->SetReadRows(taskTableStats.SelectRowRows + taskTableStats.SelectRangeRows);
- protoTable->SetReadBytes(taskTableStats.SelectRowBytes + taskTableStats.SelectRangeBytes);
- protoTable->SetWriteRows(taskTableStats.NUpdateRow);
- protoTable->SetWriteBytes(taskTableStats.UpdateRowBytes);
- protoTable->SetEraseRows(taskTableStats.NEraseRow);
-
- { // KQP Extra Stats
- NKqpProto::TKqpShardTableExtraStats tableExtraStats;
- tableExtraStats.SetShardId(dataShard.TabletID());
- // tableExtraStats.SetShardCpuTimeUs(...); // TODO: take it from TTxStats
- protoTable->MutableExtra()->PackFrom(tableExtraStats);
- }
-
- minFirstRowTimeMs = std::min(minFirstRowTimeMs, protoTask->GetFirstRowTimeMs());
- maxFinishTimeMs = std::max(maxFinishTimeMs, protoTask->GetFinishTimeMs());
-
- computeActorStats.SetCpuTimeUs(computeActorStats.GetCpuTimeUs() + protoTask->GetCpuTimeUs());
- }
-
- if (maxFinishTimeMs >= minFirstRowTimeMs) {
- computeActorStats.SetDurationUs((maxFinishTimeMs - minFirstRowTimeMs) * 1'000);
- }
-
- // TODO: fill profile stats
+ NMiniKQL::TKqpDatashardComputeContext& computeCtx, const NYql::NDqProto::EDqStatsMode& statsMode,
+ TEvDataShard::TEvProposeTransactionResult& result)
+{
+ Y_VERIFY(dataShard.GetUserTables().size() == 1, "TODO: Fix handling of collocated tables");
+ auto tableInfo = dataShard.GetUserTables().begin();
+
+ bool withProfileStats = statsMode >= NYql::NDqProto::DQ_STATS_MODE_PROFILE;
+
+ auto& computeActorStats = *result.Record.MutableComputeActorStats();
+
+ ui64 minFirstRowTimeMs = std::numeric_limits<ui64>::max();
+ ui64 maxFinishTimeMs = 0;
+
+ for (auto& [taskId, taskStats] : tasksRunner.GetTasksStats()) {
+ auto* protoTask = computeActorStats.AddTasks();
+ auto stageId = tasksRunner.GetTask(taskId).GetStageId();
+ NYql::NDq::FillTaskRunnerStats(taskId, stageId, *taskStats, protoTask, withProfileStats);
+
+ auto taskTableStats = computeCtx.GetTaskCounters(taskId);
+
+ auto* protoTable = protoTask->AddTables();
+ protoTable->SetTablePath(tableInfo->second->Path);
+ protoTable->SetReadRows(taskTableStats.SelectRowRows + taskTableStats.SelectRangeRows);
+ protoTable->SetReadBytes(taskTableStats.SelectRowBytes + taskTableStats.SelectRangeBytes);
+ protoTable->SetWriteRows(taskTableStats.NUpdateRow);
+ protoTable->SetWriteBytes(taskTableStats.UpdateRowBytes);
+ protoTable->SetEraseRows(taskTableStats.NEraseRow);
+
+ { // KQP Extra Stats
+ NKqpProto::TKqpShardTableExtraStats tableExtraStats;
+ tableExtraStats.SetShardId(dataShard.TabletID());
+ // tableExtraStats.SetShardCpuTimeUs(...); // TODO: take it from TTxStats
+ protoTable->MutableExtra()->PackFrom(tableExtraStats);
+ }
+
+ minFirstRowTimeMs = std::min(minFirstRowTimeMs, protoTask->GetFirstRowTimeMs());
+ maxFinishTimeMs = std::max(maxFinishTimeMs, protoTask->GetFinishTimeMs());
+
+ computeActorStats.SetCpuTimeUs(computeActorStats.GetCpuTimeUs() + protoTask->GetCpuTimeUs());
+ }
+
+ if (maxFinishTimeMs >= minFirstRowTimeMs) {
+ computeActorStats.SetDurationUs((maxFinishTimeMs - minFirstRowTimeMs) * 1'000);
+ }
+
+ // TODO: fill profile stats
}
-NYql::NDq::TDqTaskRunnerMemoryLimits DefaultKqpDataReqMemoryLimits() {
- NYql::NDq::TDqTaskRunnerMemoryLimits memoryLimits;
- // Data queries require output channel to be drained only once, and it must contain complete result
- // (i.e. channel must be Finished).
- // So we have to set such a big buffer size.
+NYql::NDq::TDqTaskRunnerMemoryLimits DefaultKqpDataReqMemoryLimits() {
+ NYql::NDq::TDqTaskRunnerMemoryLimits memoryLimits;
+ // Data queries require output channel to be drained only once, and it must contain complete result
+ // (i.e. channel must be Finished).
+ // So we have to set such a big buffer size.
// @link https://a.yandex-team.ru/arc/trunk/arcadia/ydb/core/tx/datashard/datashard_kqp.cpp?rev=6199480#L196-197
- memoryLimits.ChannelBufferSize = std::numeric_limits<ui32>::max();
- memoryLimits.OutputChunkMaxSize = std::numeric_limits<ui32>::max();
-
- return memoryLimits;
-}
-
-namespace {
-
-class TKqpTaskRunnerExecutionContext : public NDq::IDqTaskRunnerExecutionContext {
-public:
- NDq::IDqOutputConsumer::TPtr CreateOutputConsumer(const NDqProto::TTaskOutput& outputDesc,
- const NMiniKQL::TType* type, NUdf::IApplyContext* applyCtx, const NMiniKQL::TTypeEnvironment& typeEnv,
+ memoryLimits.ChannelBufferSize = std::numeric_limits<ui32>::max();
+ memoryLimits.OutputChunkMaxSize = std::numeric_limits<ui32>::max();
+
+ return memoryLimits;
+}
+
+namespace {
+
+class TKqpTaskRunnerExecutionContext : public NDq::IDqTaskRunnerExecutionContext {
+public:
+ NDq::IDqOutputConsumer::TPtr CreateOutputConsumer(const NDqProto::TTaskOutput& outputDesc,
+ const NMiniKQL::TType* type, NUdf::IApplyContext* applyCtx, const NMiniKQL::TTypeEnvironment& typeEnv,
TVector<NDq::IDqOutput::TPtr>&& outputs) const override
- {
+ {
return NKqp::KqpBuildOutputConsumer(outputDesc, type, applyCtx, typeEnv, std::move(outputs));
- }
-
- NDq::IDqChannelStorage::TPtr CreateChannelStorage(ui64 /* channelId */) const override {
- return {};
- }
-};
-
-} // anonymous namespace
-
-THolder<NYql::NDq::IDqTaskRunnerExecutionContext> DefaultKqpExecutionContext() {
- return THolder(new TKqpTaskRunnerExecutionContext);
-}
-
+ }
+
+ NDq::IDqChannelStorage::TPtr CreateChannelStorage(ui64 /* channelId */) const override {
+ return {};
+ }
+};
+
+} // anonymous namespace
+
+THolder<NYql::NDq::IDqTaskRunnerExecutionContext> DefaultKqpExecutionContext() {
+ return THolder(new TKqpTaskRunnerExecutionContext);
+}
+
} // namespace NDataShard
} // namespace NKikimr
diff --git a/ydb/core/tx/datashard/datashard_kqp.h b/ydb/core/tx/datashard/datashard_kqp.h
index fef355bda3..228f71fd37 100644
--- a/ydb/core/tx/datashard/datashard_kqp.h
+++ b/ydb/core/tx/datashard/datashard_kqp.h
@@ -11,44 +11,44 @@
namespace NKikimr {
namespace NDataShard {
-bool KqpValidateTransaction(const NKikimrTxDataShard::TKqpTransaction& tx, bool isImmediate,
- ui64 txId, const TActorContext& ctx, bool& hasPersistentChannels);
-
-void KqpSetTxKeys(ui64 tabletId, ui64 taskId, const TUserTable* tableInfo,
- const NKikimrTxDataShard::TKqpTransaction_TDataTaskMeta& meta, const NScheme::TTypeRegistry& typeRegistry,
- const TActorContext& ctx, TEngineBay& engineBay);
-
-void KqpSetTxLocksKeys(const NKikimrTxDataShard::TKqpLocks& locks, const TSysLocks& sysLocks, TEngineBay& engineBay);
-
+bool KqpValidateTransaction(const NKikimrTxDataShard::TKqpTransaction& tx, bool isImmediate,
+ ui64 txId, const TActorContext& ctx, bool& hasPersistentChannels);
+
+void KqpSetTxKeys(ui64 tabletId, ui64 taskId, const TUserTable* tableInfo,
+ const NKikimrTxDataShard::TKqpTransaction_TDataTaskMeta& meta, const NScheme::TTypeRegistry& typeRegistry,
+ const TActorContext& ctx, TEngineBay& engineBay);
+
+void KqpSetTxLocksKeys(const NKikimrTxDataShard::TKqpLocks& locks, const TSysLocks& sysLocks, TEngineBay& engineBay);
+
NYql::NDq::ERunStatus KqpRunTransaction(const TActorContext& ctx, ui64 txId,
- const google::protobuf::RepeatedPtrField<NYql::NDqProto::TDqTask>& tasks, NKqp::TKqpTasksRunner& tasksRunner);
+ const google::protobuf::RepeatedPtrField<NYql::NDqProto::TDqTask>& tasks, NKqp::TKqpTasksRunner& tasksRunner);
THolder<TEvDataShard::TEvProposeTransactionResult> KqpCompleteTransaction(const TActorContext& ctx,
- ui64 origin, ui64 txId, const TInputOpData::TInReadSets* inReadSets,
- const google::protobuf::RepeatedPtrField<NYql::NDqProto::TDqTask>& tasks, NKqp::TKqpTasksRunner& tasksRunner,
- const NMiniKQL::TKqpDatashardComputeContext& computeCtx);
+ ui64 origin, ui64 txId, const TInputOpData::TInReadSets* inReadSets,
+ const google::protobuf::RepeatedPtrField<NYql::NDqProto::TDqTask>& tasks, NKqp::TKqpTasksRunner& tasksRunner,
+ const NMiniKQL::TKqpDatashardComputeContext& computeCtx);
-void KqpFillOutReadSets(TOutputOpData::TOutReadSets& outReadSets, const NKikimrTxDataShard::TKqpTransaction& kqpTx,
- NKqp::TKqpTasksRunner& tasksRunner, TSysLocks& sysLocks, ui64 tabletId);
+void KqpFillOutReadSets(TOutputOpData::TOutReadSets& outReadSets, const NKikimrTxDataShard::TKqpTransaction& kqpTx,
+ NKqp::TKqpTasksRunner& tasksRunner, TSysLocks& sysLocks, ui64 tabletId);
void KqpPrepareInReadsets(TInputOpData::TInReadSets& inReadSets,
- const NKikimrTxDataShard::TKqpTransaction& kqpTx, ui64 tabletId);
+ const NKikimrTxDataShard::TKqpTransaction& kqpTx, ui64 tabletId);
+
+bool KqpValidateLocks(ui64 origin, TActiveTransaction* tx, TSysLocks& sysLocks);
-bool KqpValidateLocks(ui64 origin, TActiveTransaction* tx, TSysLocks& sysLocks);
+void KqpEraseLocks(ui64 origin, TActiveTransaction* tx, TSysLocks& sysLocks);
-void KqpEraseLocks(ui64 origin, TActiveTransaction* tx, TSysLocks& sysLocks);
-
void KqpUpdateDataShardStatCounters(TDataShard& dataShard, const NMiniKQL::TEngineHostCounters& counters);
void KqpFillTxStats(TDataShard& dataShard, const NMiniKQL::TEngineHostCounters& counters,
- TEvDataShard::TEvProposeTransactionResult& result);
-
+ TEvDataShard::TEvProposeTransactionResult& result);
+
void KqpFillStats(TDataShard& dataShard, const NKqp::TKqpTasksRunner& tasksRunner,
- NMiniKQL::TKqpDatashardComputeContext& computeCtx, const NYql::NDqProto::EDqStatsMode& statsMode,
- TEvDataShard::TEvProposeTransactionResult& result);
-
-NYql::NDq::TDqTaskRunnerMemoryLimits DefaultKqpDataReqMemoryLimits();
-THolder<NYql::NDq::IDqTaskRunnerExecutionContext> DefaultKqpExecutionContext();
-
+ NMiniKQL::TKqpDatashardComputeContext& computeCtx, const NYql::NDqProto::EDqStatsMode& statsMode,
+ TEvDataShard::TEvProposeTransactionResult& result);
+
+NYql::NDq::TDqTaskRunnerMemoryLimits DefaultKqpDataReqMemoryLimits();
+THolder<NYql::NDq::IDqTaskRunnerExecutionContext> DefaultKqpExecutionContext();
+
} // namespace NDataShard
} // namespace NKikimr
diff --git a/ydb/core/tx/datashard/datashard_kqp_compute.cpp b/ydb/core/tx/datashard/datashard_kqp_compute.cpp
index a7a40727f0..edd6ca8e19 100644
--- a/ydb/core/tx/datashard/datashard_kqp_compute.cpp
+++ b/ydb/core/tx/datashard/datashard_kqp_compute.cpp
@@ -1,5 +1,5 @@
#include "datashard_kqp_compute.h"
-#include "range_ops.h"
+#include "range_ops.h"
#include <ydb/core/kqp/runtime/kqp_transport.h>
#include <ydb/core/kqp/runtime/kqp_read_table.h>
@@ -61,7 +61,7 @@ TComputationNodeFactory GetKqpDatashardComputeFactory(TKqpDatashardComputeContex
};
typedef IComputationNode* (*TCallableScanBuilderFunc)(TCallable& callable,
- const TComputationNodeFactoryContext& ctx, TKqpScanComputeContext& computeCtx);
+ const TComputationNodeFactoryContext& ctx, TKqpScanComputeContext& computeCtx);
struct TKqpScanComputationMap {
TKqpScanComputationMap() {
@@ -183,18 +183,18 @@ TRowVersion TKqpDatashardComputeContext::GetReadVersion() const {
return ReadVersion;
}
-void TKqpDatashardComputeContext::SetTaskOutputChannel(ui64 taskId, ui64 channelId, TActorId actorId) {
- OutputChannels.emplace(std::make_pair(taskId, channelId), actorId);
-}
-
-TActorId TKqpDatashardComputeContext::GetTaskOutputChannel(ui64 taskId, ui64 channelId) const {
- auto it = OutputChannels.find(std::make_pair(taskId, channelId));
- if (it != OutputChannels.end()) {
- return it->second;
- }
- return TActorId();
-}
-
+void TKqpDatashardComputeContext::SetTaskOutputChannel(ui64 taskId, ui64 channelId, TActorId actorId) {
+ OutputChannels.emplace(std::make_pair(taskId, channelId), actorId);
+}
+
+TActorId TKqpDatashardComputeContext::GetTaskOutputChannel(ui64 taskId, ui64 channelId) const {
+ auto it = OutputChannels.find(std::make_pair(taskId, channelId));
+ if (it != OutputChannels.end()) {
+ return it->second;
+ }
+ return TActorId();
+}
+
void TKqpDatashardComputeContext::Clear() {
Database = nullptr;
LockTxId = 0;
@@ -218,62 +218,62 @@ bool TKqpDatashardComputeContext::PinPages(const TVector<IEngineFlat::TValidated
}
};
- bool ret = true;
- auto& scheme = Database->GetScheme();
-
- for (const auto& vKey : keys) {
- const TKeyDesc& key = *vKey.Key;
-
- if (TSysTables::IsSystemTable(key.TableId)) {
- continue;
- }
-
- if (key.RowOperation != TKeyDesc::ERowOperation::Read) {
- continue;
- }
-
- ui64 localTid = GetLocalTableId(key.TableId);
- Y_VERIFY(localTid, "table not exist");
-
- auto* tableInfo = scheme.GetTableInfo(localTid);
- TSmallVec<TRawTypeValue> from;
- TSmallVec<TRawTypeValue> to;
- ConvertTableKeys(scheme, tableInfo, key.Range.From, from, nullptr);
- if (!key.Range.Point) {
- ConvertTableKeys(scheme, tableInfo, key.Range.To, to, nullptr);
- }
-
- TSmallVec<NTable::TTag> columnTags;
- for (const auto& column : key.Columns) {
- if (Y_LIKELY(column.Operation == TKeyDesc::EColumnOperation::Read)) {
- columnTags.push_back(column.Column);
- }
- }
- Y_VERIFY_DEBUG(!columnTags.empty());
-
- bool ready = Database->Precharge(localTid,
- from,
- key.Range.Point ? from : to,
- columnTags,
- 0 /* readFlags */,
+ bool ret = true;
+ auto& scheme = Database->GetScheme();
+
+ for (const auto& vKey : keys) {
+ const TKeyDesc& key = *vKey.Key;
+
+ if (TSysTables::IsSystemTable(key.TableId)) {
+ continue;
+ }
+
+ if (key.RowOperation != TKeyDesc::ERowOperation::Read) {
+ continue;
+ }
+
+ ui64 localTid = GetLocalTableId(key.TableId);
+ Y_VERIFY(localTid, "table not exist");
+
+ auto* tableInfo = scheme.GetTableInfo(localTid);
+ TSmallVec<TRawTypeValue> from;
+ TSmallVec<TRawTypeValue> to;
+ ConvertTableKeys(scheme, tableInfo, key.Range.From, from, nullptr);
+ if (!key.Range.Point) {
+ ConvertTableKeys(scheme, tableInfo, key.Range.To, to, nullptr);
+ }
+
+ TSmallVec<NTable::TTag> columnTags;
+ for (const auto& column : key.Columns) {
+ if (Y_LIKELY(column.Operation == TKeyDesc::EColumnOperation::Read)) {
+ columnTags.push_back(column.Column);
+ }
+ }
+ Y_VERIFY_DEBUG(!columnTags.empty());
+
+ bool ready = Database->Precharge(localTid,
+ from,
+ key.Range.Point ? from : to,
+ columnTags,
+ 0 /* readFlags */,
adjustLimit(key.RangeLimits.ItemsLimit),
adjustLimit(key.RangeLimits.BytesLimit),
- key.Reverse ? NTable::EDirection::Reverse : NTable::EDirection::Forward,
+ key.Reverse ? NTable::EDirection::Reverse : NTable::EDirection::Forward,
GetReadVersion());
-
- LOG_TRACE_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Run precharge on table " << tableInfo->Name
- << ", columns: [" << JoinSeq(", ", columnTags) << "]"
- << ", range: " << DebugPrintRange(key.KeyColumnTypes, key.Range, *AppData()->TypeRegistry)
- << ", itemsLimit: " << key.RangeLimits.ItemsLimit
- << ", bytesLimit: " << key.RangeLimits.BytesLimit
- << ", reverse: " << key.Reverse
- << ", result: " << ready);
-
- ret &= ready;
- }
-
- return ret;
-}
-
-} // namespace NMiniKQL
+
+ LOG_TRACE_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Run precharge on table " << tableInfo->Name
+ << ", columns: [" << JoinSeq(", ", columnTags) << "]"
+ << ", range: " << DebugPrintRange(key.KeyColumnTypes, key.Range, *AppData()->TypeRegistry)
+ << ", itemsLimit: " << key.RangeLimits.ItemsLimit
+ << ", bytesLimit: " << key.RangeLimits.BytesLimit
+ << ", reverse: " << key.Reverse
+ << ", result: " << ready);
+
+ ret &= ready;
+ }
+
+ return ret;
+}
+
+} // namespace NMiniKQL
} // namespace NKikimr
diff --git a/ydb/core/tx/datashard/datashard_kqp_compute.h b/ydb/core/tx/datashard/datashard_kqp_compute.h
index dda946e7c5..4bbb8fa0de 100644
--- a/ydb/core/tx/datashard/datashard_kqp_compute.h
+++ b/ydb/core/tx/datashard/datashard_kqp_compute.h
@@ -18,13 +18,13 @@ namespace NMiniKQL {
class IEngineFlatHost;
-using TKqpTableStats = TEngineHostCounters;
+using TKqpTableStats = TEngineHostCounters;
class TKqpDatashardComputeContext : public TKqpComputeContextBase {
public:
TKqpDatashardComputeContext(NDataShard::TDataShard* shard, TEngineHostCounters& counters, TInstant now)
: Shard(shard)
- , DatashardCounters(counters)
+ , DatashardCounters(counters)
, Now(now) {}
ui64 GetLocalTableId(const TTableId& tableId) const;
@@ -39,38 +39,38 @@ public:
TVector<std::pair<NScheme::TTypeId, TString>> GetKeyColumnsInfo(const TTableId &tableId) const;
THashMap<TString, NScheme::TTypeId> GetKeyColumnsMap(const TTableId &tableId) const;
- void SetHasPersistentChannels(bool value) { PersistentChannels = value; }
- bool HasPersistentChannels() const { return PersistentChannels; }
-
- void SetTaskOutputChannel(ui64 taskId, ui64 channelId, TActorId actorId);
- TActorId GetTaskOutputChannel(ui64 taskId, ui64 channelId) const;
-
+ void SetHasPersistentChannels(bool value) { PersistentChannels = value; }
+ bool HasPersistentChannels() const { return PersistentChannels; }
+
+ void SetTaskOutputChannel(ui64 taskId, ui64 channelId, TActorId actorId);
+ TActorId GetTaskOutputChannel(ui64 taskId, ui64 channelId) const;
+
bool PinPages(const TVector<IEngineFlat::TValidatedKey>& keys, ui64 pageFaultCount = 0);
-
+
void Clear();
void SetReadVersion(TRowVersion readVersion);
TRowVersion GetReadVersion() const;
- TEngineHostCounters& GetTaskCounters(ui64 taskId) { return TaskCounters[taskId]; }
- TEngineHostCounters& GetDatashardCounters() { return DatashardCounters; }
+ TEngineHostCounters& GetTaskCounters(ui64 taskId) { return TaskCounters[taskId]; }
+ TEngineHostCounters& GetDatashardCounters() { return DatashardCounters; }
+
+ void SetTabletNotReady() { Y_VERIFY_DEBUG(!TabletNotReady); TabletNotReady = true; };
+ bool IsTabletNotReady() const { return TabletNotReady; }
- void SetTabletNotReady() { Y_VERIFY_DEBUG(!TabletNotReady); TabletNotReady = true; };
- bool IsTabletNotReady() const { return TabletNotReady; }
-
public:
NTable::TDatabase* Database = nullptr;
private:
NDataShard::TDataShard* Shard;
- std::unordered_map<ui64, TEngineHostCounters> TaskCounters;
- TEngineHostCounters& DatashardCounters;
+ std::unordered_map<ui64, TEngineHostCounters> TaskCounters;
+ TEngineHostCounters& DatashardCounters;
TInstant Now;
ui64 LockTxId = 0;
- bool PersistentChannels = false;
- bool TabletNotReady = false;
+ bool PersistentChannels = false;
+ bool TabletNotReady = false;
TRowVersion ReadVersion = TRowVersion::Min();
- THashMap<std::pair<ui64, ui64>, TActorId> OutputChannels;
+ THashMap<std::pair<ui64, ui64>, TActorId> OutputChannels;
};
class TKqpDatashardApplyContext : public NUdf::IApplyContext {
@@ -84,10 +84,10 @@ bool TryFetchRow(NTable::TTableIt& iterator, NYql::NUdf::TUnboxedValue& row, TCo
TKqpTableStats& tableStats, const TKqpDatashardComputeContext& computeCtx,
const TSmallVec<NTable::TTag>& systemColumnTags, const TSmallVec<bool>& skipNullKeys);
-bool TryFetchRow(NTable::TTableReverseIt& iterator, NYql::NUdf::TUnboxedValue& row, TComputationContext& ctx,
- TKqpTableStats& tableStats, const TKqpDatashardComputeContext& computeCtx,
- const TSmallVec<NTable::TTag>& systemColumnTags, const TSmallVec<bool>& skipNullKeys);
-
+bool TryFetchRow(NTable::TTableReverseIt& iterator, NYql::NUdf::TUnboxedValue& row, TComputationContext& ctx,
+ TKqpTableStats& tableStats, const TKqpDatashardComputeContext& computeCtx,
+ const TSmallVec<NTable::TTag>& systemColumnTags, const TSmallVec<bool>& skipNullKeys);
+
void FetchRow(const TDbTupleRef& dbTuple, NYql::NUdf::TUnboxedValue& row, TComputationContext& ctx,
TKqpTableStats& tableStats, const TKqpDatashardComputeContext& computeCtx,
const TSmallVec<NTable::TTag>& systemColumnTags);
diff --git a/ydb/core/tx/datashard/datashard_kqp_delete_rows.cpp b/ydb/core/tx/datashard/datashard_kqp_delete_rows.cpp
index f8a42e68da..6a6c3d8231 100644
--- a/ydb/core/tx/datashard/datashard_kqp_delete_rows.cpp
+++ b/ydb/core/tx/datashard/datashard_kqp_delete_rows.cpp
@@ -48,13 +48,13 @@ public:
return;
}
- ui64 nEraseRow = Owner.ShardTableStats.NEraseRow;
-
+ ui64 nEraseRow = Owner.ShardTableStats.NEraseRow;
+
engineCtx.Host->EraseRow(Owner.TableId, keyTuple);
-
- if (i64 delta = Owner.ShardTableStats.NEraseRow - nEraseRow; delta > 0) {
- Owner.TaskTableStats.NEraseRow += delta;
- }
+
+ if (i64 delta = Owner.ShardTableStats.NEraseRow - nEraseRow; delta > 0) {
+ Owner.TaskTableStats.NEraseRow += delta;
+ }
};
private:
@@ -93,7 +93,7 @@ public:
}
public:
- TKqpDeleteRowsWrapper(TComputationMutables& mutables, TKqpDatashardComputeContext& computeCtx,
+ TKqpDeleteRowsWrapper(TComputationMutables& mutables, TKqpDatashardComputeContext& computeCtx,
const TTableId& tableId, IComputationNode* rowsNode, TVector<NUdf::TDataTypeId> rowTypes, TVector<ui32> keyIndices, const TTypeEnvironment& env)
: TBase(mutables)
, TableId(tableId)
@@ -101,8 +101,8 @@ public:
, RowTypes(std::move(rowTypes))
, KeyIndices(std::move(keyIndices))
, Env(env)
- , ShardTableStats(computeCtx.GetDatashardCounters())
- , TaskTableStats(computeCtx.GetTaskCounters(computeCtx.GetCurrentTaskId())) {}
+ , ShardTableStats(computeCtx.GetDatashardCounters())
+ , TaskTableStats(computeCtx.GetTaskCounters(computeCtx.GetCurrentTaskId())) {}
private:
void RegisterDependencies() const final {
@@ -115,8 +115,8 @@ private:
const TVector<NUdf::TDataTypeId> RowTypes;
const TVector<ui32> KeyIndices;
const TTypeEnvironment& Env;
- TKqpTableStats& ShardTableStats;
- TKqpTableStats& TaskTableStats;
+ TKqpTableStats& ShardTableStats;
+ TKqpTableStats& TaskTableStats;
};
} // namespace
diff --git a/ydb/core/tx/datashard/datashard_kqp_lookup_table.cpp b/ydb/core/tx/datashard/datashard_kqp_lookup_table.cpp
index 7973bb2feb..24582c2664 100644
--- a/ydb/core/tx/datashard/datashard_kqp_lookup_table.cpp
+++ b/ydb/core/tx/datashard/datashard_kqp_lookup_table.cpp
@@ -63,13 +63,13 @@ TParseLookupTableResult ParseLookupTable(TCallable& callable) {
auto keyTypes = AS_TYPE(TStructType, AS_TYPE(TStreamType, keysNode.GetStaticType())->GetItemType());
result.KeyTypes.resize(keyTypes->GetMembersCount());
for (ui32 i = 0; i < result.KeyTypes.size(); ++i) {
- if (keyTypes->GetMemberType(i)->IsOptional()) {
- auto type = AS_TYPE(TDataType, AS_TYPE(TOptionalType, keyTypes->GetMemberType(i))->GetItemType());
- result.KeyTypes[i] = type->GetSchemeType();
- } else {
- auto type = AS_TYPE(TDataType, keyTypes->GetMemberType(i));
- result.KeyTypes[i] = type->GetSchemeType();
- }
+ if (keyTypes->GetMemberType(i)->IsOptional()) {
+ auto type = AS_TYPE(TDataType, AS_TYPE(TOptionalType, keyTypes->GetMemberType(i))->GetItemType());
+ result.KeyTypes[i] = type->GetSchemeType();
+ } else {
+ auto type = AS_TYPE(TDataType, keyTypes->GetMemberType(i));
+ result.KeyTypes[i] = type->GetSchemeType();
+ }
}
ParseReadColumns(callable.GetType()->GetReturnType(), tagsNode, result.Columns, result.SystemColumns);
@@ -77,8 +77,8 @@ TParseLookupTableResult ParseLookupTable(TCallable& callable) {
return result;
}
-class TKqpLookupRowsWrapper : public TStatelessFlowComputationNode<TKqpLookupRowsWrapper> {
- using TBase = TStatelessFlowComputationNode<TKqpLookupRowsWrapper>;
+class TKqpLookupRowsWrapper : public TStatelessFlowComputationNode<TKqpLookupRowsWrapper> {
+ using TBase = TStatelessFlowComputationNode<TKqpLookupRowsWrapper>;
public:
TKqpLookupRowsWrapper(TComputationMutables& mutables, TKqpDatashardComputeContext& computeCtx,
@@ -91,8 +91,8 @@ public:
, LocalTid(ComputeCtx.GetLocalTableId(ParseResult.TableId))
, ColumnTags(ExtractTags(ParseResult.Columns))
, SystemColumnTags(ExtractTags(ParseResult.SystemColumns))
- , ShardTableStats(ComputeCtx.GetDatashardCounters())
- , TaskTableStats(ComputeCtx.GetTaskCounters(ComputeCtx.GetCurrentTaskId()))
+ , ShardTableStats(ComputeCtx.GetDatashardCounters())
+ , TaskTableStats(ComputeCtx.GetTaskCounters(ComputeCtx.GetCurrentTaskId()))
, TableInfo(ComputeCtx.Database->GetScheme().GetTableInfo(LocalTid))
{
MKQL_ENSURE_S(TableInfo);
@@ -135,8 +135,8 @@ public:
switch (ready) {
case EReady::Page:
- ComputeCtx.SetTabletNotReady();
- return TUnboxedValue::MakeYield();
+ ComputeCtx.SetTabletNotReady();
+ return TUnboxedValue::MakeYield();
case EReady::Gone:
continue;
case EReady::Data:
@@ -149,17 +149,17 @@ public:
TDbTupleRef dbTuple(CellTypes.data(), (*dbRow).data(), dbRow.Size());
TUnboxedValue result;
- TKqpTableStats tableStats;
- FetchRow(dbTuple, result, ctx, tableStats, ComputeCtx, SystemColumnTags);
-
- ShardTableStats.NSelectRow++;
- ShardTableStats.SelectRowRows++;
- ShardTableStats.SelectRowBytes += tableStats.SelectRowBytes;
-
- TaskTableStats.NSelectRow++;
- TaskTableStats.SelectRowRows++;
- TaskTableStats.SelectRowBytes += tableStats.SelectRowBytes;
-
+ TKqpTableStats tableStats;
+ FetchRow(dbTuple, result, ctx, tableStats, ComputeCtx, SystemColumnTags);
+
+ ShardTableStats.NSelectRow++;
+ ShardTableStats.SelectRowRows++;
+ ShardTableStats.SelectRowBytes += tableStats.SelectRowBytes;
+
+ TaskTableStats.NSelectRow++;
+ TaskTableStats.SelectRowRows++;
+ TaskTableStats.SelectRowBytes += tableStats.SelectRowBytes;
+
return result;
}
@@ -190,15 +190,15 @@ private:
ui64 LocalTid;
TSmallVec<TTag> ColumnTags;
TSmallVec<TTag> SystemColumnTags;
- TKqpTableStats& ShardTableStats;
- TKqpTableStats& TaskTableStats;
+ TKqpTableStats& ShardTableStats;
+ TKqpTableStats& TaskTableStats;
const NTable::TScheme::TTableInfo* TableInfo;
TSmallVec<NScheme::TTypeId> CellTypes;
mutable bool Finished = false;
};
-class TKqpLookupTableWrapper : public TStatelessFlowComputationNode<TKqpLookupTableWrapper> {
- using TBase = TStatelessFlowComputationNode<TKqpLookupTableWrapper>;
+class TKqpLookupTableWrapper : public TStatelessFlowComputationNode<TKqpLookupTableWrapper> {
+ using TBase = TStatelessFlowComputationNode<TKqpLookupTableWrapper>;
public:
TKqpLookupTableWrapper(TComputationMutables& mutables, TKqpDatashardComputeContext& computeCtx,
@@ -211,8 +211,8 @@ public:
, LocalTid(ComputeCtx.GetLocalTableId(ParseResult.TableId))
, ColumnTags(ExtractTags(ParseResult.Columns))
, SystemColumnTags(ExtractTags(ParseResult.SystemColumns))
- , ShardTableStats(ComputeCtx.GetDatashardCounters())
- , TaskTableStats(ComputeCtx.GetTaskCounters(computeCtx.GetCurrentTaskId()))
+ , ShardTableStats(ComputeCtx.GetDatashardCounters())
+ , TaskTableStats(ComputeCtx.GetTaskCounters(computeCtx.GetCurrentTaskId()))
, TableInfo(ComputeCtx.Database->GetScheme().GetTableInfo(LocalTid))
{
MKQL_ENSURE_S(TableInfo);
@@ -261,32 +261,32 @@ public:
}
TUnboxedValue result;
- TKqpTableStats tableStats;
- auto fetched = TryFetchRow(*Iterator, result, ctx, tableStats, ComputeCtx, SystemColumnTags, ParseResult.SkipNullKeys);
-
- if (Iterator->Stats.InvisibleRowSkips) {
+ TKqpTableStats tableStats;
+ auto fetched = TryFetchRow(*Iterator, result, ctx, tableStats, ComputeCtx, SystemColumnTags, ParseResult.SkipNullKeys);
+
+ if (Iterator->Stats.InvisibleRowSkips) {
ComputeCtx.BreakSetLocks();
- }
-
- ShardTableStats += tableStats;
- TaskTableStats += tableStats;
-
- ui64 deletedRowSkips = std::exchange(Iterator->Stats.DeletedRowSkips, 0);
- ui64 invisibleRowSkips = std::exchange(Iterator->Stats.InvisibleRowSkips, 0);
-
- ShardTableStats.SelectRangeDeletedRowSkips += deletedRowSkips;
- ShardTableStats.InvisibleRowSkips += invisibleRowSkips;
-
- TaskTableStats.SelectRangeDeletedRowSkips += deletedRowSkips;
- TaskTableStats.InvisibleRowSkips += invisibleRowSkips;
-
+ }
+
+ ShardTableStats += tableStats;
+ TaskTableStats += tableStats;
+
+ ui64 deletedRowSkips = std::exchange(Iterator->Stats.DeletedRowSkips, 0);
+ ui64 invisibleRowSkips = std::exchange(Iterator->Stats.InvisibleRowSkips, 0);
+
+ ShardTableStats.SelectRangeDeletedRowSkips += deletedRowSkips;
+ ShardTableStats.InvisibleRowSkips += invisibleRowSkips;
+
+ TaskTableStats.SelectRangeDeletedRowSkips += deletedRowSkips;
+ TaskTableStats.InvisibleRowSkips += invisibleRowSkips;
+
if (fetched) {
return result;
}
if (Iterator->Last() == NTable::EReady::Page) {
- ComputeCtx.SetTabletNotReady();
- return TUnboxedValue::MakeYield();
+ ComputeCtx.SetTabletNotReady();
+ return TUnboxedValue::MakeYield();
}
Iterator = nullptr;
@@ -308,8 +308,8 @@ private:
ui64 LocalTid;
TSmallVec<TTag> ColumnTags;
TSmallVec<TTag> SystemColumnTags;
- TKqpTableStats& ShardTableStats;
- TKqpTableStats& TaskTableStats;
+ TKqpTableStats& ShardTableStats;
+ TKqpTableStats& TaskTableStats;
const NTable::TScheme::TTableInfo* TableInfo;
mutable TAutoPtr<NTable::TTableIt> Iterator;
mutable bool Finished = false;
@@ -329,9 +329,9 @@ IComputationNode* WrapKqpLookupTableInternal(TCallable& callable, const TComputa
MKQL_ENSURE_S(tableInfo);
if (tableInfo->KeyColumns.size() == parseResult.KeyIndices.size()) {
- return new TKqpLookupRowsWrapper(ctx.Mutables, computeCtx, ctx.Env, parseResult, lookupKeysNode);
+ return new TKqpLookupRowsWrapper(ctx.Mutables, computeCtx, ctx.Env, parseResult, lookupKeysNode);
} else {
- return new TKqpLookupTableWrapper(ctx.Mutables, computeCtx, ctx.Env, parseResult, lookupKeysNode);
+ return new TKqpLookupTableWrapper(ctx.Mutables, computeCtx, ctx.Env, parseResult, lookupKeysNode);
}
}
@@ -340,7 +340,7 @@ IComputationNode* WrapKqpLookupTableInternal(TCallable& callable, const TComputa
IComputationNode* WrapKqpLookupTable(TCallable& callable, const TComputationNodeFactoryContext& ctx,
TKqpDatashardComputeContext& computeCtx)
{
- return WrapKqpLookupTableInternal(callable, ctx, computeCtx);
+ return WrapKqpLookupTableInternal(callable, ctx, computeCtx);
}
} // namespace NMiniKQL
diff --git a/ydb/core/tx/datashard/datashard_kqp_read_table.cpp b/ydb/core/tx/datashard/datashard_kqp_read_table.cpp
index 9a05e9efaa..9fbd8ef0ad 100644
--- a/ydb/core/tx/datashard/datashard_kqp_read_table.cpp
+++ b/ydb/core/tx/datashard/datashard_kqp_read_table.cpp
@@ -14,8 +14,8 @@ namespace NMiniKQL {
using namespace NTable;
using namespace NUdf;
-namespace {
-
+namespace {
+
void ValidateKeyType(const TType* keyType, const std::pair<NScheme::TTypeId, TString>& keyColumn) {
auto type = keyType;
@@ -192,7 +192,7 @@ void CreateRangePoints(ui64 localTid, const TSerializedTableRange& serializedTab
ConvertTableKeys(computeCtx.Database->GetScheme(), tableInfo, tableRange.To, to, nullptr);
}
-template <bool IsReverse>
+template <bool IsReverse>
class TKqpWideReadTableWrapperBase : public TStatelessWideFlowCodegeneratorNode<TKqpWideReadTableWrapperBase<IsReverse>> {
public:
TKqpWideReadTableWrapperBase(TKqpDatashardComputeContext& computeCtx, const TTypeEnvironment& typeEnv,
@@ -202,9 +202,9 @@ public:
, TypeEnv(typeEnv)
, SystemColumnTags(systemColumnTags)
, SkipNullKeys(skipNullKeys)
- , ShardTableStats(ComputeCtx.GetDatashardCounters())
+ , ShardTableStats(ComputeCtx.GetDatashardCounters())
, TaskTableStats(ComputeCtx.GetTaskCounters(ComputeCtx.GetCurrentTaskId())) {
- }
+ }
EFetchResult DoCalculate(TComputationContext& ctx, NUdf::TUnboxedValue* const* output) const {
return this->ReadValue(ctx, output);
@@ -223,20 +223,20 @@ protected:
EFetchResult ReadNext(NUdf::TUnboxedValue* const* output) const {
bool breakLocks = false;
while (Iterator->Next(NTable::ENext::Data) == NTable::EReady::Data) {
- if (!breakLocks && (breakLocks = bool(Iterator->Stats.InvisibleRowSkips))) {
+ if (!breakLocks && (breakLocks = bool(Iterator->Stats.InvisibleRowSkips))) {
ComputeCtx.BreakSetLocks();
- }
+ }
TDbTupleRef rowKey = Iterator->GetKey();
- ui64 deletedRowSkips = std::exchange(Iterator->Stats.DeletedRowSkips, 0);
- ui64 invisibleRowSkips = std::exchange(Iterator->Stats.InvisibleRowSkips, 0);
-
- ShardTableStats.SelectRangeDeletedRowSkips += deletedRowSkips;
- ShardTableStats.InvisibleRowSkips += invisibleRowSkips;
-
- TaskTableStats.SelectRangeDeletedRowSkips += deletedRowSkips;
- TaskTableStats.InvisibleRowSkips += invisibleRowSkips;
-
+ ui64 deletedRowSkips = std::exchange(Iterator->Stats.DeletedRowSkips, 0);
+ ui64 invisibleRowSkips = std::exchange(Iterator->Stats.InvisibleRowSkips, 0);
+
+ ShardTableStats.SelectRangeDeletedRowSkips += deletedRowSkips;
+ ShardTableStats.InvisibleRowSkips += invisibleRowSkips;
+
+ TaskTableStats.SelectRangeDeletedRowSkips += deletedRowSkips;
+ TaskTableStats.InvisibleRowSkips += invisibleRowSkips;
+
Y_VERIFY(SkipNullKeys.size() <= rowKey.ColumnCount);
bool skipRow = false;
for (ui32 i = 0; i < SkipNullKeys.size(); ++i) {
@@ -279,35 +279,35 @@ protected:
}
}
- if (Remains) {
- Remains = *Remains - 1;
- }
-
- ShardTableStats.SelectRangeRows++;
- ShardTableStats.SelectRangeBytes += rowSize;
-
- TaskTableStats.SelectRangeRows++;
- TaskTableStats.SelectRangeBytes += rowSize;
-
+ if (Remains) {
+ Remains = *Remains - 1;
+ }
+
+ ShardTableStats.SelectRangeRows++;
+ ShardTableStats.SelectRangeBytes += rowSize;
+
+ TaskTableStats.SelectRangeRows++;
+ TaskTableStats.SelectRangeBytes += rowSize;
+
return EFetchResult::One;
}
- if (!breakLocks && bool(Iterator->Stats.InvisibleRowSkips)) {
+ if (!breakLocks && bool(Iterator->Stats.InvisibleRowSkips)) {
ComputeCtx.BreakSetLocks();
- }
+ }
+
+ auto deletedRowSkips = std::exchange(Iterator->Stats.DeletedRowSkips, 0);
+ auto invisibleRowSkips = std::exchange(Iterator->Stats.InvisibleRowSkips, 0);
+
+ ShardTableStats.SelectRangeDeletedRowSkips += deletedRowSkips;
+ ShardTableStats.InvisibleRowSkips += invisibleRowSkips;
- auto deletedRowSkips = std::exchange(Iterator->Stats.DeletedRowSkips, 0);
- auto invisibleRowSkips = std::exchange(Iterator->Stats.InvisibleRowSkips, 0);
+ TaskTableStats.SelectRangeDeletedRowSkips += deletedRowSkips;
+ TaskTableStats.InvisibleRowSkips += invisibleRowSkips;
- ShardTableStats.SelectRangeDeletedRowSkips += deletedRowSkips;
- ShardTableStats.InvisibleRowSkips += invisibleRowSkips;
-
- TaskTableStats.SelectRangeDeletedRowSkips += deletedRowSkips;
- TaskTableStats.InvisibleRowSkips += invisibleRowSkips;
-
if (Iterator->Last() == NTable::EReady::Page) {
- ComputeCtx.SetTabletNotReady();
- return EFetchResult::Yield;
+ ComputeCtx.SetTabletNotReady();
+ return EFetchResult::Yield;
}
return EFetchResult::Finish;
@@ -379,18 +379,18 @@ private:
private:
void RegisterDependencies() const final {
- this->FlowDependsOn(FromNode);
- this->FlowDependsOn(ToNode);
+ this->FlowDependsOn(FromNode);
+ this->FlowDependsOn(ToNode);
}
private:
TParseReadTableResult ParseResult;
IComputationNode* FromNode;
IComputationNode* ToNode;
- IComputationNode* ItemsLimit;
+ IComputationNode* ItemsLimit;
ui64 LocalTid;
TSmallVec<TTag> ColumnTags;
- ui64 TaskId;
+ ui64 TaskId;
};
template <bool IsReverse>
@@ -478,7 +478,7 @@ void FetchRowImpl(const TDbTupleRef& dbTuple, TUnboxedValue& row, TComputationCo
const TKqpDatashardComputeContext& computeCtx, const TSmallVec<TTag>& systemColumnTags)
{
size_t columnsCount = dbTuple.ColumnCount + systemColumnTags.size();
-
+
TUnboxedValue* rowItems = nullptr;
row = ctx.HolderFactory.CreateDirectArrayHolder(columnsCount, rowItems);
@@ -501,13 +501,13 @@ void FetchRowImpl(const TDbTupleRef& dbTuple, TUnboxedValue& row, TComputationCo
}
}
- tableStats.NSelectRow++;
- tableStats.SelectRowRows++;
- tableStats.SelectRowBytes += rowSize;
+ tableStats.NSelectRow++;
+ tableStats.SelectRowRows++;
+ tableStats.SelectRowBytes += rowSize;
}
-template <typename TTableIterator>
-bool TryFetchRowImpl(TTableIterator& iterator, TUnboxedValue& row, TComputationContext& ctx, TKqpTableStats& tableStats,
+template <typename TTableIterator>
+bool TryFetchRowImpl(TTableIterator& iterator, TUnboxedValue& row, TComputationContext& ctx, TKqpTableStats& tableStats,
const TKqpDatashardComputeContext& computeCtx, const TSmallVec<TTag>& systemColumnTags,
const TSmallVec<bool>& skipNullKeys)
{
@@ -535,22 +535,22 @@ bool TryFetchRowImpl(TTableIterator& iterator, TUnboxedValue& row, TComputationC
return false;
}
-} // namespace
-
-bool TryFetchRow(TTableIt& iterator, TUnboxedValue& row, TComputationContext& ctx, TKqpTableStats& tableStats,
- const TKqpDatashardComputeContext& computeCtx, const TSmallVec<TTag>& systemColumnTags,
- const TSmallVec<bool>& skipNullKeys)
-{
- return TryFetchRowImpl(iterator, row, ctx, tableStats, computeCtx, systemColumnTags, skipNullKeys);
-}
-
-bool TryFetchRow(TTableReverseIt& iterator, TUnboxedValue& row, TComputationContext& ctx, TKqpTableStats& tableStats,
- const TKqpDatashardComputeContext& computeCtx, const TSmallVec<TTag>& systemColumnTags,
- const TSmallVec<bool>& skipNullKeys)
-{
- return TryFetchRowImpl(iterator, row, ctx, tableStats, computeCtx, systemColumnTags, skipNullKeys);
-}
-
+} // namespace
+
+bool TryFetchRow(TTableIt& iterator, TUnboxedValue& row, TComputationContext& ctx, TKqpTableStats& tableStats,
+ const TKqpDatashardComputeContext& computeCtx, const TSmallVec<TTag>& systemColumnTags,
+ const TSmallVec<bool>& skipNullKeys)
+{
+ return TryFetchRowImpl(iterator, row, ctx, tableStats, computeCtx, systemColumnTags, skipNullKeys);
+}
+
+bool TryFetchRow(TTableReverseIt& iterator, TUnboxedValue& row, TComputationContext& ctx, TKqpTableStats& tableStats,
+ const TKqpDatashardComputeContext& computeCtx, const TSmallVec<TTag>& systemColumnTags,
+ const TSmallVec<bool>& skipNullKeys)
+{
+ return TryFetchRowImpl(iterator, row, ctx, tableStats, computeCtx, systemColumnTags, skipNullKeys);
+}
+
void FetchRow(const TDbTupleRef& dbTuple, TUnboxedValue& row, TComputationContext& ctx, TKqpTableStats& tableStats,
const TKqpDatashardComputeContext& computeCtx, const TSmallVec<TTag>& systemColumnTags)
{
@@ -593,16 +593,16 @@ IComputationNode* WrapKqpWideReadTable(TCallable& callable, const TComputationNo
ValidateKeyTuple(parseResult.FromTuple->GetType(), keyColumns);
ValidateKeyTuple(parseResult.ToTuple->GetType(), keyColumns);
- IComputationNode* itemsLimit = nullptr;
- if (parseResult.ItemsLimit) {
- itemsLimit = LocateNode(ctx.NodeLocator, *parseResult.ItemsLimit);
- }
-
- if (parseResult.Reverse) {
- return new TKqpWideReadTableWrapper<true>(computeCtx, ctx.Env, parseResult, fromNode, toNode, itemsLimit);
- }
-
- return new TKqpWideReadTableWrapper<false>(computeCtx, ctx.Env, parseResult, fromNode, toNode, itemsLimit);
+ IComputationNode* itemsLimit = nullptr;
+ if (parseResult.ItemsLimit) {
+ itemsLimit = LocateNode(ctx.NodeLocator, *parseResult.ItemsLimit);
+ }
+
+ if (parseResult.Reverse) {
+ return new TKqpWideReadTableWrapper<true>(computeCtx, ctx.Env, parseResult, fromNode, toNode, itemsLimit);
+ }
+
+ return new TKqpWideReadTableWrapper<false>(computeCtx, ctx.Env, parseResult, fromNode, toNode, itemsLimit);
}
} // namespace NMiniKQL
diff --git a/ydb/core/tx/datashard/datashard_kqp_upsert_rows.cpp b/ydb/core/tx/datashard/datashard_kqp_upsert_rows.cpp
index a8463aa2d1..a2dfa61bbf 100644
--- a/ydb/core/tx/datashard/datashard_kqp_upsert_rows.cpp
+++ b/ydb/core/tx/datashard/datashard_kqp_upsert_rows.cpp
@@ -73,21 +73,21 @@ public:
}
// NOTE: We have to copy values here as some values inlined in TUnboxedValue
- // cannot be inlined in TCell.
+ // cannot be inlined in TCell.
command.Value = MakeCell(type, value, Owner.Env, true);
commands.emplace_back(std::move(command));
}
- ui64 nUpdateRow = Owner.ShardTableStats.NUpdateRow;
- ui64 updateRowBytes = Owner.ShardTableStats.UpdateRowBytes;
-
+ ui64 nUpdateRow = Owner.ShardTableStats.NUpdateRow;
+ ui64 updateRowBytes = Owner.ShardTableStats.UpdateRowBytes;
+
dsApplyCtx.Host->UpdateRow(Owner.TableId, keyTuple, commands);
-
- if (i64 delta = Owner.ShardTableStats.NUpdateRow - nUpdateRow; delta > 0) {
- Owner.TaskTableStats.NUpdateRow += delta;
- Owner.TaskTableStats.UpdateRowBytes += Owner.ShardTableStats.UpdateRowBytes - updateRowBytes;
- }
+
+ if (i64 delta = Owner.ShardTableStats.NUpdateRow - nUpdateRow; delta > 0) {
+ Owner.TaskTableStats.NUpdateRow += delta;
+ Owner.TaskTableStats.UpdateRowBytes += Owner.ShardTableStats.UpdateRowBytes - updateRowBytes;
+ }
};
private:
@@ -126,7 +126,7 @@ public:
}
public:
- TKqpUpsertRowsWrapper(TComputationMutables& mutables, TKqpDatashardComputeContext& computeCtx,
+ TKqpUpsertRowsWrapper(TComputationMutables& mutables, TKqpDatashardComputeContext& computeCtx,
const TTableId& tableId, IComputationNode* rowsNode, TVector<NUdf::TDataTypeId>&& rowTypes,
TVector<ui32>&& keyIndices, TVector<TUpsertColumn>&& upsertColumns, const TTypeEnvironment& env)
: TBase(mutables)
@@ -135,9 +135,9 @@ public:
, RowTypes(std::move(rowTypes))
, KeyIndices(std::move(keyIndices))
, UpsertColumns(std::move(upsertColumns))
- , Env(env)
- , ShardTableStats(computeCtx.GetDatashardCounters())
- , TaskTableStats(computeCtx.GetTaskCounters(computeCtx.GetCurrentTaskId())) {}
+ , Env(env)
+ , ShardTableStats(computeCtx.GetDatashardCounters())
+ , TaskTableStats(computeCtx.GetTaskCounters(computeCtx.GetCurrentTaskId())) {}
private:
void RegisterDependencies() const final {
@@ -151,8 +151,8 @@ private:
TVector<ui32> KeyIndices;
TVector<TUpsertColumn> UpsertColumns;
const TTypeEnvironment& Env;
- TKqpTableStats& ShardTableStats;
- TKqpTableStats& TaskTableStats;
+ TKqpTableStats& ShardTableStats;
+ TKqpTableStats& TaskTableStats;
};
} // namespace
diff --git a/ydb/core/tx/datashard/datashard_pipeline.cpp b/ydb/core/tx/datashard/datashard_pipeline.cpp
index d9b6455c07..3498162c17 100644
--- a/ydb/core/tx/datashard/datashard_pipeline.cpp
+++ b/ydb/core/tx/datashard/datashard_pipeline.cpp
@@ -1213,7 +1213,7 @@ TOperation::TPtr TPipeline::BuildOperation(TEvDataShard::TEvProposeTransaction::
} else {
Y_VERIFY(tx->IsReadTable() || tx->IsDataTx());
auto dataTx = tx->BuildDataTx(Self, txc, ctx);
- if (dataTx->Ready() && (dataTx->ProgramSize() || dataTx->IsKqpDataTx()))
+ if (dataTx->Ready() && (dataTx->ProgramSize() || dataTx->IsKqpDataTx()))
dataTx->ExtractKeys(true);
if (!dataTx->Ready() && !dataTx->RequirePrepare()) {
@@ -1237,7 +1237,7 @@ TOperation::TPtr TPipeline::BuildOperation(TEvDataShard::TEvProposeTransaction::
tx->SetReadOnlyFlag();
if (dataTx->NeedDiagnostics())
tx->SetNeedDiagnosticsFlag();
- if (dataTx->IsKqpDataTx())
+ if (dataTx->IsKqpDataTx())
tx->SetKqpDataTransactionFlag();
if (dataTx->IsKqpScanTx()) {
tx->SetKqpScanTransactionFlag();
@@ -1332,7 +1332,7 @@ void TPipeline::BuildDataTx(TActiveTransaction *tx, TTransactionContext &txc, co
Y_VERIFY(dataTx->Ready());
// TODO: we should have no requirement to have keys
// for restarted immediate tx.
- if (dataTx->ProgramSize() || dataTx->IsKqpDataTx())
+ if (dataTx->ProgramSize() || dataTx->IsKqpDataTx())
dataTx->ExtractKeys(false);
}
diff --git a/ydb/core/tx/datashard/datashard_pipeline.h b/ydb/core/tx/datashard/datashard_pipeline.h
index 9715125f74..67c35260b2 100644
--- a/ydb/core/tx/datashard/datashard_pipeline.h
+++ b/ydb/core/tx/datashard/datashard_pipeline.h
@@ -307,19 +307,19 @@ public:
void StartStreamingTx(ui64 txId, ui32 count) {
ActiveStreamingTxs[txId] += count;
- }
-
- bool FinishStreamingTx(ui64 txId) {
- if (auto it = ActiveStreamingTxs.find(txId); it != ActiveStreamingTxs.end()) {
- it->second--;
- if (it->second == 0) {
- ActiveStreamingTxs.erase(it);
- }
- return true;
- }
- return false;
- }
-
+ }
+
+ bool FinishStreamingTx(ui64 txId) {
+ if (auto it = ActiveStreamingTxs.find(txId); it != ActiveStreamingTxs.end()) {
+ it->second--;
+ if (it->second == 0) {
+ ActiveStreamingTxs.erase(it);
+ }
+ return true;
+ }
+ return false;
+ }
+
bool HasWaitingSchemeOps() const { return !WaitingSchemeOps.empty(); }
bool AddWaitingSchemeOp(const TOperation::TPtr& op);
@@ -449,7 +449,7 @@ private:
TOperation::TPtr NextActiveOp;
// Slow operation profiles.
TList<TStoredExecutionProfile> SlowOpProfiles;
- TMap<ui64, ui32> ActiveStreamingTxs;
+ TMap<ui64, ui32> ActiveStreamingTxs;
typedef TList<TOperation::TPtr> TWaitingSchemeOpsOrder;
typedef THashMap<TOperation::TPtr, TWaitingSchemeOpsOrder::iterator> TWaitingSchemeOps;
diff --git a/ydb/core/tx/datashard/datashard_ut_common.h b/ydb/core/tx/datashard/datashard_ut_common.h
index 935aea61d1..4b5d7fe4ea 100644
--- a/ydb/core/tx/datashard/datashard_ut_common.h
+++ b/ydb/core/tx/datashard/datashard_ut_common.h
@@ -105,7 +105,7 @@ private:
TActorId Sender;
ui64 LastTxId;
ui64 LastStep;
- TMockDbSchemeResolver DbSchemeResolver;
+ TMockDbSchemeResolver DbSchemeResolver;
TString DispatchName = "NONE";
bool AllowIncompleteResult = false;
bool* ActiveZone = nullptr;
@@ -437,20 +437,20 @@ struct TShardedTableOptions {
#define Y_UNIT_TEST_WITH_MVCC(N) Y_UNIT_TEST_WITH_MVCC_IMPL(N, UseMvcc)
#define WithMvcc UseMvcc
-#define Y_UNIT_TEST_QUAD(N, OPT1, OPT2) \
- template<bool OPT1, bool OPT2> void N(NUnitTest::TTestContext&); \
- struct TTestRegistration##N { \
- TTestRegistration##N() { \
- TCurrentTest::AddTest(#N "-" #OPT1 "-" #OPT2, static_cast<void (*)(NUnitTest::TTestContext&)>(&N<false, false>), false); \
- TCurrentTest::AddTest(#N "+" #OPT1 "-" #OPT2, static_cast<void (*)(NUnitTest::TTestContext&)>(&N<true, false>), false); \
- TCurrentTest::AddTest(#N "-" #OPT1 "+" #OPT2, static_cast<void (*)(NUnitTest::TTestContext&)>(&N<false, true>), false); \
- TCurrentTest::AddTest(#N "+" #OPT1 "+" #OPT2, static_cast<void (*)(NUnitTest::TTestContext&)>(&N<true, true>), false); \
- } \
- }; \
- static TTestRegistration##N testRegistration##N; \
- template<bool OPT1, bool OPT2> \
- void N(NUnitTest::TTestContext&)
-
+#define Y_UNIT_TEST_QUAD(N, OPT1, OPT2) \
+ template<bool OPT1, bool OPT2> void N(NUnitTest::TTestContext&); \
+ struct TTestRegistration##N { \
+ TTestRegistration##N() { \
+ TCurrentTest::AddTest(#N "-" #OPT1 "-" #OPT2, static_cast<void (*)(NUnitTest::TTestContext&)>(&N<false, false>), false); \
+ TCurrentTest::AddTest(#N "+" #OPT1 "-" #OPT2, static_cast<void (*)(NUnitTest::TTestContext&)>(&N<true, false>), false); \
+ TCurrentTest::AddTest(#N "-" #OPT1 "+" #OPT2, static_cast<void (*)(NUnitTest::TTestContext&)>(&N<false, true>), false); \
+ TCurrentTest::AddTest(#N "+" #OPT1 "+" #OPT2, static_cast<void (*)(NUnitTest::TTestContext&)>(&N<true, true>), false); \
+ } \
+ }; \
+ static TTestRegistration##N testRegistration##N; \
+ template<bool OPT1, bool OPT2> \
+ void N(NUnitTest::TTestContext&)
+
void CreateShardedTable(Tests::TServer::TPtr server,
TActorId sender,
const TString &root,
diff --git a/ydb/core/tx/datashard/datashard_ut_common_kqp.h b/ydb/core/tx/datashard/datashard_ut_common_kqp.h
index 4972cfa77d..b8a7873652 100644
--- a/ydb/core/tx/datashard/datashard_ut_common_kqp.h
+++ b/ydb/core/tx/datashard/datashard_ut_common_kqp.h
@@ -124,21 +124,21 @@ namespace NKqpHelpers {
return ev->Get()->Record.GetResponse().GetSessionId();
}
- inline THolder<NKqp::TEvKqp::TEvQueryRequest> MakeStreamRequest(
+ inline THolder<NKqp::TEvKqp::TEvQueryRequest> MakeStreamRequest(
const TActorId sender,
- const TString& sql,
- const bool collectStats = false)
- {
- auto request = MakeHolder<NKqp::TEvKqp::TEvQueryRequest>();
- request->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE);
+ const TString& sql,
+ const bool collectStats = false)
+ {
+ auto request = MakeHolder<NKqp::TEvKqp::TEvQueryRequest>();
+ request->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE);
request->Record.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_SQL_SCAN);
- request->Record.MutableRequest()->SetKeepSession(false);
- request->Record.MutableRequest()->SetQuery(sql);
- request->Record.MutableRequest()->SetProfile(collectStats);
+ request->Record.MutableRequest()->SetKeepSession(false);
+ request->Record.MutableRequest()->SetQuery(sql);
+ request->Record.MutableRequest()->SetProfile(collectStats);
ActorIdToProto(sender, request->Record.MutableRequestActorId());
- return request;
- }
-
+ return request;
+ }
+
} // namespace NKqpHelpers
} // namespace NDataShard
} // namespace NKikimr
diff --git a/ydb/core/tx/datashard/datashard_ut_kqp.cpp b/ydb/core/tx/datashard/datashard_ut_kqp.cpp
index c3dfec2321..d3145951a4 100644
--- a/ydb/core/tx/datashard/datashard_ut_kqp.cpp
+++ b/ydb/core/tx/datashard/datashard_ut_kqp.cpp
@@ -1,484 +1,484 @@
-#include "datashard_ut_common.h"
-#include "datashard_ut_common_kqp.h"
-
+#include "datashard_ut_common.h"
+#include "datashard_ut_common_kqp.h"
+
#include <ydb/core/kqp/executer/kqp_executer.h>
#include <ydb/core/testlib/tenant_runtime.h>
#include <ydb/public/sdk/cpp/client/ydb_result/result.h>
#include <ydb/public/lib/yson_value/ydb_yson_value.h>
-
+
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h>
-
-using namespace NKikimr;
+
+using namespace NKikimr;
using namespace NKikimr::NDataShard;
using namespace NKikimr::NDataShard::NKqpHelpers;
-using namespace Tests;
-using namespace NKqp;
-using namespace NYql;
-using namespace NYql::NDq;
-
-
-namespace {
-
-TString ReformatYson(const TString& yson) {
- TStringStream ysonInput(yson);
- TStringStream output;
+using namespace Tests;
+using namespace NKqp;
+using namespace NYql;
+using namespace NYql::NDq;
+
+
+namespace {
+
+TString ReformatYson(const TString& yson) {
+ TStringStream ysonInput(yson);
+ TStringStream output;
NYson::ReformatYsonStream(&ysonInput, &output, NYson::EYsonFormat::Text);
- return output.Str();
-}
-
-ui32 CalcDrops(const NDqProto::TDqExecutionStats& profile) {
- ui32 count = 0;
- for (auto& stage : profile.GetStages()) {
- for (auto& ca : stage.GetComputeActors()) {
- for (auto& task: ca.GetTasks()) {
- for (auto& inputChannels: task.GetInputChannels()) {
- count += inputChannels.GetResentMessages();
- }
- for (auto& outputChannels: task.GetOutputChannels()) {
- count += outputChannels.GetResentMessages();
- }
- }
- }
- }
- return count;
-}
-
-} // namespace
-
-class KqpStabilityTests : public TTestBase {
-public:
- void SetUp() override;
-
- void TearDown() override {
- GetDqExecutionSettingsForTests().Reset();
- }
-
- void BeforeTest(const char* test) {
- Cerr << "-- Before test " << test << Endl;
- AppCfg.MutableTableServiceConfig()->MutableResourceManager()->SetChannelBufferSize(1);
+ return output.Str();
+}
+
+ui32 CalcDrops(const NDqProto::TDqExecutionStats& profile) {
+ ui32 count = 0;
+ for (auto& stage : profile.GetStages()) {
+ for (auto& ca : stage.GetComputeActors()) {
+ for (auto& task: ca.GetTasks()) {
+ for (auto& inputChannels: task.GetInputChannels()) {
+ count += inputChannels.GetResentMessages();
+ }
+ for (auto& outputChannels: task.GetOutputChannels()) {
+ count += outputChannels.GetResentMessages();
+ }
+ }
+ }
+ }
+ return count;
+}
+
+} // namespace
+
+class KqpStabilityTests : public TTestBase {
+public:
+ void SetUp() override;
+
+ void TearDown() override {
+ GetDqExecutionSettingsForTests().Reset();
+ }
+
+ void BeforeTest(const char* test) {
+ Cerr << "-- Before test " << test << Endl;
+ AppCfg.MutableTableServiceConfig()->MutableResourceManager()->SetChannelBufferSize(1);
if (test == "AbortOnDisconnect"sv) {
- AppCfg.MutableTableServiceConfig()->MutableResourceManager()->SetScanBufferSize(1);
- }
- TTestBase::BeforeTest(test);
- }
-
+ AppCfg.MutableTableServiceConfig()->MutableResourceManager()->SetScanBufferSize(1);
+ }
+ TTestBase::BeforeTest(test);
+ }
+
// disable due to https://a.yandex-team.ru/arc/trunk/arcadia/ydb/core/testlib/test_client.cpp#L448
- UNIT_TEST_SUITE(KqpStabilityTests);
-// UNIT_TEST(DropChannelDataWithUnreachable);
-// UNIT_TEST(DropChannelDataWithUnknownActor);
-// UNIT_TEST(DropChannelDataWithDisconnected); // TEvKqpCompute::TEvChannelsInfo can be lost
-// UNIT_TEST(DropChannelDataAckWithUnreachable);
-// UNIT_TEST(DropChannelDataAckWithUnknownActor);
-// UNIT_TEST(DropChannelDataAckWithDisconnected); // TEvKqpCompute::TEvChannelsInfo can be lost
-// UNIT_TEST(AbortOnDisconnect);
- UNIT_TEST_SUITE_END();
-
- void DropChannelDataWithUnreachable();
- void DropChannelDataWithUnknownActor();
- void DropChannelDataWithDisconnected();
- void DropChannelDataAckWithUnreachable();
- void DropChannelDataAckWithUnknownActor();
- void DropChannelDataAckWithDisconnected();
- void AbortOnDisconnect();
-
-private:
- struct TDropInfo {
- TString EventName;
- ui32 EventType = 0;
- ui64 ChannelId = 0;
- ui64 SeqNo = 0;
- TMaybe<TEvents::TEvUndelivered::EReason> Reason;
- std::function<ui64(NActors::IEventHandle*)> SeqNoExtractor;
- i32 Counter = 1;
- ui32 DroppedEvents = 0;
-
- ui32 StartedScans = 0;
- bool ActionDone = false;
-
- bool Matches(TAutoPtr<NActors::IEventHandle>& event) const {
- return event->GetTypeRewrite() == EventType
- && event->Cookie == ChannelId
- && (!SeqNoExtractor || SeqNoExtractor(event.Get()) == SeqNo);
- }
-
- void Attach(NActors::TTestActorRuntime* runtime) {
- runtime->SetObserverFunc(
- [this](NActors::TTestActorRuntimeBase& rt, TAutoPtr<NActors::IEventHandle>& event) {
-
- if (event->GetTypeRewrite() == TEvDataShard::TEvProposeTransactionResult::EventType) {
- auto status = event.Get()->Get<TEvDataShard::TEvProposeTransactionResult>()->GetStatus();
- if (status == NKikimrTxDataShard::TEvProposeTransactionResult::COMPLETE) {
- StartedScans++;
- if (StartedScans == 3 && !ActionDone && Counter <= 0 && !Reason) {
- auto edge = rt.AllocateEdgeActor(0);
- rt.Send(new IEventHandle(rt.GetInterconnectProxy(0, 1), edge,
- new TEvInterconnect::TEvPoisonSession), 0, true);
- ActionDone = true;
- }
- }
- }
-
- if (Counter > 0 && Matches(event)) {
- Cerr << "-- DROP " << EventName << ", channel: " << ChannelId << ", seqNo: " << SeqNo << Endl;
-
- ++DroppedEvents;
- Counter--;
-
- if (Reason) {
- auto evUndelivered = new TEvents::TEvUndelivered(event->GetTypeRewrite(), *Reason);
+ UNIT_TEST_SUITE(KqpStabilityTests);
+// UNIT_TEST(DropChannelDataWithUnreachable);
+// UNIT_TEST(DropChannelDataWithUnknownActor);
+// UNIT_TEST(DropChannelDataWithDisconnected); // TEvKqpCompute::TEvChannelsInfo can be lost
+// UNIT_TEST(DropChannelDataAckWithUnreachable);
+// UNIT_TEST(DropChannelDataAckWithUnknownActor);
+// UNIT_TEST(DropChannelDataAckWithDisconnected); // TEvKqpCompute::TEvChannelsInfo can be lost
+// UNIT_TEST(AbortOnDisconnect);
+ UNIT_TEST_SUITE_END();
+
+ void DropChannelDataWithUnreachable();
+ void DropChannelDataWithUnknownActor();
+ void DropChannelDataWithDisconnected();
+ void DropChannelDataAckWithUnreachable();
+ void DropChannelDataAckWithUnknownActor();
+ void DropChannelDataAckWithDisconnected();
+ void AbortOnDisconnect();
+
+private:
+ struct TDropInfo {
+ TString EventName;
+ ui32 EventType = 0;
+ ui64 ChannelId = 0;
+ ui64 SeqNo = 0;
+ TMaybe<TEvents::TEvUndelivered::EReason> Reason;
+ std::function<ui64(NActors::IEventHandle*)> SeqNoExtractor;
+ i32 Counter = 1;
+ ui32 DroppedEvents = 0;
+
+ ui32 StartedScans = 0;
+ bool ActionDone = false;
+
+ bool Matches(TAutoPtr<NActors::IEventHandle>& event) const {
+ return event->GetTypeRewrite() == EventType
+ && event->Cookie == ChannelId
+ && (!SeqNoExtractor || SeqNoExtractor(event.Get()) == SeqNo);
+ }
+
+ void Attach(NActors::TTestActorRuntime* runtime) {
+ runtime->SetObserverFunc(
+ [this](NActors::TTestActorRuntimeBase& rt, TAutoPtr<NActors::IEventHandle>& event) {
+
+ if (event->GetTypeRewrite() == TEvDataShard::TEvProposeTransactionResult::EventType) {
+ auto status = event.Get()->Get<TEvDataShard::TEvProposeTransactionResult>()->GetStatus();
+ if (status == NKikimrTxDataShard::TEvProposeTransactionResult::COMPLETE) {
+ StartedScans++;
+ if (StartedScans == 3 && !ActionDone && Counter <= 0 && !Reason) {
+ auto edge = rt.AllocateEdgeActor(0);
+ rt.Send(new IEventHandle(rt.GetInterconnectProxy(0, 1), edge,
+ new TEvInterconnect::TEvPoisonSession), 0, true);
+ ActionDone = true;
+ }
+ }
+ }
+
+ if (Counter > 0 && Matches(event)) {
+ Cerr << "-- DROP " << EventName << ", channel: " << ChannelId << ", seqNo: " << SeqNo << Endl;
+
+ ++DroppedEvents;
+ Counter--;
+
+ if (Reason) {
+ auto evUndelivered = new TEvents::TEvUndelivered(event->GetTypeRewrite(), *Reason);
auto handle = new IEventHandle(event->Sender, TActorId(), evUndelivered, 0, event->Cookie);
- rt.Send(handle, 0, true);
- ActionDone = true;
- } else {
- if (Counter <= 0 && StartedScans == 3 && !ActionDone) {
- auto edge = rt.AllocateEdgeActor(0);
- rt.Send(new IEventHandle(rt.GetInterconnectProxy(0, 1), edge,
- new TEvInterconnect::TEvPoisonSession), 0, true);
- ActionDone = true;
- }
- }
-
- return NActors::TTestActorRuntime::EEventAction::DROP;
- }
-
- if (Matches(event) && Counter <= 0 && !ActionDone) {
- return NActors::TTestActorRuntime::EEventAction::DROP;
- }
-
- return NActors::TTestActorRuntime::EEventAction::PROCESS;
- });
- }
- };
-
- void DoRun(TDropInfo& dropInfo, TMaybe<TString> error) {
- dropInfo.Attach(Server->GetRuntime());
-
- TMaybe<NDqProto::TDqExecutionStats> profile;
-
- auto client = Server->GetRuntime()->AllocateEdgeActor();
- SendRequest(*Server->GetRuntime(), client, MakeStreamRequest(client, Query, true));
-
- TStringStream out;
+ rt.Send(handle, 0, true);
+ ActionDone = true;
+ } else {
+ if (Counter <= 0 && StartedScans == 3 && !ActionDone) {
+ auto edge = rt.AllocateEdgeActor(0);
+ rt.Send(new IEventHandle(rt.GetInterconnectProxy(0, 1), edge,
+ new TEvInterconnect::TEvPoisonSession), 0, true);
+ ActionDone = true;
+ }
+ }
+
+ return NActors::TTestActorRuntime::EEventAction::DROP;
+ }
+
+ if (Matches(event) && Counter <= 0 && !ActionDone) {
+ return NActors::TTestActorRuntime::EEventAction::DROP;
+ }
+
+ return NActors::TTestActorRuntime::EEventAction::PROCESS;
+ });
+ }
+ };
+
+ void DoRun(TDropInfo& dropInfo, TMaybe<TString> error) {
+ dropInfo.Attach(Server->GetRuntime());
+
+ TMaybe<NDqProto::TDqExecutionStats> profile;
+
+ auto client = Server->GetRuntime()->AllocateEdgeActor();
+ SendRequest(*Server->GetRuntime(), client, MakeStreamRequest(client, Query, true));
+
+ TStringStream out;
NYson::TYsonWriter writer(&out, NYson::EYsonFormat::Text, ::NYson::EYsonType::Node, true);
- writer.OnBeginList();
-
- while (true) {
- TAutoPtr<IEventHandle> handle;
- auto replies = Server->GetRuntime()->GrabEdgeEventsRethrow<TEvKqp::TEvQueryResponse, TEvKqp::TEvAbortExecution,
- TEvKqpExecuter::TEvStreamData, TEvKqpExecuter::TEvStreamProfile>(handle);
-
- if (auto* ev = std::get<TEvKqp::TEvQueryResponse*>(replies)) {
- auto& response = ev->Record.GetRef();
-
- if (!error) {
- UNIT_ASSERT_EQUAL_C(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS, response.Utf8DebugString());
- UNIT_ASSERT(response.GetResponse().GetResults().empty());
-
- writer.OnEndList();
- UNIT_ASSERT_STRINGS_EQUAL(ReformatYson(R"(
- [[[1u];[1u]];
- [[10u];[1u]];
- [[100u];[1u]];
- [[1000u];[1u]];
- [[4000000001u];[1u]];
- [[4000000011u];[1u]];
- [[4000000101u];[1u]];
- [[4000001001u];[1u]]]
- )"), ReformatYson(out.Str()));
-
- UNIT_ASSERT(profile.Defined());
- UNIT_ASSERT(dropInfo.DroppedEvents > 0);
- UNIT_ASSERT(CalcDrops(*profile) > 0);
- //UNIT_ASSERT_C(dropInfo.DroppedEvents >= CalcDrops(*profile), dropCount << " !>= " << CalcDrops(*profile));
- } else {
- UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::GENERIC_ERROR);
- UNIT_ASSERT_STRINGS_EQUAL(response.GetResponse().GetQueryIssues()[0].issues()[0].message(), *error);
- }
- break;
- }
-
- if (auto* ev = std::get<TEvKqp::TEvAbortExecution*>(replies)) {
- UNIT_FAIL(ev->Record.GetMessage());
- }
-
- if (auto* ev = std::get<TEvKqpExecuter::TEvStreamData*>(replies)) {
- NYdb::TResultSet result = ev->Record.resultset();
- auto columns = result.GetColumnsMeta();
-
- NYdb::TResultSetParser parser(result);
- while (parser.TryNextRow()) {
- writer.OnListItem();
- writer.OnBeginList();
- for (ui32 i = 0; i < columns.size(); ++i) {
- writer.OnListItem();
- FormatValueYson(parser.GetValue(i), writer);
- }
- writer.OnEndList();
- }
-
- continue;
- }
-
- if (auto* ev = std::get<TEvKqpExecuter::TEvStreamProfile*>(replies)) {
- profile = ev->Record.profile();
- continue;
- }
- }
- }
-
- NKikimrConfig::TAppConfig AppCfg;
- TPortManager PortManager;
- TServer::TPtr Server;
- TString Query;
-};
-UNIT_TEST_SUITE_REGISTRATION(KqpStabilityTests);
-
-void KqpStabilityTests::SetUp() {
- Cerr << "-- SetUp\n";
- auto serverSettings = TServerSettings(PortManager.GetPort(2134))
- .SetDomainName("Root")
- .SetNodeCount(2)
- .SetUseRealThreads(false)
- .SetAppConfig(AppCfg);
-
- Server = new Tests::TServer(serverSettings);
- auto& runtime = *Server->GetRuntime();
-
-// runtime.SetLogPriority(NKikimrServices::KQP_COMPUTE, NActors::NLog::PRI_DEBUG);
-// runtime.SetLogPriority(NKikimrServices::KQP_EXECUTER, NActors::NLog::PRI_DEBUG);
-// runtime.SetLogPriority(NKikimrServices::KQP_WORKER, NActors::NLog::PRI_DEBUG);
-// runtime.SetLogPriority(NKikimrServices::KQP_GATEWAY, NActors::NLog::PRI_DEBUG);
-// runtime.SetLogPriority(NKikimrServices::RPC_REQUEST, NActors::NLog::PRI_DEBUG);
-// runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_DEBUG);
-// runtime.SetLogPriority(NKikimrServices::KQP_RESOURCES_MANAGER, NActors::NLog::PRI_DEBUG);
-
- auto sender = runtime.AllocateEdgeActor();
- InitRoot(Server, sender);
-
- CreateShardedTable(Server, sender, "/Root", "table-1", 2);
- CreateShardedTable(Server, sender, "/Root", "table-2", 1);
-
- ExecSQL(Server, sender, R"(
- UPSERT INTO [/Root/table-1] (key, value) VALUES
- (1, 1), (2, 2), (3, 3),
- (10, 1), (20, 2), (30, 3),
- (100, 1), (200, 2), (300, 3),
- (1000, 1), (2000, 2), (3000, 3),
- (4000000001u, 1), (4000000002u, 2), (4000000003u, 3),
- (4000000011u, 1), (4000000012u, 2), (4000000013u, 3),
- (4000000101u, 1), (4000000102u, 2), (4000000103u, 3),
- (4000001001u, 1), (4000001002u, 2), (4000001003u, 3)
- )");
- ExecSQL(Server, sender, R"(
- UPSERT INTO [/Root/table-2] (key, value) VALUES
- (1, 1), (5, 5), (7, 7), (11, 11)
- )");
-
- GetDqExecutionSettingsForTests().FlowControl.MaxOutputChunkSize = 1;
- GetDqExecutionSettingsForTests().FlowControl.InFlightBytesOvercommit = 1.0f;
-
- // Query Plan
- //
- // +---------------------+
- // | Scan: /Root/table-2 |
- // | Stage: #0 |
- // | Task: #1 |
- // +---------------------+
- // | (ch: #1, map) (ch: #3, bcast)
- // ↓ +-------------------------------------+
- // +---------------+ | |
- // | Compute Actor |---------------+ ↓
- // | Stage: #2 | +---------------------+ +---------------------+
- // | Task: #2 | | Scan: /Root/table-1 | | Scan: /Root/table-1 |
- // | Union |------------------->| Stage: #1 | | Stage: #1 |
- // +---------------+ (ch: #2, bcast) | Task: #3 | | Task: #4 |
- // | MapJoin | | MapJoin |
- // +---------------------+ +---------------------+
- // (ch: #4, map) | | (ch: #5, map)
- // ↓ ↓
- // +----------------+
- // | ComputeActor |
- // | Stage: #3 |
- // | Task: #5 |
- // | Union + Sort |
- // +----------------+
- // | (ch: #6, map)
- // ↓
- // +----------------+
- // | ComputeActor |
- // | Stage: #4 |
- // | Task: #6 |
- // | Union |
- // +----------------+
- // | (ch: 7, map)
- // ↓
- // +------------+
- // | Result |
- // +------------+
-
- Query = R"(
- SELECT t1.key AS a, t2.key AS b
- FROM [/Root/table-1] AS t1 JOIN [/Root/table-2] AS t2 ON t1.value = t2.key
- ORDER BY a, b
- )";
-}
-
-void KqpStabilityTests::DropChannelDataWithUnreachable() {
- for (ui64 channelId = 1; channelId < 8; ++channelId) {
- ui64 maxSeqNo = channelId < 4 ? 3 : 1;
- for (ui64 seqNo = 1; seqNo <= maxSeqNo; ++seqNo) {
- TDropInfo drop;
- drop.EventName = "TEvChannelData";
- drop.EventType = NYql::NDq::TEvDqCompute::TEvChannelData::EventType;
- drop.ChannelId = channelId;
- drop.SeqNo = seqNo;
- drop.Reason = TEvents::TEvUndelivered::EReason::Disconnected;
- drop.SeqNoExtractor = [](NActors::IEventHandle* handle) {
- return handle->Get<NYql::NDq::TEvDqCompute::TEvChannelData>()->Record.GetSeqNo();
- };
- drop.Counter = 3;
-
- // Cerr << "-- " << __func__ << " channelId: " << channelId << ", seqNo: " << seqNo << Endl;
- DoRun(drop, Nothing());
- }
- }
-}
-
-void KqpStabilityTests::DropChannelDataWithUnknownActor() {
- for (ui64 channelId = 1; channelId < 8; ++channelId) {
- ui64 maxSeqNo = channelId < 4 ? 3 : 1;
- for (ui64 seqNo = 1; seqNo <= maxSeqNo; ++seqNo) {
- TDropInfo drop;
- drop.EventName = "TEvChannelData";
- drop.EventType = NYql::NDq::TEvDqCompute::TEvChannelData::EventType;
- drop.ChannelId = channelId;
- drop.SeqNo = seqNo;
- drop.Reason = TEvents::TEvUndelivered::EReason::ReasonActorUnknown;
- drop.SeqNoExtractor = [](NActors::IEventHandle* handle) {
- return handle->Get<NYql::NDq::TEvDqCompute::TEvChannelData>()->Record.GetSeqNo();
- };
- drop.Counter = 3;
-
- // Cerr << "-- " << __func__ << " channelId: " << channelId << ", seqNo: " << seqNo << Endl;
- DoRun(drop, "Internal error while executing transaction.");
- }
- }
-}
-
-void KqpStabilityTests::DropChannelDataAckWithUnreachable() {
- for (ui64 channelId = 1; channelId < 7 /* TODO: add support for EXECUTOR<->CA */; ++channelId) {
- ui64 maxSeqNo = channelId < 4 ? 3 : 1;
- for (ui64 seqNo = 1; seqNo <= maxSeqNo; ++seqNo) {
- TDropInfo drop;
- drop.EventName = "TEvChannelDataAck";
- drop.EventType = NYql::NDq::TEvDqCompute::TEvChannelDataAck::EventType;
- drop.ChannelId = channelId;
- drop.SeqNo = seqNo;
- drop.Reason = TEvents::TEvUndelivered::EReason::Disconnected;
- drop.SeqNoExtractor = [](NActors::IEventHandle* handle) {
- return handle->Get<NYql::NDq::TEvDqCompute::TEvChannelDataAck>()->Record.GetSeqNo();
- };
- drop.Counter = 3;
-
- Cerr << "-- " << __func__ << " channelId: " << channelId << ", seqNo: " << seqNo << Endl;
- DoRun(drop, Nothing());
- }
- }
-}
-
-void KqpStabilityTests::DropChannelDataAckWithUnknownActor() {
- for (ui64 channelId = 1; channelId < 7 /* TODO: add support for EXECUTOR<->CA */; ++channelId) {
- ui64 maxSeqNo = channelId < 4 ? 3 : 1;
- for (ui64 seqNo = 1; seqNo <= maxSeqNo; ++seqNo) {
- TDropInfo drop;
- drop.EventName = "TEvChannelDataAck";
- drop.EventType = NYql::NDq::TEvDqCompute::TEvChannelDataAck::EventType;
- drop.ChannelId = channelId;
- drop.SeqNo = seqNo;
- drop.Reason = TEvents::TEvUndelivered::EReason::ReasonActorUnknown;
- drop.SeqNoExtractor = [](NActors::IEventHandle* handle) {
- return handle->Get<NYql::NDq::TEvDqCompute::TEvChannelDataAck>()->Record.GetSeqNo();
- };
- drop.Counter = 3;
-
- Cerr << "-- " << __func__ << " channelId: " << channelId << ", seqNo: " << seqNo << Endl;
- DoRun(drop, "Internal error while executing transaction.");
- }
- }
-}
-
-void KqpStabilityTests::DropChannelDataWithDisconnected() {
- GetDqExecutionSettingsForTests().FlowControl.InFlightBytesOvercommit = 100.0f;
- Y_DEFER {
- GetDqExecutionSettingsForTests().FlowControl.InFlightBytesOvercommit = 1.0f;
- };
-
- TDropInfo drop;
- drop.EventName = "TEvChannelData";
- drop.EventType = NYql::NDq::TEvDqCompute::TEvChannelData::EventType;
- drop.ChannelId = 1;
- drop.Counter = 3; // drop 3 messages and then kill interconnect session
-
- DoRun(drop, Nothing());
-}
-
-void KqpStabilityTests::DropChannelDataAckWithDisconnected() {
- GetDqExecutionSettingsForTests().FlowControl.InFlightBytesOvercommit = 100.0f;
- Y_DEFER {
- GetDqExecutionSettingsForTests().FlowControl.InFlightBytesOvercommit = 1.0f;
- };
-
- TDropInfo drop;
- drop.EventName = "TEvChannelDataAck";
- drop.EventType = NYql::NDq::TEvDqCompute::TEvChannelDataAck::EventType;
- drop.ChannelId = 1;
- drop.Counter = 2; // drop 3 messages and then kill interconnect session
-
- DoRun(drop, Nothing());
-}
-
-void KqpStabilityTests::AbortOnDisconnect() {
- GetDqExecutionSettingsForTests().FlowControl.InFlightBytesOvercommit = 100.0f;
- Y_DEFER {
- GetDqExecutionSettingsForTests().FlowControl.InFlightBytesOvercommit = 1.0f;
- };
-
- TDropInfo drop;
- drop.EventName = "TEvChannelData";
- drop.EventType = NYql::NDq::TEvDqCompute::TEvChannelData::EventType;
- drop.ChannelId = 1;
- drop.Counter = 3; // drop 3 messages and then kill interconnect session
-
- drop.Attach(Server->GetRuntime());
-
- auto client = Server->GetRuntime()->AllocateEdgeActor();
- SendRequest(*Server->GetRuntime(), client, MakeStreamRequest(client, Query, true));
-
- while (true) {
- TAutoPtr<IEventHandle> handle;
- auto replies = Server->GetRuntime()->GrabEdgeEventsRethrow<TEvKqp::TEvQueryResponse, TEvKqp::TEvAbortExecution,
- TEvKqpExecuter::TEvStreamData, TEvKqpExecuter::TEvStreamProfile>(handle);
-
- if (auto* ev = std::get<TEvKqp::TEvQueryResponse*>(replies)) {
- auto& response = ev->Record.GetRef();
-
- UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::ABORTED);
- UNIT_ASSERT_STRINGS_EQUAL(response.GetResponse().GetQueryIssues()[0].issues()[0].message(), "Table /Root/table-1 scan failed, reason: 2");
- return;
- }
-
- if (auto* ev = std::get<TEvKqp::TEvAbortExecution*>(replies)) {
- UNIT_FAIL(ev->Record.GetMessage());
- }
-
- if (std::get<TEvKqpExecuter::TEvStreamData*>(replies)) {
- UNIT_FAIL("unexpected");
- }
-
- if (std::get<TEvKqpExecuter::TEvStreamProfile*>(replies)) {
- UNIT_FAIL("unexpected");
- }
- }
-}
-
+ writer.OnBeginList();
+
+ while (true) {
+ TAutoPtr<IEventHandle> handle;
+ auto replies = Server->GetRuntime()->GrabEdgeEventsRethrow<TEvKqp::TEvQueryResponse, TEvKqp::TEvAbortExecution,
+ TEvKqpExecuter::TEvStreamData, TEvKqpExecuter::TEvStreamProfile>(handle);
+
+ if (auto* ev = std::get<TEvKqp::TEvQueryResponse*>(replies)) {
+ auto& response = ev->Record.GetRef();
+
+ if (!error) {
+ UNIT_ASSERT_EQUAL_C(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS, response.Utf8DebugString());
+ UNIT_ASSERT(response.GetResponse().GetResults().empty());
+
+ writer.OnEndList();
+ UNIT_ASSERT_STRINGS_EQUAL(ReformatYson(R"(
+ [[[1u];[1u]];
+ [[10u];[1u]];
+ [[100u];[1u]];
+ [[1000u];[1u]];
+ [[4000000001u];[1u]];
+ [[4000000011u];[1u]];
+ [[4000000101u];[1u]];
+ [[4000001001u];[1u]]]
+ )"), ReformatYson(out.Str()));
+
+ UNIT_ASSERT(profile.Defined());
+ UNIT_ASSERT(dropInfo.DroppedEvents > 0);
+ UNIT_ASSERT(CalcDrops(*profile) > 0);
+ //UNIT_ASSERT_C(dropInfo.DroppedEvents >= CalcDrops(*profile), dropCount << " !>= " << CalcDrops(*profile));
+ } else {
+ UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::GENERIC_ERROR);
+ UNIT_ASSERT_STRINGS_EQUAL(response.GetResponse().GetQueryIssues()[0].issues()[0].message(), *error);
+ }
+ break;
+ }
+
+ if (auto* ev = std::get<TEvKqp::TEvAbortExecution*>(replies)) {
+ UNIT_FAIL(ev->Record.GetMessage());
+ }
+
+ if (auto* ev = std::get<TEvKqpExecuter::TEvStreamData*>(replies)) {
+ NYdb::TResultSet result = ev->Record.resultset();
+ auto columns = result.GetColumnsMeta();
+
+ NYdb::TResultSetParser parser(result);
+ while (parser.TryNextRow()) {
+ writer.OnListItem();
+ writer.OnBeginList();
+ for (ui32 i = 0; i < columns.size(); ++i) {
+ writer.OnListItem();
+ FormatValueYson(parser.GetValue(i), writer);
+ }
+ writer.OnEndList();
+ }
+
+ continue;
+ }
+
+ if (auto* ev = std::get<TEvKqpExecuter::TEvStreamProfile*>(replies)) {
+ profile = ev->Record.profile();
+ continue;
+ }
+ }
+ }
+
+ NKikimrConfig::TAppConfig AppCfg;
+ TPortManager PortManager;
+ TServer::TPtr Server;
+ TString Query;
+};
+UNIT_TEST_SUITE_REGISTRATION(KqpStabilityTests);
+
+void KqpStabilityTests::SetUp() {
+ Cerr << "-- SetUp\n";
+ auto serverSettings = TServerSettings(PortManager.GetPort(2134))
+ .SetDomainName("Root")
+ .SetNodeCount(2)
+ .SetUseRealThreads(false)
+ .SetAppConfig(AppCfg);
+
+ Server = new Tests::TServer(serverSettings);
+ auto& runtime = *Server->GetRuntime();
+
+// runtime.SetLogPriority(NKikimrServices::KQP_COMPUTE, NActors::NLog::PRI_DEBUG);
+// runtime.SetLogPriority(NKikimrServices::KQP_EXECUTER, NActors::NLog::PRI_DEBUG);
+// runtime.SetLogPriority(NKikimrServices::KQP_WORKER, NActors::NLog::PRI_DEBUG);
+// runtime.SetLogPriority(NKikimrServices::KQP_GATEWAY, NActors::NLog::PRI_DEBUG);
+// runtime.SetLogPriority(NKikimrServices::RPC_REQUEST, NActors::NLog::PRI_DEBUG);
+// runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_DEBUG);
+// runtime.SetLogPriority(NKikimrServices::KQP_RESOURCES_MANAGER, NActors::NLog::PRI_DEBUG);
+
+ auto sender = runtime.AllocateEdgeActor();
+ InitRoot(Server, sender);
+
+ CreateShardedTable(Server, sender, "/Root", "table-1", 2);
+ CreateShardedTable(Server, sender, "/Root", "table-2", 1);
+
+ ExecSQL(Server, sender, R"(
+ UPSERT INTO [/Root/table-1] (key, value) VALUES
+ (1, 1), (2, 2), (3, 3),
+ (10, 1), (20, 2), (30, 3),
+ (100, 1), (200, 2), (300, 3),
+ (1000, 1), (2000, 2), (3000, 3),
+ (4000000001u, 1), (4000000002u, 2), (4000000003u, 3),
+ (4000000011u, 1), (4000000012u, 2), (4000000013u, 3),
+ (4000000101u, 1), (4000000102u, 2), (4000000103u, 3),
+ (4000001001u, 1), (4000001002u, 2), (4000001003u, 3)
+ )");
+ ExecSQL(Server, sender, R"(
+ UPSERT INTO [/Root/table-2] (key, value) VALUES
+ (1, 1), (5, 5), (7, 7), (11, 11)
+ )");
+
+ GetDqExecutionSettingsForTests().FlowControl.MaxOutputChunkSize = 1;
+ GetDqExecutionSettingsForTests().FlowControl.InFlightBytesOvercommit = 1.0f;
+
+ // Query Plan
+ //
+ // +---------------------+
+ // | Scan: /Root/table-2 |
+ // | Stage: #0 |
+ // | Task: #1 |
+ // +---------------------+
+ // | (ch: #1, map) (ch: #3, bcast)
+ // ↓ +-------------------------------------+
+ // +---------------+ | |
+ // | Compute Actor |---------------+ ↓
+ // | Stage: #2 | +---------------------+ +---------------------+
+ // | Task: #2 | | Scan: /Root/table-1 | | Scan: /Root/table-1 |
+ // | Union |------------------->| Stage: #1 | | Stage: #1 |
+ // +---------------+ (ch: #2, bcast) | Task: #3 | | Task: #4 |
+ // | MapJoin | | MapJoin |
+ // +---------------------+ +---------------------+
+ // (ch: #4, map) | | (ch: #5, map)
+ // ↓ ↓
+ // +----------------+
+ // | ComputeActor |
+ // | Stage: #3 |
+ // | Task: #5 |
+ // | Union + Sort |
+ // +----------------+
+ // | (ch: #6, map)
+ // ↓
+ // +----------------+
+ // | ComputeActor |
+ // | Stage: #4 |
+ // | Task: #6 |
+ // | Union |
+ // +----------------+
+ // | (ch: 7, map)
+ // ↓
+ // +------------+
+ // | Result |
+ // +------------+
+
+ Query = R"(
+ SELECT t1.key AS a, t2.key AS b
+ FROM [/Root/table-1] AS t1 JOIN [/Root/table-2] AS t2 ON t1.value = t2.key
+ ORDER BY a, b
+ )";
+}
+
+void KqpStabilityTests::DropChannelDataWithUnreachable() {
+ for (ui64 channelId = 1; channelId < 8; ++channelId) {
+ ui64 maxSeqNo = channelId < 4 ? 3 : 1;
+ for (ui64 seqNo = 1; seqNo <= maxSeqNo; ++seqNo) {
+ TDropInfo drop;
+ drop.EventName = "TEvChannelData";
+ drop.EventType = NYql::NDq::TEvDqCompute::TEvChannelData::EventType;
+ drop.ChannelId = channelId;
+ drop.SeqNo = seqNo;
+ drop.Reason = TEvents::TEvUndelivered::EReason::Disconnected;
+ drop.SeqNoExtractor = [](NActors::IEventHandle* handle) {
+ return handle->Get<NYql::NDq::TEvDqCompute::TEvChannelData>()->Record.GetSeqNo();
+ };
+ drop.Counter = 3;
+
+ // Cerr << "-- " << __func__ << " channelId: " << channelId << ", seqNo: " << seqNo << Endl;
+ DoRun(drop, Nothing());
+ }
+ }
+}
+
+void KqpStabilityTests::DropChannelDataWithUnknownActor() {
+ for (ui64 channelId = 1; channelId < 8; ++channelId) {
+ ui64 maxSeqNo = channelId < 4 ? 3 : 1;
+ for (ui64 seqNo = 1; seqNo <= maxSeqNo; ++seqNo) {
+ TDropInfo drop;
+ drop.EventName = "TEvChannelData";
+ drop.EventType = NYql::NDq::TEvDqCompute::TEvChannelData::EventType;
+ drop.ChannelId = channelId;
+ drop.SeqNo = seqNo;
+ drop.Reason = TEvents::TEvUndelivered::EReason::ReasonActorUnknown;
+ drop.SeqNoExtractor = [](NActors::IEventHandle* handle) {
+ return handle->Get<NYql::NDq::TEvDqCompute::TEvChannelData>()->Record.GetSeqNo();
+ };
+ drop.Counter = 3;
+
+ // Cerr << "-- " << __func__ << " channelId: " << channelId << ", seqNo: " << seqNo << Endl;
+ DoRun(drop, "Internal error while executing transaction.");
+ }
+ }
+}
+
+void KqpStabilityTests::DropChannelDataAckWithUnreachable() {
+ for (ui64 channelId = 1; channelId < 7 /* TODO: add support for EXECUTOR<->CA */; ++channelId) {
+ ui64 maxSeqNo = channelId < 4 ? 3 : 1;
+ for (ui64 seqNo = 1; seqNo <= maxSeqNo; ++seqNo) {
+ TDropInfo drop;
+ drop.EventName = "TEvChannelDataAck";
+ drop.EventType = NYql::NDq::TEvDqCompute::TEvChannelDataAck::EventType;
+ drop.ChannelId = channelId;
+ drop.SeqNo = seqNo;
+ drop.Reason = TEvents::TEvUndelivered::EReason::Disconnected;
+ drop.SeqNoExtractor = [](NActors::IEventHandle* handle) {
+ return handle->Get<NYql::NDq::TEvDqCompute::TEvChannelDataAck>()->Record.GetSeqNo();
+ };
+ drop.Counter = 3;
+
+ Cerr << "-- " << __func__ << " channelId: " << channelId << ", seqNo: " << seqNo << Endl;
+ DoRun(drop, Nothing());
+ }
+ }
+}
+
+void KqpStabilityTests::DropChannelDataAckWithUnknownActor() {
+ for (ui64 channelId = 1; channelId < 7 /* TODO: add support for EXECUTOR<->CA */; ++channelId) {
+ ui64 maxSeqNo = channelId < 4 ? 3 : 1;
+ for (ui64 seqNo = 1; seqNo <= maxSeqNo; ++seqNo) {
+ TDropInfo drop;
+ drop.EventName = "TEvChannelDataAck";
+ drop.EventType = NYql::NDq::TEvDqCompute::TEvChannelDataAck::EventType;
+ drop.ChannelId = channelId;
+ drop.SeqNo = seqNo;
+ drop.Reason = TEvents::TEvUndelivered::EReason::ReasonActorUnknown;
+ drop.SeqNoExtractor = [](NActors::IEventHandle* handle) {
+ return handle->Get<NYql::NDq::TEvDqCompute::TEvChannelDataAck>()->Record.GetSeqNo();
+ };
+ drop.Counter = 3;
+
+ Cerr << "-- " << __func__ << " channelId: " << channelId << ", seqNo: " << seqNo << Endl;
+ DoRun(drop, "Internal error while executing transaction.");
+ }
+ }
+}
+
+void KqpStabilityTests::DropChannelDataWithDisconnected() {
+ GetDqExecutionSettingsForTests().FlowControl.InFlightBytesOvercommit = 100.0f;
+ Y_DEFER {
+ GetDqExecutionSettingsForTests().FlowControl.InFlightBytesOvercommit = 1.0f;
+ };
+
+ TDropInfo drop;
+ drop.EventName = "TEvChannelData";
+ drop.EventType = NYql::NDq::TEvDqCompute::TEvChannelData::EventType;
+ drop.ChannelId = 1;
+ drop.Counter = 3; // drop 3 messages and then kill interconnect session
+
+ DoRun(drop, Nothing());
+}
+
+void KqpStabilityTests::DropChannelDataAckWithDisconnected() {
+ GetDqExecutionSettingsForTests().FlowControl.InFlightBytesOvercommit = 100.0f;
+ Y_DEFER {
+ GetDqExecutionSettingsForTests().FlowControl.InFlightBytesOvercommit = 1.0f;
+ };
+
+ TDropInfo drop;
+ drop.EventName = "TEvChannelDataAck";
+ drop.EventType = NYql::NDq::TEvDqCompute::TEvChannelDataAck::EventType;
+ drop.ChannelId = 1;
+ drop.Counter = 2; // drop 3 messages and then kill interconnect session
+
+ DoRun(drop, Nothing());
+}
+
+void KqpStabilityTests::AbortOnDisconnect() {
+ GetDqExecutionSettingsForTests().FlowControl.InFlightBytesOvercommit = 100.0f;
+ Y_DEFER {
+ GetDqExecutionSettingsForTests().FlowControl.InFlightBytesOvercommit = 1.0f;
+ };
+
+ TDropInfo drop;
+ drop.EventName = "TEvChannelData";
+ drop.EventType = NYql::NDq::TEvDqCompute::TEvChannelData::EventType;
+ drop.ChannelId = 1;
+ drop.Counter = 3; // drop 3 messages and then kill interconnect session
+
+ drop.Attach(Server->GetRuntime());
+
+ auto client = Server->GetRuntime()->AllocateEdgeActor();
+ SendRequest(*Server->GetRuntime(), client, MakeStreamRequest(client, Query, true));
+
+ while (true) {
+ TAutoPtr<IEventHandle> handle;
+ auto replies = Server->GetRuntime()->GrabEdgeEventsRethrow<TEvKqp::TEvQueryResponse, TEvKqp::TEvAbortExecution,
+ TEvKqpExecuter::TEvStreamData, TEvKqpExecuter::TEvStreamProfile>(handle);
+
+ if (auto* ev = std::get<TEvKqp::TEvQueryResponse*>(replies)) {
+ auto& response = ev->Record.GetRef();
+
+ UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::ABORTED);
+ UNIT_ASSERT_STRINGS_EQUAL(response.GetResponse().GetQueryIssues()[0].issues()[0].message(), "Table /Root/table-1 scan failed, reason: 2");
+ return;
+ }
+
+ if (auto* ev = std::get<TEvKqp::TEvAbortExecution*>(replies)) {
+ UNIT_FAIL(ev->Record.GetMessage());
+ }
+
+ if (std::get<TEvKqpExecuter::TEvStreamData*>(replies)) {
+ UNIT_FAIL("unexpected");
+ }
+
+ if (std::get<TEvKqpExecuter::TEvStreamProfile*>(replies)) {
+ UNIT_FAIL("unexpected");
+ }
+ }
+}
+
diff --git a/ydb/core/tx/datashard/datashard_ut_kqp_errors.cpp b/ydb/core/tx/datashard/datashard_ut_kqp_errors.cpp
index 47ea36d5f6..dcbae112e8 100644
--- a/ydb/core/tx/datashard/datashard_ut_kqp_errors.cpp
+++ b/ydb/core/tx/datashard/datashard_ut_kqp_errors.cpp
@@ -4,358 +4,358 @@
#include <ydb/core/tx/datashard/datashard_ut_common_kqp.h>
#include <ydb/core/base/tablet_pipecache.h>
#include <ydb/core/tx/scheme_cache/scheme_cache.h>
-
-namespace NKikimr {
-namespace NKqp {
-
-using namespace Ydb;
-using namespace NYql;
-using namespace Tests;
+
+namespace NKikimr {
+namespace NKqp {
+
+using namespace Ydb;
+using namespace NYql;
+using namespace Tests;
using namespace NKikimr::NDataShard::NKqpHelpers;
-using namespace NKikimrTxDataShard;
-
-namespace {
-
-bool HasIssue(const TIssues& issues, ui32 code, TStringBuf message, std::function<bool(const TIssue& issue)> predicate = {}) {
- bool hasIssue = false;
-
- for (auto& issue : issues) {
- WalkThroughIssues(issue, false, [&] (const TIssue& issue, int) {
- if (!hasIssue && issue.GetCode() == code && (!message || message == issue.Message)) {
- hasIssue = !predicate || predicate(issue);
- }
- });
- }
-
- return hasIssue;
-}
-
-} // anonymous namespace
-
-class TFixture : public NUnitTest::TBaseFixture {
-public:
+using namespace NKikimrTxDataShard;
+
+namespace {
+
+bool HasIssue(const TIssues& issues, ui32 code, TStringBuf message, std::function<bool(const TIssue& issue)> predicate = {}) {
+ bool hasIssue = false;
+
+ for (auto& issue : issues) {
+ WalkThroughIssues(issue, false, [&] (const TIssue& issue, int) {
+ if (!hasIssue && issue.GetCode() == code && (!message || message == issue.Message)) {
+ hasIssue = !predicate || predicate(issue);
+ }
+ });
+ }
+
+ return hasIssue;
+}
+
+} // anonymous namespace
+
+class TFixture : public NUnitTest::TBaseFixture {
+public:
void SetUp(NUnitTest::TTestContext&) override {
- TPortManager pm;
- TServerSettings serverSettings(pm.GetPort(2134));
- serverSettings.SetDomainName("Root")
- .SetNodeCount(2)
- .SetUseRealThreads(false);
-
- Server = new TServer(serverSettings);
- Runtime = Server->GetRuntime();
-
- Runtime->SetLogPriority(NKikimrServices::KQP_RESOURCE_MANAGER, NLog::PRI_DEBUG);
-
- TDispatchOptions rmReady;
- rmReady.CustomFinalCondition = [this] {
- for (ui32 i = 0; i < Runtime->GetNodeCount(); ++i) {
- ui32 nodeId = Runtime->GetNodeId(i);
- if (TryGetKqpResourceManager(nodeId) == nullptr) {
- Cerr << "... wait for RM on node " << nodeId << Endl;
- return false;
- }
- }
- return true;
- };
- Runtime->DispatchEvents(rmReady);
-
- Runtime->SetLogPriority(NKikimrServices::KQP_RESOURCE_MANAGER, NLog::PRI_NOTICE);
-// Runtime->SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG);
-// Runtime->SetLogPriority(NKikimrServices::TX_PROXY, NLog::PRI_DEBUG);
-// Runtime->SetLogPriority(NKikimrServices::KQP_EXECUTER, NActors::NLog::PRI_TRACE);
-
- auto sender = Runtime->AllocateEdgeActor();
- InitRoot(Server, sender);
- CreateShardedTable(Server, sender, "/Root", "table-1", 4);
- ExecSQL(Server, sender, "UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1), (2, 2), (3, 3)");
-
- Client = Runtime->AllocateEdgeActor();
- }
-
+ TPortManager pm;
+ TServerSettings serverSettings(pm.GetPort(2134));
+ serverSettings.SetDomainName("Root")
+ .SetNodeCount(2)
+ .SetUseRealThreads(false);
+
+ Server = new TServer(serverSettings);
+ Runtime = Server->GetRuntime();
+
+ Runtime->SetLogPriority(NKikimrServices::KQP_RESOURCE_MANAGER, NLog::PRI_DEBUG);
+
+ TDispatchOptions rmReady;
+ rmReady.CustomFinalCondition = [this] {
+ for (ui32 i = 0; i < Runtime->GetNodeCount(); ++i) {
+ ui32 nodeId = Runtime->GetNodeId(i);
+ if (TryGetKqpResourceManager(nodeId) == nullptr) {
+ Cerr << "... wait for RM on node " << nodeId << Endl;
+ return false;
+ }
+ }
+ return true;
+ };
+ Runtime->DispatchEvents(rmReady);
+
+ Runtime->SetLogPriority(NKikimrServices::KQP_RESOURCE_MANAGER, NLog::PRI_NOTICE);
+// Runtime->SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG);
+// Runtime->SetLogPriority(NKikimrServices::TX_PROXY, NLog::PRI_DEBUG);
+// Runtime->SetLogPriority(NKikimrServices::KQP_EXECUTER, NActors::NLog::PRI_TRACE);
+
+ auto sender = Runtime->AllocateEdgeActor();
+ InitRoot(Server, sender);
+ CreateShardedTable(Server, sender, "/Root", "table-1", 4);
+ ExecSQL(Server, sender, "UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1), (2, 2), (3, 3)");
+
+ Client = Runtime->AllocateEdgeActor();
+ }
+
void TearDown(NUnitTest::TTestContext&) override {
Server.Reset();
}
- Tests::TServer::TPtr Server;
- TTestActorRuntime* Runtime;
- TActorId Client;
-};
-
-Y_UNIT_TEST_SUITE_F(KqpErrors, TFixture) {
-
-Y_UNIT_TEST(ResolveTableError) {
- int skip = 1; // compile
- auto mitm = [&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) {
- if (ev->GetTypeRewrite() == TEvTxProxySchemeCache::TEvNavigateKeySetResult::EventType) {
- if (skip-- == 0) {
- // fail on execution phase (kqp_table_resolver.cpp)
- auto event = ev.Get()->Get<TEvTxProxySchemeCache::TEvNavigateKeySetResult>();
- event->Request->ErrorCount = 1;
- auto& entries = event->Request->ResultSet;
- entries[0].Status = NSchemeCache::TSchemeCacheNavigate::EStatus::PathErrorUnknown;
- }
- }
- return TTestActorRuntime::EEventAction::PROCESS;
- };
- Runtime->SetObserverFunc(mitm);
-
- SendRequest(*Runtime, Client, MakeSQLRequest("pragma kikimr.UseNewEngine='true'; select * from `/Root/table-1`"));
-
- auto ev = Runtime->GrabEdgeEventRethrow<NKqp::TEvKqp::TEvQueryResponse>(Client);
- auto& record = ev->Get()->Record.GetRef();
-
- // Cerr << record.DebugString() << Endl;
-
- UNIT_ASSERT_VALUES_EQUAL_C(record.GetYdbStatus(), Ydb::StatusIds::UNAVAILABLE, record.DebugString());
-
- TIssues issues;
- IssuesFromMessage(record.GetResponse().GetQueryIssues(), issues);
- UNIT_ASSERT_C(HasIssue(issues, NYql::TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE,
- "Failed to resolve table `Root/table-1`: PathErrorUnknown."), record.GetResponse().DebugString());
-}
-
-Y_UNIT_TEST_NEW_ENGINE(ProposeError) {
- THashSet<TActorId> knownExecuters;
-
- using TMod = std::function<void(NKikimrTxDataShard::TEvProposeTransactionResult&)>;
-
+ Tests::TServer::TPtr Server;
+ TTestActorRuntime* Runtime;
+ TActorId Client;
+};
+
+Y_UNIT_TEST_SUITE_F(KqpErrors, TFixture) {
+
+Y_UNIT_TEST(ResolveTableError) {
+ int skip = 1; // compile
+ auto mitm = [&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) {
+ if (ev->GetTypeRewrite() == TEvTxProxySchemeCache::TEvNavigateKeySetResult::EventType) {
+ if (skip-- == 0) {
+ // fail on execution phase (kqp_table_resolver.cpp)
+ auto event = ev.Get()->Get<TEvTxProxySchemeCache::TEvNavigateKeySetResult>();
+ event->Request->ErrorCount = 1;
+ auto& entries = event->Request->ResultSet;
+ entries[0].Status = NSchemeCache::TSchemeCacheNavigate::EStatus::PathErrorUnknown;
+ }
+ }
+ return TTestActorRuntime::EEventAction::PROCESS;
+ };
+ Runtime->SetObserverFunc(mitm);
+
+ SendRequest(*Runtime, Client, MakeSQLRequest("pragma kikimr.UseNewEngine='true'; select * from `/Root/table-1`"));
+
+ auto ev = Runtime->GrabEdgeEventRethrow<NKqp::TEvKqp::TEvQueryResponse>(Client);
+ auto& record = ev->Get()->Record.GetRef();
+
+ // Cerr << record.DebugString() << Endl;
+
+ UNIT_ASSERT_VALUES_EQUAL_C(record.GetYdbStatus(), Ydb::StatusIds::UNAVAILABLE, record.DebugString());
+
+ TIssues issues;
+ IssuesFromMessage(record.GetResponse().GetQueryIssues(), issues);
+ UNIT_ASSERT_C(HasIssue(issues, NYql::TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE,
+ "Failed to resolve table `Root/table-1`: PathErrorUnknown."), record.GetResponse().DebugString());
+}
+
+Y_UNIT_TEST_NEW_ENGINE(ProposeError) {
+ THashSet<TActorId> knownExecuters;
+
+ using TMod = std::function<void(NKikimrTxDataShard::TEvProposeTransactionResult&)>;
+
auto test = [&](auto proposeStatus, auto ydbStatus, auto issue, auto issueMessage, TMod mod = {}) {
- auto client = Runtime->AllocateEdgeActor();
-
- bool done = false;
- auto mitm = [&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) {
- if (!done && ev->GetTypeRewrite() == TEvDataShard::TEvProposeTransactionResult::EventType &&
- !knownExecuters.contains(ev->Recipient))
- {
- auto event = ev.Get()->Get<TEvDataShard::TEvProposeTransactionResult>();
- event->Record.SetStatus(proposeStatus);
- if (mod) {
- mod(event->Record);
- }
- knownExecuters.insert(ev->Recipient);
- done = true;
- }
- return TTestActorRuntime::EEventAction::PROCESS;
- };
- Runtime->SetObserverFunc(mitm);
-
- SendRequest(*Runtime, client, MakeSQLRequest(Q_("select * from `/Root/table-1`")));
-
- auto ev = Runtime->GrabEdgeEventRethrow<NKqp::TEvKqp::TEvQueryResponse>(client);
- auto& record = ev->Get()->Record.GetRef();
- UNIT_ASSERT_VALUES_EQUAL_C(record.GetYdbStatus(), ydbStatus, record.DebugString());
-
- // Cerr << record.DebugString() << Endl;
-
- TIssues issues;
- IssuesFromMessage(record.GetResponse().GetQueryIssues(), issues);
- UNIT_ASSERT_C(HasIssue(issues, issue, issueMessage), "issue not found, issue: " << (int) issue
- << ", message: " << issueMessage << ", response: " << record.GetResponse().DebugString());
- };
-
- test(TEvProposeTransactionResult::OVERLOADED, // propose error
- Ydb::StatusIds::OVERLOADED, // ydb status
- NYql::TIssuesIds::KIKIMR_OVERLOADED, // issue status
- "Kikimr cluster or one of its subsystems is overloaded."); // main issue message (more detailed info can be in subissues)
-
- test(TEvProposeTransactionResult::ABORTED,
- Ydb::StatusIds::ABORTED,
- NYql::TIssuesIds::KIKIMR_OPERATION_ABORTED,
- "Operation aborted.");
-
- test(TEvProposeTransactionResult::TRY_LATER,
- Ydb::StatusIds::UNAVAILABLE,
- NYql::TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE,
- "Kikimr cluster or one of its subsystems was unavailable.");
-
- test(TEvProposeTransactionResult::RESULT_UNAVAILABLE,
- Ydb::StatusIds::UNDETERMINED,
- NYql::TIssuesIds::KIKIMR_RESULT_UNAVAILABLE,
- "Result of Kikimr query didn't meet requirements and isn't available");
-
- test(TEvProposeTransactionResult::CANCELLED,
- Ydb::StatusIds::CANCELLED,
- NYql::TIssuesIds::KIKIMR_OPERATION_CANCELLED,
- "Operation cancelled.");
-
- test(TEvProposeTransactionResult::BAD_REQUEST,
- Ydb::StatusIds::BAD_REQUEST,
- NYql::TIssuesIds::KIKIMR_BAD_REQUEST,
- "Bad request.");
-
- test(TEvProposeTransactionResult::ERROR,
- Ydb::StatusIds::UNAVAILABLE,
- NYql::TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE,
- "Kikimr cluster or one of its subsystems was unavailable.");
-
- test(TEvProposeTransactionResult::ERROR,
- Ydb::StatusIds::ABORTED,
- NYql::TIssuesIds::KIKIMR_SCHEME_MISMATCH,
- UseNewEngine ? "blah-blah-blah" : "Scheme mismatch found while executing query.",
- [](NKikimrTxDataShard::TEvProposeTransactionResult& x) {
- auto* error = x.MutableError()->Add();
- error->SetKind(NKikimrTxDataShard::TError::SCHEME_CHANGED);
- error->SetReason("blah-blah-blah");
- });
-
- if (UseNewEngine) {
- test(TEvProposeTransactionResult::ERROR,
- Ydb::StatusIds::SCHEME_ERROR,
- NYql::TIssuesIds::KIKIMR_SCHEME_ERROR,
- "blah-blah-blah",
- [](NKikimrTxDataShard::TEvProposeTransactionResult& x) {
- auto* error = x.MutableError()->Add();
- error->SetKind(NKikimrTxDataShard::TError::SCHEME_ERROR);
- error->SetReason("blah-blah-blah");
- });
- } else {
- test(TEvProposeTransactionResult::ERROR,
- Ydb::StatusIds::UNAVAILABLE,
- NYql::TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE,
- "Kikimr cluster or one of its subsystems was unavailable.");
- }
-
- test(TEvProposeTransactionResult::EXEC_ERROR,
- Ydb::StatusIds::GENERIC_ERROR,
- NYql::TIssuesIds::DEFAULT_ERROR,
- UseNewEngine ? "Error executing transaction (ExecError): Execution failed" : "Default error");
-
- if (UseNewEngine) {
- test(TEvProposeTransactionResult::EXEC_ERROR,
- Ydb::StatusIds::PRECONDITION_FAILED,
- NYql::TIssuesIds::KIKIMR_PRECONDITION_FAILED,
- "Kikimr precondition failed",
- [](NKikimrTxDataShard::TEvProposeTransactionResult& x) {
- auto* error = x.MutableError()->Add();
- error->SetKind(NKikimrTxDataShard::TError::PROGRAM_ERROR);
- error->SetReason("blah-blah-blah");
- });
-
- // verify in old engine
- test(TEvProposeTransactionResult::RESPONSE_DATA,
- Ydb::StatusIds::GENERIC_ERROR,
- NYql::TIssuesIds::DEFAULT_ERROR,
- "Error executing transaction: transaction failed.");
- }
-}
-
-Y_UNIT_TEST_NEW_ENGINE(ProposeRequestUndelivered) {
- auto mitm = [&](TTestActorRuntimeBase& rt, TAutoPtr<IEventHandle> &ev) {
- if (ev->GetTypeRewrite() == TEvPipeCache::TEvForward::EventType) {
- auto forwardEvent = ev.Get()->Get<TEvPipeCache::TEvForward>();
- if (forwardEvent->Ev->Type() == TEvDataShard::TEvProposeTransaction::EventType) {
- rt.Send(new IEventHandle(ev->Sender, ev->Recipient,
- new TEvPipeCache::TEvDeliveryProblem(forwardEvent->TabletId, /* NotDelivered */ true)));
- return TTestActorRuntime::EEventAction::DROP;
- }
- }
- return TTestActorRuntime::EEventAction::PROCESS;
- };
- Runtime->SetObserverFunc(mitm);
-
- SendRequest(*Runtime, Client, MakeSQLRequest(Q_("select * from `/Root/table-1`")));
-
- auto ev = Runtime->GrabEdgeEventRethrow<NKqp::TEvKqp::TEvQueryResponse>(Client);
- auto& record = ev->Get()->Record.GetRef();
- UNIT_ASSERT_VALUES_EQUAL_C(record.GetYdbStatus(), Ydb::StatusIds::UNAVAILABLE, record.DebugString());
-
- // Cerr << record.DebugString() << Endl;
-
- TIssues issues;
- IssuesFromMessage(record.GetResponse().GetQueryIssues(), issues);
- UNIT_ASSERT_C(HasIssue(issues, NYql::TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE,
- "Kikimr cluster or one of its subsystems was unavailable."), record.GetResponse().DebugString());
-
- UNIT_ASSERT_C(HasIssue(issues, NYql::TIssuesIds::DEFAULT_ERROR, "", [] (const TIssue& issue) {
- return UseNewEngine
- ? issue.Message.StartsWith("Could not deliver program to shard ")
- : issue.Message == "Error executing transaction (ProxyShardNotAvailable): One or more of affected datashards not available, request execution cancelled";
- }), record.GetResponse().DebugString());
-}
-
-void TestProposeResultLost(TTestActorRuntime& runtime, TActorId client, const TString& query,
- std::function<void(const NKikimrKqp::TEvQueryResponse& resp)> fn)
-{
- TActorId executer;
- ui32 droppedEvents = 0;
-
- runtime.SetObserverFunc([&](TTestActorRuntimeBase& rt, TAutoPtr<IEventHandle>& ev) {
- if (ev->GetTypeRewrite() == TEvPipeCache::TEvForward::EventType) {
- auto* fe = ev.Get()->Get<TEvPipeCache::TEvForward>();
- if (fe->Ev->Type() == TEvDataShard::TEvProposeTransaction::EventType) {
- executer = ((TEvDataShard::TEvProposeTransaction*) fe->Ev.Get())->GetSource();
- // Cerr << "-- executer: " << executer << Endl;
- return TTestActorRuntime::EEventAction::PROCESS;
- }
- }
-
- if (ev->GetTypeRewrite() == TEvDataShard::TEvProposeTransactionResult::EventType) {
- auto* msg = ev.Get()->Get<TEvDataShard::TEvProposeTransactionResult>();
- if (msg->Record.GetStatus() == NKikimrTxDataShard::TEvProposeTransactionResult::PREPARED) {
- if (ev->Sender.NodeId() == executer.NodeId()) {
- ++droppedEvents;
- // Cerr << "-- send undelivery to " << ev->Recipient << ", executer: " << executer << Endl;
- rt.Send(new IEventHandle(executer, ev->Sender,
- new TEvPipeCache::TEvDeliveryProblem(msg->GetOrigin(), /* NotDelivered */ false)));
- return TTestActorRuntime::EEventAction::DROP;
- }
- }
- }
-
- return TTestActorRuntime::EEventAction::PROCESS;
- });
- SendRequest(runtime, client, MakeSQLRequest(query));
-
- auto ev = runtime.GrabEdgeEventRethrow<NKqp::TEvKqp::TEvQueryResponse>(client);
- UNIT_ASSERT(droppedEvents > 0 && droppedEvents < 4);
-
- auto& record = ev->Get()->Record.GetRef();
- // Cerr << record.DebugString() << Endl;
- fn(record);
-}
-
-Y_UNIT_TEST_NEW_ENGINE(ProposeResultLost_RoTx) {
- TestProposeResultLost(*Runtime, Client,
- Q_("select * from `/Root/table-1`"),
- [](const NKikimrKqp::TEvQueryResponse& record) {
- UNIT_ASSERT_VALUES_EQUAL_C(record.GetYdbStatus(), Ydb::StatusIds::UNAVAILABLE, record.DebugString());
-
- TIssues issues;
- IssuesFromMessage(record.GetResponse().GetQueryIssues(), issues);
- UNIT_ASSERT_C(HasIssue(issues, NYql::TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE,
- "Kikimr cluster or one of its subsystems was unavailable."), record.GetResponse().DebugString());
-
- UNIT_ASSERT_C(HasIssue(issues, NKikimrIssues::TIssuesIds::TX_STATE_UNKNOWN, "", [] (const TIssue& issue) {
- return UseNewEngine
- ? issue.Message.StartsWith("Tx state unknown for shard ")
- : issue.Message.StartsWith("tx state unknown for shard ");
- }), record.GetResponse().DebugString());
- });
-}
-
-Y_UNIT_TEST_NEW_ENGINE(ProposeResultLost_RwTx) {
- TestProposeResultLost(*Runtime, Client,
- Q_(R"(
- upsert into `/Root/table-1` (key, value) VALUES
- (1, 1), (1073741823, 1073741823), (2147483647, 2147483647), (4294967295, 4294967295)
- )"),
- [](const NKikimrKqp::TEvQueryResponse& record) {
- UNIT_ASSERT_VALUES_EQUAL_C(record.GetYdbStatus(), Ydb::StatusIds::UNDETERMINED, record.DebugString());
-
- TIssues issues;
- IssuesFromMessage(record.GetResponse().GetQueryIssues(), issues);
- UNIT_ASSERT_C(HasIssue(issues, NYql::TIssuesIds::KIKIMR_OPERATION_STATE_UNKNOWN,
- "State of operation is unknown."), record.GetResponse().DebugString());
-
- UNIT_ASSERT_C(HasIssue(issues, NKikimrIssues::TIssuesIds::TX_STATE_UNKNOWN, "", [] (const TIssue& issue) {
- return UseNewEngine
- ? issue.Message.StartsWith("Tx state unknown for shard ")
- : issue.Message.StartsWith("tx state unknown for shard ");
- }), record.GetResponse().DebugString());
- });
-}
-
-} // suite
-
-} // namespace NKqp
-} // namespace NKikimr
+ auto client = Runtime->AllocateEdgeActor();
+
+ bool done = false;
+ auto mitm = [&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) {
+ if (!done && ev->GetTypeRewrite() == TEvDataShard::TEvProposeTransactionResult::EventType &&
+ !knownExecuters.contains(ev->Recipient))
+ {
+ auto event = ev.Get()->Get<TEvDataShard::TEvProposeTransactionResult>();
+ event->Record.SetStatus(proposeStatus);
+ if (mod) {
+ mod(event->Record);
+ }
+ knownExecuters.insert(ev->Recipient);
+ done = true;
+ }
+ return TTestActorRuntime::EEventAction::PROCESS;
+ };
+ Runtime->SetObserverFunc(mitm);
+
+ SendRequest(*Runtime, client, MakeSQLRequest(Q_("select * from `/Root/table-1`")));
+
+ auto ev = Runtime->GrabEdgeEventRethrow<NKqp::TEvKqp::TEvQueryResponse>(client);
+ auto& record = ev->Get()->Record.GetRef();
+ UNIT_ASSERT_VALUES_EQUAL_C(record.GetYdbStatus(), ydbStatus, record.DebugString());
+
+ // Cerr << record.DebugString() << Endl;
+
+ TIssues issues;
+ IssuesFromMessage(record.GetResponse().GetQueryIssues(), issues);
+ UNIT_ASSERT_C(HasIssue(issues, issue, issueMessage), "issue not found, issue: " << (int) issue
+ << ", message: " << issueMessage << ", response: " << record.GetResponse().DebugString());
+ };
+
+ test(TEvProposeTransactionResult::OVERLOADED, // propose error
+ Ydb::StatusIds::OVERLOADED, // ydb status
+ NYql::TIssuesIds::KIKIMR_OVERLOADED, // issue status
+ "Kikimr cluster or one of its subsystems is overloaded."); // main issue message (more detailed info can be in subissues)
+
+ test(TEvProposeTransactionResult::ABORTED,
+ Ydb::StatusIds::ABORTED,
+ NYql::TIssuesIds::KIKIMR_OPERATION_ABORTED,
+ "Operation aborted.");
+
+ test(TEvProposeTransactionResult::TRY_LATER,
+ Ydb::StatusIds::UNAVAILABLE,
+ NYql::TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE,
+ "Kikimr cluster or one of its subsystems was unavailable.");
+
+ test(TEvProposeTransactionResult::RESULT_UNAVAILABLE,
+ Ydb::StatusIds::UNDETERMINED,
+ NYql::TIssuesIds::KIKIMR_RESULT_UNAVAILABLE,
+ "Result of Kikimr query didn't meet requirements and isn't available");
+
+ test(TEvProposeTransactionResult::CANCELLED,
+ Ydb::StatusIds::CANCELLED,
+ NYql::TIssuesIds::KIKIMR_OPERATION_CANCELLED,
+ "Operation cancelled.");
+
+ test(TEvProposeTransactionResult::BAD_REQUEST,
+ Ydb::StatusIds::BAD_REQUEST,
+ NYql::TIssuesIds::KIKIMR_BAD_REQUEST,
+ "Bad request.");
+
+ test(TEvProposeTransactionResult::ERROR,
+ Ydb::StatusIds::UNAVAILABLE,
+ NYql::TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE,
+ "Kikimr cluster or one of its subsystems was unavailable.");
+
+ test(TEvProposeTransactionResult::ERROR,
+ Ydb::StatusIds::ABORTED,
+ NYql::TIssuesIds::KIKIMR_SCHEME_MISMATCH,
+ UseNewEngine ? "blah-blah-blah" : "Scheme mismatch found while executing query.",
+ [](NKikimrTxDataShard::TEvProposeTransactionResult& x) {
+ auto* error = x.MutableError()->Add();
+ error->SetKind(NKikimrTxDataShard::TError::SCHEME_CHANGED);
+ error->SetReason("blah-blah-blah");
+ });
+
+ if (UseNewEngine) {
+ test(TEvProposeTransactionResult::ERROR,
+ Ydb::StatusIds::SCHEME_ERROR,
+ NYql::TIssuesIds::KIKIMR_SCHEME_ERROR,
+ "blah-blah-blah",
+ [](NKikimrTxDataShard::TEvProposeTransactionResult& x) {
+ auto* error = x.MutableError()->Add();
+ error->SetKind(NKikimrTxDataShard::TError::SCHEME_ERROR);
+ error->SetReason("blah-blah-blah");
+ });
+ } else {
+ test(TEvProposeTransactionResult::ERROR,
+ Ydb::StatusIds::UNAVAILABLE,
+ NYql::TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE,
+ "Kikimr cluster or one of its subsystems was unavailable.");
+ }
+
+ test(TEvProposeTransactionResult::EXEC_ERROR,
+ Ydb::StatusIds::GENERIC_ERROR,
+ NYql::TIssuesIds::DEFAULT_ERROR,
+ UseNewEngine ? "Error executing transaction (ExecError): Execution failed" : "Default error");
+
+ if (UseNewEngine) {
+ test(TEvProposeTransactionResult::EXEC_ERROR,
+ Ydb::StatusIds::PRECONDITION_FAILED,
+ NYql::TIssuesIds::KIKIMR_PRECONDITION_FAILED,
+ "Kikimr precondition failed",
+ [](NKikimrTxDataShard::TEvProposeTransactionResult& x) {
+ auto* error = x.MutableError()->Add();
+ error->SetKind(NKikimrTxDataShard::TError::PROGRAM_ERROR);
+ error->SetReason("blah-blah-blah");
+ });
+
+ // verify in old engine
+ test(TEvProposeTransactionResult::RESPONSE_DATA,
+ Ydb::StatusIds::GENERIC_ERROR,
+ NYql::TIssuesIds::DEFAULT_ERROR,
+ "Error executing transaction: transaction failed.");
+ }
+}
+
+Y_UNIT_TEST_NEW_ENGINE(ProposeRequestUndelivered) {
+ auto mitm = [&](TTestActorRuntimeBase& rt, TAutoPtr<IEventHandle> &ev) {
+ if (ev->GetTypeRewrite() == TEvPipeCache::TEvForward::EventType) {
+ auto forwardEvent = ev.Get()->Get<TEvPipeCache::TEvForward>();
+ if (forwardEvent->Ev->Type() == TEvDataShard::TEvProposeTransaction::EventType) {
+ rt.Send(new IEventHandle(ev->Sender, ev->Recipient,
+ new TEvPipeCache::TEvDeliveryProblem(forwardEvent->TabletId, /* NotDelivered */ true)));
+ return TTestActorRuntime::EEventAction::DROP;
+ }
+ }
+ return TTestActorRuntime::EEventAction::PROCESS;
+ };
+ Runtime->SetObserverFunc(mitm);
+
+ SendRequest(*Runtime, Client, MakeSQLRequest(Q_("select * from `/Root/table-1`")));
+
+ auto ev = Runtime->GrabEdgeEventRethrow<NKqp::TEvKqp::TEvQueryResponse>(Client);
+ auto& record = ev->Get()->Record.GetRef();
+ UNIT_ASSERT_VALUES_EQUAL_C(record.GetYdbStatus(), Ydb::StatusIds::UNAVAILABLE, record.DebugString());
+
+ // Cerr << record.DebugString() << Endl;
+
+ TIssues issues;
+ IssuesFromMessage(record.GetResponse().GetQueryIssues(), issues);
+ UNIT_ASSERT_C(HasIssue(issues, NYql::TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE,
+ "Kikimr cluster or one of its subsystems was unavailable."), record.GetResponse().DebugString());
+
+ UNIT_ASSERT_C(HasIssue(issues, NYql::TIssuesIds::DEFAULT_ERROR, "", [] (const TIssue& issue) {
+ return UseNewEngine
+ ? issue.Message.StartsWith("Could not deliver program to shard ")
+ : issue.Message == "Error executing transaction (ProxyShardNotAvailable): One or more of affected datashards not available, request execution cancelled";
+ }), record.GetResponse().DebugString());
+}
+
+void TestProposeResultLost(TTestActorRuntime& runtime, TActorId client, const TString& query,
+ std::function<void(const NKikimrKqp::TEvQueryResponse& resp)> fn)
+{
+ TActorId executer;
+ ui32 droppedEvents = 0;
+
+ runtime.SetObserverFunc([&](TTestActorRuntimeBase& rt, TAutoPtr<IEventHandle>& ev) {
+ if (ev->GetTypeRewrite() == TEvPipeCache::TEvForward::EventType) {
+ auto* fe = ev.Get()->Get<TEvPipeCache::TEvForward>();
+ if (fe->Ev->Type() == TEvDataShard::TEvProposeTransaction::EventType) {
+ executer = ((TEvDataShard::TEvProposeTransaction*) fe->Ev.Get())->GetSource();
+ // Cerr << "-- executer: " << executer << Endl;
+ return TTestActorRuntime::EEventAction::PROCESS;
+ }
+ }
+
+ if (ev->GetTypeRewrite() == TEvDataShard::TEvProposeTransactionResult::EventType) {
+ auto* msg = ev.Get()->Get<TEvDataShard::TEvProposeTransactionResult>();
+ if (msg->Record.GetStatus() == NKikimrTxDataShard::TEvProposeTransactionResult::PREPARED) {
+ if (ev->Sender.NodeId() == executer.NodeId()) {
+ ++droppedEvents;
+ // Cerr << "-- send undelivery to " << ev->Recipient << ", executer: " << executer << Endl;
+ rt.Send(new IEventHandle(executer, ev->Sender,
+ new TEvPipeCache::TEvDeliveryProblem(msg->GetOrigin(), /* NotDelivered */ false)));
+ return TTestActorRuntime::EEventAction::DROP;
+ }
+ }
+ }
+
+ return TTestActorRuntime::EEventAction::PROCESS;
+ });
+ SendRequest(runtime, client, MakeSQLRequest(query));
+
+ auto ev = runtime.GrabEdgeEventRethrow<NKqp::TEvKqp::TEvQueryResponse>(client);
+ UNIT_ASSERT(droppedEvents > 0 && droppedEvents < 4);
+
+ auto& record = ev->Get()->Record.GetRef();
+ // Cerr << record.DebugString() << Endl;
+ fn(record);
+}
+
+Y_UNIT_TEST_NEW_ENGINE(ProposeResultLost_RoTx) {
+ TestProposeResultLost(*Runtime, Client,
+ Q_("select * from `/Root/table-1`"),
+ [](const NKikimrKqp::TEvQueryResponse& record) {
+ UNIT_ASSERT_VALUES_EQUAL_C(record.GetYdbStatus(), Ydb::StatusIds::UNAVAILABLE, record.DebugString());
+
+ TIssues issues;
+ IssuesFromMessage(record.GetResponse().GetQueryIssues(), issues);
+ UNIT_ASSERT_C(HasIssue(issues, NYql::TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE,
+ "Kikimr cluster or one of its subsystems was unavailable."), record.GetResponse().DebugString());
+
+ UNIT_ASSERT_C(HasIssue(issues, NKikimrIssues::TIssuesIds::TX_STATE_UNKNOWN, "", [] (const TIssue& issue) {
+ return UseNewEngine
+ ? issue.Message.StartsWith("Tx state unknown for shard ")
+ : issue.Message.StartsWith("tx state unknown for shard ");
+ }), record.GetResponse().DebugString());
+ });
+}
+
+Y_UNIT_TEST_NEW_ENGINE(ProposeResultLost_RwTx) {
+ TestProposeResultLost(*Runtime, Client,
+ Q_(R"(
+ upsert into `/Root/table-1` (key, value) VALUES
+ (1, 1), (1073741823, 1073741823), (2147483647, 2147483647), (4294967295, 4294967295)
+ )"),
+ [](const NKikimrKqp::TEvQueryResponse& record) {
+ UNIT_ASSERT_VALUES_EQUAL_C(record.GetYdbStatus(), Ydb::StatusIds::UNDETERMINED, record.DebugString());
+
+ TIssues issues;
+ IssuesFromMessage(record.GetResponse().GetQueryIssues(), issues);
+ UNIT_ASSERT_C(HasIssue(issues, NYql::TIssuesIds::KIKIMR_OPERATION_STATE_UNKNOWN,
+ "State of operation is unknown."), record.GetResponse().DebugString());
+
+ UNIT_ASSERT_C(HasIssue(issues, NKikimrIssues::TIssuesIds::TX_STATE_UNKNOWN, "", [] (const TIssue& issue) {
+ return UseNewEngine
+ ? issue.Message.StartsWith("Tx state unknown for shard ")
+ : issue.Message.StartsWith("tx state unknown for shard ");
+ }), record.GetResponse().DebugString());
+ });
+}
+
+} // suite
+
+} // namespace NKqp
+} // namespace NKikimr
diff --git a/ydb/core/tx/datashard/datashard_ut_kqp_scan.cpp b/ydb/core/tx/datashard/datashard_ut_kqp_scan.cpp
index bf34e1d525..b2b14de190 100644
--- a/ydb/core/tx/datashard/datashard_ut_kqp_scan.cpp
+++ b/ydb/core/tx/datashard/datashard_ut_kqp_scan.cpp
@@ -28,12 +28,12 @@ namespace {
void EnableLogging(TTestActorRuntime& runtime) {
runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG);
- //runtime.SetLogPriority(NKikimrServices::TX_PROXY, NLog::PRI_DEBUG);
- runtime.SetLogPriority(NKikimrServices::KQP_EXECUTER, NActors::NLog::PRI_TRACE);
- //runtime.SetLogPriority(NKikimrServices::KQP_WORKER, NActors::NLog::PRI_DEBUG);
- //runtime.SetLogPriority(NKikimrServices::KQP_RESOURCE_MANAGER, NActors::NLog::PRI_DEBUG);
- //runtime.SetLogPriority(NKikimrServices::KQP_NODE, NActors::NLog::PRI_DEBUG);
- runtime.SetLogPriority(NKikimrServices::KQP_COMPUTE, NActors::NLog::PRI_TRACE);
+ //runtime.SetLogPriority(NKikimrServices::TX_PROXY, NLog::PRI_DEBUG);
+ runtime.SetLogPriority(NKikimrServices::KQP_EXECUTER, NActors::NLog::PRI_TRACE);
+ //runtime.SetLogPriority(NKikimrServices::KQP_WORKER, NActors::NLog::PRI_DEBUG);
+ //runtime.SetLogPriority(NKikimrServices::KQP_RESOURCE_MANAGER, NActors::NLog::PRI_DEBUG);
+ //runtime.SetLogPriority(NKikimrServices::KQP_NODE, NActors::NLog::PRI_DEBUG);
+ runtime.SetLogPriority(NKikimrServices::KQP_COMPUTE, NActors::NLog::PRI_TRACE);
}
}
@@ -74,7 +74,7 @@ Y_UNIT_TEST_SUITE(KqpScan) {
ExecSQL(server, sender, FillTableQuery());
TSet<TActorId> scans;
- TSet<TActorId> killedTablets;
+ TSet<TActorId> killedTablets;
ui64 result = 0;
@@ -87,16 +87,16 @@ Y_UNIT_TEST_SUITE(KqpScan) {
case NKqp::TKqpExecuterEvents::EvShardsResolveStatus: {
auto* msg = ev->Get<NKqp::TEvKqpExecuter::TEvShardsResolveStatus>();
for (auto& [shardId, nodeId]: msg->ShardNodes) {
- Cerr << "-- nodeId: " << nodeId << Endl;
- Cerr.Flush();
- nodeId = runtime.GetNodeId(0);
+ Cerr << "-- nodeId: " << nodeId << Endl;
+ Cerr.Flush();
+ nodeId = runtime.GetNodeId(0);
}
break;
}
case TEvDataShard::EvKqpScan: {
Cerr << (TStringBuilder() << "-- EvScan " << ev->Sender << " -> " << ev->Recipient << Endl);
- Cerr.Flush();
+ Cerr.Flush();
break;
}
@@ -108,7 +108,7 @@ Y_UNIT_TEST_SUITE(KqpScan) {
auto& record = ev->Get<NKqp::TEvKqpExecuter::TEvStreamData>()->Record;
Cerr << (TStringBuilder() << "-- EvStreamData: " << record.AsJSON() << Endl);
- Cerr.Flush();
+ Cerr.Flush();
Y_ASSERT(record.GetResultSet().rows().size() == 1);
Y_ASSERT(record.GetResultSet().rows().at(0).items().size() == 1);
@@ -125,16 +125,16 @@ Y_UNIT_TEST_SUITE(KqpScan) {
/* Drop message and kill tablet if we already had seen this tablet */
case NKqp::TKqpComputeEvents::EvScanData: {
if (scans.contains(ev->Sender)) {
- if (killedTablets.empty()) { // do only 1 kill per test
- runtime.Send(new IEventHandle(ev->Sender, ev->Sender, new NKqp::TEvKqpCompute::TEvKillScanTablet));
- Cerr << (TStringBuilder() << "-- EvScanData from " << ev->Sender << ": hijack event, kill tablet " << ev->Sender << Endl);
- Cerr.Flush();
- }
+ if (killedTablets.empty()) { // do only 1 kill per test
+ runtime.Send(new IEventHandle(ev->Sender, ev->Sender, new NKqp::TEvKqpCompute::TEvKillScanTablet));
+ Cerr << (TStringBuilder() << "-- EvScanData from " << ev->Sender << ": hijack event, kill tablet " << ev->Sender << Endl);
+ Cerr.Flush();
+ }
} else {
scans.insert(ev->Sender);
- runtime.EnableScheduleForActor(ev->Sender);
+ runtime.EnableScheduleForActor(ev->Sender);
Cerr << (TStringBuilder() << "-- EvScanData from " << ev->Sender << ": pass" << Endl);
- Cerr.Flush();
+ Cerr.Flush();
}
break;
@@ -178,9 +178,9 @@ Y_UNIT_TEST_SUITE(KqpScan) {
CreateShardedTable(server, sender, "/Root", "table-1", 7);
ExecSQL(server, sender, FillTableQuery());
- ui64 firstNodeId = server->GetRuntime()->GetNodeId(0);
- // ui64 secondNodeId = server->GetRuntime()->GetNodeId(1);
-
+ ui64 firstNodeId = server->GetRuntime()->GetNodeId(0);
+ // ui64 secondNodeId = server->GetRuntime()->GetNodeId(1);
+
bool remoteScanDetected = false;
ui64 result = 0;
@@ -194,7 +194,7 @@ Y_UNIT_TEST_SUITE(KqpScan) {
case NKqp::TKqpExecuterEvents::EvShardsResolveStatus: {
auto* msg = ev->Get<NKqp::TEvKqpExecuter::TEvShardsResolveStatus>();
for (auto& [shardId, nodeId]: msg->ShardNodes) {
- nodeId = firstNodeId;
+ nodeId = firstNodeId;
}
break;
}
@@ -266,17 +266,17 @@ Y_UNIT_TEST_SUITE(KqpScan) {
// EnableLogging(runtime);
- SetSplitMergePartCountLimit(&runtime, -1);
-
+ SetSplitMergePartCountLimit(&runtime, -1);
+
InitRoot(server, sender);
CreateShardedTable(server, sender, "/Root", "table-1", 1);
ExecSQL(server, sender, FillTableQuery());
auto shards = GetTableShards(server, sender, "/Root/table-1");
- for (const auto& shard: shards) {
- Cerr << (TStringBuilder() << "-- shardId=" << shard << Endl);
- Cerr.Flush();
- }
+ for (const auto& shard: shards) {
+ Cerr << (TStringBuilder() << "-- shardId=" << shard << Endl);
+ Cerr.Flush();
+ }
TSet<TActorId> scans;
TActorId firstScanActor;
@@ -284,21 +284,21 @@ Y_UNIT_TEST_SUITE(KqpScan) {
ui64 result = 0;
- auto captureEvents = [&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) {
+ auto captureEvents = [&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) {
switch (ev->GetTypeRewrite()) {
case NKqp::TKqpExecuterEvents::EvShardsResolveStatus: {
auto* msg = ev->Get<NKqp::TEvKqpExecuter::TEvShardsResolveStatus>();
for (auto& [shardId, nodeId]: msg->ShardNodes) {
tabletId = shardId;
Cerr << (TStringBuilder() << "-- tabletId= " << tabletId << Endl);
- Cerr.Flush();
+ Cerr.Flush();
}
break;
}
case TEvDataShard::EvKqpScan: {
Cerr << (TStringBuilder() << "-- EvScan " << ev->Sender << " -> " << ev->Recipient << Endl);
- Cerr.Flush();
+ Cerr.Flush();
break;
}
@@ -310,7 +310,7 @@ Y_UNIT_TEST_SUITE(KqpScan) {
auto& record = ev->Get<NKqp::TEvKqpExecuter::TEvStreamData>()->Record;
Cerr << (TStringBuilder() << "-- EvStreamData: " << record.AsJSON() << Endl);
- Cerr.Flush();
+ Cerr.Flush();
Y_ASSERT(record.GetResultSet().rows().size() == 1);
Y_ASSERT(record.GetResultSet().rows().at(0).items().size() == 1);
@@ -345,16 +345,16 @@ Y_UNIT_TEST_SUITE(KqpScan) {
runtime.Send(new IEventHandle(MakeTxProxyID(), senderSplit, request.Release()));
Cerr << (TStringBuilder() << "-- EvScanData from old tablet " << ev->Sender << ": pass and split" << Endl);
- Cerr.Flush();
+ Cerr.Flush();
} else if (firstScanActor == ev->Sender) {
// data from old table scan, drop it
Cerr << (TStringBuilder() << "-- EvScanData from old tablet " << ev->Sender << ": drop" << Endl);
- Cerr.Flush();
- return TTestActorRuntime::EEventAction::DROP;
+ Cerr.Flush();
+ return TTestActorRuntime::EEventAction::DROP;
} else {
// data from new tablet scan, pass it
Cerr << (TStringBuilder() << "-- EvScanData from new tablet" << ev->Sender << ": pass" << Endl);
- Cerr.Flush();
+ Cerr.Flush();
}
break;
@@ -403,7 +403,7 @@ Y_UNIT_TEST_SUITE(KqpScan) {
ExecSQL(server, sender, FillTableQuery());
TSet<TActorId> scans;
- TSet<TActorId> killedTablets;
+ TSet<TActorId> killedTablets;
ui64 result = 0;
ui64 incomingRangesSize = 0;
@@ -463,15 +463,15 @@ Y_UNIT_TEST_SUITE(KqpScan) {
/* Drop message and kill tablet if we already had seen this tablet */
case NKqp::TKqpComputeEvents::EvScanData: {
if (scans.contains(ev->Sender)) {
- if (killedTablets.empty()) { // do only 1 kill per test
- runtime.Send(new IEventHandle(ev->Sender, ev->Sender, new NKqp::TEvKqpCompute::TEvKillScanTablet));
- Cerr << (TStringBuilder() << "-- EvScanData from " << ev->Sender << ": hijack event, kill tablet " << ev->Sender << Endl);
- Cerr.Flush();
- }
+ if (killedTablets.empty()) { // do only 1 kill per test
+ runtime.Send(new IEventHandle(ev->Sender, ev->Sender, new NKqp::TEvKqpCompute::TEvKillScanTablet));
+ Cerr << (TStringBuilder() << "-- EvScanData from " << ev->Sender << ": hijack event, kill tablet " << ev->Sender << Endl);
+ Cerr.Flush();
+ }
} else {
scans.insert(ev->Sender);
- runtime.EnableScheduleForActor(ev->Sender);
-
+ runtime.EnableScheduleForActor(ev->Sender);
+
Cerr << (TStringBuilder() << "-- EvScanData from " << ev->Sender << ": pass" << Endl);
auto scanEvent = ev->Get<NKqp::TEvKqpCompute::TEvScanData>();
diff --git a/ydb/core/tx/datashard/datashard_ut_order.cpp b/ydb/core/tx/datashard/datashard_ut_order.cpp
index ecfc9d21fd..32472dce59 100644
--- a/ydb/core/tx/datashard/datashard_ut_order.cpp
+++ b/ydb/core/tx/datashard/datashard_ut_order.cpp
@@ -1229,7 +1229,7 @@ Y_UNIT_TEST_WITH_MVCC(ScanFollowedByUpdate) {
proxy.ExecQueue();
}
-Y_UNIT_TEST_QUAD(TestDelayedTxWaitsForWriteActiveTxOnly, UseMvcc, UseNewEngine) {
+Y_UNIT_TEST_QUAD(TestDelayedTxWaitsForWriteActiveTxOnly, UseMvcc, UseNewEngine) {
TPortManager pm;
TServerSettings serverSettings(pm.GetPort(2134));
serverSettings.SetDomainName("Root")
@@ -1242,11 +1242,11 @@ Y_UNIT_TEST_QUAD(TestDelayedTxWaitsForWriteActiveTxOnly, UseMvcc, UseNewEngine)
TAutoPtr<IEventHandle> handle;
runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG);
- if (UseNewEngine) {
- runtime.SetLogPriority(NKikimrServices::KQP_EXECUTER, NLog::PRI_DEBUG);
- } else {
- runtime.SetLogPriority(NKikimrServices::KQP_PROXY, NLog::PRI_DEBUG);
- }
+ if (UseNewEngine) {
+ runtime.SetLogPriority(NKikimrServices::KQP_EXECUTER, NLog::PRI_DEBUG);
+ } else {
+ runtime.SetLogPriority(NKikimrServices::KQP_PROXY, NLog::PRI_DEBUG);
+ }
runtime.SetLogPriority(NKikimrServices::MINIKQL_ENGINE, NActors::NLog::PRI_DEBUG);
InitRoot(server, sender);
@@ -1254,8 +1254,8 @@ Y_UNIT_TEST_QUAD(TestDelayedTxWaitsForWriteActiveTxOnly, UseMvcc, UseNewEngine)
CreateShardedTable(server, sender, "/Root", "table-1", 1);
CreateShardedTable(server, sender, "/Root", "table-2", 1);
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1), (3, 3);"));
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 2);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1), (3, 3);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 2);"));
ui64 shard2 = GetTableShards(server, sender, "/Root/table-2")[0];
@@ -1287,8 +1287,8 @@ Y_UNIT_TEST_QUAD(TestDelayedTxWaitsForWriteActiveTxOnly, UseMvcc, UseNewEngine)
}
// Copy data from table-1 to table-3. Txs should hang due to dropped RS.
- SendSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) SELECT key, value FROM `/Root/table-1` WHERE key = 1"));
- SendSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) SELECT key, value FROM `/Root/table-1` WHERE key = 3"));
+ SendSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) SELECT key, value FROM `/Root/table-1` WHERE key = 1"));
+ SendSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) SELECT key, value FROM `/Root/table-1` WHERE key = 3"));
{
TDispatchOptions options;
options.FinalEvents.emplace_back(IsTxResultComplete(), 4);
@@ -1299,13 +1299,13 @@ Y_UNIT_TEST_QUAD(TestDelayedTxWaitsForWriteActiveTxOnly, UseMvcc, UseNewEngine)
// because transactions above are stuck before performing any writes. Make sure it's
// forced to wait for above transactions by commiting a write that is guaranteed
// to "happen" after transactions above.
- ExecSQL(server, sender, Q_(R"(
- UPSERT INTO `/Root/table-1` (key, value) VALUES (4, 4);
- UPSERT INTO `/Root/table-2` (key, value) VALUES (5, 5);
- )"));
+ ExecSQL(server, sender, Q_(R"(
+ UPSERT INTO `/Root/table-1` (key, value) VALUES (4, 4);
+ UPSERT INTO `/Root/table-2` (key, value) VALUES (5, 5);
+ )"));
// This immediate tx should be delayed due to conflict with upserts.
- SendSQL(server, sender, Q_("SELECT * FROM `/Root/table-2`"));
+ SendSQL(server, sender, Q_("SELECT * FROM `/Root/table-2`"));
{
TDispatchOptions options;
options.FinalEvents.emplace_back(TEvDataShard::EvProposeTransaction, 1);
@@ -1325,7 +1325,7 @@ Y_UNIT_TEST_QUAD(TestDelayedTxWaitsForWriteActiveTxOnly, UseMvcc, UseNewEngine)
}
}
-Y_UNIT_TEST_QUAD(TestOnlyDataTxLagCausesRejects, UseMvcc, UseNewEngine) {
+Y_UNIT_TEST_QUAD(TestOnlyDataTxLagCausesRejects, UseMvcc, UseNewEngine) {
TPortManager pm;
TServerSettings serverSettings(pm.GetPort(2134));
serverSettings.SetDomainName("Root")
@@ -1338,7 +1338,7 @@ Y_UNIT_TEST_QUAD(TestOnlyDataTxLagCausesRejects, UseMvcc, UseNewEngine) {
TAutoPtr<IEventHandle> handle;
runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
- runtime.SetLogPriority(UseNewEngine ? NKikimrServices::KQP_EXECUTER : NKikimrServices::KQP_PROXY, NLog::PRI_DEBUG);
+ runtime.SetLogPriority(UseNewEngine ? NKikimrServices::KQP_EXECUTER : NKikimrServices::KQP_PROXY, NLog::PRI_DEBUG);
runtime.SetLogPriority(NKikimrServices::MINIKQL_ENGINE, NActors::NLog::PRI_DEBUG);
InitRoot(server, sender);
@@ -1346,7 +1346,7 @@ Y_UNIT_TEST_QUAD(TestOnlyDataTxLagCausesRejects, UseMvcc, UseNewEngine) {
CreateShardedTable(server, sender, "/Root", "table-1", 2);
//auto shards = GetTableShards(server, sender, "/Root/table-1");
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 3000000001), (3000000003, 3)"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 3000000001), (3000000003, 3)"));
// Send ReadTable requests and wait until they hang waiting for quota.
for (int i = 0; i < 2; ++i) {
@@ -1367,7 +1367,7 @@ Y_UNIT_TEST_QUAD(TestOnlyDataTxLagCausesRejects, UseMvcc, UseNewEngine) {
runtime.DispatchEvents(options);
}
- ExecSQL(server, sender, Q_("SELECT COUNT(*) FROM `/Root/table-1`"));
+ ExecSQL(server, sender, Q_("SELECT COUNT(*) FROM `/Root/table-1`"));
// Send SQL request which should hang due to lost RS.
auto captureRS = [](TTestActorRuntimeBase&,
@@ -1378,7 +1378,7 @@ Y_UNIT_TEST_QUAD(TestOnlyDataTxLagCausesRejects, UseMvcc, UseNewEngine) {
};
runtime.SetObserverFunc(captureRS);
- SendSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) SELECT value, key FROM `/Root/table-1`"));
+ SendSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) SELECT value, key FROM `/Root/table-1`"));
{
TDispatchOptions options;
options.FinalEvents.emplace_back(IsTxResultComplete(), 2);
@@ -1394,14 +1394,14 @@ Y_UNIT_TEST_QUAD(TestOnlyDataTxLagCausesRejects, UseMvcc, UseNewEngine) {
runtime.DispatchEvents(options);
}
- ExecSQL(server, sender, Q_("SELECT COUNT(*) FROM `/Root/table-1`"), true, Ydb::StatusIds::UNAVAILABLE);
+ ExecSQL(server, sender, Q_("SELECT COUNT(*) FROM `/Root/table-1`"), true, Ydb::StatusIds::UNAVAILABLE);
}
}
Y_UNIT_TEST_SUITE(DataShardOutOfOrder) {
-Y_UNIT_TEST_QUAD(TestOutOfOrderLockLost, UseMvcc, UseNewEngine) {
+Y_UNIT_TEST_QUAD(TestOutOfOrderLockLost, UseMvcc, UseNewEngine) {
TPortManager pm;
TServerSettings serverSettings(pm.GetPort(2134));
serverSettings.SetDomainName("Root")
@@ -1413,24 +1413,24 @@ Y_UNIT_TEST_QUAD(TestOutOfOrderLockLost, UseMvcc, UseNewEngine) {
auto sender = runtime.AllocateEdgeActor();
runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
- runtime.SetLogPriority(UseNewEngine ? NKikimrServices::KQP_EXECUTER : NKikimrServices::TX_PROXY, NLog::PRI_DEBUG);
+ runtime.SetLogPriority(UseNewEngine ? NKikimrServices::KQP_EXECUTER : NKikimrServices::TX_PROXY, NLog::PRI_DEBUG);
InitRoot(server, sender);
CreateShardedTable(server, sender, "/Root", "table-1", 1);
CreateShardedTable(server, sender, "/Root", "table-2", 1);
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1);"));
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 1);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 1);"));
TString sessionId = CreateSession(runtime, sender);
TString txId;
{
- auto ev = ExecRequest(runtime, sender, MakeBeginRequest(sessionId, Q_(R"(
- SELECT * FROM `/Root/table-1`
- UNION ALL
- SELECT * FROM `/Root/table-2`)")));
+ auto ev = ExecRequest(runtime, sender, MakeBeginRequest(sessionId, Q_(R"(
+ SELECT * FROM `/Root/table-1`
+ UNION ALL
+ SELECT * FROM `/Root/table-2`)")));
auto& response = ev->Get()->Record.GetRef();
UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS);
txId = response.GetResponse().GetTxMeta().id();
@@ -1451,9 +1451,9 @@ Y_UNIT_TEST_QUAD(TestOutOfOrderLockLost, UseMvcc, UseNewEngine) {
// Send a commit request, it would block on readset exchange
auto sender2 = runtime.AllocateEdgeActor();
- SendRequest(runtime, sender2, MakeCommitRequest(sessionId, txId, Q_(R"(
- UPSERT INTO `/Root/table-1` (key, value) VALUES (3, 2);
- UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2))")));
+ SendRequest(runtime, sender2, MakeCommitRequest(sessionId, txId, Q_(R"(
+ UPSERT INTO `/Root/table-1` (key, value) VALUES (3, 2);
+ UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2))")));
// Wait until we captured both readsets
{
@@ -1470,8 +1470,8 @@ Y_UNIT_TEST_QUAD(TestOutOfOrderLockLost, UseMvcc, UseNewEngine) {
// It would have broken locks if executed before the above commit
// However the above commit must succeed (readsets are already being exchanged)
auto sender3 = runtime.AllocateEdgeActor();
- SendRequest(runtime, sender3, MakeSimpleRequest(Q_(
- "UPSERT INTO `/Root/table-1` (key, value) VALUES (5, 3)")));
+ SendRequest(runtime, sender3, MakeSimpleRequest(Q_(
+ "UPSERT INTO `/Root/table-1` (key, value) VALUES (5, 3)")));
// Schedule a simple timer to simulate some time passing
{
@@ -1514,13 +1514,13 @@ Y_UNIT_TEST_QUAD(TestOutOfOrderLockLost, UseMvcc, UseNewEngine) {
// Select keys 3 and 4 from both tables, either both or none should be inserted
{
auto sender5 = runtime.AllocateEdgeActor();
- auto ev = ExecRequest(runtime, sender5, MakeSimpleRequest(Q_(R"(
- $rows = (
- SELECT key, value FROM `/Root/table-1` WHERE key = 3
- UNION ALL
- SELECT key, value FROM `/Root/table-2` WHERE key = 4
- );
- SELECT key, value FROM $rows ORDER BY key)")));
+ auto ev = ExecRequest(runtime, sender5, MakeSimpleRequest(Q_(R"(
+ $rows = (
+ SELECT key, value FROM `/Root/table-1` WHERE key = 3
+ UNION ALL
+ SELECT key, value FROM `/Root/table-2` WHERE key = 4
+ );
+ SELECT key, value FROM $rows ORDER BY key)")));
auto& response = ev->Get()->Record.GetRef();
UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS);
UNIT_ASSERT_VALUES_EQUAL(response.GetResponse().GetResults().size(), 1u);
@@ -1534,7 +1534,7 @@ Y_UNIT_TEST_QUAD(TestOutOfOrderLockLost, UseMvcc, UseNewEngine) {
}
}
-Y_UNIT_TEST_NEW_ENGINE(TestMvccReadDoesntBlockWrites) {
+Y_UNIT_TEST_NEW_ENGINE(TestMvccReadDoesntBlockWrites) {
TPortManager pm;
TServerSettings serverSettings(pm.GetPort(2134));
serverSettings.SetDomainName("Root")
@@ -1549,24 +1549,24 @@ Y_UNIT_TEST_NEW_ENGINE(TestMvccReadDoesntBlockWrites) {
auto sender3 = runtime.AllocateEdgeActor();
runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
- runtime.SetLogPriority(UseNewEngine ? NKikimrServices::KQP_EXECUTER : NKikimrServices::TX_PROXY, NLog::PRI_DEBUG);
+ runtime.SetLogPriority(UseNewEngine ? NKikimrServices::KQP_EXECUTER : NKikimrServices::TX_PROXY, NLog::PRI_DEBUG);
InitRoot(server, sender);
CreateShardedTable(server, sender, "/Root", "table-1", 1);
CreateShardedTable(server, sender, "/Root", "table-2", 1);
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1);"));
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 2);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 2);"));
TString sessionId = CreateSession(runtime, sender);
TString txId;
{
- auto ev = ExecRequest(runtime, sender, MakeBeginRequest(sessionId, Q_(R"(
- SELECT * FROM `/Root/table-1`
- UNION ALL
- SELECT * FROM `/Root/table-2`)")));
+ auto ev = ExecRequest(runtime, sender, MakeBeginRequest(sessionId, Q_(R"(
+ SELECT * FROM `/Root/table-1`
+ UNION ALL
+ SELECT * FROM `/Root/table-2`)")));
auto& response = ev->Get()->Record.GetRef();
UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS);
txId = response.GetResponse().GetTxMeta().id();
@@ -1586,9 +1586,9 @@ Y_UNIT_TEST_NEW_ENGINE(TestMvccReadDoesntBlockWrites) {
auto prevObserverFunc = runtime.SetObserverFunc(captureRS);
// Send a commit request, it would block on readset exchange
- SendRequest(runtime, sender2, MakeCommitRequest(sessionId, txId, Q_(R"(
- UPSERT INTO `/Root/table-1` (key, value) VALUES (3, 2);
- UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2);)")));
+ SendRequest(runtime, sender2, MakeCommitRequest(sessionId, txId, Q_(R"(
+ UPSERT INTO `/Root/table-1` (key, value) VALUES (3, 2);
+ UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2);)")));
// Wait until we captured both readsets
{
@@ -1603,13 +1603,13 @@ Y_UNIT_TEST_NEW_ENGINE(TestMvccReadDoesntBlockWrites) {
runtime.SetObserverFunc(prevObserverFunc);
// it will be blocked by previous transaction that is waiting for its readsets
- SendRequest(runtime, sender, MakeSimpleRequest(Q_(R"(
- $rows = (
- SELECT * FROM `/Root/table-1` WHERE key = 3 OR key = 5
- UNION ALL
- SELECT * FROM `/Root/table-2` WHERE key = 4 OR key = 6
- );
- SELECT key, value FROM $rows ORDER BY key)")));
+ SendRequest(runtime, sender, MakeSimpleRequest(Q_(R"(
+ $rows = (
+ SELECT * FROM `/Root/table-1` WHERE key = 3 OR key = 5
+ UNION ALL
+ SELECT * FROM `/Root/table-2` WHERE key = 4 OR key = 6
+ );
+ SELECT key, value FROM $rows ORDER BY key)")));
// wait for the tx is planned
TDispatchOptions opts;
@@ -1618,9 +1618,9 @@ Y_UNIT_TEST_NEW_ENGINE(TestMvccReadDoesntBlockWrites) {
{
// despite it's writing into the key that previous transaction reads this write should finish successfully
- auto ev = ExecRequest(runtime, sender3, MakeSimpleRequest(Q_(R"(
- UPSERT INTO `/Root/table-1` (key, value) VALUES (5, 10);
- UPSERT INTO `/Root/table-2` (key, value) VALUES (6, 10))")));
+ auto ev = ExecRequest(runtime, sender3, MakeSimpleRequest(Q_(R"(
+ UPSERT INTO `/Root/table-1` (key, value) VALUES (5, 10);
+ UPSERT INTO `/Root/table-2` (key, value) VALUES (6, 10))")));
auto& response = ev->Get()->Record.GetRef();
UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS);
}
@@ -1652,13 +1652,13 @@ Y_UNIT_TEST_NEW_ENGINE(TestMvccReadDoesntBlockWrites) {
{
// Now we see the write
- auto ev = ExecRequest(runtime, sender, MakeSimpleRequest(Q_(R"(
- $rows = (
- SELECT * FROM `/Root/table-1` WHERE key = 3 OR key = 5
- UNION ALL
- SELECT * FROM `/Root/table-2` WHERE key = 4 OR key = 6
- );
- SELECT key, value FROM $rows ORDER BY key)")));
+ auto ev = ExecRequest(runtime, sender, MakeSimpleRequest(Q_(R"(
+ $rows = (
+ SELECT * FROM `/Root/table-1` WHERE key = 3 OR key = 5
+ UNION ALL
+ SELECT * FROM `/Root/table-2` WHERE key = 4 OR key = 6
+ );
+ SELECT key, value FROM $rows ORDER BY key)")));
auto& response = ev->Get()->Record.GetRef();
UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS);
UNIT_ASSERT_VALUES_EQUAL(response.GetResponse().GetResults().size(), 1u);
@@ -1672,7 +1672,7 @@ Y_UNIT_TEST_NEW_ENGINE(TestMvccReadDoesntBlockWrites) {
}
}
-Y_UNIT_TEST_QUAD(TestOutOfOrderReadOnlyAllowed, UseMvcc, UseNewEngine) {
+Y_UNIT_TEST_QUAD(TestOutOfOrderReadOnlyAllowed, UseMvcc, UseNewEngine) {
TPortManager pm;
TServerSettings serverSettings(pm.GetPort(2134));
serverSettings.SetDomainName("Root")
@@ -1684,24 +1684,24 @@ Y_UNIT_TEST_QUAD(TestOutOfOrderReadOnlyAllowed, UseMvcc, UseNewEngine) {
auto sender = runtime.AllocateEdgeActor();
runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
- runtime.SetLogPriority(UseNewEngine ? NKikimrServices::KQP_EXECUTER : NKikimrServices::TX_PROXY, NLog::PRI_DEBUG);
+ runtime.SetLogPriority(UseNewEngine ? NKikimrServices::KQP_EXECUTER : NKikimrServices::TX_PROXY, NLog::PRI_DEBUG);
InitRoot(server, sender);
CreateShardedTable(server, sender, "/Root", "table-1", 1);
CreateShardedTable(server, sender, "/Root", "table-2", 1);
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1);"));
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 1);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 1);"));
TString sessionId = CreateSession(runtime, sender);
TString txId;
{
- auto ev = ExecRequest(runtime, sender, MakeBeginRequest(sessionId, Q_(R"(
- SELECT * FROM `/Root/table-1`
- UNION ALL
- SELECT * FROM `/Root/table-2`)")));
+ auto ev = ExecRequest(runtime, sender, MakeBeginRequest(sessionId, Q_(R"(
+ SELECT * FROM `/Root/table-1`
+ UNION ALL
+ SELECT * FROM `/Root/table-2`)")));
auto& response = ev->Get()->Record.GetRef();
UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS);
txId = response.GetResponse().GetTxMeta().id();
@@ -1722,9 +1722,9 @@ Y_UNIT_TEST_QUAD(TestOutOfOrderReadOnlyAllowed, UseMvcc, UseNewEngine) {
// Send a commit request, it would block on readset exchange
auto sender2 = runtime.AllocateEdgeActor();
- SendRequest(runtime, sender2, MakeCommitRequest(sessionId, txId, Q_(R"(
- UPSERT INTO `/Root/table-1` (key, value) VALUES (3, 2);
- UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2))")));
+ SendRequest(runtime, sender2, MakeCommitRequest(sessionId, txId, Q_(R"(
+ UPSERT INTO `/Root/table-1` (key, value) VALUES (3, 2);
+ UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2))")));
// Wait until we captured both readsets
{
@@ -1741,8 +1741,8 @@ Y_UNIT_TEST_QUAD(TestOutOfOrderReadOnlyAllowed, UseMvcc, UseNewEngine) {
// Since it's readonly it cannot affect inflight transaction and shouled be allowed
{
auto sender3 = runtime.AllocateEdgeActor();
- auto ev = ExecRequest(runtime, sender3, MakeSimpleRequest(Q_(
- "SELECT key, value FROM `/Root/table-1` ORDER BY key")));
+ auto ev = ExecRequest(runtime, sender3, MakeSimpleRequest(Q_(
+ "SELECT key, value FROM `/Root/table-1` ORDER BY key")));
auto& response = ev->Get()->Record.GetRef();
UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS);
UNIT_ASSERT_VALUES_EQUAL(response.GetResponse().GetResults().size(), 1u);
@@ -1767,13 +1767,13 @@ Y_UNIT_TEST_QUAD(TestOutOfOrderReadOnlyAllowed, UseMvcc, UseNewEngine) {
// Select keys 3 and 4 from both tables, both should have been be inserted
{
auto sender4 = runtime.AllocateEdgeActor();
- auto ev = ExecRequest(runtime, sender4, MakeSimpleRequest(Q_(R"(
- $rows = (
- SELECT key, value FROM `/Root/table-1` WHERE key = 3
- UNION ALL
- SELECT key, value FROM `/Root/table-2` WHERE key = 4
- );
- SELECT key, value FROM $rows ORDER BY key)")));
+ auto ev = ExecRequest(runtime, sender4, MakeSimpleRequest(Q_(R"(
+ $rows = (
+ SELECT key, value FROM `/Root/table-1` WHERE key = 3
+ UNION ALL
+ SELECT key, value FROM `/Root/table-2` WHERE key = 4
+ );
+ SELECT key, value FROM $rows ORDER BY key)")));
auto& response = ev->Get()->Record.GetRef();
UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS);
UNIT_ASSERT_VALUES_EQUAL(response.GetResponse().GetResults().size(), 1u);
@@ -1782,7 +1782,7 @@ Y_UNIT_TEST_QUAD(TestOutOfOrderReadOnlyAllowed, UseMvcc, UseNewEngine) {
}
}
-Y_UNIT_TEST_QUAD(TestOutOfOrderNonConflictingWrites, UseMvcc, UseNewEngine) {
+Y_UNIT_TEST_QUAD(TestOutOfOrderNonConflictingWrites, UseMvcc, UseNewEngine) {
TPortManager pm;
TServerSettings serverSettings(pm.GetPort(2134));
serverSettings.SetDomainName("Root")
@@ -1794,24 +1794,24 @@ Y_UNIT_TEST_QUAD(TestOutOfOrderNonConflictingWrites, UseMvcc, UseNewEngine) {
auto sender = runtime.AllocateEdgeActor();
runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
- runtime.SetLogPriority(UseNewEngine ? NKikimrServices::KQP_EXECUTER : NKikimrServices::TX_PROXY, NLog::PRI_DEBUG);
+ runtime.SetLogPriority(UseNewEngine ? NKikimrServices::KQP_EXECUTER : NKikimrServices::TX_PROXY, NLog::PRI_DEBUG);
InitRoot(server, sender);
CreateShardedTable(server, sender, "/Root", "table-1", 1);
CreateShardedTable(server, sender, "/Root", "table-2", 1);
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1);"));
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 1);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 1);"));
TString sessionId = CreateSession(runtime, sender);
TString txId;
{
- auto ev = ExecRequest(runtime, sender, MakeBeginRequest(sessionId, Q_(R"(
- SELECT * FROM `/Root/table-1` WHERE key = 1
- UNION ALL
- SELECT * FROM `/Root/table-2` WHERE key = 2)")));
+ auto ev = ExecRequest(runtime, sender, MakeBeginRequest(sessionId, Q_(R"(
+ SELECT * FROM `/Root/table-1` WHERE key = 1
+ UNION ALL
+ SELECT * FROM `/Root/table-2` WHERE key = 2)")));
auto& response = ev->Get()->Record.GetRef();
UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS);
txId = response.GetResponse().GetTxMeta().id();
@@ -1832,9 +1832,9 @@ Y_UNIT_TEST_QUAD(TestOutOfOrderNonConflictingWrites, UseMvcc, UseNewEngine) {
// Send a commit request, it would block on readset exchange
auto sender2 = runtime.AllocateEdgeActor();
- SendRequest(runtime, sender2, MakeCommitRequest(sessionId, txId, Q_(R"(
- UPSERT INTO `/Root/table-1` (key, value) VALUES (3, 2);
- UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2))")));
+ SendRequest(runtime, sender2, MakeCommitRequest(sessionId, txId, Q_(R"(
+ UPSERT INTO `/Root/table-1` (key, value) VALUES (3, 2);
+ UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2))")));
// Wait until we captured both readsets
if (readSets.size() < 2) {
@@ -1850,9 +1850,9 @@ Y_UNIT_TEST_QUAD(TestOutOfOrderNonConflictingWrites, UseMvcc, UseNewEngine) {
// Now send non-conflicting upsert to both tables
{
auto sender3 = runtime.AllocateEdgeActor();
- auto ev = ExecRequest(runtime, sender3, MakeSimpleRequest(Q_(R"(
- UPSERT INTO `/Root/table-1` (key, value) VALUES (5, 3);
- UPSERT INTO `/Root/table-2` (key, value) VALUES (6, 3))")));
+ auto ev = ExecRequest(runtime, sender3, MakeSimpleRequest(Q_(R"(
+ UPSERT INTO `/Root/table-1` (key, value) VALUES (5, 3);
+ UPSERT INTO `/Root/table-2` (key, value) VALUES (6, 3))")));
auto& response = ev->Get()->Record.GetRef();
UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS);
}
@@ -1860,8 +1860,8 @@ Y_UNIT_TEST_QUAD(TestOutOfOrderNonConflictingWrites, UseMvcc, UseNewEngine) {
// Check that immediate non-conflicting upsert is working too
{
auto sender4 = runtime.AllocateEdgeActor();
- auto ev = ExecRequest(runtime, sender4, MakeSimpleRequest(Q_(
- "UPSERT INTO `/Root/table-1` (key, value) VALUES (7, 4)")));
+ auto ev = ExecRequest(runtime, sender4, MakeSimpleRequest(Q_(
+ "UPSERT INTO `/Root/table-1` (key, value) VALUES (7, 4)")));
auto& response = ev->Get()->Record.GetRef();
UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS);
}
@@ -1883,13 +1883,13 @@ Y_UNIT_TEST_QUAD(TestOutOfOrderNonConflictingWrites, UseMvcc, UseNewEngine) {
// Select keys 3 and 4 from both tables, both should have been inserted
{
auto sender5 = runtime.AllocateEdgeActor();
- auto ev = ExecRequest(runtime, sender5, MakeSimpleRequest(Q_(R"(
- $rows = (
- SELECT key, value FROM `/Root/table-1` WHERE key = 3
- UNION ALL
- SELECT key, value FROM `/Root/table-2` WHERE key = 4
- );
- SELECT key, value FROM $rows ORDER BY key)")));
+ auto ev = ExecRequest(runtime, sender5, MakeSimpleRequest(Q_(R"(
+ $rows = (
+ SELECT key, value FROM `/Root/table-1` WHERE key = 3
+ UNION ALL
+ SELECT key, value FROM `/Root/table-2` WHERE key = 4
+ );
+ SELECT key, value FROM $rows ORDER BY key)")));
auto& response = ev->Get()->Record.GetRef();
UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS);
UNIT_ASSERT_VALUES_EQUAL(response.GetResponse().GetResults().size(), 1u);
@@ -1898,7 +1898,7 @@ Y_UNIT_TEST_QUAD(TestOutOfOrderNonConflictingWrites, UseMvcc, UseNewEngine) {
}
}
-Y_UNIT_TEST_NEW_ENGINE(TestOutOfOrderRestartLocksSingleWithoutBarrier) {
+Y_UNIT_TEST_NEW_ENGINE(TestOutOfOrderRestartLocksSingleWithoutBarrier) {
TPortManager pm;
TServerSettings serverSettings(pm.GetPort(2134));
serverSettings.SetDomainName("Root")
@@ -1913,7 +1913,7 @@ Y_UNIT_TEST_NEW_ENGINE(TestOutOfOrderRestartLocksSingleWithoutBarrier) {
runtime.GetAppData().FeatureFlags.SetDisableDataShardBarrier(true);
runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
- runtime.SetLogPriority(UseNewEngine ? NKikimrServices::KQP_EXECUTER : NKikimrServices::TX_PROXY, NLog::PRI_DEBUG);
+ runtime.SetLogPriority(UseNewEngine ? NKikimrServices::KQP_EXECUTER : NKikimrServices::TX_PROXY, NLog::PRI_DEBUG);
InitRoot(server, sender);
@@ -1922,18 +1922,18 @@ Y_UNIT_TEST_NEW_ENGINE(TestOutOfOrderRestartLocksSingleWithoutBarrier) {
auto table1shards = GetTableShards(server, sender, "/Root/table-1");
auto table2shards = GetTableShards(server, sender, "/Root/table-2");
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1);"));
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 1);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 1);"));
TString sessionId = CreateSession(runtime, sender);
TString txId;
{
auto sender1 = runtime.AllocateEdgeActor();
- auto ev = ExecRequest(runtime, sender1, MakeBeginRequest(sessionId, Q_(R"(
- SELECT * FROM `/Root/table-1` WHERE key = 1
- UNION ALL
- SELECT * FROM `/Root/table-2` WHERE key = 2)")));
+ auto ev = ExecRequest(runtime, sender1, MakeBeginRequest(sessionId, Q_(R"(
+ SELECT * FROM `/Root/table-1` WHERE key = 1
+ UNION ALL
+ SELECT * FROM `/Root/table-2` WHERE key = 2)")));
auto& response = ev->Get()->Record.GetRef();
UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS);
txId = response.GetResponse().GetTxMeta().id();
@@ -1954,9 +1954,9 @@ Y_UNIT_TEST_NEW_ENGINE(TestOutOfOrderRestartLocksSingleWithoutBarrier) {
// Send a commit request, it would block on readset exchange
auto sender2 = runtime.AllocateEdgeActor();
- SendRequest(runtime, sender2, MakeCommitRequest(sessionId, txId, Q_(R"(
- UPSERT INTO `/Root/table-1` (key, value) VALUES (3, 2);
- UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2))")));
+ SendRequest(runtime, sender2, MakeCommitRequest(sessionId, txId, Q_(R"(
+ UPSERT INTO `/Root/table-1` (key, value) VALUES (3, 2);
+ UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2))")));
// Wait until we captured both readsets
if (readSets.size() < 2) {
@@ -1989,8 +1989,8 @@ Y_UNIT_TEST_NEW_ENGINE(TestOutOfOrderRestartLocksSingleWithoutBarrier) {
// executes before that waiting transaction (no key 3 yet).
{
auto sender3 = runtime.AllocateEdgeActor();
- auto ev = ExecRequest(runtime, sender3, MakeSimpleRequest(Q_(
- "SELECT key, value FROM `/Root/table-1` WHERE key = 1 OR key = 3;")));
+ auto ev = ExecRequest(runtime, sender3, MakeSimpleRequest(Q_(
+ "SELECT key, value FROM `/Root/table-1` WHERE key = 1 OR key = 3;")));
auto& response = ev->Get()->Record.GetRef();
UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS);
UNIT_ASSERT_VALUES_EQUAL(response.GetResponse().GetResults().size(), 1u);
@@ -2003,7 +2003,7 @@ Y_UNIT_TEST_NEW_ENGINE(TestOutOfOrderRestartLocksSingleWithoutBarrier) {
// pretend some other conflicting write happened before that tx completes.
{
auto sender4 = runtime.AllocateEdgeActor();
- auto req = MakeSimpleRequest(Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 3);"));
+ auto req = MakeSimpleRequest(Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 3);"));
req->Record.MutableRequest()->SetCancelAfterMs(1000);
req->Record.MutableRequest()->SetTimeoutMs(1000);
auto ev = ExecRequest(runtime, sender4, std::move(req));
@@ -2016,7 +2016,7 @@ Y_UNIT_TEST_NEW_ENGINE(TestOutOfOrderRestartLocksSingleWithoutBarrier) {
// barrier.
{
auto sender4 = runtime.AllocateEdgeActor();
- auto req = MakeSimpleRequest(Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (5, 3);"));
+ auto req = MakeSimpleRequest(Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (5, 3);"));
req->Record.MutableRequest()->SetCancelAfterMs(1000);
req->Record.MutableRequest()->SetTimeoutMs(1000);
auto ev = ExecRequest(runtime, sender4, std::move(req));
@@ -2033,8 +2033,8 @@ Y_UNIT_TEST_NEW_ENGINE(TestOutOfOrderRestartLocksSingleWithoutBarrier) {
// Select key 3, we expect a success
{
auto sender9 = runtime.AllocateEdgeActor();
- auto ev = ExecRequest(runtime, sender9, MakeSimpleRequest(Q_(
- "SELECT key, value FROM `/Root/table-1` WHERE key = 3;")));
+ auto ev = ExecRequest(runtime, sender9, MakeSimpleRequest(Q_(
+ "SELECT key, value FROM `/Root/table-1` WHERE key = 3;")));
auto& response = ev->Get()->Record.GetRef();
UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS);
UNIT_ASSERT_VALUES_EQUAL(response.GetResponse().GetResults().size(), 1u);
@@ -2043,7 +2043,7 @@ Y_UNIT_TEST_NEW_ENGINE(TestOutOfOrderRestartLocksSingleWithoutBarrier) {
}
}
-Y_UNIT_TEST_NEW_ENGINE(MvccTestOutOfOrderRestartLocksSingleWithoutBarrier) {
+Y_UNIT_TEST_NEW_ENGINE(MvccTestOutOfOrderRestartLocksSingleWithoutBarrier) {
TPortManager pm;
TServerSettings serverSettings(pm.GetPort(2134));
serverSettings.SetDomainName("Root")
@@ -2058,7 +2058,7 @@ Y_UNIT_TEST_NEW_ENGINE(MvccTestOutOfOrderRestartLocksSingleWithoutBarrier) {
runtime.GetAppData().FeatureFlags.SetDisableDataShardBarrier(true);
runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
- runtime.SetLogPriority(UseNewEngine ? NKikimrServices::KQP_EXECUTER : NKikimrServices::TX_PROXY, NLog::PRI_DEBUG);
+ runtime.SetLogPriority(UseNewEngine ? NKikimrServices::KQP_EXECUTER : NKikimrServices::TX_PROXY, NLog::PRI_DEBUG);
InitRoot(server, sender);
@@ -2067,18 +2067,18 @@ Y_UNIT_TEST_NEW_ENGINE(MvccTestOutOfOrderRestartLocksSingleWithoutBarrier) {
auto table1shards = GetTableShards(server, sender, "/Root/table-1");
auto table2shards = GetTableShards(server, sender, "/Root/table-2");
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1);"));
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 1);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 1);"));
TString sessionId = CreateSession(runtime, sender);
TString txId;
{
auto sender1 = runtime.AllocateEdgeActor();
- auto ev = ExecRequest(runtime, sender1, MakeBeginRequest(sessionId, Q_(R"(
- SELECT * FROM `/Root/table-1` WHERE key = 1
- UNION ALL
- SELECT * FROM `/Root/table-2` WHERE key = 2)")));
+ auto ev = ExecRequest(runtime, sender1, MakeBeginRequest(sessionId, Q_(R"(
+ SELECT * FROM `/Root/table-1` WHERE key = 1
+ UNION ALL
+ SELECT * FROM `/Root/table-2` WHERE key = 2)")));
auto& response = ev->Get()->Record.GetRef();
UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS);
txId = response.GetResponse().GetTxMeta().id();
@@ -2099,9 +2099,9 @@ Y_UNIT_TEST_NEW_ENGINE(MvccTestOutOfOrderRestartLocksSingleWithoutBarrier) {
// Send a commit request, it would block on readset exchange
auto sender2 = runtime.AllocateEdgeActor();
- SendRequest(runtime, sender2, MakeCommitRequest(sessionId, txId, Q_(R"(
- UPSERT INTO `/Root/table-1` (key, value) VALUES (3, 2);
- UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2))")));
+ SendRequest(runtime, sender2, MakeCommitRequest(sessionId, txId, Q_(R"(
+ UPSERT INTO `/Root/table-1` (key, value) VALUES (3, 2);
+ UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2))")));
// Wait until we captured both readsets
if (readSets.size() < 2) {
@@ -2134,8 +2134,8 @@ Y_UNIT_TEST_NEW_ENGINE(MvccTestOutOfOrderRestartLocksSingleWithoutBarrier) {
// executes before that waiting transaction (no key 3 yet).
{
auto sender3 = runtime.AllocateEdgeActor();
- auto ev = ExecRequest(runtime, sender3, MakeSimpleRequest(Q_(
- "SELECT key, value FROM `/Root/table-1` WHERE key = 1 OR key = 3;")));
+ auto ev = ExecRequest(runtime, sender3, MakeSimpleRequest(Q_(
+ "SELECT key, value FROM `/Root/table-1` WHERE key = 1 OR key = 3;")));
auto& response = ev->Get()->Record.GetRef();
UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS);
UNIT_ASSERT_VALUES_EQUAL(response.GetResponse().GetResults().size(), 1u);
@@ -2146,7 +2146,7 @@ Y_UNIT_TEST_NEW_ENGINE(MvccTestOutOfOrderRestartLocksSingleWithoutBarrier) {
// Upsert key 1, we expect this immediate tx to be executed successfully because it lies to the right on the global timeline
{
auto sender4 = runtime.AllocateEdgeActor();
- auto req = MakeSimpleRequest(Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 3);"));
+ auto req = MakeSimpleRequest(Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 3);"));
req->Record.MutableRequest()->SetCancelAfterMs(1000);
req->Record.MutableRequest()->SetTimeoutMs(1000);
auto ev = ExecRequest(runtime, sender4, std::move(req));
@@ -2157,7 +2157,7 @@ Y_UNIT_TEST_NEW_ENGINE(MvccTestOutOfOrderRestartLocksSingleWithoutBarrier) {
// Upsert key 5, this immediate tx should be executed successfully too
{
auto sender4 = runtime.AllocateEdgeActor();
- auto req = MakeSimpleRequest(Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (5, 3);"));
+ auto req = MakeSimpleRequest(Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (5, 3);"));
req->Record.MutableRequest()->SetCancelAfterMs(1000);
req->Record.MutableRequest()->SetTimeoutMs(1000);
auto ev = ExecRequest(runtime, sender4, std::move(req));
@@ -2174,8 +2174,8 @@ Y_UNIT_TEST_NEW_ENGINE(MvccTestOutOfOrderRestartLocksSingleWithoutBarrier) {
// Select key 3, we expect a success
{
auto sender9 = runtime.AllocateEdgeActor();
- auto ev = ExecRequest(runtime, sender9, MakeSimpleRequest(Q_(
- "SELECT key, value FROM `/Root/table-1` WHERE key = 3;")));
+ auto ev = ExecRequest(runtime, sender9, MakeSimpleRequest(Q_(
+ "SELECT key, value FROM `/Root/table-1` WHERE key = 3;")));
auto& response = ev->Get()->Record.GetRef();
UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS);
UNIT_ASSERT_VALUES_EQUAL(response.GetResponse().GetResults().size(), 1u);
@@ -2184,7 +2184,7 @@ Y_UNIT_TEST_NEW_ENGINE(MvccTestOutOfOrderRestartLocksSingleWithoutBarrier) {
}
}
-Y_UNIT_TEST_QUAD(TestOutOfOrderRestartLocksReorderedWithoutBarrier, UseMvcc, UseNewEngine) {
+Y_UNIT_TEST_QUAD(TestOutOfOrderRestartLocksReorderedWithoutBarrier, UseMvcc, UseNewEngine) {
TPortManager pm;
TServerSettings serverSettings(pm.GetPort(2134));
serverSettings.SetDomainName("Root")
@@ -2199,7 +2199,7 @@ Y_UNIT_TEST_QUAD(TestOutOfOrderRestartLocksReorderedWithoutBarrier, UseMvcc, Use
runtime.GetAppData().FeatureFlags.SetDisableDataShardBarrier(true);
runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
- runtime.SetLogPriority(UseNewEngine ? NKikimrServices::KQP_EXECUTER : NKikimrServices::TX_PROXY, NLog::PRI_DEBUG);
+ runtime.SetLogPriority(UseNewEngine ? NKikimrServices::KQP_EXECUTER : NKikimrServices::TX_PROXY, NLog::PRI_DEBUG);
InitRoot(server, sender);
@@ -2208,18 +2208,18 @@ Y_UNIT_TEST_QUAD(TestOutOfOrderRestartLocksReorderedWithoutBarrier, UseMvcc, Use
auto table1shards = GetTableShards(server, sender, "/Root/table-1");
auto table2shards = GetTableShards(server, sender, "/Root/table-2");
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1);"));
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 1);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 1);"));
TString sessionId = CreateSession(runtime, sender);
TString txId;
{
auto sender1 = runtime.AllocateEdgeActor();
- auto ev = ExecRequest(runtime, sender1, MakeBeginRequest(sessionId, Q_(R"(
- SELECT * FROM `/Root/table-1` WHERE key = 1
- UNION ALL
- SELECT * FROM `/Root/table-2` WHERE key = 2)")));
+ auto ev = ExecRequest(runtime, sender1, MakeBeginRequest(sessionId, Q_(R"(
+ SELECT * FROM `/Root/table-1` WHERE key = 1
+ UNION ALL
+ SELECT * FROM `/Root/table-2` WHERE key = 2)")));
auto& response = ev->Get()->Record.GetRef();
UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS);
txId = response.GetResponse().GetTxMeta().id();
@@ -2240,9 +2240,9 @@ Y_UNIT_TEST_QUAD(TestOutOfOrderRestartLocksReorderedWithoutBarrier, UseMvcc, Use
// Send a commit request, it would block on readset exchange
auto sender2 = runtime.AllocateEdgeActor();
- SendRequest(runtime, sender2, MakeCommitRequest(sessionId, txId, Q_(R"(
- UPSERT INTO `/Root/table-1` (key, value) VALUES (3, 2);
- UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2))")));
+ SendRequest(runtime, sender2, MakeCommitRequest(sessionId, txId, Q_(R"(
+ UPSERT INTO `/Root/table-1` (key, value) VALUES (3, 2);
+ UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2))")));
// Wait until we captured both readsets
if (readSets.size() < 2) {
@@ -2256,15 +2256,15 @@ Y_UNIT_TEST_QUAD(TestOutOfOrderRestartLocksReorderedWithoutBarrier, UseMvcc, Use
UNIT_ASSERT_VALUES_EQUAL(readSets.size(), 2u);
// Execute some out-of-order upserts before rebooting
- ExecSQL(server, sender, Q_(R"(
- UPSERT INTO `/Root/table-1` (key, value) VALUES (5, 3);
- UPSERT INTO `/Root/table-2` (key, value) VALUES (6, 3))"));
+ ExecSQL(server, sender, Q_(R"(
+ UPSERT INTO `/Root/table-1` (key, value) VALUES (5, 3);
+ UPSERT INTO `/Root/table-2` (key, value) VALUES (6, 3))"));
// Select key 3, we expect a timeout, because logically writes
// to 3 and 5 already happened, but physically write to 3 is still waiting.
{
auto sender3 = runtime.AllocateEdgeActor();
- auto req = MakeSimpleRequest(Q_("SELECT key, value FROM `/Root/table-1` WHERE key = 3;"));
+ auto req = MakeSimpleRequest(Q_("SELECT key, value FROM `/Root/table-1` WHERE key = 3;"));
req->Record.MutableRequest()->SetCancelAfterMs(1000);
req->Record.MutableRequest()->SetTimeoutMs(1000);
auto ev = ExecRequest(runtime, sender3, std::move(req));
@@ -2291,7 +2291,7 @@ Y_UNIT_TEST_QUAD(TestOutOfOrderRestartLocksReorderedWithoutBarrier, UseMvcc, Use
// Select key 3, we still expect a timeout
{
auto sender4 = runtime.AllocateEdgeActor();
- auto req = MakeSimpleRequest(Q_("SELECT key, value FROM `/Root/table-1` WHERE key = 3;"));
+ auto req = MakeSimpleRequest(Q_("SELECT key, value FROM `/Root/table-1` WHERE key = 3;"));
req->Record.MutableRequest()->SetCancelAfterMs(1000);
req->Record.MutableRequest()->SetTimeoutMs(1000);
auto ev = ExecRequest(runtime, sender4, std::move(req));
@@ -2302,8 +2302,8 @@ Y_UNIT_TEST_QUAD(TestOutOfOrderRestartLocksReorderedWithoutBarrier, UseMvcc, Use
// Select key 5, it shouldn't pose any problems
{
auto sender5 = runtime.AllocateEdgeActor();
- auto ev = ExecRequest(runtime, sender5, MakeSimpleRequest(Q_(
- "SELECT key, value FROM `/Root/table-1` WHERE key = 5;")));
+ auto ev = ExecRequest(runtime, sender5, MakeSimpleRequest(Q_(
+ "SELECT key, value FROM `/Root/table-1` WHERE key = 5;")));
auto& response = ev->Get()->Record.GetRef();
UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS);
UNIT_ASSERT_VALUES_EQUAL(response.GetResponse().GetResults().size(), 1u);
@@ -2320,8 +2320,8 @@ Y_UNIT_TEST_QUAD(TestOutOfOrderRestartLocksReorderedWithoutBarrier, UseMvcc, Use
// Select key 3, we expect a success
{
auto sender6 = runtime.AllocateEdgeActor();
- auto ev = ExecRequest(runtime, sender6, MakeSimpleRequest(Q_(
- "SELECT key, value FROM `/Root/table-1` WHERE key = 3;")));
+ auto ev = ExecRequest(runtime, sender6, MakeSimpleRequest(Q_(
+ "SELECT key, value FROM `/Root/table-1` WHERE key = 3;")));
auto& response = ev->Get()->Record.GetRef();
UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS);
UNIT_ASSERT_VALUES_EQUAL(response.GetResponse().GetResults().size(), 1u);
@@ -2330,7 +2330,7 @@ Y_UNIT_TEST_QUAD(TestOutOfOrderRestartLocksReorderedWithoutBarrier, UseMvcc, Use
}
}
-Y_UNIT_TEST_QUAD(TestOutOfOrderNoBarrierRestartImmediateLongTail, UseMvcc, UseNewEngine) {
+Y_UNIT_TEST_QUAD(TestOutOfOrderNoBarrierRestartImmediateLongTail, UseMvcc, UseNewEngine) {
TPortManager pm;
TServerSettings serverSettings(pm.GetPort(2134));
serverSettings.SetDomainName("Root")
@@ -2345,7 +2345,7 @@ Y_UNIT_TEST_QUAD(TestOutOfOrderNoBarrierRestartImmediateLongTail, UseMvcc, UseNe
runtime.GetAppData().FeatureFlags.SetDisableDataShardBarrier(true);
runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
- runtime.SetLogPriority(UseNewEngine ? NKikimrServices::KQP_EXECUTER : NKikimrServices::TX_PROXY, NLog::PRI_DEBUG);
+ runtime.SetLogPriority(UseNewEngine ? NKikimrServices::KQP_EXECUTER : NKikimrServices::TX_PROXY, NLog::PRI_DEBUG);
InitRoot(server, sender);
@@ -2354,18 +2354,18 @@ Y_UNIT_TEST_QUAD(TestOutOfOrderNoBarrierRestartImmediateLongTail, UseMvcc, UseNe
auto table1shards = GetTableShards(server, sender, "/Root/table-1");
auto table2shards = GetTableShards(server, sender, "/Root/table-2");
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1);"));
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 1);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 1);"));
TString sessionId = CreateSession(runtime, sender);
TString txId;
{
auto sender1 = runtime.AllocateEdgeActor();
- auto ev = ExecRequest(runtime, sender1, MakeBeginRequest(sessionId, Q_(R"(
- SELECT * FROM `/Root/table-1` WHERE key = 1
- UNION ALL
- SELECT * FROM `/Root/table-2` WHERE key = 2)")));
+ auto ev = ExecRequest(runtime, sender1, MakeBeginRequest(sessionId, Q_(R"(
+ SELECT * FROM `/Root/table-1` WHERE key = 1
+ UNION ALL
+ SELECT * FROM `/Root/table-2` WHERE key = 2)")));
auto& response = ev->Get()->Record.GetRef();
UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS);
txId = response.GetResponse().GetTxMeta().id();
@@ -2413,9 +2413,9 @@ Y_UNIT_TEST_QUAD(TestOutOfOrderNoBarrierRestartImmediateLongTail, UseMvcc, UseNe
// Send a commit request, it would block on readset exchange
auto sender2 = runtime.AllocateEdgeActor();
- SendRequest(runtime, sender2, MakeCommitRequest(sessionId, txId, Q_(R"(
- UPSERT INTO `/Root/table-1` (key, value) VALUES (3, 2);
- UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2))")));
+ SendRequest(runtime, sender2, MakeCommitRequest(sessionId, txId, Q_(R"(
+ UPSERT INTO `/Root/table-1` (key, value) VALUES (3, 2);
+ UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2))")));
// Wait until we captured both readsets
if (readSets.size() < 2) {
@@ -2430,24 +2430,24 @@ Y_UNIT_TEST_QUAD(TestOutOfOrderNoBarrierRestartImmediateLongTail, UseMvcc, UseNe
// Send some more requests that form a staircase, they would all be blocked as well
auto sender3 = runtime.AllocateEdgeActor();
- SendRequest(runtime, sender3, MakeSimpleRequest(Q_(R"(
- UPSERT INTO `/Root/table-1` (key, value) VALUES (3, 3), (5, 3);
- UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 3), (6, 3))")));
+ SendRequest(runtime, sender3, MakeSimpleRequest(Q_(R"(
+ UPSERT INTO `/Root/table-1` (key, value) VALUES (3, 3), (5, 3);
+ UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 3), (6, 3))")));
SimulateSleep(server, TDuration::Seconds(1));
- SendRequest(runtime, sender3, MakeSimpleRequest(Q_(R"(
- UPSERT INTO `/Root/table-1` (key, value) VALUES (5, 4), (7, 4);
- UPSERT INTO `/Root/table-2` (key, value) VALUES (6, 4), (8, 4))")));
+ SendRequest(runtime, sender3, MakeSimpleRequest(Q_(R"(
+ UPSERT INTO `/Root/table-1` (key, value) VALUES (5, 4), (7, 4);
+ UPSERT INTO `/Root/table-2` (key, value) VALUES (6, 4), (8, 4))")));
SimulateSleep(server, TDuration::Seconds(1));
// One more request that would be executed out of order
- ExecSQL(server, sender, Q_(R"(
- UPSERT INTO `/Root/table-1` (key, value) VALUES (11, 5);
- UPSERT INTO `/Root/table-2` (key, value) VALUES (12, 5))"));
+ ExecSQL(server, sender, Q_(R"(
+ UPSERT INTO `/Root/table-1` (key, value) VALUES (11, 5);
+ UPSERT INTO `/Root/table-2` (key, value) VALUES (12, 5))"));
// Select key 7, we expect a timeout, because logically a write to it already happened
{
auto sender4 = runtime.AllocateEdgeActor();
- auto req = MakeSimpleRequest(Q_("SELECT key, value FROM `/Root/table-1` WHERE key = 7;"));
+ auto req = MakeSimpleRequest(Q_("SELECT key, value FROM `/Root/table-1` WHERE key = 7;"));
req->Record.MutableRequest()->SetCancelAfterMs(1000);
req->Record.MutableRequest()->SetTimeoutMs(1000);
auto ev = ExecRequest(runtime, sender4, std::move(req));
@@ -2489,7 +2489,7 @@ Y_UNIT_TEST_QUAD(TestOutOfOrderNoBarrierRestartImmediateLongTail, UseMvcc, UseNe
// Select key 7 again, we still expect a timeout, because logically a write to it already happened
{
auto sender5 = runtime.AllocateEdgeActor();
- auto req = MakeSimpleRequest(Q_("SELECT key, value FROM `/Root/table-1` WHERE key = 7;"));
+ auto req = MakeSimpleRequest(Q_("SELECT key, value FROM `/Root/table-1` WHERE key = 7;"));
req->Record.MutableRequest()->SetCancelAfterMs(1000);
req->Record.MutableRequest()->SetTimeoutMs(1000);
auto ev = ExecRequest(runtime, sender5, std::move(req));
@@ -2509,7 +2509,7 @@ Y_UNIT_TEST_QUAD(TestOutOfOrderNoBarrierRestartImmediateLongTail, UseMvcc, UseNe
// Select key 7 again, this time is should succeed
{
auto sender6 = runtime.AllocateEdgeActor();
- auto req = MakeSimpleRequest(Q_("SELECT key, value FROM `/Root/table-1` WHERE key = 7;"));
+ auto req = MakeSimpleRequest(Q_("SELECT key, value FROM `/Root/table-1` WHERE key = 7;"));
auto ev = ExecRequest(runtime, sender6, std::move(req));
auto& response = ev->Get()->Record.GetRef();
UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS);
@@ -2546,7 +2546,7 @@ namespace {
}
}
-Y_UNIT_TEST_QUAD(TestCopyTableNoDeadlock, UseMvcc, UseNewEngine) {
+Y_UNIT_TEST_QUAD(TestCopyTableNoDeadlock, UseMvcc, UseNewEngine) {
TPortManager pm;
TServerSettings serverSettings(pm.GetPort(2134));
serverSettings.SetDomainName("Root")
@@ -2558,24 +2558,24 @@ Y_UNIT_TEST_QUAD(TestCopyTableNoDeadlock, UseMvcc, UseNewEngine) {
auto sender = runtime.AllocateEdgeActor();
runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
- runtime.SetLogPriority(UseNewEngine ? NKikimrServices::KQP_EXECUTER : NKikimrServices::TX_PROXY, NLog::PRI_DEBUG);
+ runtime.SetLogPriority(UseNewEngine ? NKikimrServices::KQP_EXECUTER : NKikimrServices::TX_PROXY, NLog::PRI_DEBUG);
InitRoot(server, sender);
CreateShardedTable(server, sender, "/Root", "table-1", 1);
CreateShardedTable(server, sender, "/Root", "table-2", 1);
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1);"));
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 1);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 1);"));
TString sessionId = CreateSession(runtime, sender);
TString txId;
{
- auto ev = ExecRequest(runtime, sender, MakeBeginRequest(sessionId,Q_(R"(
- SELECT * FROM `/Root/table-1`
- UNION ALL
- SELECT * FROM `/Root/table-2`)")));
+ auto ev = ExecRequest(runtime, sender, MakeBeginRequest(sessionId,Q_(R"(
+ SELECT * FROM `/Root/table-1`
+ UNION ALL
+ SELECT * FROM `/Root/table-2`)")));
auto& response = ev->Get()->Record.GetRef();
UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS);
txId = response.GetResponse().GetTxMeta().id();
@@ -2617,9 +2617,9 @@ Y_UNIT_TEST_QUAD(TestCopyTableNoDeadlock, UseMvcc, UseNewEngine) {
// Send a commit request, it would block on readset exchange
auto senderCommit = runtime.AllocateEdgeActor();
- SendRequest(runtime, senderCommit, MakeCommitRequest(sessionId, txId, Q_(R"(
- UPSERT INTO `/Root/table-1` (key, value) VALUES (3, 2);
- UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2))")));
+ SendRequest(runtime, senderCommit, MakeCommitRequest(sessionId, txId, Q_(R"(
+ UPSERT INTO `/Root/table-1` (key, value) VALUES (3, 2);
+ UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2))")));
// Wait until we captured both readsets
if (readSets.size() < 2) {
@@ -2636,10 +2636,10 @@ Y_UNIT_TEST_QUAD(TestCopyTableNoDeadlock, UseMvcc, UseNewEngine) {
// Now we send a distributed read, while stopping coordinator proposals
auto senderRead = runtime.AllocateEdgeActor();
- SendRequest(runtime, senderRead, MakeSimpleRequest(Q_(R"(
- SELECT * FROM `/Root/table-1`
- UNION ALL
- SELECT * FROM `/Root/table-2`)")));
+ SendRequest(runtime, senderRead, MakeSimpleRequest(Q_(R"(
+ SELECT * FROM `/Root/table-1`
+ UNION ALL
+ SELECT * FROM `/Root/table-2`)")));
// Wait until we capture the propose request
if (txProposes.size() < 1) {
@@ -2730,7 +2730,7 @@ Y_UNIT_TEST_QUAD(TestCopyTableNoDeadlock, UseMvcc, UseNewEngine) {
}
}
-Y_UNIT_TEST_NEW_ENGINE(TestPlannedCancelSplit) {
+Y_UNIT_TEST_NEW_ENGINE(TestPlannedCancelSplit) {
TPortManager pm;
TServerSettings serverSettings(pm.GetPort(2134));
serverSettings.SetDomainName("Root")
@@ -2741,15 +2741,15 @@ Y_UNIT_TEST_NEW_ENGINE(TestPlannedCancelSplit) {
auto sender = runtime.AllocateEdgeActor();
runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
- runtime.SetLogPriority(UseNewEngine ? NKikimrServices::KQP_EXECUTER : NKikimrServices::TX_PROXY, NLog::PRI_DEBUG);
+ runtime.SetLogPriority(UseNewEngine ? NKikimrServices::KQP_EXECUTER : NKikimrServices::TX_PROXY, NLog::PRI_DEBUG);
InitRoot(server, sender);
CreateShardedTable(server, sender, "/Root", "table-1", 1);
CreateShardedTable(server, sender, "/Root", "table-2", 1);
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1);"));
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 1);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 1);"));
auto shards1 = GetTableShards(server, sender, "/Root/table-1");
UNIT_ASSERT_VALUES_EQUAL(shards1.size(), 1u);
@@ -2799,10 +2799,10 @@ Y_UNIT_TEST_NEW_ENGINE(TestPlannedCancelSplit) {
// Send a distributed read while capturing propose results
captureTxProposeResult = true;
auto senderRead1 = runtime.AllocateEdgeActor();
- SendRequest(runtime, senderRead1, MakeSimpleRequest(Q_(R"(
- SELECT * FROM `/Root/table-1`
- UNION ALL
- SELECT * FROM `/Root/table-2`)")));
+ SendRequest(runtime, senderRead1, MakeSimpleRequest(Q_(R"(
+ SELECT * FROM `/Root/table-1`
+ UNION ALL
+ SELECT * FROM `/Root/table-2`)")));
if (txProposeResults.size() < 2) {
TDispatchOptions options;
options.FinalEvents.emplace_back(
@@ -2839,10 +2839,10 @@ Y_UNIT_TEST_NEW_ENGINE(TestPlannedCancelSplit) {
// Send a distributed read again, while blocking propose messages
captureTxPropose = true;
auto senderRead2 = runtime.AllocateEdgeActor();
- SendRequest(runtime, senderRead2, MakeSimpleRequest(Q_(R"(
- SELECT * FROM `/Root/table-1`
- UNION ALL
- SELECT * FROM `/Root/table-2`)")));
+ SendRequest(runtime, senderRead2, MakeSimpleRequest(Q_(R"(
+ SELECT * FROM `/Root/table-1`
+ UNION ALL
+ SELECT * FROM `/Root/table-2`)")));
if (txProposes.size() < 2) {
TDispatchOptions options;
options.FinalEvents.emplace_back(
@@ -2936,8 +2936,8 @@ Y_UNIT_TEST_QUAD(TestPlannedTimeoutSplit, UseMvcc, UseNewEngine) {
CreateShardedTable(server, sender, "/Root", "table-1", 1);
CreateShardedTable(server, sender, "/Root", "table-2", 1);
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1);"));
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 1);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 1);"));
auto shards1 = GetTableShards(server, sender, "/Root/table-1");
UNIT_ASSERT_VALUES_EQUAL(shards1.size(), 1u);
@@ -3055,8 +3055,8 @@ Y_UNIT_TEST_QUAD(TestPlannedHalfOverloadedSplit, UseMvcc, UseNewEngine) {
CreateShardedTable(server, sender, "/Root", "table-1", 1);
CreateShardedTable(server, sender, "/Root", "table-2", 1);
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1);"));
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 1);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 1);"));
auto shards1 = GetTableShards(server, sender, "/Root/table-1");
UNIT_ASSERT_VALUES_EQUAL(shards1.size(), 1u);
@@ -3203,7 +3203,7 @@ namespace {
/**
* Regression test for KIKIMR-7751, designed to crash under asan
*/
-Y_UNIT_TEST_NEW_ENGINE(TestReadTableWriteConflict) {
+Y_UNIT_TEST_NEW_ENGINE(TestReadTableWriteConflict) {
TPortManager pm;
TServerSettings serverSettings(pm.GetPort(2134));
serverSettings.SetDomainName("Root")
@@ -3222,17 +3222,17 @@ Y_UNIT_TEST_NEW_ENGINE(TestReadTableWriteConflict) {
CreateShardedTable(server, sender, "/Root", "table-1", 2);
CreateShardedTable(server, sender, "/Root", "table-2", 1);
- ExecSQL(server, sender, Q_("UPSERT INTO [/Root/table-1] (key, value) VALUES (1, 1);"));
- ExecSQL(server, sender, Q_("UPSERT INTO [/Root/table-2] (key, value) VALUES (2, 1);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO [/Root/table-1] (key, value) VALUES (1, 1);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO [/Root/table-2] (key, value) VALUES (2, 1);"));
TString sessionId = CreateSession(runtime, sender);
TString txId;
{
- auto ev = ExecRequest(runtime, sender, MakeBeginRequest(sessionId, Q_(
+ auto ev = ExecRequest(runtime, sender, MakeBeginRequest(sessionId, Q_(
"SELECT * FROM [/Root/table-1] "
"UNION ALL "
- "SELECT * FROM [/Root/table-2]")));
+ "SELECT * FROM [/Root/table-2]")));
auto& response = ev->Get()->Record.GetRef();
UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS);
txId = response.GetResponse().GetTxMeta().id();
@@ -3266,9 +3266,9 @@ Y_UNIT_TEST_NEW_ENGINE(TestReadTableWriteConflict) {
// Send a commit request, it would block on readset exchange
auto senderCommit = runtime.AllocateEdgeActor();
- SendRequest(runtime, senderCommit, MakeCommitRequest(sessionId, txId, Q_(
- "UPSERT INTO [/Root/table-1] (key, value) VALUES (3, 2); "
- "UPSERT INTO [/Root/table-2] (key, value) VALUES (4, 2)")));
+ SendRequest(runtime, senderCommit, MakeCommitRequest(sessionId, txId, Q_(
+ "UPSERT INTO [/Root/table-1] (key, value) VALUES (3, 2); "
+ "UPSERT INTO [/Root/table-2] (key, value) VALUES (4, 2)")));
// Wait until we captured all readsets
if (readSets.size() < 4) {
@@ -3300,14 +3300,14 @@ Y_UNIT_TEST_NEW_ENGINE(TestReadTableWriteConflict) {
// Start an immediate write to table-1, it won't be able to start
// because it arrived after the read table and they block each other
auto senderWriteImm = runtime.AllocateEdgeActor();
- SendRequest(runtime, senderWriteImm, MakeSimpleRequest(Q_(
- "UPSERT INTO [/Root/table-1] (key, value) VALUES (5, 3)")));
+ SendRequest(runtime, senderWriteImm, MakeSimpleRequest(Q_(
+ "UPSERT INTO [/Root/table-1] (key, value) VALUES (5, 3)")));
// Start a planned write to both tables, wait for its plan step too
auto senderWriteDist = runtime.AllocateEdgeActor();
- SendRequest(runtime, senderWriteDist, MakeSimpleRequest(Q_(
+ SendRequest(runtime, senderWriteDist, MakeSimpleRequest(Q_(
"UPSERT INTO [/Root/table-1] (key, value) VALUES (7, 4); "
- "UPSERT INTO [/Root/table-2] (key, value) VALUES (8, 4)")));
+ "UPSERT INTO [/Root/table-2] (key, value) VALUES (8, 4)")));
if (seenPlanSteps < 2) {
TDispatchOptions options;
options.FinalEvents.emplace_back(
@@ -3353,7 +3353,7 @@ Y_UNIT_TEST_NEW_ENGINE(TestReadTableWriteConflict) {
/**
* Regression test for KIKIMR-7903
*/
-Y_UNIT_TEST_NEW_ENGINE(TestReadTableImmediateWriteBlock) {
+Y_UNIT_TEST_NEW_ENGINE(TestReadTableImmediateWriteBlock) {
TPortManager pm;
TServerSettings serverSettings(pm.GetPort(2134));
serverSettings.SetDomainName("Root")
@@ -3365,14 +3365,14 @@ Y_UNIT_TEST_NEW_ENGINE(TestReadTableImmediateWriteBlock) {
runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
runtime.SetLogPriority(NKikimrServices::TX_PROXY, NLog::PRI_DEBUG);
- runtime.SetLogPriority(NKikimrServices::KQP_EXECUTER, NLog::PRI_DEBUG);
+ runtime.SetLogPriority(NKikimrServices::KQP_EXECUTER, NLog::PRI_DEBUG);
InitRoot(server, sender);
// NOTE: table-1 has 2 shards so ReadTable is not immediate
CreateShardedTable(server, sender, "/Root", "table-1", 2);
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1);"));
// Capture and block all readset messages
size_t seenPlanSteps = 0;
@@ -3412,8 +3412,8 @@ Y_UNIT_TEST_NEW_ENGINE(TestReadTableImmediateWriteBlock) {
// Start an immediate write to table-1, it should be able to complete
auto senderWriteImm = runtime.AllocateEdgeActor();
- SendRequest(runtime, senderWriteImm, MakeSimpleRequest(Q_(
- "UPSERT INTO `/Root/table-1` (key, value) VALUES (5, 3)")));
+ SendRequest(runtime, senderWriteImm, MakeSimpleRequest(Q_(
+ "UPSERT INTO `/Root/table-1` (key, value) VALUES (5, 3)")));
// Wait for immediate write to succeed
{
@@ -3423,7 +3423,7 @@ Y_UNIT_TEST_NEW_ENGINE(TestReadTableImmediateWriteBlock) {
}
}
-Y_UNIT_TEST_QUAD(TestReadTableSingleShardImmediate, WithMvcc, UseNewEngine) {
+Y_UNIT_TEST_QUAD(TestReadTableSingleShardImmediate, WithMvcc, UseNewEngine) {
TPortManager pm;
TServerSettings serverSettings(pm.GetPort(2134));
serverSettings.SetDomainName("Root")
@@ -3442,7 +3442,7 @@ Y_UNIT_TEST_QUAD(TestReadTableSingleShardImmediate, WithMvcc, UseNewEngine) {
CreateShardedTable(server, sender, "/Root", "table-1", 1);
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1);"));
// Capture and block all readset messages
size_t seenPlanSteps = 0;
@@ -3471,7 +3471,7 @@ Y_UNIT_TEST_QUAD(TestReadTableSingleShardImmediate, WithMvcc, UseNewEngine) {
UNIT_ASSERT_VALUES_EQUAL(seenPlanSteps, 0u);
}
-Y_UNIT_TEST_NEW_ENGINE(TestImmediateQueueThenSplit) {
+Y_UNIT_TEST_NEW_ENGINE(TestImmediateQueueThenSplit) {
TPortManager pm;
TServerSettings serverSettings(pm.GetPort(2134));
serverSettings.SetDomainName("Root")
@@ -3482,7 +3482,7 @@ Y_UNIT_TEST_NEW_ENGINE(TestImmediateQueueThenSplit) {
auto sender = runtime.AllocateEdgeActor();
runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
- runtime.SetLogPriority(UseNewEngine ? NKikimrServices::KQP_EXECUTER : NKikimrServices::TX_PROXY, NLog::PRI_DEBUG);
+ runtime.SetLogPriority(UseNewEngine ? NKikimrServices::KQP_EXECUTER : NKikimrServices::TX_PROXY, NLog::PRI_DEBUG);
InitRoot(server, sender);
@@ -3491,7 +3491,7 @@ Y_UNIT_TEST_NEW_ENGINE(TestImmediateQueueThenSplit) {
auto tablets = GetTableShards(server, sender, "/Root/table-1");
// We need shard to have some data (otherwise it would die too quickly)
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (0, 0);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (0, 0);"));
bool captureSplit = true;
bool captureSplitChanged = true;
@@ -3545,7 +3545,7 @@ Y_UNIT_TEST_NEW_ENGINE(TestImmediateQueueThenSplit) {
// Send a lot of write requests in parallel (so there's a large propose queue)
for (int i = 0; i < totalWrites; ++i) {
auto writeSender = runtime.AllocateEdgeActor();
- SendSQL(server, writeSender, Q_(Sprintf("UPSERT INTO `/Root/table-1` (key, value) VALUES (%d, %d);", i, i)));
+ SendSQL(server, writeSender, Q_(Sprintf("UPSERT INTO `/Root/table-1` (key, value) VALUES (%d, %d);", i, i)));
writeSenders.push_back(writeSender);
}
@@ -3640,52 +3640,52 @@ Y_UNIT_TEST_NEW_ENGINE(TestImmediateQueueThenSplit) {
}
void TestLateKqpQueryAfterColumnDrop(bool dataQuery, const TString& query, bool enableMvcc = false) {
- TPortManager pm;
- TServerSettings serverSettings(pm.GetPort(2134));
- serverSettings.SetDomainName("Root")
+ TPortManager pm;
+ TServerSettings serverSettings(pm.GetPort(2134));
+ serverSettings.SetDomainName("Root")
.SetEnableMvcc(enableMvcc)
- .SetUseRealThreads(false);
-
- if (dataQuery) {
- NKikimrKqp::TKqpSetting setting;
- setting.SetName("_KqpAllowNewEngine");
- setting.SetValue("true");
- serverSettings.KqpSettings.push_back(setting);
- }
-
- Tests::TServer::TPtr server = new TServer(serverSettings);
- auto &runtime = *server->GetRuntime();
- auto sender = runtime.AllocateEdgeActor();
- auto streamSender = runtime.AllocateEdgeActor();
-
- runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
- runtime.SetLogPriority(NKikimrServices::TX_PROXY, NLog::PRI_DEBUG);
- runtime.SetLogPriority(NKikimrServices::KQP_PROXY, NLog::PRI_DEBUG);
- runtime.SetLogPriority(NKikimrServices::KQP_WORKER, NLog::PRI_DEBUG);
- runtime.SetLogPriority(NKikimrServices::KQP_TASKS_RUNNER, NLog::PRI_DEBUG);
- runtime.SetLogPriority(NKikimrServices::KQP_YQL, NLog::PRI_DEBUG);
- runtime.SetLogPriority(NKikimrServices::KQP_EXECUTER, NLog::PRI_DEBUG);
- runtime.SetLogPriority(NKikimrServices::KQP_COMPUTE, NLog::PRI_DEBUG);
- runtime.SetLogPriority(NKikimrServices::KQP_RESOURCE_MANAGER, NLog::PRI_DEBUG);
-
- InitRoot(server, sender);
-
- CreateShardedTable(server, sender, "/Root", "table-1", TShardedTableOptions()
- .Columns({
+ .SetUseRealThreads(false);
+
+ if (dataQuery) {
+ NKikimrKqp::TKqpSetting setting;
+ setting.SetName("_KqpAllowNewEngine");
+ setting.SetValue("true");
+ serverSettings.KqpSettings.push_back(setting);
+ }
+
+ Tests::TServer::TPtr server = new TServer(serverSettings);
+ auto &runtime = *server->GetRuntime();
+ auto sender = runtime.AllocateEdgeActor();
+ auto streamSender = runtime.AllocateEdgeActor();
+
+ runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
+ runtime.SetLogPriority(NKikimrServices::TX_PROXY, NLog::PRI_DEBUG);
+ runtime.SetLogPriority(NKikimrServices::KQP_PROXY, NLog::PRI_DEBUG);
+ runtime.SetLogPriority(NKikimrServices::KQP_WORKER, NLog::PRI_DEBUG);
+ runtime.SetLogPriority(NKikimrServices::KQP_TASKS_RUNNER, NLog::PRI_DEBUG);
+ runtime.SetLogPriority(NKikimrServices::KQP_YQL, NLog::PRI_DEBUG);
+ runtime.SetLogPriority(NKikimrServices::KQP_EXECUTER, NLog::PRI_DEBUG);
+ runtime.SetLogPriority(NKikimrServices::KQP_COMPUTE, NLog::PRI_DEBUG);
+ runtime.SetLogPriority(NKikimrServices::KQP_RESOURCE_MANAGER, NLog::PRI_DEBUG);
+
+ InitRoot(server, sender);
+
+ CreateShardedTable(server, sender, "/Root", "table-1", TShardedTableOptions()
+ .Columns({
{"key", "Uint32", true, false},
{"value1", "Uint32", false, false},
{"value2", "Uint32", false, false}}));
-
- ExecSQL(server, sender, "UPSERT INTO `/Root/table-1` (key, value1, value2) VALUES (1, 1, 10), (2, 2, 20);");
-
- bool capturePropose = true;
- TVector<THolder<IEventHandle>> eventsPropose;
- auto captureEvents = [&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) -> auto {
- // if (ev->GetRecipientRewrite() == streamSender) {
- // Cerr << "Stream sender got " << ev->GetTypeRewrite() << " " << ev->GetBase()->ToStringHeader() << Endl;
- // }
- switch (ev->GetTypeRewrite()) {
- case TEvDataShard::EvProposeTransaction: {
+
+ ExecSQL(server, sender, "UPSERT INTO `/Root/table-1` (key, value1, value2) VALUES (1, 1, 10), (2, 2, 20);");
+
+ bool capturePropose = true;
+ TVector<THolder<IEventHandle>> eventsPropose;
+ auto captureEvents = [&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) -> auto {
+ // if (ev->GetRecipientRewrite() == streamSender) {
+ // Cerr << "Stream sender got " << ev->GetTypeRewrite() << " " << ev->GetBase()->ToStringHeader() << Endl;
+ // }
+ switch (ev->GetTypeRewrite()) {
+ case TEvDataShard::EvProposeTransaction: {
auto &rec = ev->Get<TEvDataShard::TEvProposeTransaction>()->Record;
if (capturePropose && rec.GetTxKind() != NKikimrTxDataShard::TX_KIND_SNAPSHOT) {
Cerr << "---- capture EvProposeTransaction ---- type=" << rec.GetTxKind() << Endl;
@@ -3696,73 +3696,73 @@ void TestLateKqpQueryAfterColumnDrop(bool dataQuery, const TString& query, bool
}
case TEvDataShard::EvKqpScan: {
- if (capturePropose) {
+ if (capturePropose) {
Cerr << "---- capture EvKqpScan ----" << Endl;
- eventsPropose.emplace_back(ev.Release());
- return TTestActorRuntime::EEventAction::DROP;
- }
- break;
- }
- default:
- break;
- }
- return TTestActorRuntime::EEventAction::PROCESS;
- };
- auto prevObserverFunc = runtime.SetObserverFunc(captureEvents);
-
- if (dataQuery) {
- Cerr << "--- sending data query request" << Endl;
- SendRequest(runtime, streamSender, MakeSimpleRequest(query));
- } else {
- Cerr << "--- sending stream request" << Endl;
- SendRequest(runtime, streamSender, MakeStreamRequest(streamSender, query, false));
- }
-
- // Wait until there's exactly one propose message at our datashard
- if (eventsPropose.size() < 1) {
- TDispatchOptions options;
- options.CustomFinalCondition = [&]() {
- return eventsPropose.size() >= 1;
- };
- runtime.DispatchEvents(options);
- }
- UNIT_ASSERT_VALUES_EQUAL(eventsPropose.size(), 1u);
- Cerr << "--- captured scan tx proposal" << Endl;
- capturePropose = false;
-
- // Drop column value2 and wait for drop to finish
- auto dropTxId = AsyncAlterDropColumn(server, "/Root", "table-1", "value2");
- WaitTxNotification(server, dropTxId);
-
- // Resend delayed propose messages
- Cerr << "--- resending captured proposals" << Endl;
- for (auto& ev : eventsPropose) {
- runtime.Send(ev.Release(), 0, true);
- }
- eventsPropose.clear();
-
- Cerr << "--- waiting for result" << Endl;
- auto ev = runtime.GrabEdgeEventRethrow<NKqp::TEvKqp::TEvQueryResponse>(streamSender);
- auto& response = ev->Get()->Record.GetRef();
- Cerr << response.DebugString() << Endl;
- UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::ABORTED);
- auto& issue = response.GetResponse().GetQueryIssues(0).Getissues(0);
- UNIT_ASSERT_VALUES_EQUAL(issue.issue_code(), (int) NYql::TIssuesIds::KIKIMR_SCHEME_MISMATCH);
- UNIT_ASSERT_STRINGS_EQUAL(issue.message(), "Table \'/Root/table-1\' scheme changed.");
-}
-
+ eventsPropose.emplace_back(ev.Release());
+ return TTestActorRuntime::EEventAction::DROP;
+ }
+ break;
+ }
+ default:
+ break;
+ }
+ return TTestActorRuntime::EEventAction::PROCESS;
+ };
+ auto prevObserverFunc = runtime.SetObserverFunc(captureEvents);
+
+ if (dataQuery) {
+ Cerr << "--- sending data query request" << Endl;
+ SendRequest(runtime, streamSender, MakeSimpleRequest(query));
+ } else {
+ Cerr << "--- sending stream request" << Endl;
+ SendRequest(runtime, streamSender, MakeStreamRequest(streamSender, query, false));
+ }
+
+ // Wait until there's exactly one propose message at our datashard
+ if (eventsPropose.size() < 1) {
+ TDispatchOptions options;
+ options.CustomFinalCondition = [&]() {
+ return eventsPropose.size() >= 1;
+ };
+ runtime.DispatchEvents(options);
+ }
+ UNIT_ASSERT_VALUES_EQUAL(eventsPropose.size(), 1u);
+ Cerr << "--- captured scan tx proposal" << Endl;
+ capturePropose = false;
+
+ // Drop column value2 and wait for drop to finish
+ auto dropTxId = AsyncAlterDropColumn(server, "/Root", "table-1", "value2");
+ WaitTxNotification(server, dropTxId);
+
+ // Resend delayed propose messages
+ Cerr << "--- resending captured proposals" << Endl;
+ for (auto& ev : eventsPropose) {
+ runtime.Send(ev.Release(), 0, true);
+ }
+ eventsPropose.clear();
+
+ Cerr << "--- waiting for result" << Endl;
+ auto ev = runtime.GrabEdgeEventRethrow<NKqp::TEvKqp::TEvQueryResponse>(streamSender);
+ auto& response = ev->Get()->Record.GetRef();
+ Cerr << response.DebugString() << Endl;
+ UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::ABORTED);
+ auto& issue = response.GetResponse().GetQueryIssues(0).Getissues(0);
+ UNIT_ASSERT_VALUES_EQUAL(issue.issue_code(), (int) NYql::TIssuesIds::KIKIMR_SCHEME_MISMATCH);
+ UNIT_ASSERT_STRINGS_EQUAL(issue.message(), "Table \'/Root/table-1\' scheme changed.");
+}
+
Y_UNIT_TEST_WITH_MVCC(TestLateKqpScanAfterColumnDrop) {
TestLateKqpQueryAfterColumnDrop(false, "SELECT SUM(value2) FROM `/Root/table-1`", WithMvcc);
-}
-
+}
+
Y_UNIT_TEST_WITH_MVCC(TestLateKqpDataReadAfterColumnDrop) {
- TestLateKqpQueryAfterColumnDrop(true, R"(
- PRAGMA kikimr.UseNewEngine = "true";
- SELECT SUM(value2) FROM `/Root/table-1`
+ TestLateKqpQueryAfterColumnDrop(true, R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ SELECT SUM(value2) FROM `/Root/table-1`
)", WithMvcc);
-}
-
-Y_UNIT_TEST_NEW_ENGINE(MvccTestSnapshotRead) {
+}
+
+Y_UNIT_TEST_NEW_ENGINE(MvccTestSnapshotRead) {
TPortManager pm;
TServerSettings serverSettings(pm.GetPort(2134));
serverSettings.SetDomainName("Root")
@@ -3774,13 +3774,13 @@ Y_UNIT_TEST_NEW_ENGINE(MvccTestSnapshotRead) {
auto sender = runtime.AllocateEdgeActor();
runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
- runtime.SetLogPriority(UseNewEngine ? NKikimrServices::KQP_EXECUTER : NKikimrServices::TX_PROXY, NLog::PRI_DEBUG);
+ runtime.SetLogPriority(UseNewEngine ? NKikimrServices::KQP_EXECUTER : NKikimrServices::TX_PROXY, NLog::PRI_DEBUG);
InitRoot(server, sender);
CreateShardedTable(server, sender, "/Root", "table-1", 1);
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (0, 0), (1, 1), (2, 2), (3, 3);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (0, 0), (1, 1), (2, 2), (3, 3);"));
auto waitFor = [&](const auto& condition, const TString& description) {
if (!condition()) {
@@ -3842,7 +3842,7 @@ Y_UNIT_TEST_NEW_ENGINE(MvccTestSnapshotRead) {
// future snapshot
snapshot = TRowVersion(lastStep + 1000, Max<ui64>());
- SendRequest(runtime, sender, MakeSimpleRequest(Q_("SELECT key, value FROM `/Root/table-1` WHERE key = 2 ORDER BY key")));
+ SendRequest(runtime, sender, MakeSimpleRequest(Q_("SELECT key, value FROM `/Root/table-1` WHERE key = 2 ORDER BY key")));
waitFor([&]{ return rewritten; }, "EvProposeTransaction rewritten");
@@ -3863,10 +3863,10 @@ Y_UNIT_TEST_NEW_ENGINE(MvccTestSnapshotRead) {
// check transaction reads from snapshot
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (2, 10);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (2, 10);"));
{
- auto ev = ExecRequest(runtime, sender, MakeSimpleRequest(Q_("SELECT key, value FROM `/Root/table-1` WHERE key = 2 ORDER BY key")));
+ auto ev = ExecRequest(runtime, sender, MakeSimpleRequest(Q_("SELECT key, value FROM `/Root/table-1` WHERE key = 2 ORDER BY key")));
auto& response = ev->Get()->Record.GetRef();
UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS);
UNIT_ASSERT_VALUES_EQUAL(response.GetResponse().GetResults().size(), 1u);
@@ -3878,7 +3878,7 @@ Y_UNIT_TEST_NEW_ENGINE(MvccTestSnapshotRead) {
rescheduled = false;
{
- auto ev = ExecRequest(runtime, sender, MakeSimpleRequest(Q_("SELECT key, value FROM `/Root/table-1` WHERE key = 2 ORDER BY key")));
+ auto ev = ExecRequest(runtime, sender, MakeSimpleRequest(Q_("SELECT key, value FROM `/Root/table-1` WHERE key = 2 ORDER BY key")));
auto& response = ev->Get()->Record.GetRef();
UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS);
UNIT_ASSERT_VALUES_EQUAL(response.GetResponse().GetResults().size(), 1u);
@@ -3889,7 +3889,7 @@ Y_UNIT_TEST_NEW_ENGINE(MvccTestSnapshotRead) {
UNIT_ASSERT(!rescheduled);
}
-Y_UNIT_TEST_NEW_ENGINE(TestSecondaryClearanceAfterShardRestartRace) {
+Y_UNIT_TEST_NEW_ENGINE(TestSecondaryClearanceAfterShardRestartRace) {
TPortManager pm;
TServerSettings serverSettings(pm.GetPort(2134));
serverSettings.SetDomainName("Root")
@@ -3900,14 +3900,14 @@ Y_UNIT_TEST_NEW_ENGINE(TestSecondaryClearanceAfterShardRestartRace) {
auto sender = runtime.AllocateEdgeActor();
runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
- runtime.SetLogPriority(UseNewEngine ? NKikimrServices::KQP_EXECUTER : NKikimrServices::TX_PROXY, NLog::PRI_TRACE);
+ runtime.SetLogPriority(UseNewEngine ? NKikimrServices::KQP_EXECUTER : NKikimrServices::TX_PROXY, NLog::PRI_TRACE);
InitRoot(server, sender);
CreateShardedTable(server, sender, "/Root", "table-1", 2);
auto shards = GetTableShards(server, sender, "/Root/table-1");
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1), (2, 2), (3, 3);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1), (2, 2), (3, 3);"));
auto waitFor = [&](const auto& condition, const TString& description) {
if (!condition()) {
@@ -3971,7 +3971,7 @@ Y_UNIT_TEST_NEW_ENGINE(TestSecondaryClearanceAfterShardRestartRace) {
// We expect this upsert to complete successfully
// When there's a bug it will get stuck due to readtable before barrier
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (4, 4);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (4, 4);"));
}
Y_UNIT_TEST_QUAD(TestShardRestartNoUndeterminedImmediate, UseMvcc, UseNewEngine) {
@@ -3985,13 +3985,13 @@ Y_UNIT_TEST_QUAD(TestShardRestartNoUndeterminedImmediate, UseMvcc, UseNewEngine)
auto &runtime = *server->GetRuntime();
auto sender = runtime.AllocateEdgeActor();
-// runtime.SetLogPriority(NKikimrServices::TABLET_MAIN, NLog::PRI_TRACE);
-// runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
-// if (UseNewEngine) {
-// runtime.SetLogPriority(NKikimrServices::KQP_EXECUTER, NLog::PRI_TRACE);
-// } else {
-// runtime.SetLogPriority(NKikimrServices::TX_PROXY, NLog::PRI_TRACE);
-// }
+// runtime.SetLogPriority(NKikimrServices::TABLET_MAIN, NLog::PRI_TRACE);
+// runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
+// if (UseNewEngine) {
+// runtime.SetLogPriority(NKikimrServices::KQP_EXECUTER, NLog::PRI_TRACE);
+// } else {
+// runtime.SetLogPriority(NKikimrServices::TX_PROXY, NLog::PRI_TRACE);
+// }
InitRoot(server, sender);
@@ -3999,17 +3999,17 @@ Y_UNIT_TEST_QUAD(TestShardRestartNoUndeterminedImmediate, UseMvcc, UseNewEngine)
CreateShardedTable(server, sender, "/Root", "table-2", 1);
auto table1shards = GetTableShards(server, sender, "/Root/table-1");
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1);"));
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 1);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1);"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 1);"));
TString sessionId = CreateSession(runtime, sender);
TString txId;
{
- auto ev = ExecRequest(runtime, sender, MakeBeginRequest(sessionId, Q_(R"(
- SELECT * FROM `/Root/table-1`
- UNION ALL
- SELECT * FROM `/Root/table-2`)")));
+ auto ev = ExecRequest(runtime, sender, MakeBeginRequest(sessionId, Q_(R"(
+ SELECT * FROM `/Root/table-1`
+ UNION ALL
+ SELECT * FROM `/Root/table-2`)")));
auto& response = ev->Get()->Record.GetRef();
UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS);
txId = response.GetResponse().GetTxMeta().id();
@@ -4048,9 +4048,9 @@ Y_UNIT_TEST_QUAD(TestShardRestartNoUndeterminedImmediate, UseMvcc, UseNewEngine)
// Send a commit request, it would block on readset exchange
auto sender2 = runtime.AllocateEdgeActor();
- SendRequest(runtime, sender2, MakeCommitRequest(sessionId, txId, Q_(R"(
- UPSERT INTO `/Root/table-1` (key, value) VALUES (3, 2);
- UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2))")));
+ SendRequest(runtime, sender2, MakeCommitRequest(sessionId, txId, Q_(R"(
+ UPSERT INTO `/Root/table-1` (key, value) VALUES (3, 2);
+ UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2))")));
// Wait until we captured both readsets
waitFor([&]{ return readSets.size() >= 2; }, "commit read sets");
@@ -4060,7 +4060,7 @@ Y_UNIT_TEST_QUAD(TestShardRestartNoUndeterminedImmediate, UseMvcc, UseNewEngine)
delayedProposeCount = 0;
auto sender3 = runtime.AllocateEdgeActor();
Cerr << "... sending immediate upsert" << Endl;
- SendRequest(runtime, sender3, MakeSimpleRequest(Q_(R"(
+ SendRequest(runtime, sender3, MakeSimpleRequest(Q_(R"(
UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 42), (3, 51))")));
// Wait unti that propose starts to execute
@@ -4083,8 +4083,8 @@ Y_UNIT_TEST_QUAD(TestShardRestartNoUndeterminedImmediate, UseMvcc, UseNewEngine)
// Select key 1 and verify its value was not updated
{
auto sender4 = runtime.AllocateEdgeActor();
- auto ev = ExecRequest(runtime, sender4, MakeSimpleRequest(Q_(R"(
- SELECT key, value FROM `/Root/table-1` WHERE key = 1 ORDER BY key)")));
+ auto ev = ExecRequest(runtime, sender4, MakeSimpleRequest(Q_(R"(
+ SELECT key, value FROM `/Root/table-1` WHERE key = 1 ORDER BY key)")));
auto& response = ev->Get()->Record.GetRef();
UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS);
UNIT_ASSERT_VALUES_EQUAL(response.GetResponse().GetResults().size(), 1u);
@@ -4093,7 +4093,7 @@ Y_UNIT_TEST_QUAD(TestShardRestartNoUndeterminedImmediate, UseMvcc, UseNewEngine)
}
}
-Y_UNIT_TEST_QUAD(TestShardRestartPlannedCommitShouldSucceed, UseMvcc, UseNewEngine) {
+Y_UNIT_TEST_QUAD(TestShardRestartPlannedCommitShouldSucceed, UseMvcc, UseNewEngine) {
TPortManager pm;
TServerSettings serverSettings(pm.GetPort(2134));
serverSettings.SetDomainName("Root")
@@ -4104,13 +4104,13 @@ Y_UNIT_TEST_QUAD(TestShardRestartPlannedCommitShouldSucceed, UseMvcc, UseNewEngi
auto &runtime = *server->GetRuntime();
auto sender = runtime.AllocateEdgeActor();
-// runtime.SetLogPriority(NKikimrServices::TABLET_MAIN, NLog::PRI_TRACE);
-// runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
-// if (UseNewEngine) {
-// runtime.SetLogPriority(NKikimrServices::KQP_EXECUTER, NLog::PRI_TRACE);
-// } else {
-// runtime.SetLogPriority(NKikimrServices::TX_PROXY, NLog::PRI_TRACE);
-// }
+// runtime.SetLogPriority(NKikimrServices::TABLET_MAIN, NLog::PRI_TRACE);
+// runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
+// if (UseNewEngine) {
+// runtime.SetLogPriority(NKikimrServices::KQP_EXECUTER, NLog::PRI_TRACE);
+// } else {
+// runtime.SetLogPriority(NKikimrServices::TX_PROXY, NLog::PRI_TRACE);
+// }
InitRoot(server, sender);
@@ -4118,17 +4118,17 @@ Y_UNIT_TEST_QUAD(TestShardRestartPlannedCommitShouldSucceed, UseMvcc, UseNewEngi
CreateShardedTable(server, sender, "/Root", "table-2", 1);
auto table1shards = GetTableShards(server, sender, "/Root/table-1");
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1)"));
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 1)"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1)"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 1)"));
TString sessionId = CreateSession(runtime, sender);
TString txId;
{
- auto ev = ExecRequest(runtime, sender, MakeBeginRequest(sessionId, Q_(R"(
- SELECT * FROM `/Root/table-1`
- UNION ALL
- SELECT * FROM `/Root/table-2`)")));
+ auto ev = ExecRequest(runtime, sender, MakeBeginRequest(sessionId, Q_(R"(
+ SELECT * FROM `/Root/table-1`
+ UNION ALL
+ SELECT * FROM `/Root/table-2`)")));
auto& response = ev->Get()->Record.GetRef();
UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS);
txId = response.GetResponse().GetTxMeta().id();
@@ -4163,9 +4163,9 @@ Y_UNIT_TEST_QUAD(TestShardRestartPlannedCommitShouldSucceed, UseMvcc, UseNewEngi
// Send a commit request, it would block on readset exchange
auto sender2 = runtime.AllocateEdgeActor();
- SendRequest(runtime, sender2, MakeCommitRequest(sessionId, txId, Q_(R"(
- UPSERT INTO `/Root/table-1` (key, value) VALUES (3, 2);
- UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2))")));
+ SendRequest(runtime, sender2, MakeCommitRequest(sessionId, txId, Q_(R"(
+ UPSERT INTO `/Root/table-1` (key, value) VALUES (3, 2);
+ UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2))")));
// Wait until we captured both readsets
waitFor([&]{ return readSets.size() >= 2; }, "commit read sets");
@@ -4186,8 +4186,8 @@ Y_UNIT_TEST_QUAD(TestShardRestartPlannedCommitShouldSucceed, UseMvcc, UseNewEngi
// Select key 3 and verify its value was updated
{
auto sender4 = runtime.AllocateEdgeActor();
- auto ev = ExecRequest(runtime, sender4, MakeSimpleRequest(Q_(R"(
- SELECT key, value FROM `/Root/table-1` WHERE key = 3 ORDER BY key)")));
+ auto ev = ExecRequest(runtime, sender4, MakeSimpleRequest(Q_(R"(
+ SELECT key, value FROM `/Root/table-1` WHERE key = 3 ORDER BY key)")));
auto& response = ev->Get()->Record.GetRef();
UNIT_ASSERT_VALUES_EQUAL(response.GetYdbStatus(), Ydb::StatusIds::SUCCESS);
UNIT_ASSERT_VALUES_EQUAL(response.GetResponse().GetResults().size(), 1u);
@@ -4196,7 +4196,7 @@ Y_UNIT_TEST_QUAD(TestShardRestartPlannedCommitShouldSucceed, UseMvcc, UseNewEngi
}
}
-Y_UNIT_TEST_NEW_ENGINE(TestShardSnapshotReadNoEarlyReply) {
+Y_UNIT_TEST_NEW_ENGINE(TestShardSnapshotReadNoEarlyReply) {
TPortManager pm;
TServerSettings serverSettings(pm.GetPort(2134));
serverSettings.SetDomainName("Root")
@@ -4213,8 +4213,8 @@ Y_UNIT_TEST_NEW_ENGINE(TestShardSnapshotReadNoEarlyReply) {
CreateShardedTable(server, sender, "/Root", "table-1", 1);
CreateShardedTable(server, sender, "/Root", "table-2", 1);
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1)"));
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 2)"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1)"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 2)"));
auto table1shards = GetTableShards(server, sender, "/Root/table-1");
auto table2shards = GetTableShards(server, sender, "/Root/table-2");
auto isTableShard = [&](ui64 tabletId) -> bool {
@@ -4268,16 +4268,16 @@ Y_UNIT_TEST_NEW_ENGINE(TestShardSnapshotReadNoEarlyReply) {
auto sender2 = runtime.AllocateEdgeActor();
TString sessionId2 = CreateSession(runtime, sender2, "/Root");
- SendRequest(runtime, sender1, MakeBeginRequest(sessionId1, Q_(R"(
+ SendRequest(runtime, sender1, MakeBeginRequest(sessionId1, Q_(R"(
SELECT * FROM `/Root/table-1`
UNION ALL
SELECT * FROM `/Root/table-2`
- )"), "/Root"));
- SendRequest(runtime, sender2, MakeBeginRequest(sessionId2, Q_(R"(
+ )"), "/Root"));
+ SendRequest(runtime, sender2, MakeBeginRequest(sessionId2, Q_(R"(
SELECT * FROM `/Root/table-1`
UNION ALL
SELECT * FROM `/Root/table-2`
- )"), "/Root"));
+ )"), "/Root"));
waitFor([&]{ return blockedCommits.size() >= 2; }, "at least 2 blocked commits");
@@ -4310,16 +4310,16 @@ Y_UNIT_TEST_NEW_ENGINE(TestShardSnapshotReadNoEarlyReply) {
// Start blocking commits again and try performing new writes
prevObserver = runtime.SetObserverFunc(blockCommits);
- SendRequest(runtime, sender, MakeSimpleRequest(Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (3, 3)"), "/Root"));
- SendRequest(runtime, sender, MakeSimpleRequest(Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 4)"), "/Root"));
+ SendRequest(runtime, sender, MakeSimpleRequest(Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (3, 3)"), "/Root"));
+ SendRequest(runtime, sender, MakeSimpleRequest(Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 4)"), "/Root"));
waitFor([&]{ return blockedCommits.size() >= 2; }, "at least 2 blocked commits");
// Send an additional read request, it must not be blocked
- SendRequest(runtime, sender1, MakeContinueRequest(sessionId1, txId1, Q_(R"(
+ SendRequest(runtime, sender1, MakeContinueRequest(sessionId1, txId1, Q_(R"(
SELECT * FROM `/Root/table-1`
UNION ALL
SELECT * FROM `/Root/table-2`
- )"), "/Root"));
+ )"), "/Root"));
{
auto ev = runtime.GrabEdgeEventRethrow<NKqp::TEvKqp::TEvQueryResponse>(sender1);
@@ -4345,8 +4345,8 @@ Y_UNIT_TEST_TWIN(TestSnapshotReadAfterBrokenLock, UseNewEngine) {
CreateShardedTable(server, sender, "/Root", "table-1", 1);
CreateShardedTable(server, sender, "/Root", "table-2", 1);
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1)"));
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 2)"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1)"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 2)"));
SimulateSleep(server, TDuration::Seconds(1));
@@ -4369,7 +4369,7 @@ Y_UNIT_TEST_TWIN(TestSnapshotReadAfterBrokenLock, UseNewEngine) {
SimulateSleep(server, TDuration::Seconds(1));
// Perform immediate write, which would not break the above lock
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (3, 3)"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (3, 3)"));
// Perform an additional read, it would mark transaction as write-broken
{
@@ -4417,8 +4417,8 @@ Y_UNIT_TEST_TWIN(TestSnapshotReadAfterBrokenLockOutOfOrder, UseNewEngine) {
CreateShardedTable(server, sender, "/Root", "table-1", 1);
CreateShardedTable(server, sender, "/Root", "table-2", 1);
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1)"));
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 2)"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1)"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 2)"));
SimulateSleep(server, TDuration::Seconds(1));
@@ -4494,7 +4494,7 @@ Y_UNIT_TEST_TWIN(TestSnapshotReadAfterBrokenLockOutOfOrder, UseNewEngine) {
// Perform immediate write, which would break the above lock
Cerr << "... performing an upsert" << Endl;
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 3)"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 3)"));
// Perform an additional read, it would mark transaction as write-broken for the first time
{
@@ -4547,8 +4547,8 @@ Y_UNIT_TEST_TWIN(TestSnapshotReadAfterStuckRW, UseNewEngine) {
CreateShardedTable(server, sender, "/Root", "table-1", 1);
CreateShardedTable(server, sender, "/Root", "table-2", 1);
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1)"));
- ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 2)"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) VALUES (1, 1)"));
+ ExecSQL(server, sender, Q_("UPSERT INTO `/Root/table-2` (key, value) VALUES (2, 2)"));
SimulateSleep(server, TDuration::Seconds(1));
diff --git a/ydb/core/tx/datashard/execute_data_tx_unit.cpp b/ydb/core/tx/datashard/execute_data_tx_unit.cpp
index 136bce8e3f..b32f635cc0 100644
--- a/ydb/core/tx/datashard/execute_data_tx_unit.cpp
+++ b/ydb/core/tx/datashard/execute_data_tx_unit.cpp
@@ -1,4 +1,4 @@
-#include "datashard_kqp.h"
+#include "datashard_kqp.h"
#include "execution_unit_ctors.h"
#include "setup_sys_locks.h"
@@ -138,9 +138,9 @@ EExecutionStatus TExecuteDataTxUnit::Execute(TOperation::TPtr op,
<< " and requests " << txc.GetMemoryLimit() * MEMORY_REQUEST_FACTOR
<< " more for the next try");
- txc.NotEnoughMemory();
- DataShard.IncCounter(DataShard.NotEnoughMemoryCounter(txc.GetNotEnoughMemoryCount()));
-
+ txc.NotEnoughMemory();
+ DataShard.IncCounter(DataShard.NotEnoughMemoryCounter(txc.GetNotEnoughMemoryCount()));
+
engine->ReleaseUnusedMemory();
txc.RequestMemory(txc.GetMemoryLimit() * MEMORY_REQUEST_FACTOR);
@@ -148,11 +148,11 @@ EExecutionStatus TExecuteDataTxUnit::Execute(TOperation::TPtr op,
return EExecutionStatus::Restart;
} catch (const TNotReadyTabletException&) {
- LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, "Tablet " << DataShard.TabletID()
+ LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, "Tablet " << DataShard.TabletID()
<< " is not ready for " << *op << " execution");
- DataShard.IncCounter(COUNTER_TX_TABLET_NOT_READY);
-
+ DataShard.IncCounter(COUNTER_TX_TABLET_NOT_READY);
+
tx->ReleaseTxData(txc, ctx);
return EExecutionStatus::Restart;
@@ -258,14 +258,14 @@ void TExecuteDataTxUnit::ExecuteDataTx(TOperation::TPtr op,
"Datashard execution counters for " << *op << " at "
<< DataShard.TabletID() << ": " << counters.ToString());
- KqpUpdateDataShardStatCounters(DataShard, counters);
- if (tx->GetDataTx()->CollectStats()) {
- KqpFillTxStats(DataShard, counters, *result);
- }
+ KqpUpdateDataShardStatCounters(DataShard, counters);
+ if (tx->GetDataTx()->CollectStats()) {
+ KqpFillTxStats(DataShard, counters, *result);
+ }
- if (counters.InvisibleRowSkips) {
+ if (counters.InvisibleRowSkips) {
DataShard.SysLocksTable().BreakSetLocks(op->LockTxId());
- }
+ }
AddLocksToResult(op);
diff --git a/ydb/core/tx/datashard/execute_kqp_data_tx_unit.cpp b/ydb/core/tx/datashard/execute_kqp_data_tx_unit.cpp
index 3d6abd5c62..70e7be3662 100644
--- a/ydb/core/tx/datashard/execute_kqp_data_tx_unit.cpp
+++ b/ydb/core/tx/datashard/execute_kqp_data_tx_unit.cpp
@@ -12,12 +12,12 @@ namespace NDataShard {
using namespace NMiniKQL;
-#define LOG_T(stream) LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, stream)
-#define LOG_D(stream) LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, stream)
-#define LOG_E(stream) LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, stream)
-#define LOG_C(stream) LOG_CRIT_S(ctx, NKikimrServices::TX_DATASHARD, stream)
-#define LOG_W(stream) LOG_WARN_S(ctx, NKikimrServices::TX_DATASHARD, stream)
-
+#define LOG_T(stream) LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, stream)
+#define LOG_D(stream) LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, stream)
+#define LOG_E(stream) LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, stream)
+#define LOG_C(stream) LOG_CRIT_S(ctx, NKikimrServices::TX_DATASHARD, stream)
+#define LOG_W(stream) LOG_WARN_S(ctx, NKikimrServices::TX_DATASHARD, stream)
+
class TExecuteKqpDataTxUnit : public TExecutionUnit {
public:
TExecuteKqpDataTxUnit(TDataShard& dataShard, TPipeline& pipeline);
@@ -28,9 +28,9 @@ public:
void Complete(TOperation::TPtr op, const TActorContext& ctx) override;
private:
- void AddLocksToResult(TOperation::TPtr op, const TActorContext& ctx);
- EExecutionStatus OnTabletNotReady(TActiveTransaction& tx, TValidatedDataTx& dataTx, TTransactionContext& txc,
- const TActorContext& ctx);
+ void AddLocksToResult(TOperation::TPtr op, const TActorContext& ctx);
+ EExecutionStatus OnTabletNotReady(TActiveTransaction& tx, TValidatedDataTx& dataTx, TTransactionContext& txc,
+ const TActorContext& ctx);
};
TExecuteKqpDataTxUnit::TExecuteKqpDataTxUnit(TDataShard& dataShard, TPipeline& pipeline)
@@ -58,9 +58,9 @@ EExecutionStatus TExecuteKqpDataTxUnit::Execute(TOperation::TPtr op, TTransactio
return EExecutionStatus::Executed;
}
- // We remember current time now, but will only count it when transaction succeeds
- TDuration waitExecuteLatency = op->GetCurrentElapsed();
- TDuration waitTotalLatency = op->GetTotalElapsed();
+ // We remember current time now, but will only count it when transaction succeeds
+ TDuration waitExecuteLatency = op->GetCurrentElapsed();
+ TDuration waitTotalLatency = op->GetTotalElapsed();
if (op->IsImmediate()) {
// Every time we execute immediate transaction we may choose a new mvcc version
@@ -84,76 +84,76 @@ EExecutionStatus TExecuteKqpDataTxUnit::Execute(TOperation::TPtr op, TTransactio
}
}
- ui64 tabletId = DataShard.TabletID();
- const TValidatedDataTx::TPtr& dataTx = tx->GetDataTx();
-
- if (dataTx->CheckCancelled()) {
- tx->ReleaseTxData(txc, ctx);
- BuildResult(op, NKikimrTxDataShard::TEvProposeTransactionResult::CANCELLED)
- ->AddError(NKikimrTxDataShard::TError::EXECUTION_CANCELLED, "Tx was cancelled");
-
- DataShard.IncCounter(op->IsImmediate() ? COUNTER_IMMEDIATE_TX_CANCELLED : COUNTER_PLANNED_TX_CANCELLED);
-
- return EExecutionStatus::Executed;
- }
-
+ ui64 tabletId = DataShard.TabletID();
+ const TValidatedDataTx::TPtr& dataTx = tx->GetDataTx();
+
+ if (dataTx->CheckCancelled()) {
+ tx->ReleaseTxData(txc, ctx);
+ BuildResult(op, NKikimrTxDataShard::TEvProposeTransactionResult::CANCELLED)
+ ->AddError(NKikimrTxDataShard::TError::EXECUTION_CANCELLED, "Tx was cancelled");
+
+ DataShard.IncCounter(op->IsImmediate() ? COUNTER_IMMEDIATE_TX_CANCELLED : COUNTER_PLANNED_TX_CANCELLED);
+
+ return EExecutionStatus::Executed;
+ }
+
try {
- auto& kqpTx = dataTx->GetKqpTransaction();
-
- if (!KqpValidateLocks(tabletId, tx, DataShard.SysLocksTable())) {
- KqpEraseLocks(tabletId, tx, DataShard.SysLocksTable());
- DataShard.SysLocksTable().ApplyLocks();
- return EExecutionStatus::Executed;
+ auto& kqpTx = dataTx->GetKqpTransaction();
+
+ if (!KqpValidateLocks(tabletId, tx, DataShard.SysLocksTable())) {
+ KqpEraseLocks(tabletId, tx, DataShard.SysLocksTable());
+ DataShard.SysLocksTable().ApplyLocks();
+ return EExecutionStatus::Executed;
}
- auto& tasksRunner = dataTx->GetKqpTasksRunner();
-
- auto allocGuard = tasksRunner.BindAllocator(txc.GetMemoryLimit() - dataTx->GetTxSize());
-
- NKqp::NRm::TKqpResourcesRequest req;
- req.MemoryPool = NKqp::NRm::EKqpMemoryPool::DataQuery;
- req.Memory = txc.GetMemoryLimit();
- ui64 taskId = kqpTx.GetTasks().empty() ? std::numeric_limits<ui64>::max() : kqpTx.GetTasks()[0].GetId();
- NKqp::GetKqpResourceManager()->NotifyExternalResourcesAllocated(tx->GetTxId(), taskId, req);
-
- Y_DEFER {
- NKqp::GetKqpResourceManager()->NotifyExternalResourcesFreed(tx->GetTxId(), taskId);
- };
-
- LOG_T("Operation " << *op << " (execute_kqp_data_tx) at " << tabletId
- << " set memory limit " << (txc.GetMemoryLimit() - dataTx->GetTxSize()));
-
- auto execCtx = DefaultKqpExecutionContext();
- tasksRunner.Prepare(DefaultKqpDataReqMemoryLimits(), *execCtx);
-
+ auto& tasksRunner = dataTx->GetKqpTasksRunner();
+
+ auto allocGuard = tasksRunner.BindAllocator(txc.GetMemoryLimit() - dataTx->GetTxSize());
+
+ NKqp::NRm::TKqpResourcesRequest req;
+ req.MemoryPool = NKqp::NRm::EKqpMemoryPool::DataQuery;
+ req.Memory = txc.GetMemoryLimit();
+ ui64 taskId = kqpTx.GetTasks().empty() ? std::numeric_limits<ui64>::max() : kqpTx.GetTasks()[0].GetId();
+ NKqp::GetKqpResourceManager()->NotifyExternalResourcesAllocated(tx->GetTxId(), taskId, req);
+
+ Y_DEFER {
+ NKqp::GetKqpResourceManager()->NotifyExternalResourcesFreed(tx->GetTxId(), taskId);
+ };
+
+ LOG_T("Operation " << *op << " (execute_kqp_data_tx) at " << tabletId
+ << " set memory limit " << (txc.GetMemoryLimit() - dataTx->GetTxSize()));
+
+ auto execCtx = DefaultKqpExecutionContext();
+ tasksRunner.Prepare(DefaultKqpDataReqMemoryLimits(), *execCtx);
+
auto [readVersion, writeVersion] = DataShard.GetReadWriteVersions(tx);
dataTx->SetReadVersion(readVersion);
dataTx->SetWriteVersion(writeVersion);
- auto& computeCtx = tx->GetDataTx()->GetKqpComputeCtx();
-
- auto result = KqpCompleteTransaction(ctx, tabletId, op->GetTxId(),
- op->HasKqpAttachedRSFlag() ? nullptr : &op->InReadSets(), dataTx->GetKqpTasks(), tasksRunner, computeCtx);
-
- if (!result && computeCtx.IsTabletNotReady()) {
- return OnTabletNotReady(*tx, *dataTx, txc, ctx);
- }
-
- Y_VERIFY(result);
- op->Result().Swap(result);
+ auto& computeCtx = tx->GetDataTx()->GetKqpComputeCtx();
+
+ auto result = KqpCompleteTransaction(ctx, tabletId, op->GetTxId(),
+ op->HasKqpAttachedRSFlag() ? nullptr : &op->InReadSets(), dataTx->GetKqpTasks(), tasksRunner, computeCtx);
+
+ if (!result && computeCtx.IsTabletNotReady()) {
+ return OnTabletNotReady(*tx, *dataTx, txc, ctx);
+ }
+
+ Y_VERIFY(result);
+ op->Result().Swap(result);
op->SetKqpAttachedRSFlag();
- KqpEraseLocks(tabletId, tx, DataShard.SysLocksTable());
-
- if (dataTx->GetCounters().InvisibleRowSkips) {
- DataShard.SysLocksTable().BreakSetLocks(op->LockTxId());
- }
-
- AddLocksToResult(op, ctx);
-
- if (op->IsImmediate() && !op->IsReadOnly()) {
+ KqpEraseLocks(tabletId, tx, DataShard.SysLocksTable());
+
+ if (dataTx->GetCounters().InvisibleRowSkips) {
+ DataShard.SysLocksTable().BreakSetLocks(op->LockTxId());
+ }
+
+ AddLocksToResult(op, ctx);
+
+ if (op->IsImmediate() && !op->IsReadOnly()) {
DataShard.PromoteCompleteEdge(writeVersion.Step, txc);
- }
+ }
if (auto changes = dataTx->GetCollectedChanges()) {
op->ChangeRecords().reserve(changes.size());
@@ -162,52 +162,52 @@ EExecutionStatus TExecuteKqpDataTxUnit::Execute(TOperation::TPtr op, TTransactio
}
}
- KqpUpdateDataShardStatCounters(DataShard, dataTx->GetCounters());
- auto statsMode = kqpTx.GetRuntimeSettings().GetStatsMode();
- if (statsMode >= NYql::NDqProto::DQ_STATS_MODE_BASIC) {
- KqpFillTxStats(DataShard, dataTx->GetCounters(), *op->Result());
- KqpFillStats(DataShard, tasksRunner, computeCtx, statsMode, *op->Result());
+ KqpUpdateDataShardStatCounters(DataShard, dataTx->GetCounters());
+ auto statsMode = kqpTx.GetRuntimeSettings().GetStatsMode();
+ if (statsMode >= NYql::NDqProto::DQ_STATS_MODE_BASIC) {
+ KqpFillTxStats(DataShard, dataTx->GetCounters(), *op->Result());
+ KqpFillStats(DataShard, tasksRunner, computeCtx, statsMode, *op->Result());
}
} catch (const TMemoryLimitExceededException&) {
- txc.NotEnoughMemory();
-
- LOG_T("Operation " << *op << " at " << tabletId
+ txc.NotEnoughMemory();
+
+ LOG_T("Operation " << *op << " at " << tabletId
<< " exceeded memory limit " << txc.GetMemoryLimit()
<< " and requests " << txc.GetMemoryLimit() * MEMORY_REQUEST_FACTOR
- << " more for the next try (" << txc.GetNotEnoughMemoryCount() << ")");
+ << " more for the next try (" << txc.GetNotEnoughMemoryCount() << ")");
+
+ DataShard.IncCounter(DataShard.NotEnoughMemoryCounter(txc.GetNotEnoughMemoryCount()));
- DataShard.IncCounter(DataShard.NotEnoughMemoryCounter(txc.GetNotEnoughMemoryCount()));
-
txc.RequestMemory(txc.GetMemoryLimit() * MEMORY_REQUEST_FACTOR);
tx->ReleaseTxData(txc, ctx);
return EExecutionStatus::Restart;
- } catch (const TNotReadyTabletException&) {
- return OnTabletNotReady(*tx, *dataTx, txc, ctx);
+ } catch (const TNotReadyTabletException&) {
+ return OnTabletNotReady(*tx, *dataTx, txc, ctx);
} catch (const yexception& e) {
- LOG_C("Exception while executing KQP transaction " << *op << " at " << tabletId << ": " << e.what());
- if (op->IsReadOnly() || op->IsImmediate()) {
- tx->ReleaseTxData(txc, ctx);
- BuildResult(op, NKikimrTxDataShard::TEvProposeTransactionResult::EXEC_ERROR)
- ->AddError(NKikimrTxDataShard::TError::UNKNOWN, TStringBuilder() << "Tx was terminated: " << e.what());
- return EExecutionStatus::Executed;
- } else {
- Y_FAIL_S("Unexpected exception in KQP transaction execution: " << e.what());
- }
+ LOG_C("Exception while executing KQP transaction " << *op << " at " << tabletId << ": " << e.what());
+ if (op->IsReadOnly() || op->IsImmediate()) {
+ tx->ReleaseTxData(txc, ctx);
+ BuildResult(op, NKikimrTxDataShard::TEvProposeTransactionResult::EXEC_ERROR)
+ ->AddError(NKikimrTxDataShard::TError::UNKNOWN, TStringBuilder() << "Tx was terminated: " << e.what());
+ return EExecutionStatus::Executed;
+ } else {
+ Y_FAIL_S("Unexpected exception in KQP transaction execution: " << e.what());
+ }
}
Pipeline.AddCommittingOp(op);
- DataShard.IncCounter(COUNTER_WAIT_EXECUTE_LATENCY_MS, waitExecuteLatency.MilliSeconds());
- DataShard.IncCounter(COUNTER_WAIT_TOTAL_LATENCY_MS, waitTotalLatency.MilliSeconds());
- op->ResetCurrentTimer();
-
- return op->IsReadOnly() ? EExecutionStatus::Executed : EExecutionStatus::ExecutedNoMoreRestarts;
+ DataShard.IncCounter(COUNTER_WAIT_EXECUTE_LATENCY_MS, waitExecuteLatency.MilliSeconds());
+ DataShard.IncCounter(COUNTER_WAIT_TOTAL_LATENCY_MS, waitTotalLatency.MilliSeconds());
+ op->ResetCurrentTimer();
+
+ return op->IsReadOnly() ? EExecutionStatus::Executed : EExecutionStatus::ExecutedNoMoreRestarts;
}
-void TExecuteKqpDataTxUnit::AddLocksToResult(TOperation::TPtr op, const TActorContext& ctx) {
+void TExecuteKqpDataTxUnit::AddLocksToResult(TOperation::TPtr op, const TActorContext& ctx) {
auto locks = DataShard.SysLocksTable().ApplyLocks();
- LOG_T("add locks to result: " << locks.size());
+ LOG_T("add locks to result: " << locks.size());
for (const auto& lock : locks) {
if (lock.IsError()) {
LOG_NOTICE_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, "Lock is not set for "
@@ -216,25 +216,25 @@ void TExecuteKqpDataTxUnit::AddLocksToResult(TOperation::TPtr op, const TActorCo
op->Result()->AddTxLock(lock.LockId, lock.DataShard, lock.Generation, lock.Counter, lock.SchemeShard,
lock.PathId);
-
- LOG_T("add lock to result: " << op->Result()->Record.GetTxLocks().rbegin()->ShortDebugString());
+
+ LOG_T("add lock to result: " << op->Result()->Record.GetTxLocks().rbegin()->ShortDebugString());
}
}
-EExecutionStatus TExecuteKqpDataTxUnit::OnTabletNotReady(TActiveTransaction& tx, TValidatedDataTx& dataTx,
- TTransactionContext& txc, const TActorContext& ctx)
-{
- LOG_T("Tablet " << DataShard.TabletID() << " is not ready for " << tx << " execution");
-
- DataShard.IncCounter(COUNTER_TX_TABLET_NOT_READY);
-
- ui64 pageFaultCount = tx.IncrementPageFaultCount();
- dataTx.GetKqpComputeCtx().PinPages(dataTx.TxInfo().Keys, pageFaultCount);
-
- tx.ReleaseTxData(txc, ctx);
- return EExecutionStatus::Restart;
-}
-
+EExecutionStatus TExecuteKqpDataTxUnit::OnTabletNotReady(TActiveTransaction& tx, TValidatedDataTx& dataTx,
+ TTransactionContext& txc, const TActorContext& ctx)
+{
+ LOG_T("Tablet " << DataShard.TabletID() << " is not ready for " << tx << " execution");
+
+ DataShard.IncCounter(COUNTER_TX_TABLET_NOT_READY);
+
+ ui64 pageFaultCount = tx.IncrementPageFaultCount();
+ dataTx.GetKqpComputeCtx().PinPages(dataTx.TxInfo().Keys, pageFaultCount);
+
+ tx.ReleaseTxData(txc, ctx);
+ return EExecutionStatus::Restart;
+}
+
void TExecuteKqpDataTxUnit::Complete(TOperation::TPtr, const TActorContext&) {}
THolder<TExecutionUnit> CreateExecuteKqpDataTxUnit(TDataShard& dataShard, TPipeline& pipeline) {
diff --git a/ydb/core/tx/datashard/execute_kqp_scan_tx_unit.cpp b/ydb/core/tx/datashard/execute_kqp_scan_tx_unit.cpp
index 610a35e66e..1362a5149f 100644
--- a/ydb/core/tx/datashard/execute_kqp_scan_tx_unit.cpp
+++ b/ydb/core/tx/datashard/execute_kqp_scan_tx_unit.cpp
@@ -30,21 +30,21 @@ public:
return !op->HasRuntimeConflicts();
}
- EExecutionStatus Execute(TOperation::TPtr op, TTransactionContext&, const TActorContext& ctx) override {
+ EExecutionStatus Execute(TOperation::TPtr op, TTransactionContext&, const TActorContext& ctx) override {
if (op->Result() || op->HasResultSentFlag() || op->IsImmediate() && CheckRejectDataTx(op, ctx)) {
return EExecutionStatus::Executed;
}
- BuildResult(op, NKikimrTxDataShard::TEvProposeTransactionResult::BAD_REQUEST)
- ->AddError(NKikimrTxDataShard::TError::BAD_TX_KIND, "Unexpected KqpScanTx");
- op->Abort();
-
- LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, "Unexpected KqpScanTx");
+ BuildResult(op, NKikimrTxDataShard::TEvProposeTransactionResult::BAD_REQUEST)
+ ->AddError(NKikimrTxDataShard::TError::BAD_TX_KIND, "Unexpected KqpScanTx");
+ op->Abort();
+
+ LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, "Unexpected KqpScanTx");
return EExecutionStatus::Executed;
}
- void Complete(TOperation::TPtr, const TActorContext&) override {
+ void Complete(TOperation::TPtr, const TActorContext&) override {
}
};
diff --git a/ydb/core/tx/datashard/prepare_kqp_data_tx_in_rs_unit.cpp b/ydb/core/tx/datashard/prepare_kqp_data_tx_in_rs_unit.cpp
index b65906baa7..05ab71fc36 100644
--- a/ydb/core/tx/datashard/prepare_kqp_data_tx_in_rs_unit.cpp
+++ b/ydb/core/tx/datashard/prepare_kqp_data_tx_in_rs_unit.cpp
@@ -45,16 +45,16 @@ EExecutionStatus TPrepareKqpDataTxInRSUnit::Execute(TOperation::TPtr op, TTransa
}
}
- if (tx->GetDataTx()->CheckCancelled()) {
- tx->ReleaseTxData(txc, ctx);
- BuildResult(op, NKikimrTxDataShard::TEvProposeTransactionResult::CANCELLED)
- ->AddError(NKikimrTxDataShard::TError::EXECUTION_CANCELLED, "Tx was cancelled");
-
- DataShard.IncCounter(op->IsImmediate() ? COUNTER_IMMEDIATE_TX_CANCELLED : COUNTER_PLANNED_TX_CANCELLED);
-
- return EExecutionStatus::Executed;
- }
-
+ if (tx->GetDataTx()->CheckCancelled()) {
+ tx->ReleaseTxData(txc, ctx);
+ BuildResult(op, NKikimrTxDataShard::TEvProposeTransactionResult::CANCELLED)
+ ->AddError(NKikimrTxDataShard::TError::EXECUTION_CANCELLED, "Tx was cancelled");
+
+ DataShard.IncCounter(op->IsImmediate() ? COUNTER_IMMEDIATE_TX_CANCELLED : COUNTER_PLANNED_TX_CANCELLED);
+
+ return EExecutionStatus::Executed;
+ }
+
try {
KqpPrepareInReadsets(op->InReadSets(), tx->GetDataTx()->GetKqpTransaction(), DataShard.TabletID());
} catch (const yexception& e) {
diff --git a/ydb/core/tx/datashard/range_ops.cpp b/ydb/core/tx/datashard/range_ops.cpp
index 1f859c4167..64abea2b60 100644
--- a/ydb/core/tx/datashard/range_ops.cpp
+++ b/ydb/core/tx/datashard/range_ops.cpp
@@ -196,13 +196,13 @@ NKikimr::TTableRange NKikimr::Intersect(TConstArrayRef<NScheme::TTypeId> types,
}
}
-TString NKikimr::DebugPrintRange(TConstArrayRef<NScheme::TTypeId> types, const NKikimr::TTableRange &range,
- const NScheme::TTypeRegistry& typeRegistry)
-{
- if (range.Point) {
- return DebugPrintPoint(types, range.From, typeRegistry);
- }
-
+TString NKikimr::DebugPrintRange(TConstArrayRef<NScheme::TTypeId> types, const NKikimr::TTableRange &range,
+ const NScheme::TTypeRegistry& typeRegistry)
+{
+ if (range.Point) {
+ return DebugPrintPoint(types, range.From, typeRegistry);
+ }
+
return TStringBuilder()
<< (range.InclusiveFrom ? "[" : "(")
<< DebugPrintPoint(types, range.From, typeRegistry)
@@ -217,23 +217,23 @@ TString NKikimr::DebugPrintPoint(TConstArrayRef<NScheme::TTypeId> types, const T
return DbgPrintTuple(pointRef, typeRegistry);
}
-
-TString NKikimr::DebugPrintPartitionInfo(const TKeyDesc::TPartitionInfo& partition,
- const TVector<NScheme::TTypeId>& keyTypes, const NScheme::TTypeRegistry& typeRegistry)
-{
- TStringBuilder range;
- if (partition.Range) {
- range << "{ EndKeyPrefix: " << DebugPrintPoint(keyTypes, partition.Range->EndKeyPrefix.GetCells(), typeRegistry)
- << ", IsInclusive: " << partition.Range->IsInclusive
- << ", IsPoint: " << partition.Range->IsPoint
- << " }";
- } else {
- range << "full";
- }
-
- return TStringBuilder()
- << "TPartitionInfo{"
- << " ShardId: " << partition.ShardId
- << ", Range: " << range
- << " }";
-}
+
+TString NKikimr::DebugPrintPartitionInfo(const TKeyDesc::TPartitionInfo& partition,
+ const TVector<NScheme::TTypeId>& keyTypes, const NScheme::TTypeRegistry& typeRegistry)
+{
+ TStringBuilder range;
+ if (partition.Range) {
+ range << "{ EndKeyPrefix: " << DebugPrintPoint(keyTypes, partition.Range->EndKeyPrefix.GetCells(), typeRegistry)
+ << ", IsInclusive: " << partition.Range->IsInclusive
+ << ", IsPoint: " << partition.Range->IsPoint
+ << " }";
+ } else {
+ range << "full";
+ }
+
+ return TStringBuilder()
+ << "TPartitionInfo{"
+ << " ShardId: " << partition.ShardId
+ << ", Range: " << range
+ << " }";
+}
diff --git a/ydb/core/tx/datashard/range_ops.h b/ydb/core/tx/datashard/range_ops.h
index 03ac904931..0ca0a6eeb8 100644
--- a/ydb/core/tx/datashard/range_ops.h
+++ b/ydb/core/tx/datashard/range_ops.h
@@ -9,7 +9,7 @@ TTableRange Intersect(TConstArrayRef<NScheme::TTypeId> types, const TTableRange&
TString DebugPrintRange(TConstArrayRef<NScheme::TTypeId> types, const TTableRange& range, const NScheme::TTypeRegistry& typeRegistry);
TString DebugPrintPoint(TConstArrayRef<NScheme::TTypeId> types, const TConstArrayRef<TCell>& point, const NScheme::TTypeRegistry& typeRegistry);
-TString DebugPrintPartitionInfo(const TKeyDesc::TPartitionInfo& partition, const TVector<NScheme::TTypeId>& keyTypes,
- const NScheme::TTypeRegistry& typeRegistry);
-
+TString DebugPrintPartitionInfo(const TKeyDesc::TPartitionInfo& partition, const TVector<NScheme::TTypeId>& keyTypes,
+ const NScheme::TTypeRegistry& typeRegistry);
+
}
diff --git a/ydb/core/viewer/json_query.h b/ydb/core/viewer/json_query.h
index dd177a7d08..600d8088e1 100644
--- a/ydb/core/viewer/json_query.h
+++ b/ydb/core/viewer/json_query.h
@@ -108,7 +108,7 @@ public:
request.SetType(NKikimrKqp::QUERY_TYPE_SQL_DML);
}
if (Stats == "profile") {
- request.SetStatsMode(NYql::NDqProto::DQ_STATS_MODE_PROFILE);
+ request.SetStatsMode(NYql::NDqProto::DQ_STATS_MODE_PROFILE);
}
if (database) {
request.SetDatabase(database);
diff --git a/ydb/core/ydb_convert/ydb_convert.cpp b/ydb/core/ydb_convert/ydb_convert.cpp
index 4ffa57ecab..11c804cff5 100644
--- a/ydb/core/ydb_convert/ydb_convert.cpp
+++ b/ydb/core/ydb_convert/ydb_convert.cpp
@@ -60,10 +60,10 @@ void ConvertMiniKQLTypeToYdbType(const NKikimrMiniKQL::TType& input, Ydb::Type&
output.set_void_type(::google::protobuf::NULL_VALUE);
break;
}
- case NKikimrMiniKQL::ETypeKind::Null: {
- output.set_null_type(::google::protobuf::NULL_VALUE);
- break;
- }
+ case NKikimrMiniKQL::ETypeKind::Null: {
+ output.set_null_type(::google::protobuf::NULL_VALUE);
+ break;
+ }
case NKikimrMiniKQL::ETypeKind::Data: {
const NKikimrMiniKQL::TDataType& protoData = input.GetData();
NUdf::TDataTypeId schemeType = protoData.GetScheme();
@@ -134,9 +134,9 @@ void ConvertYdbTypeToMiniKQLType(const Ydb::Type& input, NKikimrMiniKQL::TType&
case Ydb::Type::kVoidType:
output.SetKind(NKikimrMiniKQL::ETypeKind::Void);
break;
- case Ydb::Type::kNullType:
- output.SetKind(NKikimrMiniKQL::ETypeKind::Null);
- break;
+ case Ydb::Type::kNullType:
+ output.SetKind(NKikimrMiniKQL::ETypeKind::Null);
+ break;
case Ydb::Type::kTypeId:
output.SetKind(NKikimrMiniKQL::ETypeKind::Data);
output.MutableData()->SetScheme(input.type_id());
@@ -451,10 +451,10 @@ void ConvertMiniKQLValueToYdbValue(const NKikimrMiniKQL::TType& inputType,
case NKikimrMiniKQL::ETypeKind::Void: {
break;
}
- case NKikimrMiniKQL::ETypeKind::Null: {
- output.set_null_flag_value(::google::protobuf::NULL_VALUE);
- break;
- }
+ case NKikimrMiniKQL::ETypeKind::Null: {
+ output.set_null_flag_value(::google::protobuf::NULL_VALUE);
+ break;
+ }
case NKikimrMiniKQL::ETypeKind::Data: {
const NKikimrMiniKQL::TDataType& protoData = inputType.GetData();
const NUdf::TDataTypeId typeId = protoData.GetScheme();
diff --git a/ydb/core/yq/libs/result_formatter/result_formatter_ut.cpp b/ydb/core/yq/libs/result_formatter/result_formatter_ut.cpp
index 1887e0e8e4..bebbf68ab3 100644
--- a/ydb/core/yq/libs/result_formatter/result_formatter_ut.cpp
+++ b/ydb/core/yq/libs/result_formatter/result_formatter_ut.cpp
@@ -286,13 +286,13 @@ Y_UNIT_TEST_SUITE(ResultFormatter) {
NJson::WriteJson(&stream, &root);
//Cerr << stream.Str() << Endl;
- TString expected1 = R"___({"data":[{"column0":[["31337","113370"],["113370","31337"]]}],"columns":[{"name":"column0","type":["DictType",["DataType","Int32"],["DataType","Int64"]]}]})___";
- TString expected2 = R"___({"data":[{"column0":[["113370","31337"],["31337","113370"]]}],"columns":[{"name":"column0","type":["DictType",["DataType","Int32"],["DataType","Int64"]]}]})___";
+ TString expected1 = R"___({"data":[{"column0":[["31337","113370"],["113370","31337"]]}],"columns":[{"name":"column0","type":["DictType",["DataType","Int32"],["DataType","Int64"]]}]})___";
+ TString expected2 = R"___({"data":[{"column0":[["113370","31337"],["31337","113370"]]}],"columns":[{"name":"column0","type":["DictType",["DataType","Int32"],["DataType","Int64"]]}]})___";
- auto actual = stream.Str();
-
- UNIT_ASSERT_C(actual == expected1 || actual == expected2, "expected either " << expected1 << " or " << expected2
- << ", got " << actual);
+ auto actual = stream.Str();
+
+ UNIT_ASSERT_C(actual == expected1 || actual == expected2, "expected either " << expected1 << " or " << expected2
+ << ", got " << actual);
}
Y_UNIT_TEST(VariantTuple) {
diff --git a/ydb/library/mkql_proto/mkql_proto.cpp b/ydb/library/mkql_proto/mkql_proto.cpp
index 65d959031e..53d7722dee 100644
--- a/ydb/library/mkql_proto/mkql_proto.cpp
+++ b/ydb/library/mkql_proto/mkql_proto.cpp
@@ -129,10 +129,10 @@ void ExportTypeToProtoImpl(TType* type, NKikimrMiniKQL::TType& res) {
res.SetKind(NKikimrMiniKQL::ETypeKind::Void);
break;
- case TType::EKind::Null:
- res.SetKind(NKikimrMiniKQL::ETypeKind::Null);
- break;
-
+ case TType::EKind::Null:
+ res.SetKind(NKikimrMiniKQL::ETypeKind::Null);
+ break;
+
case TType::EKind::Data: {
auto dataType = static_cast<TDataType *>(type);
auto schemeType = dataType->GetSchemeType();
@@ -429,11 +429,11 @@ void ExportValueToProtoImpl(TType* type, const NUdf::TUnboxedValuePod& value, NK
case TType::EKind::EmptyDict:
break;
- case TType::EKind::Null: {
- res.SetNullFlagValue(::google::protobuf::NULL_VALUE);
- break;
- }
-
+ case TType::EKind::Null: {
+ res.SetNullFlagValue(::google::protobuf::NULL_VALUE);
+ break;
+ }
+
case TType::EKind::Data: {
HandleKindDataExport(type, value, res);
break;
@@ -525,11 +525,11 @@ void ExportValueToProtoImpl(TType* type, const NUdf::TUnboxedValuePod& value, Yd
case TType::EKind::EmptyDict:
break;
- case TType::EKind::Null: {
- res.set_null_flag_value(::google::protobuf::NULL_VALUE);
- break;
- }
-
+ case TType::EKind::Null: {
+ res.set_null_flag_value(::google::protobuf::NULL_VALUE);
+ break;
+ }
+
case TType::EKind::Data: {
HandleKindDataExport(type, value, res);
break;
@@ -689,24 +689,24 @@ Y_FORCE_INLINE NUdf::TUnboxedValue HandleKindDataImport(const TType* type, const
case NUdf::TDataType<NUdf::TJsonDocument>::Id:
MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kBytes);
return MakeString(value.GetBytes());
- case NUdf::TDataType<NUdf::TDecimal>::Id:
- return NUdf::TUnboxedValuePod(NYql::NDecimal::FromHalfs(value.GetLow128(), value.GetHi128()));
+ case NUdf::TDataType<NUdf::TDecimal>::Id:
+ return NUdf::TUnboxedValuePod(NYql::NDecimal::FromHalfs(value.GetLow128(), value.GetHi128()));
case NUdf::TDataType<NUdf::TDyNumber>::Id:
MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kBytes);
return MakeString(value.GetBytes());
- case NUdf::TDataType<NUdf::TUuid>::Id: {
- MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kLow128);
- union {
- ui64 half[2];
- char bytes[16];
- } buf;
- buf.half[0] = value.GetLow128();
- buf.half[1] = value.GetHi128();
- return MakeString(NUdf::TStringRef(buf.bytes, 16));
- }
+ case NUdf::TDataType<NUdf::TUuid>::Id: {
+ MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kLow128);
+ union {
+ ui64 half[2];
+ char bytes[16];
+ } buf;
+ buf.half[0] = value.GetLow128();
+ buf.half[1] = value.GetHi128();
+ return MakeString(NUdf::TStringRef(buf.bytes, 16));
+ }
default:
- MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kBytes,
- "got: " << (int) oneOfCase << ", type: " << (int) dataType->GetSchemeType());
+ MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kBytes,
+ "got: " << (int) oneOfCase << ", type: " << (int) dataType->GetSchemeType());
return MakeString(value.GetBytes());
}
}
@@ -937,9 +937,9 @@ TType* TProtoImporter::ImportTypeFromProto(const NKikimrMiniKQL::TType& type) {
case NKikimrMiniKQL::ETypeKind::Void: {
return env.GetVoid()->GetType();
}
- case NKikimrMiniKQL::ETypeKind::Null: {
- return env.GetNull()->GetType();
- }
+ case NKikimrMiniKQL::ETypeKind::Null: {
+ return env.GetNull()->GetType();
+ }
case NKikimrMiniKQL::ETypeKind::Data: {
const NKikimrMiniKQL::TDataType& protoData = type.GetData();
NUdf::TDataTypeId schemeType = protoData.GetScheme();
@@ -1013,9 +1013,9 @@ TNode* TProtoImporter::ImportNodeFromProto(TType* type, const NKikimrMiniKQL::TV
case TCallableType::EKind::Void: {
return env.GetVoid();
}
- case TCallableType::EKind::Null: {
- return env.GetNull();
- }
+ case TCallableType::EKind::Null: {
+ return env.GetNull();
+ }
case TCallableType::EKind::Data: {
TDataType* dataType = static_cast<TDataType*>(type);
TDataLiteral* dataNode = nullptr;
@@ -1189,9 +1189,9 @@ NUdf::TUnboxedValue TProtoImporter::ImportValueFromProto(const TType* type, cons
case TType::EKind::Void:
return NUdf::TUnboxedValuePod::Void();
- case TType::EKind::Null:
- return NUdf::TUnboxedValuePod();
-
+ case TType::EKind::Null:
+ return NUdf::TUnboxedValuePod();
+
case TType::EKind::Data:
return HandleKindDataImport(type, value);
@@ -1249,7 +1249,7 @@ NUdf::TUnboxedValue TProtoImporter::ImportValueFromProto(const TType* type, cons
auto dictType = static_cast<const TDictType*>(type);
const TType* keyType = dictType->GetKeyType();
const TType* payloadType = dictType->GetPayloadType();
- auto dictBuilder = factory.NewDict(dictType, NUdf::TDictFlags::EDictKind::Hashed);
+ auto dictBuilder = factory.NewDict(dictType, NUdf::TDictFlags::EDictKind::Hashed);
for (const auto& x : value.GetDict()) {
dictBuilder->Add(
diff --git a/ydb/library/mkql_proto/protos/minikql.proto b/ydb/library/mkql_proto/protos/minikql.proto
index ab684fad0a..2a4243855c 100644
--- a/ydb/library/mkql_proto/protos/minikql.proto
+++ b/ydb/library/mkql_proto/protos/minikql.proto
@@ -2,8 +2,8 @@ package NKikimrMiniKQL;
option java_package = "ru.yandex.kikimr.proto";
option cc_enable_arenas = true;
-import "google/protobuf/struct.proto";
-
+import "google/protobuf/struct.proto";
+
enum ETypeKind {
Unknown = 0;
Void = 1;
@@ -14,12 +14,12 @@ enum ETypeKind {
Struct = 6;
Dict = 7;
Variant = 8;
- Null = 9;
+ Null = 9;
Reserved_10 = 10;
Reserved_11 = 11;
Reserved_12 = 12;
- Reserved_13 = 13;
- Reserved_14 = 14;
+ Reserved_13 = 13;
+ Reserved_14 = 14;
}
message TDecimalParams {
@@ -96,7 +96,7 @@ message TValue {
string Text = 9;
TValue Optional = 10;
fixed64 Low128 = 15;
- google.protobuf.NullValue NullFlagValue = 18; // Set if current TValue is terminal Null
+ google.protobuf.NullValue NullFlagValue = 18; // Set if current TValue is terminal Null
}
// Logically part of oneof,
// but protobuf does not allow repeated fields in oneof.
diff --git a/ydb/library/yql/ast/yql_expr.cpp b/ydb/library/yql/ast/yql_expr.cpp
index 8a759aaf53..82f6c5812a 100644
--- a/ydb/library/yql/ast/yql_expr.cpp
+++ b/ydb/library/yql/ast/yql_expr.cpp
@@ -755,13 +755,13 @@ namespace {
return TAstNode::NewList(TPosition(), pool, self, itemType);
}
- case ETypeAnnotationKind::Flow:
- {
+ case ETypeAnnotationKind::Flow:
+ {
auto self = TAstNode::NewLiteralAtom(TPosition(), TStringBuf("Flow"), pool);
- auto itemType = ConvertTypeAnnotationToAst(*annotation.Cast<TFlowExprType>()->GetItemType(), pool, refAtoms);
- return TAstNode::NewList(TPosition(), pool, self, itemType);
+ auto itemType = ConvertTypeAnnotationToAst(*annotation.Cast<TFlowExprType>()->GetItemType(), pool, refAtoms);
+ return TAstNode::NewList(TPosition(), pool, self, itemType);
}
-
+
case ETypeAnnotationKind::Multi:
{
auto self = TAstNode::NewLiteralAtom(TPosition(), TStringBuf("Multi"), pool);
@@ -774,18 +774,18 @@ namespace {
return TAstNode::NewList(TPosition(), children.data(), children.size(), pool);
}
- case ETypeAnnotationKind::EmptyList:
+ case ETypeAnnotationKind::EmptyList:
{
return TAstNode::NewLiteralAtom(TPosition(), TStringBuf("EmptyList"), pool);
}
- case ETypeAnnotationKind::EmptyDict:
+ case ETypeAnnotationKind::EmptyDict:
{
return TAstNode::NewLiteralAtom(TPosition(), TStringBuf("EmptyDict"), pool);
}
- case ETypeAnnotationKind::LastType:
- YQL_ENSURE(false, "Unknown kind: " << annotation.GetKind());
-
- }
+ case ETypeAnnotationKind::LastType:
+ YQL_ENSURE(false, "Unknown kind: " << annotation.GetKind());
+
+ }
}
TAstNode* AnnotateAstNode(TAstNode* node, const TExprNode* exprNode, ui32 flags, TMemoryPool& pool, bool refAtoms) {
@@ -2565,26 +2565,26 @@ TExprNode::TPtr TExprContext::FuseLambdas(const TExprNode& outer, const TExprNod
return NewLambda(outer.Pos(), NewArguments(inner.Head().Pos(), std::move(newArgNodes)), std::move(newBody));
}
-TExprNode::TPtr TExprContext::DeepCopy(const TExprNode& node, TExprContext& nodeCtx, TNodeOnNodeOwnedMap& deepClones,
+TExprNode::TPtr TExprContext::DeepCopy(const TExprNode& node, TExprContext& nodeCtx, TNodeOnNodeOwnedMap& deepClones,
bool internStrings, bool copyTypes, bool copyResult, TCustomDeepCopier customCopier)
-{
+{
const auto ins = deepClones.emplace(&node, nullptr);
if (ins.second) {
TExprNode::TListType children;
children.reserve(node.ChildrenSize());
-
- if (customCopier && customCopier(node, children)) {
- } else {
- node.ForEachChild([&](const TExprNode& child) {
+
+ if (customCopier && customCopier(node, children)) {
+ } else {
+ node.ForEachChild([&](const TExprNode& child) {
children.emplace_back(DeepCopy(child, nodeCtx, deepClones, internStrings, copyTypes, copyResult, customCopier));
- });
- }
-
+ });
+ }
+
++NodeAllocationCounter;
- auto newNode = TExprNode::NewNode(AppendPosition(nodeCtx.GetPosition(node.Pos())), node.Type(),
- std::move(children), internStrings ? AppendString(node.Content()) : node.Content(), node.Flags(),
- AllocateNextUniqueId());
-
+ auto newNode = TExprNode::NewNode(AppendPosition(nodeCtx.GetPosition(node.Pos())), node.Type(),
+ std::move(children), internStrings ? AppendString(node.Content()) : node.Content(), node.Flags(),
+ AllocateNextUniqueId());
+
if (copyTypes && node.GetTypeAnn()) {
newNode->SetTypeAnn(node.GetTypeAnn());
}
diff --git a/ydb/library/yql/ast/yql_expr.h b/ydb/library/yql/ast/yql_expr.h
index 918f994474..d6eb544f9a 100644
--- a/ydb/library/yql/ast/yql_expr.h
+++ b/ydb/library/yql/ast/yql_expr.h
@@ -2315,12 +2315,12 @@ struct TExprContext : private TNonCopyable {
[[nodiscard]]
TExprNode::TPtr FuseLambdas(const TExprNode& outer, const TExprNode& inner);
- using TCustomDeepCopier = std::function<bool(const TExprNode& node, TExprNode::TListType& newChildren)>;
+ using TCustomDeepCopier = std::function<bool(const TExprNode& node, TExprNode::TListType& newChildren)>;
[[nodiscard]]
- TExprNode::TPtr DeepCopy(const TExprNode& node, TExprContext& nodeContext, TNodeOnNodeOwnedMap& deepClones,
+ TExprNode::TPtr DeepCopy(const TExprNode& node, TExprContext& nodeContext, TNodeOnNodeOwnedMap& deepClones,
bool internStrings, bool copyTypes, bool copyResult = false, TCustomDeepCopier customCopier = {});
-
+
[[nodiscard]]
TExprNode::TPtr SwapWithHead(const TExprNode& node);
TExprNode::TPtr ReplaceNode(TExprNode::TPtr&& start, const TExprNode& src, TExprNode::TPtr dst);
diff --git a/ydb/library/yql/core/common_opt/yql_co_extr_members.cpp b/ydb/library/yql/core/common_opt/yql_co_extr_members.cpp
index 1fd9c3225a..215a1e9f69 100644
--- a/ydb/library/yql/core/common_opt/yql_co_extr_members.cpp
+++ b/ydb/library/yql/core/common_opt/yql_co_extr_members.cpp
@@ -180,20 +180,20 @@ TExprNode::TPtr ApplyExtractMembersToSort(const TExprNode::TPtr& node, const TEx
extractFields.emplace(x->Content());
}
TSet<TStringBuf> sortKeys;
- bool fieldSubset = HaveFieldsSubset(sort.KeySelectorLambda().Body().Ptr(), sort.KeySelectorLambda().Args().Arg(0).Ref(), sortKeys, parentsMap);
+ bool fieldSubset = HaveFieldsSubset(sort.KeySelectorLambda().Body().Ptr(), sort.KeySelectorLambda().Args().Arg(0).Ref(), sortKeys, parentsMap);
bool allExist = true;
if (!sortKeys.empty()) {
for (const auto& key : sortKeys) {
- auto ret = extractFields.emplace(key);
- if (ret.second) {
+ auto ret = extractFields.emplace(key);
+ if (ret.second) {
allExist = false;
}
}
}
- if (allExist && sortKeys.size() == extractFields.size()) {
- YQL_CLOG(DEBUG, Core) << "Force `fieldSubset` for ExtractMembers over " << node->Content();
- fieldSubset = true;
- }
+ if (allExist && sortKeys.size() == extractFields.size()) {
+ YQL_CLOG(DEBUG, Core) << "Force `fieldSubset` for ExtractMembers over " << node->Content();
+ fieldSubset = true;
+ }
if (fieldSubset && allExist) {
YQL_CLOG(DEBUG, Core) << "Move ExtractMembers over " << node->Content() << logSuffix;
return ctx.Builder(sort.Pos())
diff --git a/ydb/library/yql/core/common_opt/yql_co_simple1.cpp b/ydb/library/yql/core/common_opt/yql_co_simple1.cpp
index 8f6a130de0..a32a6c3dab 100644
--- a/ydb/library/yql/core/common_opt/yql_co_simple1.cpp
+++ b/ydb/library/yql/core/common_opt/yql_co_simple1.cpp
@@ -1,5 +1,5 @@
#include "yql_co.h"
-#include "yql_co_sqlin.h"
+#include "yql_co_sqlin.h"
#include <ydb/library/yql/core/yql_atom_enums.h>
#include <ydb/library/yql/core/yql_expr_type_annotation.h>
@@ -1431,68 +1431,68 @@ TExprNode::TPtr FuseToListWithFlatmap(const TExprNode::TPtr& node, TExprContext&
return result;
}
-bool ShouldConvertSqlInToJoin(const TCoSqlIn& sqlIn, bool /* negated */) {
- bool tableSource = false;
+bool ShouldConvertSqlInToJoin(const TCoSqlIn& sqlIn, bool /* negated */) {
+ bool tableSource = false;
- for (const auto& hint : sqlIn.Options()) {
+ for (const auto& hint : sqlIn.Options()) {
if (hint.Name().Value() == TStringBuf("isCompact")) {
- return false;
+ return false;
}
if (hint.Name().Value() == TStringBuf("tableSource")) {
- tableSource = true;
- }
+ tableSource = true;
+ }
}
- return tableSource;
+ return tableSource;
}
-bool CanConvertSqlInToJoin(const TCoSqlIn& sqlIn) {
- auto leftArg = sqlIn.Lookup();
- auto leftColumnType = leftArg.Ref().GetTypeAnn();
-
- auto rightArg = sqlIn.Collection();
- auto rightArgType = rightArg.Ref().GetTypeAnn();
-
- if (rightArgType->GetKind() == ETypeAnnotationKind::List) {
- auto rightListItemType = rightArgType->Cast<TListExprType>()->GetItemType();
-
- auto isDataOrTupleOfData = [](const TTypeAnnotationNode* type) {
- if (IsDataOrOptionalOfData(type)) {
- return true;
- }
- if (type->GetKind() == ETypeAnnotationKind::Tuple) {
- return AllOf(type->Cast<TTupleExprType>()->GetItems(), [](const auto& item) {
- return IsDataOrOptionalOfData(item);
- });
- }
- return false;
- };
-
- if (rightListItemType->GetKind() == ETypeAnnotationKind::Struct) {
- auto rightStructType = rightListItemType->Cast<TStructExprType>();
- YQL_ENSURE(rightStructType->GetSize() == 1);
- auto rightColumnType = rightStructType->GetItems()[0]->GetItemType();
- return isDataOrTupleOfData(rightColumnType);
- }
-
- return isDataOrTupleOfData(rightListItemType);
- }
-
- /**
- * todo: support tuple of equal tuples
- *
- * sql expression \code{.sql} ... where (k1, k2) in ((1, 2), (2, 3), (3, 4)) \endcode
- * is equivalent to the \code{.sql} ... where (k1, k2) in AsTuple((1, 2), (2, 3), (3, 4)) \endcode
- * but not to the \code{.sql} ... where (k1, k2) in AsList((1, 2), (2, 3), (3, 4)) \endcode
- * so, it's not supported now
- */
-
- if (rightArgType->GetKind() == ETypeAnnotationKind::Dict) {
- auto rightDictType = rightArgType->Cast<TDictExprType>()->GetKeyType();
- return IsDataOrOptionalOfData(leftColumnType) && IsDataOrOptionalOfData(rightDictType);
- }
-
- return false;
+bool CanConvertSqlInToJoin(const TCoSqlIn& sqlIn) {
+ auto leftArg = sqlIn.Lookup();
+ auto leftColumnType = leftArg.Ref().GetTypeAnn();
+
+ auto rightArg = sqlIn.Collection();
+ auto rightArgType = rightArg.Ref().GetTypeAnn();
+
+ if (rightArgType->GetKind() == ETypeAnnotationKind::List) {
+ auto rightListItemType = rightArgType->Cast<TListExprType>()->GetItemType();
+
+ auto isDataOrTupleOfData = [](const TTypeAnnotationNode* type) {
+ if (IsDataOrOptionalOfData(type)) {
+ return true;
+ }
+ if (type->GetKind() == ETypeAnnotationKind::Tuple) {
+ return AllOf(type->Cast<TTupleExprType>()->GetItems(), [](const auto& item) {
+ return IsDataOrOptionalOfData(item);
+ });
+ }
+ return false;
+ };
+
+ if (rightListItemType->GetKind() == ETypeAnnotationKind::Struct) {
+ auto rightStructType = rightListItemType->Cast<TStructExprType>();
+ YQL_ENSURE(rightStructType->GetSize() == 1);
+ auto rightColumnType = rightStructType->GetItems()[0]->GetItemType();
+ return isDataOrTupleOfData(rightColumnType);
+ }
+
+ return isDataOrTupleOfData(rightListItemType);
+ }
+
+ /**
+ * todo: support tuple of equal tuples
+ *
+ * sql expression \code{.sql} ... where (k1, k2) in ((1, 2), (2, 3), (3, 4)) \endcode
+ * is equivalent to the \code{.sql} ... where (k1, k2) in AsTuple((1, 2), (2, 3), (3, 4)) \endcode
+ * but not to the \code{.sql} ... where (k1, k2) in AsList((1, 2), (2, 3), (3, 4)) \endcode
+ * so, it's not supported now
+ */
+
+ if (rightArgType->GetKind() == ETypeAnnotationKind::Dict) {
+ auto rightDictType = rightArgType->Cast<TDictExprType>()->GetKeyType();
+ return IsDataOrOptionalOfData(leftColumnType) && IsDataOrOptionalOfData(rightDictType);
+ }
+
+ return false;
}
struct TPredicateChainNode {
@@ -1507,10 +1507,10 @@ struct TPredicateChainNode {
// SqlIn params
TPositionHandle SqlInPos;
- TExprNode::TPtr Left; // used only if LeftArgColumns is empty
+ TExprNode::TPtr Left; // used only if LeftArgColumns is empty
TExprNode::TPtr Right;
- TVector<TStringBuf> LeftArgColumns; // set if left side of IN is input column reference or tuple of columns references
+ TVector<TStringBuf> LeftArgColumns; // set if left side of IN is input column reference or tuple of columns references
TVector<TString> RightArgColumns; // always set
};
@@ -1624,14 +1624,14 @@ TExprNode::TPtr BuildSqlInCollectionEmptyPred(const TCoSqlIn& sqlIn, TExprContex
return collectionEmptyPred;
}
-TPredicateChainNode ParsePredicateChainNode(const TExprNode::TPtr& predicate, const TExprNode::TPtr& topLambdaArg,
- std::function<bool(const TCoSqlIn&, bool /* negated */)> shouldConvertSqlInToJoin, TExprContext& ctx)
-{
+TPredicateChainNode ParsePredicateChainNode(const TExprNode::TPtr& predicate, const TExprNode::TPtr& topLambdaArg,
+ std::function<bool(const TCoSqlIn&, bool /* negated */)> shouldConvertSqlInToJoin, TExprContext& ctx)
+{
TPredicateChainNode result;
- result.Pred = predicate;
+ result.Pred = predicate;
- auto curr = predicate;
+ auto curr = predicate;
TExprNode::TPtr pred;
if (curr->IsCallable("Not")) {
curr = curr->HeadPtr();
@@ -1653,17 +1653,17 @@ TPredicateChainNode ParsePredicateChainNode(const TExprNode::TPtr& predicate, co
hasCoalesce = true;
}
- if (!leftArg) {
- // not SqlIn
+ if (!leftArg) {
+ // not SqlIn
+ return result;
+ }
+
+ TCoSqlIn sqlIn(curr);
+ if (!shouldConvertSqlInToJoin(sqlIn, result.Negated) || !CanConvertSqlInToJoin(sqlIn)) {
+ // not convertible to join
return result;
}
- TCoSqlIn sqlIn(curr);
- if (!shouldConvertSqlInToJoin(sqlIn, result.Negated) || !CanConvertSqlInToJoin(sqlIn)) {
- // not convertible to join
- return result;
- }
-
result.SqlInPos = sqlIn.Pos();
result.ConvertibleToJoin = true;
result.Left = leftArg;
@@ -1722,177 +1722,177 @@ TPredicateChainNode ParsePredicateChainNode(const TExprNode::TPtr& predicate, co
}
}
- auto isMemberOf = [](const TExprNode::TPtr& node, const TExprNode::TPtr& arg) {
+ auto isMemberOf = [](const TExprNode::TPtr& node, const TExprNode::TPtr& arg) {
return node->IsCallable("Member") && node->HeadPtr() == arg;
- };
-
- if (isMemberOf(leftArg, topLambdaArg)) {
+ };
+
+ if (isMemberOf(leftArg, topLambdaArg)) {
// left side of IN is column reference
- result.LeftArgColumns.emplace_back(leftArg->Child(1)->Content());
- } else if (leftArg->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Tuple) {
- // if leftArg is tuple of members then replace tuple with its members
- for (const auto& tupleItem : leftArg->Children()) {
- if (isMemberOf(tupleItem, topLambdaArg)) {
- result.LeftArgColumns.emplace_back(tupleItem->Child(1)->Content());
- } else {
- // fallback to join on whole tuple
- result.LeftArgColumns.clear();
- break;
- }
- }
- }
-
- auto rightArg = sqlIn.Collection().Ptr();
- auto rightArgType = rightArg->GetTypeAnn();
-
- if (rightArgType->GetKind() == ETypeAnnotationKind::List) {
- auto rightArgItemType = rightArgType->Cast<TListExprType>()->GetItemType();
-
- if (rightArgItemType->GetKind() == ETypeAnnotationKind::Struct) {
- auto rightStructType = rightArgItemType->Cast<TStructExprType>();
- YQL_ENSURE(rightStructType->GetSize() == 1);
-
- const TItemExprType* itemType = rightStructType->GetItems()[0];
- if (IsDataOrOptionalOfData(itemType->GetItemType())) {
- result.Right = rightArg;
- result.RightArgColumns = { ToString(itemType->GetName()) };
- return result;
- }
-
- YQL_ENSURE(itemType->GetItemType()->GetKind() == ETypeAnnotationKind::Tuple);
-
- rightArg = Build<TCoFlatMap>(ctx, rightArg->Pos())
- .Input(rightArg)
- .Lambda()
- .Args({"item"})
- .Body<TCoJust>()
- .Input<TCoMember>()
- .Struct("item")
- .Name().Build(itemType->GetName())
- .Build()
- .Build()
- .Build()
- .Done()
- .Ptr();
-
- if (!result.LeftArgColumns.empty()) {
- auto rowArg = Build<TCoArgument>(ctx, sqlIn.Pos())
- .Name("row")
- .Done();
- auto asStructBuilder = Build<TCoAsStruct>(ctx, sqlIn.Pos());
- for (size_t i = 0; i < itemType->GetItemType()->Cast<TTupleExprType>()->GetItems().size(); ++i) {
- const TString columnName = TStringBuilder() << "_yql_sqlin_tuple_" << i;
- asStructBuilder.Add<TCoNameValueTuple>()
- .Name().Build(columnName)
- .Value<TCoNth>()
- .Tuple(rowArg)
- .Index(ctx.NewAtom(sqlIn.Pos(), ToString(i)))
- .Build()
- .Build();
- result.RightArgColumns.emplace_back(columnName);
- }
- result.Right = Build<TCoMap>(ctx, sqlIn.Pos())
- .Input(rightArg)
- .Lambda()
- .Args(rowArg)
- .Body(asStructBuilder.Done())
- .Build()
- .Done()
- .Ptr();
-
- return result;
- }
-
- // fallthrough to default join by the whole tuple
- } else if (rightArgItemType->GetKind() == ETypeAnnotationKind::Tuple) {
- auto tupleItemTypes = rightArgItemType->Cast<TTupleExprType>()->GetItems();
-
- if (!result.LeftArgColumns.empty()) {
- auto rowArg = Build<TCoArgument>(ctx, sqlIn.Pos())
- .Name("row")
- .Done();
- auto asStructBuilder = Build<TCoAsStruct>(ctx, sqlIn.Pos());
- for (size_t i = 0; i < tupleItemTypes.size(); ++i) {
- const TString columnName = TStringBuilder() << "_yql_sqlin_tuple_" << i;
- asStructBuilder.Add<TCoNameValueTuple>()
- .Name().Build(columnName)
- .Value<TCoNth>()
- .Tuple(rowArg)
- .Index(ctx.NewAtom(sqlIn.Pos(), ToString(i)))
- .Build()
- .Build();
- result.RightArgColumns.emplace_back(columnName);
- }
- result.Right = Build<TCoMap>(ctx, sqlIn.Pos())
- .Input(rightArg)
- .Lambda()
- .Args(rowArg)
- .Body(asStructBuilder.Done())
- .Build()
- .Done()
- .Ptr();
- return result;
- }
-
- // fallthrough to default join by the whole tuple
- } else {
- YQL_ENSURE(IsDataOrOptionalOfData(rightArgItemType), "" << FormatType(rightArgItemType));
- }
-
- // rewrite List<DataType|Tuple> to List<Struct<key: DataType|Tuple>>
- result.Right = Build<TCoMap>(ctx, sqlIn.Pos())
- .Input(rightArg)
- .Lambda()
- .Args({"item"})
- .Body<TCoAsStruct>()
- .Add<TCoNameValueTuple>()
- .Name().Build("key")
- .Value("item")
- .Build()
- .Build()
- .Build()
- .Done()
- .Ptr();
- result.RightArgColumns = { "key" };
-
- return result;
- }
-
- YQL_ENSURE(rightArgType->GetKind() == ETypeAnnotationKind::Dict, "" << FormatType(rightArgType));
-
- auto rightDictType = rightArgType->Cast<TDictExprType>()->GetKeyType();
- YQL_ENSURE(IsDataOrOptionalOfData(rightDictType));
-
- auto dictKeys = ctx.Builder(sqlIn.Pos())
- .Callable("DictKeys")
- .Add(0, rightArg)
- .Seal()
- .Build();
-
- result.Right = Build<TCoMap>(ctx, sqlIn.Pos())
- .Input(dictKeys)
- .Lambda()
- .Args({"item"})
- .Body<TCoAsStruct>()
- .Add<TCoNameValueTuple>()
- .Name().Build("key")
- .Value("item")
- .Build()
- .Build()
- .Build()
- .Done()
- .Ptr();
- result.RightArgColumns = { "key" };
-
+ result.LeftArgColumns.emplace_back(leftArg->Child(1)->Content());
+ } else if (leftArg->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Tuple) {
+ // if leftArg is tuple of members then replace tuple with its members
+ for (const auto& tupleItem : leftArg->Children()) {
+ if (isMemberOf(tupleItem, topLambdaArg)) {
+ result.LeftArgColumns.emplace_back(tupleItem->Child(1)->Content());
+ } else {
+ // fallback to join on whole tuple
+ result.LeftArgColumns.clear();
+ break;
+ }
+ }
+ }
+
+ auto rightArg = sqlIn.Collection().Ptr();
+ auto rightArgType = rightArg->GetTypeAnn();
+
+ if (rightArgType->GetKind() == ETypeAnnotationKind::List) {
+ auto rightArgItemType = rightArgType->Cast<TListExprType>()->GetItemType();
+
+ if (rightArgItemType->GetKind() == ETypeAnnotationKind::Struct) {
+ auto rightStructType = rightArgItemType->Cast<TStructExprType>();
+ YQL_ENSURE(rightStructType->GetSize() == 1);
+
+ const TItemExprType* itemType = rightStructType->GetItems()[0];
+ if (IsDataOrOptionalOfData(itemType->GetItemType())) {
+ result.Right = rightArg;
+ result.RightArgColumns = { ToString(itemType->GetName()) };
+ return result;
+ }
+
+ YQL_ENSURE(itemType->GetItemType()->GetKind() == ETypeAnnotationKind::Tuple);
+
+ rightArg = Build<TCoFlatMap>(ctx, rightArg->Pos())
+ .Input(rightArg)
+ .Lambda()
+ .Args({"item"})
+ .Body<TCoJust>()
+ .Input<TCoMember>()
+ .Struct("item")
+ .Name().Build(itemType->GetName())
+ .Build()
+ .Build()
+ .Build()
+ .Done()
+ .Ptr();
+
+ if (!result.LeftArgColumns.empty()) {
+ auto rowArg = Build<TCoArgument>(ctx, sqlIn.Pos())
+ .Name("row")
+ .Done();
+ auto asStructBuilder = Build<TCoAsStruct>(ctx, sqlIn.Pos());
+ for (size_t i = 0; i < itemType->GetItemType()->Cast<TTupleExprType>()->GetItems().size(); ++i) {
+ const TString columnName = TStringBuilder() << "_yql_sqlin_tuple_" << i;
+ asStructBuilder.Add<TCoNameValueTuple>()
+ .Name().Build(columnName)
+ .Value<TCoNth>()
+ .Tuple(rowArg)
+ .Index(ctx.NewAtom(sqlIn.Pos(), ToString(i)))
+ .Build()
+ .Build();
+ result.RightArgColumns.emplace_back(columnName);
+ }
+ result.Right = Build<TCoMap>(ctx, sqlIn.Pos())
+ .Input(rightArg)
+ .Lambda()
+ .Args(rowArg)
+ .Body(asStructBuilder.Done())
+ .Build()
+ .Done()
+ .Ptr();
+
+ return result;
+ }
+
+ // fallthrough to default join by the whole tuple
+ } else if (rightArgItemType->GetKind() == ETypeAnnotationKind::Tuple) {
+ auto tupleItemTypes = rightArgItemType->Cast<TTupleExprType>()->GetItems();
+
+ if (!result.LeftArgColumns.empty()) {
+ auto rowArg = Build<TCoArgument>(ctx, sqlIn.Pos())
+ .Name("row")
+ .Done();
+ auto asStructBuilder = Build<TCoAsStruct>(ctx, sqlIn.Pos());
+ for (size_t i = 0; i < tupleItemTypes.size(); ++i) {
+ const TString columnName = TStringBuilder() << "_yql_sqlin_tuple_" << i;
+ asStructBuilder.Add<TCoNameValueTuple>()
+ .Name().Build(columnName)
+ .Value<TCoNth>()
+ .Tuple(rowArg)
+ .Index(ctx.NewAtom(sqlIn.Pos(), ToString(i)))
+ .Build()
+ .Build();
+ result.RightArgColumns.emplace_back(columnName);
+ }
+ result.Right = Build<TCoMap>(ctx, sqlIn.Pos())
+ .Input(rightArg)
+ .Lambda()
+ .Args(rowArg)
+ .Body(asStructBuilder.Done())
+ .Build()
+ .Done()
+ .Ptr();
+ return result;
+ }
+
+ // fallthrough to default join by the whole tuple
+ } else {
+ YQL_ENSURE(IsDataOrOptionalOfData(rightArgItemType), "" << FormatType(rightArgItemType));
+ }
+
+ // rewrite List<DataType|Tuple> to List<Struct<key: DataType|Tuple>>
+ result.Right = Build<TCoMap>(ctx, sqlIn.Pos())
+ .Input(rightArg)
+ .Lambda()
+ .Args({"item"})
+ .Body<TCoAsStruct>()
+ .Add<TCoNameValueTuple>()
+ .Name().Build("key")
+ .Value("item")
+ .Build()
+ .Build()
+ .Build()
+ .Done()
+ .Ptr();
+ result.RightArgColumns = { "key" };
+
+ return result;
+ }
+
+ YQL_ENSURE(rightArgType->GetKind() == ETypeAnnotationKind::Dict, "" << FormatType(rightArgType));
+
+ auto rightDictType = rightArgType->Cast<TDictExprType>()->GetKeyType();
+ YQL_ENSURE(IsDataOrOptionalOfData(rightDictType));
+
+ auto dictKeys = ctx.Builder(sqlIn.Pos())
+ .Callable("DictKeys")
+ .Add(0, rightArg)
+ .Seal()
+ .Build();
+
+ result.Right = Build<TCoMap>(ctx, sqlIn.Pos())
+ .Input(dictKeys)
+ .Lambda()
+ .Args({"item"})
+ .Body<TCoAsStruct>()
+ .Add<TCoNameValueTuple>()
+ .Name().Build("key")
+ .Value("item")
+ .Build()
+ .Build()
+ .Build()
+ .Done()
+ .Ptr();
+ result.RightArgColumns = { "key" };
+
return result;
}
-TExprNode::TPtr SplitPredicateChain(TExprNode::TPtr&& node, const TExprNode::TPtr& topLambdaArg,
- std::function<bool(const TCoSqlIn&, bool /* negated */)> shouldConvertSqlInToJoin, TPredicateChain& prefix,
- TExprContext& ctx)
-{
+TExprNode::TPtr SplitPredicateChain(TExprNode::TPtr&& node, const TExprNode::TPtr& topLambdaArg,
+ std::function<bool(const TCoSqlIn&, bool /* negated */)> shouldConvertSqlInToJoin, TPredicateChain& prefix,
+ TExprContext& ctx)
+{
if (!node->IsCallable("And")) {
- TPredicateChainNode curr = ParsePredicateChainNode(node, topLambdaArg, shouldConvertSqlInToJoin, ctx);
+ TPredicateChainNode curr = ParsePredicateChainNode(node, topLambdaArg, shouldConvertSqlInToJoin, ctx);
if (!prefix.empty() && prefix.back().ConvertibleToJoin != curr.ConvertibleToJoin) {
// stop splitting
return std::move(node);
@@ -1909,7 +1909,7 @@ TExprNode::TPtr SplitPredicateChain(TExprNode::TPtr&& node, const TExprNode::TPt
if (child) {
break;
}
- }
+ }
if (children.front().Get() == &node->Head()) {
return std::move(node);
@@ -1967,11 +1967,11 @@ TExprNode::TPtr BuildEquiJoinForSqlInChain(const TExprNode::TPtr& flatMapNode, c
auto inputTableAtom = ctx.NewAtom(input->Pos(), inputTable);
for (size_t i = 0; i < chain.size(); ++i) {
- const TString tableName = TStringBuilder() << "_yql_injoin_" << i;
- const TString columnName = TStringBuilder() << "_yql_injoin_column_" << i;
- const auto pos = chain[i].SqlInPos;
+ const TString tableName = TStringBuilder() << "_yql_injoin_" << i;
+ const TString columnName = TStringBuilder() << "_yql_injoin_column_" << i;
+ const auto pos = chain[i].SqlInPos;
- auto equiJoinArg = ctx.Builder(pos)
+ auto equiJoinArg = ctx.Builder(pos)
.List()
.Add(0, chain[i].Right)
.Atom(1, tableName)
@@ -1980,41 +1980,41 @@ TExprNode::TPtr BuildEquiJoinForSqlInChain(const TExprNode::TPtr& flatMapNode, c
equiJoinArgs.push_back(equiJoinArg);
- TExprNodeList leftKeys;
- if (chain[i].LeftArgColumns.empty()) {
- leftKeys.push_back(inputTableAtom);
- leftKeys.push_back(ctx.NewAtom(pos, columnName));
- } else {
- for (TStringBuf leftKey : chain[i].LeftArgColumns) {
- leftKeys.push_back(inputTableAtom);
- leftKeys.push_back(ctx.NewAtom(pos, leftKey));
- }
- }
-
- TExprNodeList rightKeys;
- for (const TString& rightKey : chain[i].RightArgColumns) {
- rightKeys.push_back(ctx.NewAtom(pos, tableName));
- rightKeys.push_back(ctx.NewAtom(pos, rightKey));
- }
-
- joinChain = ctx.Builder(pos)
+ TExprNodeList leftKeys;
+ if (chain[i].LeftArgColumns.empty()) {
+ leftKeys.push_back(inputTableAtom);
+ leftKeys.push_back(ctx.NewAtom(pos, columnName));
+ } else {
+ for (TStringBuf leftKey : chain[i].LeftArgColumns) {
+ leftKeys.push_back(inputTableAtom);
+ leftKeys.push_back(ctx.NewAtom(pos, leftKey));
+ }
+ }
+
+ TExprNodeList rightKeys;
+ for (const TString& rightKey : chain[i].RightArgColumns) {
+ rightKeys.push_back(ctx.NewAtom(pos, tableName));
+ rightKeys.push_back(ctx.NewAtom(pos, rightKey));
+ }
+
+ joinChain = ctx.Builder(pos)
.List()
.Atom(0, chain[i].Negated ? "LeftOnly" : "LeftSemi")
.Add(1, joinChain ? joinChain : inputTableAtom)
.Atom(2, tableName)
.List(3)
- .Add(std::move(leftKeys))
+ .Add(std::move(leftKeys))
.Seal()
.List(4)
- .Add(std::move(rightKeys))
+ .Add(std::move(rightKeys))
.Seal()
.List(5)
.Seal()
.Seal()
.Build();
- if (chain[i].LeftArgColumns.empty()) {
- auto rename = ctx.Builder(pos)
+ if (chain[i].LeftArgColumns.empty()) {
+ auto rename = ctx.Builder(pos)
.List()
.Atom(0, "rename")
.Atom(1, FullColumnName(inputTable, columnName))
@@ -2222,31 +2222,31 @@ TExprNode::TPtr SimpleFlatMap(const TExprNode::TPtr& node, TExprContext& ctx, TO
}
}
- // rewrite in 'canonical' way (prefer OptionalIf to ListIf)
+ // rewrite in 'canonical' way (prefer OptionalIf to ListIf)
if (self.Input().Ref().GetTypeAnn()->GetKind() != ETypeAnnotationKind::Optional && self.Lambda().Body().Maybe<TCoListIf>())
- {
+ {
YQL_CLOG(DEBUG, Core) << "Convert " << node->Content() << " lambda ListIf to OptionalIf";
- auto listIf = self.Lambda().Body().Cast<TCoListIf>();
-
- auto newLambda = Build<TCoLambda>(ctx, node->Pos())
- .Args({"item"})
- .Body<TCoOptionalIf>()
- .Predicate<TExprApplier>()
- .Apply(listIf.Predicate())
- .With(self.Lambda().Args().Arg(0), "item")
- .Build()
- .Value<TExprApplier>()
- .Apply(listIf.Value())
- .With(self.Lambda().Args().Arg(0), "item")
- .Build()
- .Build()
+ auto listIf = self.Lambda().Body().Cast<TCoListIf>();
+
+ auto newLambda = Build<TCoLambda>(ctx, node->Pos())
+ .Args({"item"})
+ .Body<TCoOptionalIf>()
+ .Predicate<TExprApplier>()
+ .Apply(listIf.Predicate())
+ .With(self.Lambda().Args().Arg(0), "item")
+ .Build()
+ .Value<TExprApplier>()
+ .Apply(listIf.Value())
+ .With(self.Lambda().Args().Arg(0), "item")
+ .Build()
+ .Build()
.Done().Ptr();
-
+
return ctx.ChangeChild(*node, 1U, std::move(newLambda));
- }
-
+ }
+
if (auto expr = TryConvertSqlInPredicatesToJoins(self, ShouldConvertSqlInToJoin, ctx)) {
- return expr;
+ return expr;
}
if (auto just = self.Lambda().Body().Maybe<TCoJust>()) {
@@ -2971,11 +2971,11 @@ TExprNode::TPtr PullAssumeColumnOrderOverEquiJoin(const TExprNode::TPtr& node, T
return node;
}
-} // namespace
-
-TExprNode::TPtr TryConvertSqlInPredicatesToJoins(const TCoFlatMapBase& flatMap,
- TShouldConvertSqlInToJoinPredicate shouldConvertSqlInToJoin, TExprContext& ctx, bool prefixOnly)
-{
+} // namespace
+
+TExprNode::TPtr TryConvertSqlInPredicatesToJoins(const TCoFlatMapBase& flatMap,
+ TShouldConvertSqlInToJoinPredicate shouldConvertSqlInToJoin, TExprContext& ctx, bool prefixOnly)
+{
// FlatMap input should be List<Struct<...>> to be accepted as EquiJoin input
auto inputType = flatMap.Input().Ref().GetTypeAnn();
if (inputType->GetKind() != ETypeAnnotationKind::List ||
@@ -2984,38 +2984,38 @@ TExprNode::TPtr TryConvertSqlInPredicatesToJoins(const TCoFlatMapBase& flatMap,
return {};
}
- TCoLambda lambda = flatMap.Lambda();
+ TCoLambda lambda = flatMap.Lambda();
if (!lambda.Body().Maybe<TCoConditionalValueBase>()) {
- return {};
- }
-
+ return {};
+ }
+
TCoConditionalValueBase conditional(lambda.Body().Ptr());
TPredicateChain chain;
auto lambdaArg = lambda.Ptr()->Head().HeadPtr();
auto sqlInTail = SplitPredicateChain(conditional.Predicate().Ptr(), lambdaArg, shouldConvertSqlInToJoin, chain, ctx);
-
- if (!chain.empty()) {
- if (chain.front().ConvertibleToJoin) {
+
+ if (!chain.empty()) {
+ if (chain.front().ConvertibleToJoin) {
return ConvertSqlInPredicatesPrefixToJoins(flatMap.Ptr(), chain, sqlInTail, ctx);
- }
-
+ }
+
if (sqlInTail && !prefixOnly) {
- YQL_CLOG(DEBUG, Core) << "FlatMapOverNonJoinableSqlInChain of size " << chain.size();
- TExprNode::TListType predicates;
- predicates.reserve(chain.size());
- for (auto& it : chain) {
- predicates.emplace_back(std::move(it.Pred));
- }
- auto prefixPred = ctx.NewCallable(flatMap.Pos(), "And", std::move(predicates));
-
+ YQL_CLOG(DEBUG, Core) << "FlatMapOverNonJoinableSqlInChain of size " << chain.size();
+ TExprNode::TListType predicates;
+ predicates.reserve(chain.size());
+ for (auto& it : chain) {
+ predicates.emplace_back(std::move(it.Pred));
+ }
+ auto prefixPred = ctx.NewCallable(flatMap.Pos(), "And", std::move(predicates));
+
auto innerFlatMap = RebuildFlatmapOverPartOfPredicate(flatMap.Ptr(), flatMap.Input().Ptr(), prefixPred, false, ctx);
auto outerFlatMap = RebuildFlatmapOverPartOfPredicate(flatMap.Ptr(), innerFlatMap, sqlInTail, true, ctx);
return ctx.RenameNode(*outerFlatMap,
outerFlatMap->Content() == "OrderedFlatMap" ? "OrderedFlatMapToEquiJoin" : "FlatMapToEquiJoin");
- }
- }
-
- return {};
+ }
+ }
+
+ return {};
}
TExprNode::TPtr FoldParseAfterSerialize(const TExprNode::TPtr& node, const TStringBuf parseUdfName, const THashSet<TStringBuf>& serializeUdfNames) {
@@ -3545,11 +3545,11 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) {
};
map["ExtractMembers"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) {
- if (IsSameAnnotation(*node->GetTypeAnn(), *node->Head().GetTypeAnn())) {
- YQL_CLOG(DEBUG, Core) << "Drop redundant ExtractMembers over " << node->Head().Content();
- return node->HeadPtr();
- }
-
+ if (IsSameAnnotation(*node->GetTypeAnn(), *node->Head().GetTypeAnn())) {
+ YQL_CLOG(DEBUG, Core) << "Drop redundant ExtractMembers over " << node->Head().Content();
+ return node->HeadPtr();
+ }
+
if (node->Head().IsCallable(node->Content())) {
YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content();
return ctx.ChangeChild(*node, 0U, node->Head().HeadPtr());
@@ -7233,4 +7233,4 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) {
};
}
-} // namespace NYql
+} // namespace NYql
diff --git a/ydb/library/yql/core/common_opt/yql_co_sqlin.h b/ydb/library/yql/core/common_opt/yql_co_sqlin.h
index a6a9faae36..38e9f0c517 100644
--- a/ydb/library/yql/core/common_opt/yql_co_sqlin.h
+++ b/ydb/library/yql/core/common_opt/yql_co_sqlin.h
@@ -1,13 +1,13 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/ast/yql_expr.h>
#include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h>
-
-namespace NYql {
-
-using TShouldConvertSqlInToJoinPredicate = std::function<bool(const NNodes::TCoSqlIn&, bool /* negated */)>;
-
-TExprNode::TPtr TryConvertSqlInPredicatesToJoins(const NNodes::TCoFlatMapBase& flatMap,
- TShouldConvertSqlInToJoinPredicate shouldConvertSqlInToJoin, TExprContext& ctx, bool prefixOnly = false);
-
-} // namespace NYql
+
+namespace NYql {
+
+using TShouldConvertSqlInToJoinPredicate = std::function<bool(const NNodes::TCoSqlIn&, bool /* negated */)>;
+
+TExprNode::TPtr TryConvertSqlInPredicatesToJoins(const NNodes::TCoFlatMapBase& flatMap,
+ TShouldConvertSqlInToJoinPredicate shouldConvertSqlInToJoin, TExprContext& ctx, bool prefixOnly = false);
+
+} // namespace NYql
diff --git a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json
index 0a9375546b..edfcaa1530 100644
--- a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json
+++ b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json
@@ -971,17 +971,17 @@
]
},
{
- "Name": "TCoSqueezeToDict",
- "Base": "TCallable",
- "Match": {"Type": "Callable", "Name": "SqueezeToDict"},
- "Children": [
- {"Index": 0, "Name": "Stream", "Type": "TExprBase"},
- {"Index": 1, "Name": "KeySelector", "Type": "TCoLambda"},
- {"Index": 2, "Name": "PayloadSelector", "Type": "TCoLambda"},
- {"Index": 3, "Name": "Settings", "Type": "TCoAtomList"}
- ]
- },
- {
+ "Name": "TCoSqueezeToDict",
+ "Base": "TCallable",
+ "Match": {"Type": "Callable", "Name": "SqueezeToDict"},
+ "Children": [
+ {"Index": 0, "Name": "Stream", "Type": "TExprBase"},
+ {"Index": 1, "Name": "KeySelector", "Type": "TCoLambda"},
+ {"Index": 2, "Name": "PayloadSelector", "Type": "TCoLambda"},
+ {"Index": 3, "Name": "Settings", "Type": "TCoAtomList"}
+ ]
+ },
+ {
"Name": "TCoSqueezeToList",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "SqueezeToList"},
@@ -991,22 +991,22 @@
]
},
{
- "Name": "TCoDictKeys",
- "Base": "TCallable",
- "Match": {"Type": "Callable", "Name": "DictKeys"},
- "Children": [
- {"Index": 0, "Name": "Dict", "Type": "TExprBase"}
- ]
- },
- {
- "Name": "TCoDictPayloads",
- "Base": "TCallable",
- "Match": {"Type": "Callable", "Name": "DictPayloads"},
- "Children": [
- {"Index": 0, "Name": "Dict", "Type": "TExprBase"}
- ]
- },
- {
+ "Name": "TCoDictKeys",
+ "Base": "TCallable",
+ "Match": {"Type": "Callable", "Name": "DictKeys"},
+ "Children": [
+ {"Index": 0, "Name": "Dict", "Type": "TExprBase"}
+ ]
+ },
+ {
+ "Name": "TCoDictPayloads",
+ "Base": "TCallable",
+ "Match": {"Type": "Callable", "Name": "DictPayloads"},
+ "Children": [
+ {"Index": 0, "Name": "Dict", "Type": "TExprBase"}
+ ]
+ },
+ {
"Name": "TCoDictItems",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "DictItems"},
@@ -1496,13 +1496,13 @@
{"Index": 1, "Name": "Members", "Type": "TCoAtomList"}
]
},
-
+
+ {
+ "Name": "TCoFlattenMembers",
+ "VarArgBase": "TCoNameValueTuple",
+ "Match": {"Type": "Callable", "Name": "FlattenMembers"}
+ },
{
- "Name": "TCoFlattenMembers",
- "VarArgBase": "TCoNameValueTuple",
- "Match": {"Type": "Callable", "Name": "FlattenMembers"}
- },
- {
"Name": "TCoAssumeUnique",
"Base": "TCoInputBase",
"Match": {"Type": "Callable", "Name": "AssumeUnique"},
@@ -1789,15 +1789,15 @@
]
},
{
- "Name": "TCoStrictCast",
- "Base": "TCallable",
- "Match": {"Type": "Callable", "Name": "StrictCast"},
- "Children": [
- {"Index": 0, "Name": "Value", "Type": "TExprBase"},
- {"Index": 1, "Name": "Type", "Type": "TExprBase"}
- ]
- },
- {
+ "Name": "TCoStrictCast",
+ "Base": "TCallable",
+ "Match": {"Type": "Callable", "Name": "StrictCast"},
+ "Children": [
+ {"Index": 0, "Name": "Value", "Type": "TExprBase"},
+ {"Index": 1, "Name": "Type", "Type": "TExprBase"}
+ ]
+ },
+ {
"Name": "TCoFromYsonSimpleType",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "FromYsonSimpleType"},
diff --git a/ydb/library/yql/core/issue/protos/issue_id.proto b/ydb/library/yql/core/issue/protos/issue_id.proto
index 8dbe4cec0c..a2ed91d640 100644
--- a/ydb/library/yql/core/issue/protos/issue_id.proto
+++ b/ydb/library/yql/core/issue/protos/issue_id.proto
@@ -70,7 +70,7 @@ message TIssuesIds {
KIKIMR_OPERATION_STATE_UNKNOWN = 2026;
KIKIMR_INDEX_IS_NOT_READY = 2027;
KIKIMR_SCHEME_MISMATCH = 2028;
- KIKIMR_PRECONDITION_FAILED = 2029;
+ KIKIMR_PRECONDITION_FAILED = 2029;
KIKIMR_UNSUPPORTED = 2030;
KIKIMR_BAD_COLUMN_TYPE = 2031;
KIKIMR_NO_COLUMN_DEFAULT_VALUE = 2032;
diff --git a/ydb/library/yql/core/issue/yql_issue.txt b/ydb/library/yql/core/issue/yql_issue.txt
index 2842061efa..897d5d6f9a 100644
--- a/ydb/library/yql/core/issue/yql_issue.txt
+++ b/ydb/library/yql/core/issue/yql_issue.txt
@@ -277,10 +277,10 @@ ids {
format: "Scheme mismatch found while executing query."
}
ids {
- code: KIKIMR_PRECONDITION_FAILED
- severity: S_ERROR
-}
-ids {
+ code: KIKIMR_PRECONDITION_FAILED
+ severity: S_ERROR
+}
+ids {
code: KIKIMR_UNSUPPORTED
severity: S_ERROR
}
diff --git a/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.cpp b/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.cpp
index 1906a86d24..ce22b12571 100644
--- a/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.cpp
+++ b/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.cpp
@@ -2698,40 +2698,40 @@ TExprNode::TPtr OptimizeLogicalDups(const TExprNode::TPtr& node, TExprContext& c
return node;
}
-TExprNode::TPtr ExpandCombineByKey(const TExprNode::TPtr& node, TExprContext& ctx) {
- const bool isStreamOrFlow = node->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Stream ||
- node->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Flow;
-
- if (!isStreamOrFlow) {
- return node;
- }
-
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << node->Content() << " over stream or flow";
-
- TCoCombineByKey combine(node);
-
- return Build<TCoCombineCore>(ctx, node->Pos())
- .Input<TCoFlatMap>()
- .Input(combine.Input())
- .Lambda()
- .Args({"arg"})
- .Body<TExprApplier>()
- .Apply(combine.PreMapLambda())
- .With(0, "arg")
- .Build()
- .Build()
- .Build()
- .KeyExtractor(combine.KeySelectorLambda())
- .InitHandler(combine.InitHandlerLambda())
- .UpdateHandler(combine.UpdateHandlerLambda())
- .FinishHandler(combine.FinishHandlerLambda())
- .MemLimit()
- .Value("0")
- .Build()
- .Done()
- .Ptr();
-}
-
+TExprNode::TPtr ExpandCombineByKey(const TExprNode::TPtr& node, TExprContext& ctx) {
+ const bool isStreamOrFlow = node->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Stream ||
+ node->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Flow;
+
+ if (!isStreamOrFlow) {
+ return node;
+ }
+
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << node->Content() << " over stream or flow";
+
+ TCoCombineByKey combine(node);
+
+ return Build<TCoCombineCore>(ctx, node->Pos())
+ .Input<TCoFlatMap>()
+ .Input(combine.Input())
+ .Lambda()
+ .Args({"arg"})
+ .Body<TExprApplier>()
+ .Apply(combine.PreMapLambda())
+ .With(0, "arg")
+ .Build()
+ .Build()
+ .Build()
+ .KeyExtractor(combine.KeySelectorLambda())
+ .InitHandler(combine.InitHandlerLambda())
+ .UpdateHandler(combine.UpdateHandlerLambda())
+ .FinishHandler(combine.FinishHandlerLambda())
+ .MemLimit()
+ .Value("0")
+ .Build()
+ .Done()
+ .Ptr();
+}
+
template<typename TRowType>
TExprNode::TPtr MakeWideMapJoinCore(const TExprNode& mapjoin, TExprNode::TPtr&& input, TExprContext& ctx) {
const auto inStructType = GetSeqItemType(mapjoin.Head().GetTypeAnn())->Cast<TRowType>();
@@ -5945,7 +5945,7 @@ IGraphTransformer::TStatus DoPeepHoleOptimizeNode(const TExprNode::TPtr& input,
template <bool EnableNewOptimizers>
IGraphTransformer::TStatus PeepHoleOptimizeNode(const TExprNode::TPtr& input, TExprNode::TPtr& output,
- TExprContext& ctx, TTypeAnnotationContext& types, IGraphTransformer* typeAnnotator,
+ TExprContext& ctx, TTypeAnnotationContext& types, IGraphTransformer* typeAnnotator,
bool& hasNonDeterministicFunctions, const TPeepholeSettings& peepholeSettings)
{
hasNonDeterministicFunctions = false;
diff --git a/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.h b/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.h
index 5a49c1f74c..a357c5b395 100644
--- a/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.h
+++ b/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.h
@@ -16,7 +16,7 @@ struct TPeepholeSettings {
template <bool EnableNewOptimizers>
IGraphTransformer::TStatus PeepHoleOptimizeNode(const TExprNode::TPtr& input, TExprNode::TPtr& output,
- TExprContext& ctx, TTypeAnnotationContext& types, IGraphTransformer* typeAnnotator,
+ TExprContext& ctx, TTypeAnnotationContext& types, IGraphTransformer* typeAnnotator,
bool& hasNonDeterministicFunctions, const TPeepholeSettings& peepholeSettings = {});
THolder<IGraphTransformer> MakePeepholeOptimization(TTypeAnnotationContextPtr typeAnnotationContext, const IPipelineConfigurator* config = nullptr);
diff --git a/ydb/library/yql/core/type_ann/type_ann_core.cpp b/ydb/library/yql/core/type_ann/type_ann_core.cpp
index 5510a7be4f..5846e6cb10 100644
--- a/ydb/library/yql/core/type_ann/type_ann_core.cpp
+++ b/ydb/library/yql/core/type_ann/type_ann_core.cpp
@@ -6397,12 +6397,12 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
}
template<bool Narrow>
- IGraphTransformer::TStatus SqueezeToDictWrapper(const TExprNode::TPtr& input, TExprNode::TPtr&, TContext& ctx) {
- if (!EnsureArgsCount(*input, 4, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- const TTypeAnnotationNode* itemType = nullptr;
+ IGraphTransformer::TStatus SqueezeToDictWrapper(const TExprNode::TPtr& input, TExprNode::TPtr&, TContext& ctx) {
+ if (!EnsureArgsCount(*input, 4, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ const TTypeAnnotationNode* itemType = nullptr;
if constexpr (Narrow) {
if (!EnsureWideFlowType(input->Head(), ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
@@ -6413,63 +6413,63 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
if (!EnsureNewSeqType<false>(input->Head(), ctx.Expr, &itemType)) {
return IGraphTransformer::TStatus::Error;
}
- }
-
+ }
+
auto& lambda1 = input->ChildRef(1);
auto& lambda2 = input->ChildRef(2);
const auto width = Narrow ? itemType->Cast<TMultiExprType>()->GetSize() : 1U;
if (const auto status = ConvertToLambda(lambda1, ctx.Expr, width); status.Level != IGraphTransformer::TStatus::Ok) {
- return status;
- }
-
+ return status;
+ }
+
if (const auto status = ConvertToLambda(lambda2, ctx.Expr, width); status.Level != IGraphTransformer::TStatus::Ok) {
- return status;
- }
-
+ return status;
+ }
+
if (const auto& items = Narrow ? itemType->Cast<TMultiExprType>()->GetItems() : TTypeAnnotationNode::TListType{itemType};
!(UpdateLambdaAllArgumentsTypes(lambda1, items, ctx.Expr) && UpdateLambdaAllArgumentsTypes(lambda2, items, ctx.Expr))) {
- return IGraphTransformer::TStatus::Error;
- }
-
- if (!lambda1->GetTypeAnn() || !lambda2->GetTypeAnn()) {
- return IGraphTransformer::TStatus::Repeat;
- }
-
- TMaybe<bool> isMany;
- TMaybe<bool> isHashed;
- TMaybe<ui64> itemsCount;
- bool isCompact;
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ if (!lambda1->GetTypeAnn() || !lambda2->GetTypeAnn()) {
+ return IGraphTransformer::TStatus::Repeat;
+ }
+
+ TMaybe<bool> isMany;
+ TMaybe<bool> isHashed;
+ TMaybe<ui64> itemsCount;
+ bool isCompact;
if (const auto error = ParseToDictSettings(*input, ctx.Expr, isMany, isHashed, itemsCount, isCompact)) {
- ctx.Expr.AddError(*error);
- return IGraphTransformer::TStatus::Error;
- }
-
- auto keyType = lambda1->GetTypeAnn();
- auto payloadType = lambda2->GetTypeAnn();
- if (*isMany) {
- payloadType = ctx.Expr.MakeType<TListExprType>(payloadType);
- }
-
- auto dictType = ctx.Expr.MakeType<TDictExprType>(keyType, payloadType);
- if (!dictType->Validate(input->Pos(), ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- if (!*isHashed && !keyType->IsComparable()) {
- ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder()
- << "Expected comparable key type for sorted dict, but got: " << *keyType));
- return IGraphTransformer::TStatus::Error;
- }
-
- if (input->Head().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Flow) {
- input->SetTypeAnn(ctx.Expr.MakeType<TFlowExprType>(dictType));
- } else {
- input->SetTypeAnn(ctx.Expr.MakeType<TStreamExprType>(dictType));
- }
- return IGraphTransformer::TStatus::Ok;
- }
-
+ ctx.Expr.AddError(*error);
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ auto keyType = lambda1->GetTypeAnn();
+ auto payloadType = lambda2->GetTypeAnn();
+ if (*isMany) {
+ payloadType = ctx.Expr.MakeType<TListExprType>(payloadType);
+ }
+
+ auto dictType = ctx.Expr.MakeType<TDictExprType>(keyType, payloadType);
+ if (!dictType->Validate(input->Pos(), ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ if (!*isHashed && !keyType->IsComparable()) {
+ ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder()
+ << "Expected comparable key type for sorted dict, but got: " << *keyType));
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ if (input->Head().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Flow) {
+ input->SetTypeAnn(ctx.Expr.MakeType<TFlowExprType>(dictType));
+ } else {
+ input->SetTypeAnn(ctx.Expr.MakeType<TStreamExprType>(dictType));
+ }
+ return IGraphTransformer::TStatus::Ok;
+ }
+
IGraphTransformer::TStatus VoidWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
Y_UNUSED(output);
if (!EnsureArgsCount(*input, 0, ctx.Expr)) {
diff --git a/ydb/library/yql/core/type_ann/type_ann_join.cpp b/ydb/library/yql/core/type_ann/type_ann_join.cpp
index b6d3fe15bc..18516a8ab8 100644
--- a/ydb/library/yql/core/type_ann/type_ann_join.cpp
+++ b/ydb/library/yql/core/type_ann/type_ann_join.cpp
@@ -346,7 +346,7 @@ namespace NTypeAnnImpl {
for (const auto& child : input->Child(3)->Children()) {
if (!GetFieldPosition(leftItemType, child->Content())) {
- ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Unknown key column: " << child->Content()));
+ ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Unknown key column: " << child->Content()));
return IGraphTransformer::TStatus::Error;
}
}
diff --git a/ydb/library/yql/core/type_ann/type_ann_list.cpp b/ydb/library/yql/core/type_ann/type_ann_list.cpp
index 94647d5570..b145cde389 100644
--- a/ydb/library/yql/core/type_ann/type_ann_list.cpp
+++ b/ydb/library/yql/core/type_ann/type_ann_list.cpp
@@ -3799,11 +3799,11 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- const TTypeAnnotationNode* rawItemType = nullptr;
- if (!EnsureNewSeqType<false, true>(input->Head(), ctx.Expr, &rawItemType)) {
+ const TTypeAnnotationNode* rawItemType = nullptr;
+ if (!EnsureNewSeqType<false, true>(input->Head(), ctx.Expr, &rawItemType)) {
return IGraphTransformer::TStatus::Error;
}
- auto inputTypeKind = input->Head().GetTypeAnn()->GetKind();
+ auto inputTypeKind = input->Head().GetTypeAnn()->GetKind();
auto status = ConvertToLambda(input->ChildRef(1), ctx.Expr, 1);
status = status.Combine(ConvertToLambda(input->ChildRef(2), ctx.Expr, 1));
@@ -3823,8 +3823,8 @@ namespace {
return IGraphTransformer::TStatus::Repeat;
}
- const TTypeAnnotationNode* itemType = nullptr;
- if (!EnsureNewSeqType<true, true>(*lambdaPreMap, ctx.Expr, &itemType)) {
+ const TTypeAnnotationNode* itemType = nullptr;
+ if (!EnsureNewSeqType<true, true>(*lambdaPreMap, ctx.Expr, &itemType)) {
return IGraphTransformer::TStatus::Error;
}
@@ -3890,12 +3890,12 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- const TTypeAnnotationNode* retItemType = nullptr;
- if (!EnsureNewSeqType<true, true>(*lambdaFinishHandler, ctx.Expr, &retItemType)) {
+ const TTypeAnnotationNode* retItemType = nullptr;
+ if (!EnsureNewSeqType<true, true>(*lambdaFinishHandler, ctx.Expr, &retItemType)) {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(MakeSequenceType(inputTypeKind, *retItemType, ctx.Expr));
+ input->SetTypeAnn(MakeSequenceType(inputTypeKind, *retItemType, ctx.Expr));
return IGraphTransformer::TStatus::Ok;
}
diff --git a/ydb/library/yql/core/yql_expr_type_annotation.cpp b/ydb/library/yql/core/yql_expr_type_annotation.cpp
index 77f2366158..f2b793af8d 100644
--- a/ydb/library/yql/core/yql_expr_type_annotation.cpp
+++ b/ydb/library/yql/core/yql_expr_type_annotation.cpp
@@ -1783,12 +1783,12 @@ bool IsDataOrOptionalOfData(const TTypeAnnotationNode* typeAnnotation, bool& isO
return IsDataOrOptionalOfData({}, typeAnnotation, isOptional, dataType, err, hasErrorType);
}
-bool IsDataOrOptionalOfData(const TTypeAnnotationNode* typeAnnotation) {
- bool isOptional;
- const TDataExprType* dataType;
- return IsDataOrOptionalOfData(typeAnnotation, isOptional, dataType);
-}
-
+bool IsDataOrOptionalOfData(const TTypeAnnotationNode* typeAnnotation) {
+ bool isOptional;
+ const TDataExprType* dataType;
+ return IsDataOrOptionalOfData(typeAnnotation, isOptional, dataType);
+}
+
bool EnsureArgsCount(const TExprNode& node, ui32 expectedArgs, TExprContext& ctx) {
if (node.ChildrenSize() != expectedArgs) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected " << expectedArgs << " argument(s), but got " <<
diff --git a/ydb/library/yql/core/yql_expr_type_annotation.h b/ydb/library/yql/core/yql_expr_type_annotation.h
index 22a0240323..a865aa27cd 100644
--- a/ydb/library/yql/core/yql_expr_type_annotation.h
+++ b/ydb/library/yql/core/yql_expr_type_annotation.h
@@ -53,7 +53,7 @@ void ClearExprTypeAnnotations(TExprNode& root);
bool AreAllNodesTypeAnnotated(const TExprNode& root);
void EnsureAllNodesTypeAnnotated(const TExprNode& root);
bool IsDataOrOptionalOfData(const TTypeAnnotationNode* typeAnnotation, bool& isOptional, const TDataExprType*& dataType);
-bool IsDataOrOptionalOfData(const TTypeAnnotationNode* typeAnnotation);
+bool IsDataOrOptionalOfData(const TTypeAnnotationNode* typeAnnotation);
bool UpdateLambdaAllArgumentsTypes(TExprNode::TPtr& lambda, const std::vector<const TTypeAnnotationNode*>& argumentsAnnotations, TExprContext& ctx);
bool UpdateLambdaArgumentsType(const TExprNode& lambda, TExprContext& ctx);
bool EnsureArgsCount(const TExprNode& node, ui32 expectedArgs, TExprContext& ctx);
diff --git a/ydb/library/yql/core/yql_opt_utils.cpp b/ydb/library/yql/core/yql_opt_utils.cpp
index c0df12a756..69aa84a500 100644
--- a/ydb/library/yql/core/yql_opt_utils.cpp
+++ b/ydb/library/yql/core/yql_opt_utils.cpp
@@ -336,11 +336,11 @@ const TTypeAnnotationNode* GetSeqItemType(const TTypeAnnotationNode* type) {
case ETypeAnnotationKind::Stream: return type->Cast<TStreamExprType>()->GetItemType();
case ETypeAnnotationKind::Optional: return type->Cast<TOptionalExprType>()->GetItemType();
default: break;
- }
+ }
+
+ THROW yexception() << "Impossible to get item type from " << *type;
+}
- THROW yexception() << "Impossible to get item type from " << *type;
-}
-
TExprNode::TPtr GetSetting(const TExprNode& settings, const TStringBuf& name) {
for (auto& setting : settings.Children()) {
if (setting->ChildrenSize() != 0 && setting->Child(0)->Content() == name) {
diff --git a/ydb/library/yql/core/yql_opt_utils.h b/ydb/library/yql/core/yql_opt_utils.h
index c42db30be9..00635fea81 100644
--- a/ydb/library/yql/core/yql_opt_utils.h
+++ b/ydb/library/yql/core/yql_opt_utils.h
@@ -39,7 +39,7 @@ bool IsEmptyContainer(const TExprNode& node);
const TTypeAnnotationNode* RemoveOptionalType(const TTypeAnnotationNode* type);
const TTypeAnnotationNode* RemoveAllOptionals(const TTypeAnnotationNode* type);
-const TTypeAnnotationNode* GetSeqItemType(const TTypeAnnotationNode* seq);
+const TTypeAnnotationNode* GetSeqItemType(const TTypeAnnotationNode* seq);
TExprNode::TPtr GetSetting(const TExprNode& settings, const TStringBuf& name);
bool HasSetting(const TExprNode& settings, const TStringBuf& name);
diff --git a/ydb/library/yql/dq/actors/compute/dq_checkpoints.h b/ydb/library/yql/dq/actors/compute/dq_checkpoints.h
index b439872db1..e6775602a4 100644
--- a/ydb/library/yql/dq/actors/compute/dq_checkpoints.h
+++ b/ydb/library/yql/dq/actors/compute/dq_checkpoints.h
@@ -1,14 +1,14 @@
#pragma once
-#include <library/cpp/actors/core/actorid.h>
-
-namespace NYql {
-namespace NDq {
-
+#include <library/cpp/actors/core/actorid.h>
+
+namespace NYql {
+namespace NDq {
+
inline static NActors::TActorId MakeCheckpointStorageID() {
const char name[12] = "cp_storage";
return NActors::TActorId(0, TStringBuf(name, 12));
}
-
-} // namespace NDq
-} // namespace NYql
+
+} // namespace NDq
+} // namespace NYql
diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor.cpp b/ydb/library/yql/dq/actors/compute/dq_compute_actor.cpp
index 2153d8c09d..c2f6c45e2b 100644
--- a/ydb/library/yql/dq/actors/compute/dq_compute_actor.cpp
+++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor.cpp
@@ -2,77 +2,77 @@
#include "dq_compute_actor.h"
#include <ydb/library/yql/dq/common/dq_common.h>
-
-namespace NYql {
-namespace NDq {
+
+namespace NYql {
+namespace NDq {
using namespace NActors;
-namespace {
-TDqExecutionSettings ExecutionSettings;
-
-bool IsDebugLogEnabled(const TActorSystem* actorSystem) {
- auto* settings = actorSystem->LoggerSettings();
- return settings && settings->Satisfies(NActors::NLog::EPriority::PRI_DEBUG, NKikimrServices::KQP_TASKS_RUNNER);
-}
-
-} // anonymous namespace
-
-const TDqExecutionSettings& GetDqExecutionSettings() {
- return ExecutionSettings;
-}
-
-TDqExecutionSettings& GetDqExecutionSettingsForTests() {
- return ExecutionSettings;
-}
-
-class TDqComputeActor : public TDqComputeActorBase<TDqComputeActor> {
- using TBase = TDqComputeActorBase<TDqComputeActor>;
+namespace {
+TDqExecutionSettings ExecutionSettings;
+
+bool IsDebugLogEnabled(const TActorSystem* actorSystem) {
+ auto* settings = actorSystem->LoggerSettings();
+ return settings && settings->Satisfies(NActors::NLog::EPriority::PRI_DEBUG, NKikimrServices::KQP_TASKS_RUNNER);
+}
+
+} // anonymous namespace
+
+const TDqExecutionSettings& GetDqExecutionSettings() {
+ return ExecutionSettings;
+}
+
+TDqExecutionSettings& GetDqExecutionSettingsForTests() {
+ return ExecutionSettings;
+}
+
+class TDqComputeActor : public TDqComputeActorBase<TDqComputeActor> {
+ using TBase = TDqComputeActorBase<TDqComputeActor>;
public:
static constexpr char ActorName[] = "DQ_COMPUTE_ACTOR";
- TDqComputeActor(const TActorId& executerId, const TTxId& txId, NDqProto::TDqTask&& task,
+ TDqComputeActor(const TActorId& executerId, const TTxId& txId, NDqProto::TDqTask&& task,
IDqSourceActorFactory::TPtr sourceActorFactory, IDqSinkActorFactory::TPtr sinkActorFactory,
- const TComputeRuntimeSettings& settings, const TComputeMemoryLimits& memoryLimits,
- const TTaskRunnerFactory& taskRunnerFactory)
- : TBase(executerId, txId, std::move(task), std::move(sourceActorFactory), std::move(sinkActorFactory), settings, memoryLimits)
+ const TComputeRuntimeSettings& settings, const TComputeMemoryLimits& memoryLimits,
+ const TTaskRunnerFactory& taskRunnerFactory)
+ : TBase(executerId, txId, std::move(task), std::move(sourceActorFactory), std::move(sinkActorFactory), settings, memoryLimits)
, TaskRunnerFactory(taskRunnerFactory)
{}
void DoBootstrap() {
- const TActorSystem* actorSystem = TlsActivationContext->ActorSystem();
-
- TLogFunc logger;
- if (IsDebugLogEnabled(actorSystem)) {
- logger = [actorSystem, txId = this->GetTxId(), taskId = GetTask().GetId()] (const TString& message) {
- LOG_DEBUG_S(*actorSystem, NKikimrServices::KQP_TASKS_RUNNER, "TxId: " << txId
- << ", task: " << taskId << ": " << message);
- };
- }
-
- auto taskRunner = TaskRunnerFactory(GetTask(), logger);
- SetTaskRunner(taskRunner);
- PrepareTaskRunner();
-
- ContinueExecute();
+ const TActorSystem* actorSystem = TlsActivationContext->ActorSystem();
+
+ TLogFunc logger;
+ if (IsDebugLogEnabled(actorSystem)) {
+ logger = [actorSystem, txId = this->GetTxId(), taskId = GetTask().GetId()] (const TString& message) {
+ LOG_DEBUG_S(*actorSystem, NKikimrServices::KQP_TASKS_RUNNER, "TxId: " << txId
+ << ", task: " << taskId << ": " << message);
+ };
+ }
+
+ auto taskRunner = TaskRunnerFactory(GetTask(), logger);
+ SetTaskRunner(taskRunner);
+ PrepareTaskRunner();
+
+ ContinueExecute();
+ }
+
+ void FillExtraStats(NDqProto::TDqComputeActorStats* /* dst */, bool /* last */) {
}
- void FillExtraStats(NDqProto::TDqComputeActorStats* /* dst */, bool /* last */) {
- }
-
private:
const TTaskRunnerFactory TaskRunnerFactory;
};
-
+
IActor* CreateDqComputeActor(const TActorId& executerId, const TTxId& txId, NYql::NDqProto::TDqTask&& task,
IDqSourceActorFactory::TPtr sourceActorFactory, IDqSinkActorFactory::TPtr sinkActorFactory,
const TComputeRuntimeSettings& settings, const TComputeMemoryLimits& memoryLimits, const TTaskRunnerFactory& taskRunnerFactory)
{
- return new TDqComputeActor(executerId, txId, std::move(task), std::move(sourceActorFactory),
- std::move(sinkActorFactory), settings, memoryLimits, taskRunnerFactory);
+ return new TDqComputeActor(executerId, txId, std::move(task), std::move(sourceActorFactory),
+ std::move(sinkActorFactory), settings, memoryLimits, taskRunnerFactory);
}
-} // namespace NDq
-} // namespace NYql
+} // namespace NDq
+} // namespace NYql
diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor.h
index fc0800b0b3..84eeda2edb 100644
--- a/ydb/library/yql/dq/actors/compute/dq_compute_actor.h
+++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor.h
@@ -13,51 +13,51 @@
#include <library/cpp/actors/core/hfunc.h>
#include <library/cpp/actors/core/log.h>
-namespace NYql {
-namespace NDq {
+namespace NYql {
+namespace NDq {
-struct TEvDqCompute {
- struct TEvState : public NActors::TEventPB<TEvState, NDqProto::TEvComputeActorState, TDqComputeEvents::EvState> {};
+struct TEvDqCompute {
+ struct TEvState : public NActors::TEventPB<TEvState, NDqProto::TEvComputeActorState, TDqComputeEvents::EvState> {};
struct TEvStateRequest : public NActors::TEventPB<TEvState, NDqProto::TEvComputeStateRequest, TDqComputeEvents::EvStateRequest> {};
- struct TEvResumeExecution : public NActors::TEventLocal<TEvResumeExecution, TDqComputeEvents::EvResumeExecution> {};
-
- struct TEvChannelsInfo : public NActors::TEventPB<TEvChannelsInfo, NDqProto::TEvChannelsInfo,
- TDqComputeEvents::EvChannelsInfo> {};
-
- struct TEvChannelData : public NActors::TEventPB<TEvChannelData, NDqProto::TEvComputeChannelData,
- TDqComputeEvents::EvChannelData> {};
-
- struct TEvChannelDataAck : public NActors::TEventPB<TEvChannelDataAck, NDqProto::TEvComputeChannelDataAck,
- TDqComputeEvents::EvChannelDataAck> {};
-
- // todo: make it private
- struct TEvRetryChannelData : public NActors::TEventLocal<TEvRetryChannelData, TDqComputeEvents::EvRetryChannelData> {
- TEvRetryChannelData(ui64 channelId, ui64 fromSeqNo, ui64 toSeqNo)
- : ChannelId(channelId)
- , FromSeqNo(fromSeqNo)
- , ToSeqNo(toSeqNo) {}
-
- const ui64 ChannelId;
- const ui64 FromSeqNo;
- const ui64 ToSeqNo;
- };
-
- // todo: make it private
- struct TEvRetryChannelDataAck : public NActors::TEventLocal<TEvRetryChannelDataAck, TDqComputeEvents::EvRetryChannelDataAck> {
- TEvRetryChannelDataAck(ui64 channelId, ui64 fromSeqNo, ui64 toSeqNo)
- : ChannelId(channelId)
- , FromSeqNo(fromSeqNo)
- , ToSeqNo(toSeqNo) {}
-
- const ui64 ChannelId;
- const ui64 FromSeqNo;
- const ui64 ToSeqNo;
- };
-
- struct TEvRun : public NActors::TEventPB<TEvRun, NDqProto::TEvRun, TDqComputeEvents::EvRun> {};
-
- struct TEvNewCheckpointCoordinator : public NActors::TEventPB<TEvNewCheckpointCoordinator,
+ struct TEvResumeExecution : public NActors::TEventLocal<TEvResumeExecution, TDqComputeEvents::EvResumeExecution> {};
+
+ struct TEvChannelsInfo : public NActors::TEventPB<TEvChannelsInfo, NDqProto::TEvChannelsInfo,
+ TDqComputeEvents::EvChannelsInfo> {};
+
+ struct TEvChannelData : public NActors::TEventPB<TEvChannelData, NDqProto::TEvComputeChannelData,
+ TDqComputeEvents::EvChannelData> {};
+
+ struct TEvChannelDataAck : public NActors::TEventPB<TEvChannelDataAck, NDqProto::TEvComputeChannelDataAck,
+ TDqComputeEvents::EvChannelDataAck> {};
+
+ // todo: make it private
+ struct TEvRetryChannelData : public NActors::TEventLocal<TEvRetryChannelData, TDqComputeEvents::EvRetryChannelData> {
+ TEvRetryChannelData(ui64 channelId, ui64 fromSeqNo, ui64 toSeqNo)
+ : ChannelId(channelId)
+ , FromSeqNo(fromSeqNo)
+ , ToSeqNo(toSeqNo) {}
+
+ const ui64 ChannelId;
+ const ui64 FromSeqNo;
+ const ui64 ToSeqNo;
+ };
+
+ // todo: make it private
+ struct TEvRetryChannelDataAck : public NActors::TEventLocal<TEvRetryChannelDataAck, TDqComputeEvents::EvRetryChannelDataAck> {
+ TEvRetryChannelDataAck(ui64 channelId, ui64 fromSeqNo, ui64 toSeqNo)
+ : ChannelId(channelId)
+ , FromSeqNo(fromSeqNo)
+ , ToSeqNo(toSeqNo) {}
+
+ const ui64 ChannelId;
+ const ui64 FromSeqNo;
+ const ui64 ToSeqNo;
+ };
+
+ struct TEvRun : public NActors::TEventPB<TEvRun, NDqProto::TEvRun, TDqComputeEvents::EvRun> {};
+
+ struct TEvNewCheckpointCoordinator : public NActors::TEventPB<TEvNewCheckpointCoordinator,
NDqProto::TEvNewCheckpointCoordinator, TDqComputeEvents::EvNewCheckpointCoordinator> {
TEvNewCheckpointCoordinator() = default;
@@ -74,7 +74,7 @@ struct TEvDqCompute {
TEvNewCheckpointCoordinatorAck() = default;
};
- struct TEvInjectCheckpoint : public NActors::TEventPB<TEvInjectCheckpoint,
+ struct TEvInjectCheckpoint : public NActors::TEventPB<TEvInjectCheckpoint,
NDqProto::TEvInjectCheckpoint, TDqComputeEvents::EvInjectCheckpoint> {
TEvInjectCheckpoint() = default;
@@ -86,7 +86,7 @@ struct TEvDqCompute {
}
};
- struct TEvSaveTaskState : public NActors::TEventLocal<TEvSaveTaskState, TDqComputeEvents::EvSaveTaskState> {
+ struct TEvSaveTaskState : public NActors::TEventLocal<TEvSaveTaskState, TDqComputeEvents::EvSaveTaskState> {
TEvSaveTaskState(TString graphId, ui64 taskId, NDqProto::TCheckpoint checkpoint)
: GraphId(std::move(graphId))
, TaskId(taskId)
@@ -95,14 +95,14 @@ struct TEvDqCompute {
const TString GraphId;
const ui64 TaskId;
- const NDqProto::TCheckpoint Checkpoint;
+ const NDqProto::TCheckpoint Checkpoint;
NDqProto::TComputeActorState State;
};
- struct TEvSaveTaskStateResult : public NActors::TEventPB<TEvSaveTaskStateResult,
- NDqProto::TEvSaveTaskStateResult, TDqComputeEvents::EvSaveTaskStateResult> {};
+ struct TEvSaveTaskStateResult : public NActors::TEventPB<TEvSaveTaskStateResult,
+ NDqProto::TEvSaveTaskStateResult, TDqComputeEvents::EvSaveTaskStateResult> {};
- struct TEvCommitState : public NActors::TEventPB<TEvCommitState,
+ struct TEvCommitState : public NActors::TEventPB<TEvCommitState,
NDqProto::TEvCommitState, TDqComputeEvents::EvCommitState> {
TEvCommitState() = default;
@@ -114,7 +114,7 @@ struct TEvDqCompute {
}
};
- struct TEvStateCommitted : public NActors::TEventPB<TEvStateCommitted,
+ struct TEvStateCommitted : public NActors::TEventPB<TEvStateCommitted,
NDqProto::TEvStateCommitted, TDqComputeEvents::EvStateCommitted> {
TEvStateCommitted() = default;
@@ -162,51 +162,51 @@ struct TEvDqCompute {
}
};
- struct TEvGetTaskState : public NActors::TEventLocal<TEvGetTaskState, TDqComputeEvents::EvGetTaskState> {
+ struct TEvGetTaskState : public NActors::TEventLocal<TEvGetTaskState, TDqComputeEvents::EvGetTaskState> {
TEvGetTaskState(TString graphId, const std::vector<ui64>& taskIds, NDqProto::TCheckpoint checkpoint, ui64 generation)
: GraphId(std::move(graphId))
, TaskIds(taskIds)
, Checkpoint(std::move(checkpoint))
- , Generation(generation) {}
+ , Generation(generation) {}
const TString GraphId;
const std::vector<ui64> TaskIds;
- const NDqProto::TCheckpoint Checkpoint;
+ const NDqProto::TCheckpoint Checkpoint;
const ui64 Generation;
};
- struct TEvGetTaskStateResult : public NActors::TEventLocal<TEvGetTaskStateResult, TDqComputeEvents::EvGetTaskStateResult> {
+ struct TEvGetTaskStateResult : public NActors::TEventLocal<TEvGetTaskStateResult, TDqComputeEvents::EvGetTaskStateResult> {
TEvGetTaskStateResult(NDqProto::TCheckpoint checkpoint, TIssues issues, ui64 generation)
: Checkpoint(std::move(checkpoint))
, Issues(std::move(issues))
- , Generation(generation) {}
+ , Generation(generation) {}
- const NDqProto::TCheckpoint Checkpoint;
+ const NDqProto::TCheckpoint Checkpoint;
std::vector<NDqProto::TComputeActorState> States;
- const TIssues Issues;
+ const TIssues Issues;
const ui64 Generation;
};
};
-struct TDqExecutionSettings {
- struct TFlowControl {
- ui64 MaxOutputChunkSize = 2_MB;
- float InFlightBytesOvercommit = 1.5f;
-
- TDuration OutputChannelDeliveryInterval = TDuration::Seconds(30);
- TDuration OutputChannelRetryInterval = TDuration::MilliSeconds(500);
- };
-
- TFlowControl FlowControl;
-
- void Reset() {
- FlowControl = TFlowControl();
- }
-};
-
-const TDqExecutionSettings& GetDqExecutionSettings();
-TDqExecutionSettings& GetDqExecutionSettingsForTests();
-
+struct TDqExecutionSettings {
+ struct TFlowControl {
+ ui64 MaxOutputChunkSize = 2_MB;
+ float InFlightBytesOvercommit = 1.5f;
+
+ TDuration OutputChannelDeliveryInterval = TDuration::Seconds(30);
+ TDuration OutputChannelRetryInterval = TDuration::MilliSeconds(500);
+ };
+
+ TFlowControl FlowControl;
+
+ void Reset() {
+ FlowControl = TFlowControl();
+ }
+};
+
+const TDqExecutionSettings& GetDqExecutionSettings();
+TDqExecutionSettings& GetDqExecutionSettingsForTests();
+
struct TReportStatsSettings {
// Min interval between stats messages.
TDuration MinInterval;
@@ -214,51 +214,51 @@ struct TReportStatsSettings {
TDuration MaxInterval;
};
-struct TComputeRuntimeSettings {
- TMaybe<TDuration> Timeout;
- NDqProto::EDqStatsMode StatsMode = NDqProto::DQ_STATS_MODE_NONE;
+struct TComputeRuntimeSettings {
+ TMaybe<TDuration> Timeout;
+ NDqProto::EDqStatsMode StatsMode = NDqProto::DQ_STATS_MODE_NONE;
TMaybe<TReportStatsSettings> ReportStatsSettings;
-
- // see kqp_rm.h
- // 0 - disable extra memory allocation
- // 1 - allocate via memory pool ScanQuery
- // 2 - allocate via memory pool DataQuery
- ui32 ExtraMemoryAllocationPool = 0;
-
- bool FailOnUndelivery = true;
- bool UseLLVM = false;
- bool UseSpilling = false;
-
- std::function<void(bool success, const TString& reason)> TerminateHandler;
+
+ // see kqp_rm.h
+ // 0 - disable extra memory allocation
+ // 1 - allocate via memory pool ScanQuery
+ // 2 - allocate via memory pool DataQuery
+ ui32 ExtraMemoryAllocationPool = 0;
+
+ bool FailOnUndelivery = true;
+ bool UseLLVM = false;
+ bool UseSpilling = false;
+
+ std::function<void(bool success, const TString& reason)> TerminateHandler;
TMaybe<NDqProto::TRlPath> RlPath;
-};
+};
using TAllocateMemoryCallback = std::function<bool(const TTxId& txId, ui64 taskId, ui64 memory)>;
using TFreeMemoryCallback = std::function<void(const TTxId& txId, ui64 taskId, ui64 memory)>;
-struct TComputeMemoryLimits {
- ui64 ChannelBufferSize = 0;
- ui64 ScanBufferSize = 0; // TODO: drop it
- ui64 MkqlLightProgramMemoryLimit = 0;
- ui64 MkqlHeavyProgramMemoryLimit = 0;
-
+struct TComputeMemoryLimits {
+ ui64 ChannelBufferSize = 0;
+ ui64 ScanBufferSize = 0; // TODO: drop it
+ ui64 MkqlLightProgramMemoryLimit = 0;
+ ui64 MkqlHeavyProgramMemoryLimit = 0;
+
TAllocateMemoryCallback AllocateMemoryFn = nullptr;
TFreeMemoryCallback FreeMemoryFn = nullptr;
ui64 MinMemAllocSize = 30_MB;
ui64 MinMemFreeSize = 30_MB;
-};
-
-using TTaskRunnerFactory = std::function<
- TIntrusivePtr<IDqTaskRunner>(const NDqProto::TDqTask& task, const TLogFunc& logFunc)
->;
-
-void FillTaskRunnerStats(ui64 taskId, ui32 stageId, const TDqTaskRunnerStats& taskStats,
- NDqProto::TDqTaskStats* protoTask, bool withProfileStats);
-
+};
+
+using TTaskRunnerFactory = std::function<
+ TIntrusivePtr<IDqTaskRunner>(const NDqProto::TDqTask& task, const TLogFunc& logFunc)
+>;
+
+void FillTaskRunnerStats(ui64 taskId, ui32 stageId, const TDqTaskRunnerStats& taskStats,
+ NDqProto::TDqTaskStats* protoTask, bool withProfileStats);
+
NActors::IActor* CreateDqComputeActor(const NActors::TActorId& executerId, const TTxId& txId, NDqProto::TDqTask&& task,
IDqSourceActorFactory::TPtr sourceActorFactory, IDqSinkActorFactory::TPtr sinkActorFactory,
- const TComputeRuntimeSettings& settings, const TComputeMemoryLimits& memoryLimits,
- const TTaskRunnerFactory& taskRunnerFactory);
+ const TComputeRuntimeSettings& settings, const TComputeMemoryLimits& memoryLimits,
+ const TTaskRunnerFactory& taskRunnerFactory);
-} // namespace NDq
-} // namespace NYql
+} // namespace NDq
+} // namespace NYql
diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.cpp b/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.cpp
index 28cd7cd9b2..1a1611ec81 100644
--- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.cpp
+++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.cpp
@@ -1,741 +1,741 @@
#include "dq_compute_actor_channels.h"
-
-#include <util/string/join.h>
-
-
-#define LOG_D(s) \
- LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "TxId: " << TxId << ", task: " << TaskId << ". " << s)
-#define LOG_I(s) \
- LOG_INFO_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "TxId: " << TxId << ", task: " << TaskId << ". " << s)
-#define LOG_E(s) \
- LOG_ERROR_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "TxId: " << TxId << ", task: " << TaskId << ". " << s)
-#define LOG_C(s) \
- LOG_CRIT_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "TxId: " << TxId << ", task: " << TaskId << ". " << s)
-
-namespace NYql::NDq {
-
-using namespace NActors;
-using namespace NKikimr;
-
-namespace {
-
-template <typename TCollection>
-TString InFlightMessagesStr(const TCollection& inFlight) {
- TVector<ui64> ids;
- ids.reserve(inFlight.size());
- for (auto& kv : inFlight) {
- ids.push_back(kv.first);
- }
- return TStringBuilder() << '[' << JoinSeq(", ", ids) << ']';
-}
-
-} // anonymous namespace
-
+
+#include <util/string/join.h>
+
+
+#define LOG_D(s) \
+ LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "TxId: " << TxId << ", task: " << TaskId << ". " << s)
+#define LOG_I(s) \
+ LOG_INFO_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "TxId: " << TxId << ", task: " << TaskId << ". " << s)
+#define LOG_E(s) \
+ LOG_ERROR_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "TxId: " << TxId << ", task: " << TaskId << ". " << s)
+#define LOG_C(s) \
+ LOG_CRIT_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "TxId: " << TxId << ", task: " << TaskId << ". " << s)
+
+namespace NYql::NDq {
+
+using namespace NActors;
+using namespace NKikimr;
+
+namespace {
+
+template <typename TCollection>
+TString InFlightMessagesStr(const TCollection& inFlight) {
+ TVector<ui64> ids;
+ ids.reserve(inFlight.size());
+ for (auto& kv : inFlight) {
+ ids.push_back(kv.first);
+ }
+ return TStringBuilder() << '[' << JoinSeq(", ", ids) << ']';
+}
+
+} // anonymous namespace
+
TDqComputeActorChannels::TDqComputeActorChannels(TActorId owner, const TTxId& txId, const NDqProto::TDqTask& task,
bool retryOnUndelivery, NDqProto::EDqStatsMode statsMode, ui64 channelBufferSize, ICallbacks* cbs, ui32 actorActivityType)
: TActor(&TDqComputeActorChannels::WorkState, actorActivityType)
- , Owner(owner)
- , TxId(txId)
- , TaskId(task.GetId())
- , RetryOnUndelivery(retryOnUndelivery)
- , Cbs(cbs)
-{
- for (ui32 i = 0; i < task.InputsSize(); ++i) {
- for (auto& channel : task.GetInputs(i).GetChannels()) {
- TInputChannelState inputChannel;
- inputChannel.ChannelId = channel.GetId();
-
- if (Y_UNLIKELY(statsMode >= NDqProto::DQ_STATS_MODE_PROFILE)) {
- inputChannel.Stats = std::make_unique<TInputChannelStats>();
- }
-
- auto result = InputChannelsMap.emplace(channel.GetId(), std::move(inputChannel));
- YQL_ENSURE(result.second);
- }
- }
-
- for (ui32 i = 0; i < task.OutputsSize(); ++i) {
- for (auto& channel : task.GetOutputs(i).GetChannels()) {
- TOutputChannelState outputChannel;
- outputChannel.ChannelId = channel.GetId();
- outputChannel.PeerState.PeerFreeSpace = channelBufferSize;
-
- if (channel.GetDstEndpoint().HasActorId()) {
+ , Owner(owner)
+ , TxId(txId)
+ , TaskId(task.GetId())
+ , RetryOnUndelivery(retryOnUndelivery)
+ , Cbs(cbs)
+{
+ for (ui32 i = 0; i < task.InputsSize(); ++i) {
+ for (auto& channel : task.GetInputs(i).GetChannels()) {
+ TInputChannelState inputChannel;
+ inputChannel.ChannelId = channel.GetId();
+
+ if (Y_UNLIKELY(statsMode >= NDqProto::DQ_STATS_MODE_PROFILE)) {
+ inputChannel.Stats = std::make_unique<TInputChannelStats>();
+ }
+
+ auto result = InputChannelsMap.emplace(channel.GetId(), std::move(inputChannel));
+ YQL_ENSURE(result.second);
+ }
+ }
+
+ for (ui32 i = 0; i < task.OutputsSize(); ++i) {
+ for (auto& channel : task.GetOutputs(i).GetChannels()) {
+ TOutputChannelState outputChannel;
+ outputChannel.ChannelId = channel.GetId();
+ outputChannel.PeerState.PeerFreeSpace = channelBufferSize;
+
+ if (channel.GetDstEndpoint().HasActorId()) {
outputChannel.Peer = ActorIdFromProto(channel.GetDstEndpoint().GetActorId());
- }
-
- if (Y_UNLIKELY(statsMode >= NDqProto::DQ_STATS_MODE_PROFILE)) {
- outputChannel.Stats = std::make_unique<TOutputChannelStats>();
- }
-
- auto result = OutputChannelsMap.emplace(channel.GetId(), std::move(outputChannel));
- YQL_ENSURE(result.second);
- }
- }
-}
-
-STATEFN(TDqComputeActorChannels::WorkState) {
- switch (ev->GetTypeRewrite()) {
- hFunc(TEvDqCompute::TEvChannelData, HandleWork);
- hFunc(TEvDqCompute::TEvChannelDataAck, HandleWork);
- hFunc(TEvDqCompute::TEvRetryChannelData, HandleWork);
- hFunc(TEvDqCompute::TEvRetryChannelDataAck, HandleWork);
- hFunc(TEvents::TEvPoison, HandlePoison);
- hFunc(TEvents::TEvUndelivered, HandleWork);
- hFunc(TEvInterconnect::TEvNodeDisconnected, HandleWork);
- IgnoreFunc(TEvInterconnect::TEvNodeConnected);
- default: {
- InternalError(TStringBuilder() << "TDqComputeActorChannels: unexpected event " << ev->GetTypeRewrite()
- << " at WorkState");
- }
- }
-}
-
-void TDqComputeActorChannels::HandleWork(TEvDqCompute::TEvChannelData::TPtr& ev) {
- auto& record = ev->Get()->Record;
- auto& channelData = record.GetChannelData();
- ui64 channelId = channelData.GetChannelId();
-
- TInputChannelState& inputChannel = InCh(channelId);
-
- LOG_D("Received input for channelId: " << channelId
- << ", seqNo: " << record.GetSeqNo()
- << ", size: " << channelData.GetData().GetRaw().size()
- << ", rows: " << channelData.GetData().GetRows()
+ }
+
+ if (Y_UNLIKELY(statsMode >= NDqProto::DQ_STATS_MODE_PROFILE)) {
+ outputChannel.Stats = std::make_unique<TOutputChannelStats>();
+ }
+
+ auto result = OutputChannelsMap.emplace(channel.GetId(), std::move(outputChannel));
+ YQL_ENSURE(result.second);
+ }
+ }
+}
+
+STATEFN(TDqComputeActorChannels::WorkState) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvDqCompute::TEvChannelData, HandleWork);
+ hFunc(TEvDqCompute::TEvChannelDataAck, HandleWork);
+ hFunc(TEvDqCompute::TEvRetryChannelData, HandleWork);
+ hFunc(TEvDqCompute::TEvRetryChannelDataAck, HandleWork);
+ hFunc(TEvents::TEvPoison, HandlePoison);
+ hFunc(TEvents::TEvUndelivered, HandleWork);
+ hFunc(TEvInterconnect::TEvNodeDisconnected, HandleWork);
+ IgnoreFunc(TEvInterconnect::TEvNodeConnected);
+ default: {
+ InternalError(TStringBuilder() << "TDqComputeActorChannels: unexpected event " << ev->GetTypeRewrite()
+ << " at WorkState");
+ }
+ }
+}
+
+void TDqComputeActorChannels::HandleWork(TEvDqCompute::TEvChannelData::TPtr& ev) {
+ auto& record = ev->Get()->Record;
+ auto& channelData = record.GetChannelData();
+ ui64 channelId = channelData.GetChannelId();
+
+ TInputChannelState& inputChannel = InCh(channelId);
+
+ LOG_D("Received input for channelId: " << channelId
+ << ", seqNo: " << record.GetSeqNo()
+ << ", size: " << channelData.GetData().GetRaw().size()
+ << ", rows: " << channelData.GetData().GetRows()
<< ", checkpoint: " << channelData.HasCheckpoint()
- << ", finished: " << channelData.GetFinished()
- << ", from: " << ev->Sender
- << ", expected seqNo: " << (inputChannel.LastRecvSeqNo + 1));
-
- if (record.GetSeqNo() != inputChannel.LastRecvSeqNo + 1) {
- LOG_E("Unexpected input channelId: " << channelId << " seqNo: " << record.GetSeqNo()
- << ", expected: " << (inputChannel.LastRecvSeqNo + 1));
- return;
- }
-
- if (record.GetSeqNo() == 1) {
- YQL_ENSURE(inputChannel.InFlight.empty(), "" << InFlightMessagesStr(inputChannel.InFlight));
- if (!inputChannel.Peer) {
- inputChannel.Peer = ev->Sender;
- }
- }
-
- YQL_ENSURE(inputChannel.Peer);
- inputChannel.LastRecvSeqNo++;
-
- // remove all ack messages with seqNo <= seqNo
- while (!inputChannel.InFlight.empty() && inputChannel.InFlight.begin()->first <= record.GetSeqNo()) {
- inputChannel.InFlight.erase(inputChannel.InFlight.begin());
- }
-
- if (inputChannel.RetryState && !inputChannel.InFlight.empty()) {
- LOG_D("Received all retried messages for input channelId: " << channelId);
- inputChannel.RetryState.reset();
- }
-
- if (inputChannel.RetryState) {
- LOG_D("Waiting for input channelId: " << channelId
- << " messages: " << InFlightMessagesStr(inputChannel.InFlight));
- }
-
- if (inputChannel.PollRequest && inputChannel.PollRequest->SeqNo <= record.GetSeqNo()) {
- if (Y_UNLIKELY(inputChannel.Stats)) {
- inputChannel.Stats->WaitTime += TInstant::Now() - *inputChannel.StartPollTime;
- inputChannel.StartPollTime.reset();
- }
- inputChannel.PollRequest.reset();
- }
-
- if (channelData.GetFinished()) {
- LOG_D("Finish input channelId: " << channelId << ", from: " << *inputChannel.Peer);
- inputChannel.Finished = true;
- }
-
+ << ", finished: " << channelData.GetFinished()
+ << ", from: " << ev->Sender
+ << ", expected seqNo: " << (inputChannel.LastRecvSeqNo + 1));
+
+ if (record.GetSeqNo() != inputChannel.LastRecvSeqNo + 1) {
+ LOG_E("Unexpected input channelId: " << channelId << " seqNo: " << record.GetSeqNo()
+ << ", expected: " << (inputChannel.LastRecvSeqNo + 1));
+ return;
+ }
+
+ if (record.GetSeqNo() == 1) {
+ YQL_ENSURE(inputChannel.InFlight.empty(), "" << InFlightMessagesStr(inputChannel.InFlight));
+ if (!inputChannel.Peer) {
+ inputChannel.Peer = ev->Sender;
+ }
+ }
+
+ YQL_ENSURE(inputChannel.Peer);
+ inputChannel.LastRecvSeqNo++;
+
+ // remove all ack messages with seqNo <= seqNo
+ while (!inputChannel.InFlight.empty() && inputChannel.InFlight.begin()->first <= record.GetSeqNo()) {
+ inputChannel.InFlight.erase(inputChannel.InFlight.begin());
+ }
+
+ if (inputChannel.RetryState && !inputChannel.InFlight.empty()) {
+ LOG_D("Received all retried messages for input channelId: " << channelId);
+ inputChannel.RetryState.reset();
+ }
+
+ if (inputChannel.RetryState) {
+ LOG_D("Waiting for input channelId: " << channelId
+ << " messages: " << InFlightMessagesStr(inputChannel.InFlight));
+ }
+
+ if (inputChannel.PollRequest && inputChannel.PollRequest->SeqNo <= record.GetSeqNo()) {
+ if (Y_UNLIKELY(inputChannel.Stats)) {
+ inputChannel.Stats->WaitTime += TInstant::Now() - *inputChannel.StartPollTime;
+ inputChannel.StartPollTime.reset();
+ }
+ inputChannel.PollRequest.reset();
+ }
+
+ if (channelData.GetFinished()) {
+ LOG_D("Finish input channelId: " << channelId << ", from: " << *inputChannel.Peer);
+ inputChannel.Finished = true;
+ }
+
Cbs->TakeInputChannelData(std::move(*record.MutableChannelData()), !record.GetNoAck());
-}
-
-void TDqComputeActorChannels::HandleWork(TEvDqCompute::TEvChannelDataAck::TPtr& ev) {
- auto& record = ev->Get()->Record;
-
- TOutputChannelState& outputChannel = OutCh(record.GetChannelId());
-
- LOG_D("Received channel data ack for channelId: " << record.GetChannelId()
- << ", seqNo: " << record.GetSeqNo()
- << ", lastSentSeqNo: " << outputChannel.LastSentSeqNo
- << ", freeSpace: " << record.GetFreeSpace()
- << ", early finish: " << record.GetFinish());
-
- YQL_ENSURE(record.GetSeqNo() <= outputChannel.LastSentSeqNo);
-
- if (record.GetFinish()) {
- outputChannel.InFlight.clear();
- outputChannel.Finished = true;
- outputChannel.EarlyFinish = true;
- Cbs->PeerFinished(record.GetChannelId());
- return;
- }
-
- // remove all messages with seqNo <= ackSeqNo
- auto it = outputChannel.InFlight.begin();
- while (it != outputChannel.InFlight.end() && it->first <= record.GetSeqNo()) {
+}
+
+void TDqComputeActorChannels::HandleWork(TEvDqCompute::TEvChannelDataAck::TPtr& ev) {
+ auto& record = ev->Get()->Record;
+
+ TOutputChannelState& outputChannel = OutCh(record.GetChannelId());
+
+ LOG_D("Received channel data ack for channelId: " << record.GetChannelId()
+ << ", seqNo: " << record.GetSeqNo()
+ << ", lastSentSeqNo: " << outputChannel.LastSentSeqNo
+ << ", freeSpace: " << record.GetFreeSpace()
+ << ", early finish: " << record.GetFinish());
+
+ YQL_ENSURE(record.GetSeqNo() <= outputChannel.LastSentSeqNo);
+
+ if (record.GetFinish()) {
+ outputChannel.InFlight.clear();
+ outputChannel.Finished = true;
+ outputChannel.EarlyFinish = true;
+ Cbs->PeerFinished(record.GetChannelId());
+ return;
+ }
+
+ // remove all messages with seqNo <= ackSeqNo
+ auto it = outputChannel.InFlight.begin();
+ while (it != outputChannel.InFlight.end() && it->first <= record.GetSeqNo()) {
Y_VERIFY_DEBUG(outputChannel.PeerState.InFlightBytes >= it->second.Data.GetData().GetRaw().size());
Y_VERIFY_DEBUG(outputChannel.PeerState.InFlightRows >= it->second.Data.GetData().GetRows());
- Y_VERIFY_DEBUG(outputChannel.PeerState.InFlightCount >= 1);
-
+ Y_VERIFY_DEBUG(outputChannel.PeerState.InFlightCount >= 1);
+
outputChannel.PeerState.InFlightBytes -= it->second.Data.GetData().GetRaw().size();
outputChannel.PeerState.InFlightRows -= it->second.Data.GetData().GetRows();
- outputChannel.PeerState.InFlightCount -= 1;
- it = outputChannel.InFlight.erase(it);
- }
-
- outputChannel.PeerState.PeerFreeSpace = record.GetFreeSpace();
-
- LOG_D("PeerState, freeSpace: " << outputChannel.PeerState.PeerFreeSpace
- << ", inflight bytes: " << outputChannel.PeerState.InFlightBytes
- << ", inflight count: " << outputChannel.PeerState.InFlightCount
- << ", sentSeqNo: " << outputChannel.LastSentSeqNo
- << ", ackSeqNo: " << record.GetSeqNo());
-
- if (outputChannel.RetryState && outputChannel.InFlight.empty()) {
- LOG_D("Received all retried messages for output channelId: " << record.GetChannelId());
- outputChannel.RetryState.reset();
- }
-
- if (outputChannel.RetryState) {
- LOG_D("Waiting for output channelId: " << record.GetChannelId()
- << " messages: " << InFlightMessagesStr(outputChannel.InFlight));
- return;
- }
-
- LOG_D("Resume compute actor");
- Cbs->ResumeExecution();
-}
-
-void TDqComputeActorChannels::HandleWork(TEvDqCompute::TEvRetryChannelData::TPtr& ev) {
- auto* msg = ev->Get();
- LOG_D("Resend channel data events for output channelId: " << msg->ChannelId
- << ", seqNo: [" << msg->FromSeqNo << ".." << msg->ToSeqNo << ']');
-
- TOutputChannelState& outputChannel = OutCh(msg->ChannelId);
-
- if (!outputChannel.RetryState) {
- LOG_D("Output channelId: " << msg->ChannelId << " does not have retry state");
- return;
- }
-
- if (!outputChannel.RetryState->RetryScheduled) {
- LOG_E("Output channelId: " << msg->ChannelId << " does not have scheduled retry event");
- return;
- }
-
- outputChannel.RetryState->RetryScheduled = false;
-
- auto now = Now();
-
- for (auto& inFlight : outputChannel.InFlight) {
- ui64 seqNo = inFlight.first;
-
- if (seqNo < msg->FromSeqNo || seqNo > msg->ToSeqNo) {
- LOG_E("Output channelId: " << msg->ChannelId << " has unexpected inflight message seqNo: " << seqNo);
- }
-
- if (Y_UNLIKELY(outputChannel.Stats)) {
- outputChannel.Stats->ResentMessages++;
- }
-
- auto retryEv = MakeHolder<TEvDqCompute::TEvChannelData>();
- retryEv->Record.SetSeqNo(seqNo);
- retryEv->Record.SetSendTime(now.MilliSeconds());
-
- auto* data = retryEv->Record.MutableChannelData();
+ outputChannel.PeerState.InFlightCount -= 1;
+ it = outputChannel.InFlight.erase(it);
+ }
+
+ outputChannel.PeerState.PeerFreeSpace = record.GetFreeSpace();
+
+ LOG_D("PeerState, freeSpace: " << outputChannel.PeerState.PeerFreeSpace
+ << ", inflight bytes: " << outputChannel.PeerState.InFlightBytes
+ << ", inflight count: " << outputChannel.PeerState.InFlightCount
+ << ", sentSeqNo: " << outputChannel.LastSentSeqNo
+ << ", ackSeqNo: " << record.GetSeqNo());
+
+ if (outputChannel.RetryState && outputChannel.InFlight.empty()) {
+ LOG_D("Received all retried messages for output channelId: " << record.GetChannelId());
+ outputChannel.RetryState.reset();
+ }
+
+ if (outputChannel.RetryState) {
+ LOG_D("Waiting for output channelId: " << record.GetChannelId()
+ << " messages: " << InFlightMessagesStr(outputChannel.InFlight));
+ return;
+ }
+
+ LOG_D("Resume compute actor");
+ Cbs->ResumeExecution();
+}
+
+void TDqComputeActorChannels::HandleWork(TEvDqCompute::TEvRetryChannelData::TPtr& ev) {
+ auto* msg = ev->Get();
+ LOG_D("Resend channel data events for output channelId: " << msg->ChannelId
+ << ", seqNo: [" << msg->FromSeqNo << ".." << msg->ToSeqNo << ']');
+
+ TOutputChannelState& outputChannel = OutCh(msg->ChannelId);
+
+ if (!outputChannel.RetryState) {
+ LOG_D("Output channelId: " << msg->ChannelId << " does not have retry state");
+ return;
+ }
+
+ if (!outputChannel.RetryState->RetryScheduled) {
+ LOG_E("Output channelId: " << msg->ChannelId << " does not have scheduled retry event");
+ return;
+ }
+
+ outputChannel.RetryState->RetryScheduled = false;
+
+ auto now = Now();
+
+ for (auto& inFlight : outputChannel.InFlight) {
+ ui64 seqNo = inFlight.first;
+
+ if (seqNo < msg->FromSeqNo || seqNo > msg->ToSeqNo) {
+ LOG_E("Output channelId: " << msg->ChannelId << " has unexpected inflight message seqNo: " << seqNo);
+ }
+
+ if (Y_UNLIKELY(outputChannel.Stats)) {
+ outputChannel.Stats->ResentMessages++;
+ }
+
+ auto retryEv = MakeHolder<TEvDqCompute::TEvChannelData>();
+ retryEv->Record.SetSeqNo(seqNo);
+ retryEv->Record.SetSendTime(now.MilliSeconds());
+
+ auto* data = retryEv->Record.MutableChannelData();
data->CopyFrom(inFlight.second.Data);
- data->SetChannelId(msg->ChannelId);
- data->SetFinished(inFlight.second.Finished);
-
- LOG_D("Resending data chunk, channelId: " << msg->ChannelId
- << ", peer: " << *outputChannel.Peer
- << ", data size: " << data->GetData().GetRaw().size()
- << ", seqNo: " << seqNo
- << ", finished: " << inFlight.second.Finished);
-
- ui32 flags = CalcMessageFlags(*outputChannel.Peer);
- Send(*outputChannel.Peer, retryEv.Release(), flags, /* cookie */ msg->ChannelId);
- }
-}
-
-void TDqComputeActorChannels::HandleWork(TEvDqCompute::TEvRetryChannelDataAck::TPtr& ev) {
- auto* msg = ev->Get();
- LOG_D("Resend channel data ack events for input channelId: " << msg->ChannelId
- << ", seqNo: [" << msg->FromSeqNo << ".." << msg->ToSeqNo << ']');
-
- TInputChannelState& inputChannel = InCh(msg->ChannelId);
-
- if (!inputChannel.RetryState) {
- LOG_E("Input channelId: " << msg->ChannelId << " does not have retry state");
- return;
- }
-
- if (!inputChannel.RetryState->RetryScheduled) {
- LOG_E("Input channelId: " << msg->ChannelId << " does not have scheduled retry event");
- return;
- }
-
- inputChannel.RetryState->RetryScheduled = false;
-
- for (auto& inFlight : inputChannel.InFlight) {
- ui64 seqNo = inFlight.first;
- if (seqNo < msg->FromSeqNo || seqNo > msg->ToSeqNo) {
- LOG_E("Input channelId: " << msg->ChannelId << " has unexpected inflight message seqNo: " << seqNo);
- }
-
- if (Y_UNLIKELY(inputChannel.Stats)) {
- inputChannel.Stats->ResentMessages++;
- }
-
- auto retryEv = MakeHolder<TEvDqCompute::TEvChannelDataAck>();
- retryEv->Record.SetSeqNo(seqNo);
- retryEv->Record.SetChannelId(msg->ChannelId);
- retryEv->Record.SetFreeSpace(Cbs->GetInputChannelFreeSpace(msg->ChannelId));
-
- LOG_D("Resending data chunk ack, channelId: " << msg->ChannelId
- << ", peer: " << *inputChannel.Peer
- << ", seqNo: " << seqNo
- << ", freeSpace: " << retryEv->Record.GetFreeSpace());
-
- ui32 flags = CalcMessageFlags(*inputChannel.Peer);
- Send(*inputChannel.Peer, retryEv.Release(), flags, /* cookie */ msg->ChannelId);
- }
-}
-
-void TDqComputeActorChannels::HandleWork(TEvents::TEvUndelivered::TPtr& ev) {
- auto sourceType = ev->Get()->SourceType;
- auto reason = ev->Get()->Reason;
-
- if (!RetryOnUndelivery) {
- auto message = TStringBuilder() << "Handle undelivered event: " << sourceType << ", cookie: " << ev->Cookie
- << ", reason: " << reason;
- return RuntimeError(message);
- }
-
- if (sourceType == TEvDqCompute::TEvChannelData::EventType) {
- return HandleUndeliveredEvChannelData(ev->Cookie, reason);
- }
-
- if (sourceType == TEvDqCompute::TEvChannelDataAck::EventType) {
- return HandleUndeliveredEvChannelDataAck(ev->Cookie, reason);
- }
-
- LOG_E("Handle undelivered event: " << sourceType << ", cookie: " << ev->Cookie << ", reason: " << reason
- << ", ignore it");
-}
-
-void TDqComputeActorChannels::HandleWork(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) {
- ui32 nodeId = ev->Get()->NodeId;
- LOG_E("Handle node disconnect: " << nodeId);
-
- if (TrackingNodes.erase(nodeId) == 0) {
- return;
- }
-
- if (!RetryOnUndelivery) {
- return RuntimeError("detected disconnected node");
- }
-
- auto now = Now();
-
- for (auto& inputChannel : InputChannelsMap) {
- if (!inputChannel.second.IsFromNode(nodeId)) {
- continue;
- }
-
- inputChannel.second.PollRequest.reset();
-
- LOG_E("Disconnected node for input channelId: " << inputChannel.first
- << ", retry inflight events: " << InFlightMessagesStr(inputChannel.second.InFlight));
-
- if (!inputChannel.second.InFlight.empty()) {
- bool success = ScheduleRetryForChannel<TInputChannelState, TEvDqCompute::TEvRetryChannelDataAck>(
- inputChannel.second, now);
- if (!success) {
- return;
- }
- }
- }
-
- for (auto& outputChannel : OutputChannelsMap) {
- if (!outputChannel.second.IsToNode(nodeId)) {
- continue;
- }
-
- LOG_E("Disconnected node for output channelId: " << outputChannel.first
- << ", retry inflight events: " << InFlightMessagesStr(outputChannel.second.InFlight));
-
- if (!outputChannel.second.InFlight.empty()) {
- bool success = ScheduleRetryForChannel<TOutputChannelState, TEvDqCompute::TEvRetryChannelData>(
- outputChannel.second, now);
- if (!success) {
- return;
- }
- }
- }
-}
-
-void TDqComputeActorChannels::HandleUndeliveredEvChannelData(ui64 channelId, TEvents::TEvUndelivered::EReason reason) {
- LOG_E("Handle undelivered event: TEvChannelData, channelId: " << channelId << ", reason: " << reason);
-
- TOutputChannelState& outputChannel = OutCh(channelId);
-
+ data->SetChannelId(msg->ChannelId);
+ data->SetFinished(inFlight.second.Finished);
+
+ LOG_D("Resending data chunk, channelId: " << msg->ChannelId
+ << ", peer: " << *outputChannel.Peer
+ << ", data size: " << data->GetData().GetRaw().size()
+ << ", seqNo: " << seqNo
+ << ", finished: " << inFlight.second.Finished);
+
+ ui32 flags = CalcMessageFlags(*outputChannel.Peer);
+ Send(*outputChannel.Peer, retryEv.Release(), flags, /* cookie */ msg->ChannelId);
+ }
+}
+
+void TDqComputeActorChannels::HandleWork(TEvDqCompute::TEvRetryChannelDataAck::TPtr& ev) {
+ auto* msg = ev->Get();
+ LOG_D("Resend channel data ack events for input channelId: " << msg->ChannelId
+ << ", seqNo: [" << msg->FromSeqNo << ".." << msg->ToSeqNo << ']');
+
+ TInputChannelState& inputChannel = InCh(msg->ChannelId);
+
+ if (!inputChannel.RetryState) {
+ LOG_E("Input channelId: " << msg->ChannelId << " does not have retry state");
+ return;
+ }
+
+ if (!inputChannel.RetryState->RetryScheduled) {
+ LOG_E("Input channelId: " << msg->ChannelId << " does not have scheduled retry event");
+ return;
+ }
+
+ inputChannel.RetryState->RetryScheduled = false;
+
+ for (auto& inFlight : inputChannel.InFlight) {
+ ui64 seqNo = inFlight.first;
+ if (seqNo < msg->FromSeqNo || seqNo > msg->ToSeqNo) {
+ LOG_E("Input channelId: " << msg->ChannelId << " has unexpected inflight message seqNo: " << seqNo);
+ }
+
+ if (Y_UNLIKELY(inputChannel.Stats)) {
+ inputChannel.Stats->ResentMessages++;
+ }
+
+ auto retryEv = MakeHolder<TEvDqCompute::TEvChannelDataAck>();
+ retryEv->Record.SetSeqNo(seqNo);
+ retryEv->Record.SetChannelId(msg->ChannelId);
+ retryEv->Record.SetFreeSpace(Cbs->GetInputChannelFreeSpace(msg->ChannelId));
+
+ LOG_D("Resending data chunk ack, channelId: " << msg->ChannelId
+ << ", peer: " << *inputChannel.Peer
+ << ", seqNo: " << seqNo
+ << ", freeSpace: " << retryEv->Record.GetFreeSpace());
+
+ ui32 flags = CalcMessageFlags(*inputChannel.Peer);
+ Send(*inputChannel.Peer, retryEv.Release(), flags, /* cookie */ msg->ChannelId);
+ }
+}
+
+void TDqComputeActorChannels::HandleWork(TEvents::TEvUndelivered::TPtr& ev) {
+ auto sourceType = ev->Get()->SourceType;
+ auto reason = ev->Get()->Reason;
+
+ if (!RetryOnUndelivery) {
+ auto message = TStringBuilder() << "Handle undelivered event: " << sourceType << ", cookie: " << ev->Cookie
+ << ", reason: " << reason;
+ return RuntimeError(message);
+ }
+
+ if (sourceType == TEvDqCompute::TEvChannelData::EventType) {
+ return HandleUndeliveredEvChannelData(ev->Cookie, reason);
+ }
+
+ if (sourceType == TEvDqCompute::TEvChannelDataAck::EventType) {
+ return HandleUndeliveredEvChannelDataAck(ev->Cookie, reason);
+ }
+
+ LOG_E("Handle undelivered event: " << sourceType << ", cookie: " << ev->Cookie << ", reason: " << reason
+ << ", ignore it");
+}
+
+void TDqComputeActorChannels::HandleWork(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) {
+ ui32 nodeId = ev->Get()->NodeId;
+ LOG_E("Handle node disconnect: " << nodeId);
+
+ if (TrackingNodes.erase(nodeId) == 0) {
+ return;
+ }
+
+ if (!RetryOnUndelivery) {
+ return RuntimeError("detected disconnected node");
+ }
+
+ auto now = Now();
+
+ for (auto& inputChannel : InputChannelsMap) {
+ if (!inputChannel.second.IsFromNode(nodeId)) {
+ continue;
+ }
+
+ inputChannel.second.PollRequest.reset();
+
+ LOG_E("Disconnected node for input channelId: " << inputChannel.first
+ << ", retry inflight events: " << InFlightMessagesStr(inputChannel.second.InFlight));
+
+ if (!inputChannel.second.InFlight.empty()) {
+ bool success = ScheduleRetryForChannel<TInputChannelState, TEvDqCompute::TEvRetryChannelDataAck>(
+ inputChannel.second, now);
+ if (!success) {
+ return;
+ }
+ }
+ }
+
+ for (auto& outputChannel : OutputChannelsMap) {
+ if (!outputChannel.second.IsToNode(nodeId)) {
+ continue;
+ }
+
+ LOG_E("Disconnected node for output channelId: " << outputChannel.first
+ << ", retry inflight events: " << InFlightMessagesStr(outputChannel.second.InFlight));
+
+ if (!outputChannel.second.InFlight.empty()) {
+ bool success = ScheduleRetryForChannel<TOutputChannelState, TEvDqCompute::TEvRetryChannelData>(
+ outputChannel.second, now);
+ if (!success) {
+ return;
+ }
+ }
+ }
+}
+
+void TDqComputeActorChannels::HandleUndeliveredEvChannelData(ui64 channelId, TEvents::TEvUndelivered::EReason reason) {
+ LOG_E("Handle undelivered event: TEvChannelData, channelId: " << channelId << ", reason: " << reason);
+
+ TOutputChannelState& outputChannel = OutCh(channelId);
+
if (outputChannel.Finished && outputChannel.EarlyFinish && !SupportCheckpoints) {
- LOG_E("Ignore undelivered TEvChannelData event due to early finish, channelId: " << channelId);
- outputChannel.InFlight.clear();
- Cbs->ResumeExecution();
- return;
- }
-
- if (reason == TEvents::TEvUndelivered::ReasonActorUnknown) {
- return RuntimeError("unknown output channel actor");
- }
-
- if (outputChannel.InFlight.empty()) {
- return;
- }
-
- ScheduleRetryForChannel<TOutputChannelState, TEvDqCompute::TEvRetryChannelData>(outputChannel, Now());
-}
-
-void TDqComputeActorChannels::HandleUndeliveredEvChannelDataAck(ui64 channelId, TEvents::TEvUndelivered::EReason reason) {
- LOG_E("Handle undelivered event: TEvChannelDataAck, channelId: " << channelId << ", reason: " << reason);
-
- TInputChannelState& inputChannel = InCh(channelId);
- inputChannel.PollRequest.reset();
-
+ LOG_E("Ignore undelivered TEvChannelData event due to early finish, channelId: " << channelId);
+ outputChannel.InFlight.clear();
+ Cbs->ResumeExecution();
+ return;
+ }
+
+ if (reason == TEvents::TEvUndelivered::ReasonActorUnknown) {
+ return RuntimeError("unknown output channel actor");
+ }
+
+ if (outputChannel.InFlight.empty()) {
+ return;
+ }
+
+ ScheduleRetryForChannel<TOutputChannelState, TEvDqCompute::TEvRetryChannelData>(outputChannel, Now());
+}
+
+void TDqComputeActorChannels::HandleUndeliveredEvChannelDataAck(ui64 channelId, TEvents::TEvUndelivered::EReason reason) {
+ LOG_E("Handle undelivered event: TEvChannelDataAck, channelId: " << channelId << ", reason: " << reason);
+
+ TInputChannelState& inputChannel = InCh(channelId);
+ inputChannel.PollRequest.reset();
+
if (inputChannel.Finished && !SupportCheckpoints) {
- LOG_I("Handle undelivered event: TEvChannelDataAck, channelId: " << channelId << ", reason: " << reason
- << ". Ignore, channel is finished.");
- inputChannel.InFlight.clear();
- Cbs->ResumeExecution();
- return;
- }
-
- if (reason == TEvents::TEvUndelivered::ReasonActorUnknown) {
- return RuntimeError("unknown input channel actor");
- }
-
- if (inputChannel.InFlight.empty()) {
- return;
- }
-
- ScheduleRetryForChannel<TInputChannelState, TEvDqCompute::TEvRetryChannelDataAck>(inputChannel, Now());
-}
-
-template <typename TChannelState, typename TRetryEvent>
-bool TDqComputeActorChannels::ScheduleRetryForChannel(TChannelState& channel, TInstant now) {
- if (!channel.RetryState) {
- channel.RetryState.emplace(now);
- }
-
- auto retryAt = channel.RetryState->CalcNextRetry(now);
- if (!retryAt) {
- LOG_E("Delivery interval exceeded for channelId: " << channel.ChannelId);
- RuntimeError("channel delivery interval exceeded");
- return false;
- }
-
- // every InFlight message can generate undelivery event,
- // but we schedule retry event only once
- if (!channel.RetryState->RetryScheduled) {
- channel.RetryState->RetryScheduled = true;
- auto retryAfter = *retryAt - now;
- LOG_D("Retry #" << channel.RetryState->AttemptNo << " scheduled on " << retryAt
- << ", deadline: " << channel.RetryState->Deadline);
- Schedule(retryAfter, new TRetryEvent(channel.ChannelId, channel.InFlight.begin()->first,
- channel.InFlight.rbegin()->first));
- }
-
- return true;
-}
-
-STATEFN(TDqComputeActorChannels::DeadState) {
- switch (ev->GetTypeRewrite()) {
- hFunc(TEvents::TEvPoison, HandlePoison);
- default: {
- LOG_E("Skip unexpected event " << ev->GetTypeRewrite() << " at DeadState");
- }
- }
-}
-
-void TDqComputeActorChannels::HandlePoison(TEvents::TEvPoison::TPtr&) {
- LOG_D("pass away");
- PassAway();
-}
-
-TInstant TDqComputeActorChannels::Now() const {
- return TInstant::Now();
-}
-
-void TDqComputeActorChannels::RuntimeError(const TString& message) {
- LOG_E(message);
-
- auto ev = TEvDq::TEvAbortExecution::Unavailable(message);
- Send(Owner, ev.Release());
-
- Become(&TDqComputeActorChannels::DeadState);
-}
-
-void TDqComputeActorChannels::InternalError(const TString& message) {
- LOG_C(message);
-
- auto ev = TEvDq::TEvAbortExecution::InternalError(message);
- Send(Owner, ev.Release());
-
- Become(&TDqComputeActorChannels::DeadState);
-}
-
-void TDqComputeActorChannels::PassAway() {
- for (ui32 nodeId : TrackingNodes) {
- Send(TActivationContext::InterconnectProxy(nodeId), new TEvents::TEvUnsubscribe());
- }
- IActor::PassAway();
-}
-
+ LOG_I("Handle undelivered event: TEvChannelDataAck, channelId: " << channelId << ", reason: " << reason
+ << ". Ignore, channel is finished.");
+ inputChannel.InFlight.clear();
+ Cbs->ResumeExecution();
+ return;
+ }
+
+ if (reason == TEvents::TEvUndelivered::ReasonActorUnknown) {
+ return RuntimeError("unknown input channel actor");
+ }
+
+ if (inputChannel.InFlight.empty()) {
+ return;
+ }
+
+ ScheduleRetryForChannel<TInputChannelState, TEvDqCompute::TEvRetryChannelDataAck>(inputChannel, Now());
+}
+
+template <typename TChannelState, typename TRetryEvent>
+bool TDqComputeActorChannels::ScheduleRetryForChannel(TChannelState& channel, TInstant now) {
+ if (!channel.RetryState) {
+ channel.RetryState.emplace(now);
+ }
+
+ auto retryAt = channel.RetryState->CalcNextRetry(now);
+ if (!retryAt) {
+ LOG_E("Delivery interval exceeded for channelId: " << channel.ChannelId);
+ RuntimeError("channel delivery interval exceeded");
+ return false;
+ }
+
+ // every InFlight message can generate undelivery event,
+ // but we schedule retry event only once
+ if (!channel.RetryState->RetryScheduled) {
+ channel.RetryState->RetryScheduled = true;
+ auto retryAfter = *retryAt - now;
+ LOG_D("Retry #" << channel.RetryState->AttemptNo << " scheduled on " << retryAt
+ << ", deadline: " << channel.RetryState->Deadline);
+ Schedule(retryAfter, new TRetryEvent(channel.ChannelId, channel.InFlight.begin()->first,
+ channel.InFlight.rbegin()->first));
+ }
+
+ return true;
+}
+
+STATEFN(TDqComputeActorChannels::DeadState) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvents::TEvPoison, HandlePoison);
+ default: {
+ LOG_E("Skip unexpected event " << ev->GetTypeRewrite() << " at DeadState");
+ }
+ }
+}
+
+void TDqComputeActorChannels::HandlePoison(TEvents::TEvPoison::TPtr&) {
+ LOG_D("pass away");
+ PassAway();
+}
+
+TInstant TDqComputeActorChannels::Now() const {
+ return TInstant::Now();
+}
+
+void TDqComputeActorChannels::RuntimeError(const TString& message) {
+ LOG_E(message);
+
+ auto ev = TEvDq::TEvAbortExecution::Unavailable(message);
+ Send(Owner, ev.Release());
+
+ Become(&TDqComputeActorChannels::DeadState);
+}
+
+void TDqComputeActorChannels::InternalError(const TString& message) {
+ LOG_C(message);
+
+ auto ev = TEvDq::TEvAbortExecution::InternalError(message);
+ Send(Owner, ev.Release());
+
+ Become(&TDqComputeActorChannels::DeadState);
+}
+
+void TDqComputeActorChannels::PassAway() {
+ for (ui32 nodeId : TrackingNodes) {
+ Send(TActivationContext::InterconnectProxy(nodeId), new TEvents::TEvUnsubscribe());
+ }
+ IActor::PassAway();
+}
+
void TDqComputeActorChannels::SetInputChannelPeer(ui64 channelId, const TActorId& peer) {
TInputChannelState& inputChannel = InCh(channelId);
inputChannel.Peer = peer;
}
-void TDqComputeActorChannels::SetOutputChannelPeer(ui64 channelId, const TActorId& peer) {
- TOutputChannelState& outputChannel = OutCh(channelId);
- outputChannel.Peer = peer;
-}
-
-bool TDqComputeActorChannels::CanSendChannelData(ui64 channelId) {
- TOutputChannelState& outputChannel = OutCh(channelId);
+void TDqComputeActorChannels::SetOutputChannelPeer(ui64 channelId, const TActorId& peer) {
+ TOutputChannelState& outputChannel = OutCh(channelId);
+ outputChannel.Peer = peer;
+}
+
+bool TDqComputeActorChannels::CanSendChannelData(ui64 channelId) {
+ TOutputChannelState& outputChannel = OutCh(channelId);
return outputChannel.Peer && (!outputChannel.Finished || SupportCheckpoints) && !outputChannel.RetryState;
-}
-
-void TDqComputeActorChannels::SendChannelData(NDqProto::TChannelData&& channelData) {
- TOutputChannelState& outputChannel = OutCh(channelData.GetChannelId());
-
+}
+
+void TDqComputeActorChannels::SendChannelData(NDqProto::TChannelData&& channelData) {
+ TOutputChannelState& outputChannel = OutCh(channelData.GetChannelId());
+
YQL_ENSURE(!outputChannel.Finished || SupportCheckpoints);
- YQL_ENSURE(!outputChannel.RetryState);
-
- ui64 seqNo = ++outputChannel.LastSentSeqNo;
- ui32 chunkBytes = channelData.GetData().GetRaw().size();
- ui32 chunkRows = channelData.GetData().GetRows();
- bool finished = channelData.GetFinished();
-
- LOG_D("SendChannelData, channelId: " << channelData.GetChannelId()
- << ", peer: " << *outputChannel.Peer
- << ", rows: " << chunkRows
- << ", bytes: " << chunkBytes
+ YQL_ENSURE(!outputChannel.RetryState);
+
+ ui64 seqNo = ++outputChannel.LastSentSeqNo;
+ ui32 chunkBytes = channelData.GetData().GetRaw().size();
+ ui32 chunkRows = channelData.GetData().GetRows();
+ bool finished = channelData.GetFinished();
+
+ LOG_D("SendChannelData, channelId: " << channelData.GetChannelId()
+ << ", peer: " << *outputChannel.Peer
+ << ", rows: " << chunkRows
+ << ", bytes: " << chunkBytes
<< ", checkpoint: " << channelData.HasCheckpoint()
- << ", seqNo: " << seqNo
- << ", finished: " << finished);
-
- auto dataEv = MakeHolder<TEvDqCompute::TEvChannelData>();
- dataEv->Record.SetSeqNo(seqNo);
- dataEv->Record.SetSendTime(Now().MilliSeconds());
- dataEv->Record.MutableChannelData()->Swap(&channelData);
-
- outputChannel.InFlight.emplace(
- seqNo,
- TOutputChannelState::TInFlightMessage(
- seqNo,
+ << ", seqNo: " << seqNo
+ << ", finished: " << finished);
+
+ auto dataEv = MakeHolder<TEvDqCompute::TEvChannelData>();
+ dataEv->Record.SetSeqNo(seqNo);
+ dataEv->Record.SetSendTime(Now().MilliSeconds());
+ dataEv->Record.MutableChannelData()->Swap(&channelData);
+
+ outputChannel.InFlight.emplace(
+ seqNo,
+ TOutputChannelState::TInFlightMessage(
+ seqNo,
NYql::NDqProto::TChannelData(dataEv->Record.GetChannelData()),
- finished
- )
- );
- outputChannel.Finished = finished;
-
- ui32 flags = CalcMessageFlags(*outputChannel.Peer);
- Send(*outputChannel.Peer, dataEv.Release(), flags, /* cookie */ outputChannel.ChannelId);
-
- outputChannel.PeerState.InFlightBytes += chunkBytes;
- outputChannel.PeerState.InFlightRows += chunkRows;
- outputChannel.PeerState.InFlightCount += 1;
-}
-
-bool TDqComputeActorChannels::PollChannel(ui64 channelId, i64 freeSpace) {
- TInputChannelState& inputChannel = InCh(channelId);
-
+ finished
+ )
+ );
+ outputChannel.Finished = finished;
+
+ ui32 flags = CalcMessageFlags(*outputChannel.Peer);
+ Send(*outputChannel.Peer, dataEv.Release(), flags, /* cookie */ outputChannel.ChannelId);
+
+ outputChannel.PeerState.InFlightBytes += chunkBytes;
+ outputChannel.PeerState.InFlightRows += chunkRows;
+ outputChannel.PeerState.InFlightCount += 1;
+}
+
+bool TDqComputeActorChannels::PollChannel(ui64 channelId, i64 freeSpace) {
+ TInputChannelState& inputChannel = InCh(channelId);
+
if (!inputChannel.Peer || (inputChannel.Finished && !SupportCheckpoints) || inputChannel.RetryState ||
- inputChannel.LastRecvSeqNo == 0 || freeSpace <= 0)
- {
- LOG_D("no poll, channelId: " << channelId << ", hasPeer: " << inputChannel.Peer.has_value()
- << ", finished: " << inputChannel.Finished << ", retry: " << inputChannel.RetryState.has_value()
- << ", lastSeqNo: " << inputChannel.LastRecvSeqNo << ", freeSpace: " << freeSpace);
- return false;
- }
-
- if (inputChannel.PollRequest) {
- if (inputChannel.PollRequest->SeqNo == inputChannel.LastRecvSeqNo &&
- inputChannel.PollRequest->FreeSpace == freeSpace)
- {
- LOG_D("no poll, channelId: " << channelId << ", has poll request"
- << " with same seqNo: " << inputChannel.PollRequest->SeqNo
- << " and freeSpace: " << inputChannel.PollRequest->FreeSpace);
- return false;
- }
- } else {
- if (!inputChannel.InFlight.empty()) {
- i64 lastFreeSpace = inputChannel.InFlight.rbegin()->second.FreeSpace;
- if (lastFreeSpace >= freeSpace) {
- LOG_D("no poll, channelId: " << channelId
- << ", has inflight messages: " << InFlightMessagesStr(inputChannel.InFlight)
- << ", last with freeSpace: " << lastFreeSpace
- << ", current freeSpace: " << freeSpace);
- return false;
- } else {
- LOG_D("do poll, channelId: " << channelId << ", last freeSpace: " << lastFreeSpace
- << ", current freeSpace: " << freeSpace << ", seqNo: " << inputChannel.LastRecvSeqNo);
- }
- }
- }
-
- inputChannel.PollRequest.emplace(inputChannel.LastRecvSeqNo, freeSpace);
-
- LOG_D("Poll input channelId: " << channelId
- << ", from: " << *inputChannel.Peer
- << ", seqNo: " << inputChannel.LastRecvSeqNo
- << ", freeSpace: " << freeSpace);
-
- if (Y_UNLIKELY(inputChannel.Stats)) {
- inputChannel.Stats->PollRequests++;
- if (!inputChannel.StartPollTime) {
- inputChannel.StartPollTime = TInstant::Now();
- }
- }
-
- SendChannelDataAck(inputChannel, freeSpace);
- return true;
-}
-
-bool TDqComputeActorChannels::CheckInFlight(const TString& prefix) {
- for (auto& inputChannel: InputChannelsMap) {
- if (!inputChannel.second.InFlight.empty()) {
- if (inputChannel.second.Finished) {
- LOG_D(prefix << ", don't wait for ack delivery in channelId: "
- << inputChannel.first << ", seqNo: " << InFlightMessagesStr(inputChannel.second.InFlight));
- continue;
- }
- LOG_D(prefix << ", waiting for ack delivery in channelId: "
- << inputChannel.first << ", seqNo: " << InFlightMessagesStr(inputChannel.second.InFlight));
- return false;
- }
- }
-
- for (auto& outputChannel : OutputChannelsMap) {
- if (!outputChannel.second.InFlight.empty()) {
- LOG_D(prefix << ", waiting for chunk delivery in channelId: "
- << outputChannel.first << ", seqNo: " << InFlightMessagesStr(outputChannel.second.InFlight));
- return false;
- }
- }
-
- LOG_D(prefix);
- return true;
-}
-
-bool TDqComputeActorChannels::FinishInputChannels() {
- bool result = true;
-
- for (auto& pair : InputChannelsMap) {
- ui64 channelId = pair.first;
- auto& inputChannel = pair.second;
-
- if (!inputChannel.Peer) {
- LOG_D("Can not finish input channelId: " << channelId << " prematurely, peer not set yet");
- result = false;
- continue;
- }
-
- if (inputChannel.Finished) {
- continue;
- }
-
- LOG_D("Finish input channelId: " << channelId << " prematurely");
-
- inputChannel.Finished = true;
-
- auto ackEv = MakeHolder<TEvDqCompute::TEvChannelDataAck>();
- ackEv->Record.SetSeqNo(inputChannel.LastRecvSeqNo);
- ackEv->Record.SetChannelId(inputChannel.ChannelId);
- ackEv->Record.SetFreeSpace(0);
- ackEv->Record.SetFinish(true);
-
+ inputChannel.LastRecvSeqNo == 0 || freeSpace <= 0)
+ {
+ LOG_D("no poll, channelId: " << channelId << ", hasPeer: " << inputChannel.Peer.has_value()
+ << ", finished: " << inputChannel.Finished << ", retry: " << inputChannel.RetryState.has_value()
+ << ", lastSeqNo: " << inputChannel.LastRecvSeqNo << ", freeSpace: " << freeSpace);
+ return false;
+ }
+
+ if (inputChannel.PollRequest) {
+ if (inputChannel.PollRequest->SeqNo == inputChannel.LastRecvSeqNo &&
+ inputChannel.PollRequest->FreeSpace == freeSpace)
+ {
+ LOG_D("no poll, channelId: " << channelId << ", has poll request"
+ << " with same seqNo: " << inputChannel.PollRequest->SeqNo
+ << " and freeSpace: " << inputChannel.PollRequest->FreeSpace);
+ return false;
+ }
+ } else {
+ if (!inputChannel.InFlight.empty()) {
+ i64 lastFreeSpace = inputChannel.InFlight.rbegin()->second.FreeSpace;
+ if (lastFreeSpace >= freeSpace) {
+ LOG_D("no poll, channelId: " << channelId
+ << ", has inflight messages: " << InFlightMessagesStr(inputChannel.InFlight)
+ << ", last with freeSpace: " << lastFreeSpace
+ << ", current freeSpace: " << freeSpace);
+ return false;
+ } else {
+ LOG_D("do poll, channelId: " << channelId << ", last freeSpace: " << lastFreeSpace
+ << ", current freeSpace: " << freeSpace << ", seqNo: " << inputChannel.LastRecvSeqNo);
+ }
+ }
+ }
+
+ inputChannel.PollRequest.emplace(inputChannel.LastRecvSeqNo, freeSpace);
+
+ LOG_D("Poll input channelId: " << channelId
+ << ", from: " << *inputChannel.Peer
+ << ", seqNo: " << inputChannel.LastRecvSeqNo
+ << ", freeSpace: " << freeSpace);
+
+ if (Y_UNLIKELY(inputChannel.Stats)) {
+ inputChannel.Stats->PollRequests++;
+ if (!inputChannel.StartPollTime) {
+ inputChannel.StartPollTime = TInstant::Now();
+ }
+ }
+
+ SendChannelDataAck(inputChannel, freeSpace);
+ return true;
+}
+
+bool TDqComputeActorChannels::CheckInFlight(const TString& prefix) {
+ for (auto& inputChannel: InputChannelsMap) {
+ if (!inputChannel.second.InFlight.empty()) {
+ if (inputChannel.second.Finished) {
+ LOG_D(prefix << ", don't wait for ack delivery in channelId: "
+ << inputChannel.first << ", seqNo: " << InFlightMessagesStr(inputChannel.second.InFlight));
+ continue;
+ }
+ LOG_D(prefix << ", waiting for ack delivery in channelId: "
+ << inputChannel.first << ", seqNo: " << InFlightMessagesStr(inputChannel.second.InFlight));
+ return false;
+ }
+ }
+
+ for (auto& outputChannel : OutputChannelsMap) {
+ if (!outputChannel.second.InFlight.empty()) {
+ LOG_D(prefix << ", waiting for chunk delivery in channelId: "
+ << outputChannel.first << ", seqNo: " << InFlightMessagesStr(outputChannel.second.InFlight));
+ return false;
+ }
+ }
+
+ LOG_D(prefix);
+ return true;
+}
+
+bool TDqComputeActorChannels::FinishInputChannels() {
+ bool result = true;
+
+ for (auto& pair : InputChannelsMap) {
+ ui64 channelId = pair.first;
+ auto& inputChannel = pair.second;
+
+ if (!inputChannel.Peer) {
+ LOG_D("Can not finish input channelId: " << channelId << " prematurely, peer not set yet");
+ result = false;
+ continue;
+ }
+
+ if (inputChannel.Finished) {
+ continue;
+ }
+
+ LOG_D("Finish input channelId: " << channelId << " prematurely");
+
+ inputChannel.Finished = true;
+
+ auto ackEv = MakeHolder<TEvDqCompute::TEvChannelDataAck>();
+ ackEv->Record.SetSeqNo(inputChannel.LastRecvSeqNo);
+ ackEv->Record.SetChannelId(inputChannel.ChannelId);
+ ackEv->Record.SetFreeSpace(0);
+ ackEv->Record.SetFinish(true);
+
ui32 flags = (inputChannel.Finished && !SupportCheckpoints) ? 0 : CalcMessageFlags(*inputChannel.Peer);
- Send(*inputChannel.Peer, ackEv.Release(), flags, /* cookie */ inputChannel.ChannelId);
- }
-
- return result;
-}
-
-const TDqComputeActorChannels::TPeerState& TDqComputeActorChannels::GetOutputChannelInFlightState(ui64 channelId) {
- TOutputChannelState& outputChannel = OutCh(channelId);
- return outputChannel.PeerState;
-}
-
-const TDqComputeActorChannels::TInputChannelStats* TDqComputeActorChannels::GetInputChannelStats(ui64 channelId) {
- return InCh(channelId).Stats.get();
-}
-
-const TDqComputeActorChannels::TOutputChannelStats* TDqComputeActorChannels::GetOutputChannelStats(ui64 channelId) {
- return OutCh(channelId).Stats.get();
-}
-
+ Send(*inputChannel.Peer, ackEv.Release(), flags, /* cookie */ inputChannel.ChannelId);
+ }
+
+ return result;
+}
+
+const TDqComputeActorChannels::TPeerState& TDqComputeActorChannels::GetOutputChannelInFlightState(ui64 channelId) {
+ TOutputChannelState& outputChannel = OutCh(channelId);
+ return outputChannel.PeerState;
+}
+
+const TDqComputeActorChannels::TInputChannelStats* TDqComputeActorChannels::GetInputChannelStats(ui64 channelId) {
+ return InCh(channelId).Stats.get();
+}
+
+const TDqComputeActorChannels::TOutputChannelStats* TDqComputeActorChannels::GetOutputChannelStats(ui64 channelId) {
+ return OutCh(channelId).Stats.get();
+}
+
void TDqComputeActorChannels::SendChannelDataAck(i64 channelId, i64 freeSpace) {
TInputChannelState& inputChannel = InCh(channelId);
SendChannelDataAck(inputChannel, freeSpace);
}
-void TDqComputeActorChannels::SendChannelDataAck(TInputChannelState& inputChannel, i64 freeSpace) {
- LOG_D("Sending channel data ack to"
- << " channelId: " << inputChannel.ChannelId
- << ", peer: " << *inputChannel.Peer
- << ", from: " << Owner
- << ", freeSpace: " << freeSpace
- << ", seqNo: " << inputChannel.LastRecvSeqNo
- << ", finished: " << inputChannel.Finished);
-
- inputChannel.InFlight.emplace(
- inputChannel.LastRecvSeqNo,
- TInputChannelState::TInFlightMessage(
- inputChannel.LastRecvSeqNo,
- freeSpace
- )
- );
-
- auto ackEv = MakeHolder<TEvDqCompute::TEvChannelDataAck>();
- ackEv->Record.SetSeqNo(inputChannel.LastRecvSeqNo);
- ackEv->Record.SetChannelId(inputChannel.ChannelId);
- ackEv->Record.SetFreeSpace(freeSpace);
-
+void TDqComputeActorChannels::SendChannelDataAck(TInputChannelState& inputChannel, i64 freeSpace) {
+ LOG_D("Sending channel data ack to"
+ << " channelId: " << inputChannel.ChannelId
+ << ", peer: " << *inputChannel.Peer
+ << ", from: " << Owner
+ << ", freeSpace: " << freeSpace
+ << ", seqNo: " << inputChannel.LastRecvSeqNo
+ << ", finished: " << inputChannel.Finished);
+
+ inputChannel.InFlight.emplace(
+ inputChannel.LastRecvSeqNo,
+ TInputChannelState::TInFlightMessage(
+ inputChannel.LastRecvSeqNo,
+ freeSpace
+ )
+ );
+
+ auto ackEv = MakeHolder<TEvDqCompute::TEvChannelDataAck>();
+ ackEv->Record.SetSeqNo(inputChannel.LastRecvSeqNo);
+ ackEv->Record.SetChannelId(inputChannel.ChannelId);
+ ackEv->Record.SetFreeSpace(freeSpace);
+
ui32 flags = (inputChannel.Finished && !SupportCheckpoints) ? 0 : CalcMessageFlags(*inputChannel.Peer);
- Send(*inputChannel.Peer, ackEv.Release(), flags, /* cookie */ inputChannel.ChannelId);
-}
-
-ui32 TDqComputeActorChannels::CalcMessageFlags(const TActorId& peer) {
- ui32 flags = IEventHandle::FlagTrackDelivery;
- if (TrackingNodes.insert(peer.NodeId()).second) {
- flags |= IEventHandle::FlagSubscribeOnSession;
- }
- return flags;
-}
-
-TDqComputeActorChannels::TInputChannelState& TDqComputeActorChannels::InCh(ui64 channelId) {
- auto ch = InputChannelsMap.FindPtr(channelId);
- YQL_ENSURE(ch, "task: " << TaskId << ", unknown input channelId: " << channelId);
- return *ch;
-}
-
-TDqComputeActorChannels::TOutputChannelState& TDqComputeActorChannels::OutCh(ui64 channelId) {
- auto ch = OutputChannelsMap.FindPtr(channelId);
- YQL_ENSURE(ch, "task: " << TaskId << ", unknown output channelId: " << channelId);
- return *ch;
-}
-
+ Send(*inputChannel.Peer, ackEv.Release(), flags, /* cookie */ inputChannel.ChannelId);
+}
+
+ui32 TDqComputeActorChannels::CalcMessageFlags(const TActorId& peer) {
+ ui32 flags = IEventHandle::FlagTrackDelivery;
+ if (TrackingNodes.insert(peer.NodeId()).second) {
+ flags |= IEventHandle::FlagSubscribeOnSession;
+ }
+ return flags;
+}
+
+TDqComputeActorChannels::TInputChannelState& TDqComputeActorChannels::InCh(ui64 channelId) {
+ auto ch = InputChannelsMap.FindPtr(channelId);
+ YQL_ENSURE(ch, "task: " << TaskId << ", unknown input channelId: " << channelId);
+ return *ch;
+}
+
+TDqComputeActorChannels::TOutputChannelState& TDqComputeActorChannels::OutCh(ui64 channelId) {
+ auto ch = OutputChannelsMap.FindPtr(channelId);
+ YQL_ENSURE(ch, "task: " << TaskId << ", unknown output channelId: " << channelId);
+ return *ch;
+}
+
void TDqComputeActorChannels::SetCheckpointsSupport() {
SupportCheckpoints = true;
}
-} // namespace NYql::NDq
+} // namespace NYql::NDq
diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.h
index 7992e5ad6c..648b1d8b56 100644
--- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.h
+++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.h
@@ -1,188 +1,188 @@
-#pragma once
-
+#pragma once
+
#include "dq_compute_actor.h"
-
+
#include <ydb/core/kqp/kqp.h>
-
+
#include <library/cpp/actors/core/actor.h>
#include <library/cpp/actors/interconnect/interconnect.h>
-
-
-namespace NYql::NDq {
-
-class TDqComputeActorChannels : public NActors::TActor<TDqComputeActorChannels> {
-public:
- struct TPeerState {
- i64 PeerFreeSpace = 0;
- ui64 InFlightBytes = 0;
- ui64 InFlightRows = 0;
- ui32 InFlightCount = 0;
- };
-
- struct ICallbacks {
- virtual i64 GetInputChannelFreeSpace(ui64 channelId) const = 0;
+
+
+namespace NYql::NDq {
+
+class TDqComputeActorChannels : public NActors::TActor<TDqComputeActorChannels> {
+public:
+ struct TPeerState {
+ i64 PeerFreeSpace = 0;
+ ui64 InFlightBytes = 0;
+ ui64 InFlightRows = 0;
+ ui32 InFlightCount = 0;
+ };
+
+ struct ICallbacks {
+ virtual i64 GetInputChannelFreeSpace(ui64 channelId) const = 0;
virtual void TakeInputChannelData(NDqProto::TChannelData&& channelData, bool ack) = 0;
- virtual void PeerFinished(ui64 channelId) = 0;
- virtual void ResumeExecution() = 0;
+ virtual void PeerFinished(ui64 channelId) = 0;
+ virtual void ResumeExecution() = 0;
virtual ~ICallbacks() = default;
- };
-
- struct TInputChannelStats {
- ui64 PollRequests = 0;
- ui64 ResentMessages = 0;
- TDuration WaitTime;
- };
-
- struct TOutputChannelStats {
- ui64 ResentMessages = 0;
- };
-
-public:
+ };
+
+ struct TInputChannelStats {
+ ui64 PollRequests = 0;
+ ui64 ResentMessages = 0;
+ TDuration WaitTime;
+ };
+
+ struct TOutputChannelStats {
+ ui64 ResentMessages = 0;
+ };
+
+public:
TDqComputeActorChannels(NActors::TActorId owner, const TTxId& txId, const NYql::NDqProto::TDqTask& task, bool retryOnUndelivery,
NDqProto::EDqStatsMode statsMode, ui64 channelBufferSize, ICallbacks* cbs, ui32 actorActivityType);
-
-private:
- STATEFN(WorkState);
- void HandleWork(TEvDqCompute::TEvChannelData::TPtr& ev);
- void HandleWork(TEvDqCompute::TEvChannelDataAck::TPtr& ev);
- void HandleWork(TEvDqCompute::TEvRetryChannelData::TPtr& ev);
- void HandleWork(TEvDqCompute::TEvRetryChannelDataAck::TPtr& ev);
- void HandleWork(NKikimr::TEvents::TEvUndelivered::TPtr& ev);
- void HandleWork(NKikimr::TEvInterconnect::TEvNodeDisconnected::TPtr& ev);
- void HandleUndeliveredEvChannelData(ui64 channelId, NKikimr::TEvents::TEvUndelivered::EReason reason);
- void HandleUndeliveredEvChannelDataAck(ui64 channelId, NKikimr::TEvents::TEvUndelivered::EReason reason);
- template <typename TChannelState, typename TRetryEvent>
- bool ScheduleRetryForChannel(TChannelState& channel, TInstant now);
-
-private:
- STATEFN(DeadState);
- void HandlePoison(NKikimr::TEvents::TEvPoison::TPtr&);
-
-private:
- TInstant Now() const;
- void RuntimeError(const TString& message);
- void InternalError(const TString& message);
- void PassAway() override;
-
-public:
+
+private:
+ STATEFN(WorkState);
+ void HandleWork(TEvDqCompute::TEvChannelData::TPtr& ev);
+ void HandleWork(TEvDqCompute::TEvChannelDataAck::TPtr& ev);
+ void HandleWork(TEvDqCompute::TEvRetryChannelData::TPtr& ev);
+ void HandleWork(TEvDqCompute::TEvRetryChannelDataAck::TPtr& ev);
+ void HandleWork(NKikimr::TEvents::TEvUndelivered::TPtr& ev);
+ void HandleWork(NKikimr::TEvInterconnect::TEvNodeDisconnected::TPtr& ev);
+ void HandleUndeliveredEvChannelData(ui64 channelId, NKikimr::TEvents::TEvUndelivered::EReason reason);
+ void HandleUndeliveredEvChannelDataAck(ui64 channelId, NKikimr::TEvents::TEvUndelivered::EReason reason);
+ template <typename TChannelState, typename TRetryEvent>
+ bool ScheduleRetryForChannel(TChannelState& channel, TInstant now);
+
+private:
+ STATEFN(DeadState);
+ void HandlePoison(NKikimr::TEvents::TEvPoison::TPtr&);
+
+private:
+ TInstant Now() const;
+ void RuntimeError(const TString& message);
+ void InternalError(const TString& message);
+ void PassAway() override;
+
+public:
void SetCheckpointsSupport(); // Finished channels will be polled for checkpoints.
void SetInputChannelPeer(ui64 channelId, const NActors::TActorId& peer);
- void SetOutputChannelPeer(ui64 channelId, const NActors::TActorId& peer);
- bool CanSendChannelData(ui64 channelId);
- void SendChannelData(NDqProto::TChannelData&& channelData);
+ void SetOutputChannelPeer(ui64 channelId, const NActors::TActorId& peer);
+ bool CanSendChannelData(ui64 channelId);
+ void SendChannelData(NDqProto::TChannelData&& channelData);
void SendChannelDataAck(i64 channelId, i64 freeSpace);
- bool PollChannel(ui64 channelId, i64 freeSpace);
- bool CheckInFlight(const TString& prefix);
- bool FinishInputChannels();
- const TPeerState& GetOutputChannelInFlightState(ui64 channelId);
- const TInputChannelStats* GetInputChannelStats(ui64 channelId);
- const TOutputChannelStats* GetOutputChannelStats(ui64 channelId);
-
-private:
- struct TChannelRetryState {
- const TInstant Deadline;
- TInstant RetryAt;
- ui32 AttemptNo = 0;
- bool RetryScheduled = false;
-
- explicit TChannelRetryState(TInstant now)
- : Deadline(now + GetDqExecutionSettings().FlowControl.OutputChannelDeliveryInterval)
- , RetryAt(now + GetDqExecutionSettings().FlowControl.OutputChannelRetryInterval)
- , AttemptNo(0) {}
-
- TMaybe<TInstant> CalcNextRetry(TInstant now) {
- if (!RetryScheduled) {
- ++AttemptNo;
- RetryAt = now + GetDqExecutionSettings().FlowControl.OutputChannelRetryInterval * AttemptNo;
- }
- if (RetryAt < Deadline) {
- return RetryAt;
- }
- return Nothing();
- }
- };
-
- struct TInputChannelState {
- ui64 ChannelId = 0;
- std::optional<NActors::TActorId> Peer;
- ui64 LastRecvSeqNo = 0;
- bool Finished = false;
-
- struct TInFlightMessage {
- TInFlightMessage(ui64 seqNo, i64 freeSpace)
- : SeqNo(seqNo)
- , FreeSpace(freeSpace) {}
-
- const ui64 SeqNo;
- const i64 FreeSpace;
- };
- TMap<ui64, TInFlightMessage> InFlight;
-
- std::optional<TChannelRetryState> RetryState;
-
- struct TPollRequest {
- TPollRequest(ui64 seqNo, i64 freeSpace)
- : SeqNo(seqNo)
- , FreeSpace(freeSpace) {}
-
- const ui64 SeqNo;
- const i64 FreeSpace;
- };
- std::optional<TPollRequest> PollRequest;
- std::optional<TInstant> StartPollTime; // used only if Stats defined
-
- bool IsFromNode(ui32 nodeId) const {
- return Peer && Peer->NodeId() == nodeId;
- }
-
- std::unique_ptr<TInputChannelStats> Stats;
- };
-
- struct TOutputChannelState {
- ui64 ChannelId = 0;
- std::optional<NActors::TActorId> Peer;
- ui64 LastSentSeqNo = 0;
- bool Finished = false;
- bool EarlyFinish = false;
-
- struct TInFlightMessage {
+ bool PollChannel(ui64 channelId, i64 freeSpace);
+ bool CheckInFlight(const TString& prefix);
+ bool FinishInputChannels();
+ const TPeerState& GetOutputChannelInFlightState(ui64 channelId);
+ const TInputChannelStats* GetInputChannelStats(ui64 channelId);
+ const TOutputChannelStats* GetOutputChannelStats(ui64 channelId);
+
+private:
+ struct TChannelRetryState {
+ const TInstant Deadline;
+ TInstant RetryAt;
+ ui32 AttemptNo = 0;
+ bool RetryScheduled = false;
+
+ explicit TChannelRetryState(TInstant now)
+ : Deadline(now + GetDqExecutionSettings().FlowControl.OutputChannelDeliveryInterval)
+ , RetryAt(now + GetDqExecutionSettings().FlowControl.OutputChannelRetryInterval)
+ , AttemptNo(0) {}
+
+ TMaybe<TInstant> CalcNextRetry(TInstant now) {
+ if (!RetryScheduled) {
+ ++AttemptNo;
+ RetryAt = now + GetDqExecutionSettings().FlowControl.OutputChannelRetryInterval * AttemptNo;
+ }
+ if (RetryAt < Deadline) {
+ return RetryAt;
+ }
+ return Nothing();
+ }
+ };
+
+ struct TInputChannelState {
+ ui64 ChannelId = 0;
+ std::optional<NActors::TActorId> Peer;
+ ui64 LastRecvSeqNo = 0;
+ bool Finished = false;
+
+ struct TInFlightMessage {
+ TInFlightMessage(ui64 seqNo, i64 freeSpace)
+ : SeqNo(seqNo)
+ , FreeSpace(freeSpace) {}
+
+ const ui64 SeqNo;
+ const i64 FreeSpace;
+ };
+ TMap<ui64, TInFlightMessage> InFlight;
+
+ std::optional<TChannelRetryState> RetryState;
+
+ struct TPollRequest {
+ TPollRequest(ui64 seqNo, i64 freeSpace)
+ : SeqNo(seqNo)
+ , FreeSpace(freeSpace) {}
+
+ const ui64 SeqNo;
+ const i64 FreeSpace;
+ };
+ std::optional<TPollRequest> PollRequest;
+ std::optional<TInstant> StartPollTime; // used only if Stats defined
+
+ bool IsFromNode(ui32 nodeId) const {
+ return Peer && Peer->NodeId() == nodeId;
+ }
+
+ std::unique_ptr<TInputChannelStats> Stats;
+ };
+
+ struct TOutputChannelState {
+ ui64 ChannelId = 0;
+ std::optional<NActors::TActorId> Peer;
+ ui64 LastSentSeqNo = 0;
+ bool Finished = false;
+ bool EarlyFinish = false;
+
+ struct TInFlightMessage {
TInFlightMessage(ui64 seqNo, NYql::NDqProto::TChannelData&& data, bool finished)
- : SeqNo(seqNo)
- , Data(std::move(data))
- , Finished(finished) {}
-
- const ui64 SeqNo;
+ : SeqNo(seqNo)
+ , Data(std::move(data))
+ , Finished(finished) {}
+
+ const ui64 SeqNo;
const NYql::NDqProto::TChannelData Data;
- const bool Finished;
- };
- TMap<ui64, TInFlightMessage> InFlight;
- TPeerState PeerState;
-
- std::optional<TChannelRetryState> RetryState;
-
- bool IsToNode(ui32 nodeId) const {
- return Peer && Peer->NodeId() == nodeId;
- }
-
- std::unique_ptr<TOutputChannelStats> Stats;
- };
-
- void SendChannelDataAck(TInputChannelState& inputChannel, i64 freeSpace);
- ui32 CalcMessageFlags(const NActors::TActorId& peer);
- TInputChannelState& InCh(ui64 channelId);
- TOutputChannelState& OutCh(ui64 channelId);
-
-private:
- const NActors::TActorId Owner;
+ const bool Finished;
+ };
+ TMap<ui64, TInFlightMessage> InFlight;
+ TPeerState PeerState;
+
+ std::optional<TChannelRetryState> RetryState;
+
+ bool IsToNode(ui32 nodeId) const {
+ return Peer && Peer->NodeId() == nodeId;
+ }
+
+ std::unique_ptr<TOutputChannelStats> Stats;
+ };
+
+ void SendChannelDataAck(TInputChannelState& inputChannel, i64 freeSpace);
+ ui32 CalcMessageFlags(const NActors::TActorId& peer);
+ TInputChannelState& InCh(ui64 channelId);
+ TOutputChannelState& OutCh(ui64 channelId);
+
+private:
+ const NActors::TActorId Owner;
const TTxId TxId;
- const ui64 TaskId;
- const bool RetryOnUndelivery;
+ const ui64 TaskId;
+ const bool RetryOnUndelivery;
bool SupportCheckpoints = false;
- ICallbacks* const Cbs;
- THashSet<ui32> TrackingNodes;
- THashMap<ui64, TInputChannelState> InputChannelsMap;
- THashMap<ui64, TOutputChannelState> OutputChannelsMap;
-};
-
-} // namespace NYql::NDq
+ ICallbacks* const Cbs;
+ THashSet<ui32> TrackingNodes;
+ THashMap<ui64, TInputChannelState> InputChannelsMap;
+ THashMap<ui64, TOutputChannelState> OutputChannelsMap;
+};
+
+} // namespace NYql::NDq
diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.cpp b/ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.cpp
index 6cb73c709c..9d943cbca3 100644
--- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.cpp
+++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.cpp
@@ -22,7 +22,7 @@
#define LOG_CP_E(s) \
LOG_E("[Checkpoint " << MakeStringForLog(*PendingCheckpoint.Checkpoint) << "] " << s)
-namespace NYql::NDq {
+namespace NYql::NDq {
using namespace NActors;
@@ -167,7 +167,7 @@ bool TDqComputeActorCheckpoints::ShouldIgnoreOldCoordinator(const E& ev, bool ve
return false;
}
-void TDqComputeActorCheckpoints::Handle(TEvDqCompute::TEvNewCheckpointCoordinator::TPtr& ev) {
+void TDqComputeActorCheckpoints::Handle(TEvDqCompute::TEvNewCheckpointCoordinator::TPtr& ev) {
if (ShouldIgnoreOldCoordinator(ev, false)) {
return;
}
@@ -200,7 +200,7 @@ void TDqComputeActorCheckpoints::Handle(TEvDqCompute::TEvNewCheckpointCoordinato
}
}
-void TDqComputeActorCheckpoints::Handle(TEvDqCompute::TEvInjectCheckpoint::TPtr& ev) {
+void TDqComputeActorCheckpoints::Handle(TEvDqCompute::TEvInjectCheckpoint::TPtr& ev) {
if (ShouldIgnoreOldCoordinator(ev)) {
return;
}
@@ -213,7 +213,7 @@ void TDqComputeActorCheckpoints::Handle(TEvDqCompute::TEvInjectCheckpoint::TPtr&
ComputeActor->ResumeExecution();
}
-void TDqComputeActorCheckpoints::Handle(TEvDqCompute::TEvSaveTaskStateResult::TPtr& ev) {
+void TDqComputeActorCheckpoints::Handle(TEvDqCompute::TEvSaveTaskStateResult::TPtr& ev) {
if (ShouldIgnoreOldCoordinator(ev)) {
return;
}
@@ -221,7 +221,7 @@ void TDqComputeActorCheckpoints::Handle(TEvDqCompute::TEvSaveTaskStateResult::TP
EventsQueue.Send(ev->Release().Release(), ev->Cookie);
}
-void TDqComputeActorCheckpoints::Handle(TEvDqCompute::TEvRestoreFromCheckpoint::TPtr& ev) {
+void TDqComputeActorCheckpoints::Handle(TEvDqCompute::TEvRestoreFromCheckpoint::TPtr& ev) {
if (ShouldIgnoreOldCoordinator(ev)) {
return;
}
@@ -273,7 +273,7 @@ void TDqComputeActorCheckpoints::Handle(TEvDqCompute::TEvRestoreFromCheckpoint::
}
}
-void TDqComputeActorCheckpoints::Handle(TEvDqCompute::TEvGetTaskStateResult::TPtr& ev) {
+void TDqComputeActorCheckpoints::Handle(TEvDqCompute::TEvGetTaskStateResult::TPtr& ev) {
if (ShouldIgnoreOldCoordinator(ev)) {
return;
}
@@ -328,7 +328,7 @@ void TDqComputeActorCheckpoints::Handle(TEvDqCompute::TEvRun::TPtr& ev) {
EventsQueue.OnEventReceived(ev);
}
-void TDqComputeActorCheckpoints::Handle(TEvDqCompute::TEvCommitState::TPtr& ev) {
+void TDqComputeActorCheckpoints::Handle(TEvDqCompute::TEvCommitState::TPtr& ev) {
if (ShouldIgnoreOldCoordinator(ev)) {
return;
}
@@ -395,10 +395,10 @@ bool TDqComputeActorCheckpoints::SaveState() {
PendingCheckpoint.Clear();
LOG_CP_E("Failed to save state: " << e.what());
- auto resultEv = MakeHolder<TEvDqCompute::TEvSaveTaskStateResult>();
+ auto resultEv = MakeHolder<TEvDqCompute::TEvSaveTaskStateResult>();
resultEv->Record.MutableCheckpoint()->CopyFrom(*PendingCheckpoint.Checkpoint);
resultEv->Record.SetTaskId(Task.GetId());
- resultEv->Record.SetStatus(NDqProto::TEvSaveTaskStateResult::INTERNAL_ERROR);
+ resultEv->Record.SetStatus(NDqProto::TEvSaveTaskStateResult::INTERNAL_ERROR);
EventsQueue.Send(std::move(resultEv));
return false;
@@ -408,7 +408,7 @@ bool TDqComputeActorCheckpoints::SaveState() {
return true;
}
-void TDqComputeActorCheckpoints::RegisterCheckpoint(const NDqProto::TCheckpoint& checkpoint, ui64 channelId) {
+void TDqComputeActorCheckpoints::RegisterCheckpoint(const NDqProto::TCheckpoint& checkpoint, ui64 channelId) {
if (!PendingCheckpoint) {
PendingCheckpoint = checkpoint;
} else {
@@ -500,4 +500,4 @@ NDqProto::ECheckpointingMode GetTaskCheckpointingMode(const NDqProto::TDqTask& t
return NDqProto::CHECKPOINTING_MODE_DISABLED;
}
-} // namespace NYql::NDq
+} // namespace NYql::NDq
diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.h
index bd5f71dca1..27fd851338 100644
--- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.h
+++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.h
@@ -18,7 +18,7 @@ namespace NYql::NDqProto {
enum ECheckpointingMode;
} // namespace NYql::NDqProto
-namespace NYql::NDq {
+namespace NYql::NDq {
NDqProto::ECheckpointingMode GetTaskCheckpointingMode(const NDqProto::TDqTask& task);
@@ -90,7 +90,7 @@ public:
bool ComputeActorStateSaved() const;
void DoCheckpoint();
bool SaveState();
- void RegisterCheckpoint(const NDqProto::TCheckpoint& checkpoint, ui64 channelId);
+ void RegisterCheckpoint(const NDqProto::TCheckpoint& checkpoint, ui64 channelId);
// Sink actor support.
void OnSinkStateSaved(NDqProto::TSinkState&& state, ui64 outputIndex, const NDqProto::TCheckpoint& checkpoint);
@@ -99,12 +99,12 @@ public:
private:
STATEFN(StateFunc);
- void Handle(TEvDqCompute::TEvNewCheckpointCoordinator::TPtr&);
- void Handle(TEvDqCompute::TEvInjectCheckpoint::TPtr&);
- void Handle(TEvDqCompute::TEvSaveTaskStateResult::TPtr&);
- void Handle(TEvDqCompute::TEvCommitState::TPtr&);
- void Handle(TEvDqCompute::TEvRestoreFromCheckpoint::TPtr&);
- void Handle(TEvDqCompute::TEvGetTaskStateResult::TPtr&);
+ void Handle(TEvDqCompute::TEvNewCheckpointCoordinator::TPtr&);
+ void Handle(TEvDqCompute::TEvInjectCheckpoint::TPtr&);
+ void Handle(TEvDqCompute::TEvSaveTaskStateResult::TPtr&);
+ void Handle(TEvDqCompute::TEvCommitState::TPtr&);
+ void Handle(TEvDqCompute::TEvRestoreFromCheckpoint::TPtr&);
+ void Handle(TEvDqCompute::TEvGetTaskStateResult::TPtr&);
void Handle(TEvDqCompute::TEvRun::TPtr& ev);
void Handle(NActors::TEvents::TEvPoison::TPtr&);
void Handle(NActors::TEvInterconnect::TEvNodeDisconnected::TPtr& ev);
@@ -119,7 +119,7 @@ private:
private:
const TTxId TxId;
- const NDqProto::TDqTask Task;
+ const NDqProto::TDqTask Task;
const bool IngressTask;
const NActors::TActorId CheckpointStorage;
@@ -135,4 +135,4 @@ private:
NYql::NDqProto::NDqStateLoadPlan::TTaskPlan TaskLoadPlan;
};
-} // namespace NYql::NDq
+} // namespace NYql::NDq
diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h
index 28e4388098..deaadefb42 100644
--- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h
+++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h
@@ -1,5 +1,5 @@
#pragma once
-
+
#include "dq_compute_actor.h"
#include "dq_compute_actor_channels.h"
#include "dq_compute_actor_checkpoints.h"
@@ -10,7 +10,7 @@
#include <ydb/core/scheme/scheme_tabledefs.h> // TODO: TTableId
#include <ydb/core/base/kikimr_issue.h>
#include <ydb/core/tablet_flat/util_basics.h> // TODO: IDestructable
-
+
#include <ydb/library/yql/dq/actors/protos/dq_events.pb.h>
#include <ydb/library/yql/dq/common/dq_common.h>
#include <ydb/library/yql/dq/proto/dq_tasks.pb.h>
@@ -19,35 +19,35 @@
#include <ydb/library/yql/public/issue/yql_issue_message.h>
#include <library/cpp/actors/core/interconnect.h>
-
-#include <util/generic/size_literals.h>
-#include <util/string/join.h>
-#include <util/system/hostname.h>
-
-#if defined CA_LOG_D || defined CA_LOG_I || defined CA_LOG_E || defined CA_LOG_C
-# error log macro definition clash
-#endif
-
-#define CA_LOG_T(s) \
- LOG_TRACE_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "SelfId: " << this->SelfId() << ", TxId: " << TxId << ", task: " << Task.GetId() << ". " << s)
-#define CA_LOG_D(s) \
- LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "SelfId: " << this->SelfId() << ", TxId: " << TxId << ", task: " << Task.GetId() << ". " << s)
-#define CA_LOG_I(s) \
- LOG_INFO_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "SelfId: " << this->SelfId() << ", TxId: " << TxId << ", task: " << Task.GetId() << ". " << s)
-#define CA_LOG_W(s) \
- LOG_WARN_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "SelfId: " << this->SelfId() << ", TxId: " << TxId << ", task: " << Task.GetId() << ". " << s)
-#define CA_LOG_N(s) \
- LOG_NOTICE_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "SelfId: " << this->SelfId() << ", TxId: " << TxId << ", task: " << Task.GetId() << ". " << s)
-#define CA_LOG_E(s) \
- LOG_ERROR_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "SelfId: " << this->SelfId() << ", TxId: " << TxId << ", task: " << Task.GetId() << ". " << s)
-#define CA_LOG_C(s) \
- LOG_CRIT_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "SelfId: " << this->SelfId() << ", TxId: " << TxId << ", task: " << Task.GetId() << ". " << s)
-#define CA_LOG(prio, s) \
- LOG_LOG_S(*NActors::TlsActivationContext, prio, NKikimrServices::KQP_COMPUTE, "SelfId: " << this->SelfId() << ", TxId: " << TxId << ", task: " << Task.GetId() << ". " << s)
-
-
-namespace NYql {
-namespace NDq {
+
+#include <util/generic/size_literals.h>
+#include <util/string/join.h>
+#include <util/system/hostname.h>
+
+#if defined CA_LOG_D || defined CA_LOG_I || defined CA_LOG_E || defined CA_LOG_C
+# error log macro definition clash
+#endif
+
+#define CA_LOG_T(s) \
+ LOG_TRACE_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "SelfId: " << this->SelfId() << ", TxId: " << TxId << ", task: " << Task.GetId() << ". " << s)
+#define CA_LOG_D(s) \
+ LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "SelfId: " << this->SelfId() << ", TxId: " << TxId << ", task: " << Task.GetId() << ". " << s)
+#define CA_LOG_I(s) \
+ LOG_INFO_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "SelfId: " << this->SelfId() << ", TxId: " << TxId << ", task: " << Task.GetId() << ". " << s)
+#define CA_LOG_W(s) \
+ LOG_WARN_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "SelfId: " << this->SelfId() << ", TxId: " << TxId << ", task: " << Task.GetId() << ". " << s)
+#define CA_LOG_N(s) \
+ LOG_NOTICE_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "SelfId: " << this->SelfId() << ", TxId: " << TxId << ", task: " << Task.GetId() << ". " << s)
+#define CA_LOG_E(s) \
+ LOG_ERROR_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "SelfId: " << this->SelfId() << ", TxId: " << TxId << ", task: " << Task.GetId() << ". " << s)
+#define CA_LOG_C(s) \
+ LOG_CRIT_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "SelfId: " << this->SelfId() << ", TxId: " << TxId << ", task: " << Task.GetId() << ". " << s)
+#define CA_LOG(prio, s) \
+ LOG_LOG_S(*NActors::TlsActivationContext, prio, NKikimrServices::KQP_COMPUTE, "SelfId: " << this->SelfId() << ", TxId: " << TxId << ", task: " << Task.GetId() << ". " << s)
+
+
+namespace NYql {
+namespace NDq {
enum : ui64 {
ComputeActorNonProtobufStateVersion = 1,
@@ -55,28 +55,28 @@ enum : ui64 {
};
constexpr ui32 IssuesBufferSize = 16;
-
-template<typename TDerived>
-class TDqComputeActorBase : public NActors::TActorBootstrapped<TDerived>
- , public TDqComputeActorChannels::ICallbacks
+
+template<typename TDerived>
+class TDqComputeActorBase : public NActors::TActorBootstrapped<TDerived>
+ , public TDqComputeActorChannels::ICallbacks
, public TDqComputeActorCheckpoints::ICallbacks
, public IDqSourceActor::ICallbacks
, public IDqSinkActor::ICallbacks
-{
+{
protected:
- enum EEvWakeupTag : ui64 {
- TimeoutTag = 1,
+ enum EEvWakeupTag : ui64 {
+ TimeoutTag = 1,
PeriodicStatsTag = 2,
RlSendAllowedTag = 101,
RlNoResourceTag = 102,
- };
-
+ };
+
public:
void Bootstrap() {
try {
CA_LOG_D("Start compute actor " << this->SelfId() << ", task: " << Task.GetId());
- Channels = new TDqComputeActorChannels(this->SelfId(), TxId, Task, !RuntimeSettings.FailOnUndelivery,
+ Channels = new TDqComputeActorChannels(this->SelfId(), TxId, Task, !RuntimeSettings.FailOnUndelivery,
RuntimeSettings.StatsMode, MemoryLimits.ChannelBufferSize, this, this->GetActivityType());
this->RegisterWithSameMailbox(Channels);
@@ -94,82 +94,82 @@ public:
if (SayHelloOnBootstrap()) {
// say "Hello" to executer
- auto ev = MakeHolder<TEvDqCompute::TEvState>();
- ev->Record.SetState(NDqProto::COMPUTE_STATE_EXECUTING);
+ auto ev = MakeHolder<TEvDqCompute::TEvState>();
+ ev->Record.SetState(NDqProto::COMPUTE_STATE_EXECUTING);
ev->Record.SetTaskId(Task.GetId());
- this->Send(ExecuterId, ev.Release(), NActors::IEventHandle::FlagTrackDelivery);
+ this->Send(ExecuterId, ev.Release(), NActors::IEventHandle::FlagTrackDelivery);
this->Become(&TDqComputeActorBase::StateFuncBase);
}
static_cast<TDerived*>(this)->DoBootstrap();
- } catch (const NKikimr::TMemoryLimitExceededException& e) {
- InternalError(TIssuesIds::KIKIMR_PRECONDITION_FAILED, TStringBuilder()
+ } catch (const NKikimr::TMemoryLimitExceededException& e) {
+ InternalError(TIssuesIds::KIKIMR_PRECONDITION_FAILED, TStringBuilder()
<< "Mkql memory limit exceeded, limit: " << MkqlMemoryLimit
<< ", host: " << HostName()
<< ", canAllocateExtraMemory: " << CanAllocateExtraMemory);
} catch (const yexception& e) {
- InternalError(TIssuesIds::DEFAULT_ERROR, e.what());
+ InternalError(TIssuesIds::DEFAULT_ERROR, e.what());
}
-
- ReportEventElapsedTime();
+
+ ReportEventElapsedTime();
}
protected:
- TDqComputeActorBase(const NActors::TActorId& executerId, const TTxId& txId, NDqProto::TDqTask&& task,
- IDqSourceActorFactory::TPtr sourceActorFactory, IDqSinkActorFactory::TPtr sinkActorFactory,
+ TDqComputeActorBase(const NActors::TActorId& executerId, const TTxId& txId, NDqProto::TDqTask&& task,
+ IDqSourceActorFactory::TPtr sourceActorFactory, IDqSinkActorFactory::TPtr sinkActorFactory,
const TComputeRuntimeSettings& settings, const TComputeMemoryLimits& memoryLimits)
- : ExecuterId(executerId)
- , TxId(txId)
- , Task(std::move(task))
- , RuntimeSettings(settings)
- , MemoryLimits(memoryLimits)
- , CanAllocateExtraMemory(RuntimeSettings.ExtraMemoryAllocationPool != 0 && MemoryLimits.AllocateMemoryFn)
+ : ExecuterId(executerId)
+ , TxId(txId)
+ , Task(std::move(task))
+ , RuntimeSettings(settings)
+ , MemoryLimits(memoryLimits)
+ , CanAllocateExtraMemory(RuntimeSettings.ExtraMemoryAllocationPool != 0 && MemoryLimits.AllocateMemoryFn)
, SourceActorFactory(std::move(sourceActorFactory))
, SinkActorFactory(std::move(sinkActorFactory))
, CheckpointingMode(GetTaskCheckpointingMode(Task))
, State(Task.GetCreateSuspended() ? NDqProto::COMPUTE_STATE_UNKNOWN : NDqProto::COMPUTE_STATE_EXECUTING)
, Running(!Task.GetCreateSuspended())
- {
- if (RuntimeSettings.StatsMode >= NDqProto::DQ_STATS_MODE_BASIC) {
- BasicStats = std::make_unique<TBasicStats>();
- }
- if (RuntimeSettings.StatsMode >= NDqProto::DQ_STATS_MODE_PROFILE) {
- ProfileStats = std::make_unique<TProfileStats>();
- }
- InitializeTask();
- }
-
- TDqComputeActorBase(const NActors::TActorId& executerId, const TTxId& txId, const NDqProto::TDqTask& task,
- IDqSourceActorFactory::TPtr sourceActorFactory, IDqSinkActorFactory::TPtr sinkActorFactory,
+ {
+ if (RuntimeSettings.StatsMode >= NDqProto::DQ_STATS_MODE_BASIC) {
+ BasicStats = std::make_unique<TBasicStats>();
+ }
+ if (RuntimeSettings.StatsMode >= NDqProto::DQ_STATS_MODE_PROFILE) {
+ ProfileStats = std::make_unique<TProfileStats>();
+ }
+ InitializeTask();
+ }
+
+ TDqComputeActorBase(const NActors::TActorId& executerId, const TTxId& txId, const NDqProto::TDqTask& task,
+ IDqSourceActorFactory::TPtr sourceActorFactory, IDqSinkActorFactory::TPtr sinkActorFactory,
const TComputeRuntimeSettings& settings, const TComputeMemoryLimits& memoryLimits)
- : ExecuterId(executerId)
- , TxId(txId)
- , Task(task)
- , RuntimeSettings(settings)
- , MemoryLimits(memoryLimits)
- , CanAllocateExtraMemory(RuntimeSettings.ExtraMemoryAllocationPool != 0 && MemoryLimits.AllocateMemoryFn)
+ : ExecuterId(executerId)
+ , TxId(txId)
+ , Task(task)
+ , RuntimeSettings(settings)
+ , MemoryLimits(memoryLimits)
+ , CanAllocateExtraMemory(RuntimeSettings.ExtraMemoryAllocationPool != 0 && MemoryLimits.AllocateMemoryFn)
, SourceActorFactory(std::move(sourceActorFactory))
, SinkActorFactory(std::move(sinkActorFactory))
, State(Task.GetCreateSuspended() ? NDqProto::COMPUTE_STATE_UNKNOWN : NDqProto::COMPUTE_STATE_EXECUTING)
, Running(!Task.GetCreateSuspended())
- {
- if (RuntimeSettings.StatsMode >= NDqProto::DQ_STATS_MODE_BASIC) {
- BasicStats = std::make_unique<TBasicStats>();
- }
- if (RuntimeSettings.StatsMode >= NDqProto::DQ_STATS_MODE_PROFILE) {
- ProfileStats = std::make_unique<TProfileStats>();
- }
- InitializeTask();
- }
-
- void ReportEventElapsedTime() {
- if (BasicStats) {
- ui64 elapsedMicros = NActors::TlsActivationContext->GetCurrentEventTicksAsSeconds() * 1'000'000;
- BasicStats->CpuTime += TDuration::MicroSeconds(elapsedMicros);
- }
- }
-
+ {
+ if (RuntimeSettings.StatsMode >= NDqProto::DQ_STATS_MODE_BASIC) {
+ BasicStats = std::make_unique<TBasicStats>();
+ }
+ if (RuntimeSettings.StatsMode >= NDqProto::DQ_STATS_MODE_PROFILE) {
+ ProfileStats = std::make_unique<TProfileStats>();
+ }
+ InitializeTask();
+ }
+
+ void ReportEventElapsedTime() {
+ if (BasicStats) {
+ ui64 elapsedMicros = NActors::TlsActivationContext->GetCurrentEventTicksAsSeconds() * 1'000'000;
+ BasicStats->CpuTime += TDuration::MicroSeconds(elapsedMicros);
+ }
+ }
+
TString GetEventTypeString(TAutoPtr<::NActors::IEventHandle>& ev) {
try {
if (NActors::IEventBase* eventBase = ev->GetBase()) {
@@ -180,56 +180,56 @@ protected:
return "Unknown type";
}
- STFUNC(StateFuncBase) {
- const bool reportTime = this->CurrentStateFunc() == &TDqComputeActorBase::StateFuncBase;
-
- try {
- switch (ev->GetTypeRewrite()) {
- hFunc(TEvDqCompute::TEvResumeExecution, HandleExecuteBase);
- hFunc(TEvDqCompute::TEvChannelsInfo, HandleExecuteBase);
- hFunc(TEvDq::TEvAbortExecution, HandleExecuteBase);
+ STFUNC(StateFuncBase) {
+ const bool reportTime = this->CurrentStateFunc() == &TDqComputeActorBase::StateFuncBase;
+
+ try {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvDqCompute::TEvResumeExecution, HandleExecuteBase);
+ hFunc(TEvDqCompute::TEvChannelsInfo, HandleExecuteBase);
+ hFunc(TEvDq::TEvAbortExecution, HandleExecuteBase);
hFunc(NActors::TEvents::TEvWakeup, HandleExecuteBase);
hFunc(NActors::TEvents::TEvUndelivered, HandleExecuteBase);
- FFunc(TEvDqCompute::TEvChannelData::EventType, Channels->Receive);
- FFunc(TEvDqCompute::TEvChannelDataAck::EventType, Channels->Receive);
- hFunc(TEvDqCompute::TEvRun, HandleExecuteBase);
+ FFunc(TEvDqCompute::TEvChannelData::EventType, Channels->Receive);
+ FFunc(TEvDqCompute::TEvChannelDataAck::EventType, Channels->Receive);
+ hFunc(TEvDqCompute::TEvRun, HandleExecuteBase);
hFunc(TEvDqCompute::TEvStateRequest, HandleExecuteBase);
- hFunc(TEvDqCompute::TEvNewCheckpointCoordinator, HandleExecuteBase);
- FFunc(TEvDqCompute::TEvInjectCheckpoint::EventType, Checkpoints->Receive);
- FFunc(TEvDqCompute::TEvCommitState::EventType, Checkpoints->Receive);
- FFunc(TEvDqCompute::TEvRestoreFromCheckpoint::EventType, Checkpoints->Receive);
+ hFunc(TEvDqCompute::TEvNewCheckpointCoordinator, HandleExecuteBase);
+ FFunc(TEvDqCompute::TEvInjectCheckpoint::EventType, Checkpoints->Receive);
+ FFunc(TEvDqCompute::TEvCommitState::EventType, Checkpoints->Receive);
+ FFunc(TEvDqCompute::TEvRestoreFromCheckpoint::EventType, Checkpoints->Receive);
hFunc(NActors::TEvInterconnect::TEvNodeDisconnected, HandleExecuteBase);
hFunc(NActors::TEvInterconnect::TEvNodeConnected, HandleExecuteBase);
- default: {
+ default: {
CA_LOG_C("TDqComputeActorBase, unexpected event: " << ev->GetTypeRewrite() << " (" << GetEventTypeString(ev) << ")");
- InternalError(TIssuesIds::DEFAULT_ERROR, "Unexpected event");
- }
- }
- } catch (const NKikimr::TMemoryLimitExceededException& e) {
- InternalError(TIssuesIds::KIKIMR_PRECONDITION_FAILED, TStringBuilder()
+ InternalError(TIssuesIds::DEFAULT_ERROR, "Unexpected event");
+ }
+ }
+ } catch (const NKikimr::TMemoryLimitExceededException& e) {
+ InternalError(TIssuesIds::KIKIMR_PRECONDITION_FAILED, TStringBuilder()
<< "Mkql memory limit exceeded, limit: " << MkqlMemoryLimit
<< ", host: " << HostName()
<< ", canAllocateExtraMemory: " << CanAllocateExtraMemory);
- } catch (const yexception& e) {
- InternalError(TIssuesIds::DEFAULT_ERROR, e.what());
- }
-
- if (reportTime) {
- ReportEventElapsedTime();
- }
- }
-
-protected:
- void DoExecute() {
+ } catch (const yexception& e) {
+ InternalError(TIssuesIds::DEFAULT_ERROR, e.what());
+ }
+
+ if (reportTime) {
+ ReportEventElapsedTime();
+ }
+ }
+
+protected:
+ void DoExecute() {
auto guard = BindAllocator();
auto* alloc = guard.GetMutex();
-
+
if (State == NDqProto::COMPUTE_STATE_FINISHED) {
DoHandleChannelsAfterFinishImpl();
} else {
DoExecuteImpl();
}
-
+
if (alloc->GetAllocated() - alloc->GetUsed() > MemoryLimits.MinMemFreeSize) {
alloc->ReleaseFreePages();
if (MemoryLimits.FreeMemoryFn) {
@@ -243,29 +243,29 @@ protected:
}
}
}
-
- auto now = TInstant::Now();
+
+ auto now = TInstant::Now();
if (Y_UNLIKELY(ProfileStats)) {
- ProfileStats->MkqlMaxUsedMemory = std::max(ProfileStats->MkqlMaxUsedMemory, alloc->GetPeakAllocated());
- CA_LOG_D("Peak memory usage: " << ProfileStats->MkqlMaxUsedMemory);
- }
+ ProfileStats->MkqlMaxUsedMemory = std::max(ProfileStats->MkqlMaxUsedMemory, alloc->GetPeakAllocated());
+ CA_LOG_D("Peak memory usage: " << ProfileStats->MkqlMaxUsedMemory);
+ }
+
+ ReportStats(now);
+ }
- ReportStats(now);
- }
-
virtual void DoExecuteImpl() {
- auto sourcesState = GetSourcesState();
-
+ auto sourcesState = GetSourcesState();
+
PollSourceActors();
ERunStatus status = TaskRunner->Run();
- CA_LOG_D("Resume execution, run status: " << status);
+ CA_LOG_D("Resume execution, run status: " << status);
+
+ if (status != ERunStatus::Finished) {
+ PollSources(std::move(sourcesState));
+ }
- if (status != ERunStatus::Finished) {
- PollSources(std::move(sourcesState));
- }
-
if ((status == ERunStatus::PendingInput || status == ERunStatus::Finished) && Checkpoints && Checkpoints->HasPendingCheckpoint() && !Checkpoints->ComputeActorStateSaved() && ReadyToCheckpoint()) {
Checkpoints->DoCheckpoint();
}
@@ -286,40 +286,40 @@ protected:
void ProcessOutputsImpl(ERunStatus status) {
ProcessOutputsState.LastRunStatus = status;
-
+
if (ProcessOutputsState.Inflight == 0) {
ProcessOutputsState = TProcessOutputsState();
}
- for (auto& entry : OutputChannelsMap) {
+ for (auto& entry : OutputChannelsMap) {
const ui64 channelId = entry.first;
- TOutputChannelInfo& outputChannel = entry.second;
-
- if (!outputChannel.HasPeer) {
- // Channel info not complete, skip until dst info is available
+ TOutputChannelInfo& outputChannel = entry.second;
+
+ if (!outputChannel.HasPeer) {
+ // Channel info not complete, skip until dst info is available
ProcessOutputsState.ChannelsReady = false;
ProcessOutputsState.HasDataToSend = true;
ProcessOutputsState.AllOutputsFinished = false;
- CA_LOG_D("Can not drain channelId: " << channelId << ", no dst actor id");
- if (Y_UNLIKELY(outputChannel.Stats)) {
- outputChannel.Stats->NoDstActorId++;
- }
- continue;
- }
+ CA_LOG_D("Can not drain channelId: " << channelId << ", no dst actor id");
+ if (Y_UNLIKELY(outputChannel.Stats)) {
+ outputChannel.Stats->NoDstActorId++;
+ }
+ continue;
+ }
if (!outputChannel.Finished || Checkpoints) {
- if (Channels->CanSendChannelData(channelId)) {
- auto peerState = Channels->GetOutputChannelInFlightState(channelId);
+ if (Channels->CanSendChannelData(channelId)) {
+ auto peerState = Channels->GetOutputChannelInFlightState(channelId);
DrainOutputChannel(outputChannel, peerState);
} else {
ProcessOutputsState.HasDataToSend |= !outputChannel.Finished;
- }
- } else {
- CA_LOG_D("Do not drain channelId: " << channelId << ", finished");
+ }
+ } else {
+ CA_LOG_D("Do not drain channelId: " << channelId << ", finished");
ProcessOutputsState.AllOutputsFinished &= outputChannel.Finished;
- }
- }
-
+ }
+ }
+
for (auto& entry : SinksMap) {
const ui64 outputIndex = entry.first;
TSinkInfo& sinkInfo = entry.second;
@@ -341,69 +341,69 @@ protected:
status = ERunStatus::Finished;
}
- if (status != ERunStatus::Finished) {
- // If the incoming channel's buffer was full at the moment when last ChannelDataAck event had been sent,
- // there will be no attempts to send a new piece of data from the other side of this channel.
- // So, if there is space in the channel buffer (and on previous step is was full), we send ChannelDataAck
- // event with the last known seqNo, and the process on the other side of this channel updates its state
- // and sends us a new batch of data.
- bool pollSent = false;
- for (auto& [channelId, inputChannel] : InputChannelsMap) {
+ if (status != ERunStatus::Finished) {
+ // If the incoming channel's buffer was full at the moment when last ChannelDataAck event had been sent,
+ // there will be no attempts to send a new piece of data from the other side of this channel.
+ // So, if there is space in the channel buffer (and on previous step is was full), we send ChannelDataAck
+ // event with the last known seqNo, and the process on the other side of this channel updates its state
+ // and sends us a new batch of data.
+ bool pollSent = false;
+ for (auto& [channelId, inputChannel] : InputChannelsMap) {
pollSent |= Channels->PollChannel(channelId, GetInputChannelFreeSpace(channelId));
- }
- if (!pollSent) {
+ }
+ if (!pollSent) {
if (ProcessOutputsState.DataWasSent) {
- ContinueExecute();
- }
- return;
- }
- }
-
- if (status == ERunStatus::PendingOutput) {
+ ContinueExecute();
+ }
+ return;
+ }
+ }
+
+ if (status == ERunStatus::PendingOutput) {
if (ProcessOutputsState.DataWasSent) {
- // we have sent some data, so we have space in output channel(s)
- ContinueExecute();
- }
+ // we have sent some data, so we have space in output channel(s)
+ ContinueExecute();
+ }
return;
}
// Handle finishing of our task.
if (status == ERunStatus::Finished && State != NDqProto::COMPUTE_STATE_FINISHED) {
if (ProcessOutputsState.HasDataToSend || !ProcessOutputsState.ChannelsReady) {
- CA_LOG_D("Continue execution, either output buffers are not empty or not all channels are ready"
+ CA_LOG_D("Continue execution, either output buffers are not empty or not all channels are ready"
<< ", hasDataToSend: " << ProcessOutputsState.HasDataToSend << ", channelsReady: " << ProcessOutputsState.ChannelsReady);
- } else {
- if (!Channels->FinishInputChannels()) {
- CA_LOG_D("Continue execution, not all input channels are initialized");
- return;
- }
- if (Channels->CheckInFlight("Tasks execution finished")) {
+ } else {
+ if (!Channels->FinishInputChannels()) {
+ CA_LOG_D("Continue execution, not all input channels are initialized");
+ return;
+ }
+ if (Channels->CheckInFlight("Tasks execution finished")) {
State = NDqProto::COMPUTE_STATE_FINISHED;
CA_LOG_D("Compute state finished. All channels finished");
ReportStateAndMaybeDie(TIssue("success"));
- }
- }
+ }
+ }
}
}
-protected:
- void Terminate(bool success, const TString& message) {
+protected:
+ void Terminate(bool success, const TString& message) {
if (MkqlMemoryLimit && MemoryLimits.FreeMemoryFn) {
MemoryLimits.FreeMemoryFn(TxId, Task.GetId(), MkqlMemoryLimit);
MkqlMemoryLimit = 0;
}
- if (Channels) {
- TAutoPtr<NActors::IEventHandle> handle = new NActors::IEventHandle(Channels->SelfId(), this->SelfId(),
+ if (Channels) {
+ TAutoPtr<NActors::IEventHandle> handle = new NActors::IEventHandle(Channels->SelfId(), this->SelfId(),
new NActors::TEvents::TEvPoison);
- Channels->Receive(handle, NActors::TActivationContext::AsActorContext());
- }
-
+ Channels->Receive(handle, NActors::TActivationContext::AsActorContext());
+ }
+
if (Checkpoints) {
- TAutoPtr<NActors::IEventHandle> handle = new NActors::IEventHandle(Checkpoints->SelfId(), this->SelfId(),
+ TAutoPtr<NActors::IEventHandle> handle = new NActors::IEventHandle(Checkpoints->SelfId(), this->SelfId(),
new NActors::TEvents::TEvPoison);
- Checkpoints->Receive(handle, NActors::TActivationContext::AsActorContext());
+ Checkpoints->Receive(handle, NActors::TActivationContext::AsActorContext());
}
for (auto& [_, source] : SourcesMap) {
@@ -418,66 +418,66 @@ protected:
}
}
- for (auto& [_, outputChannel] : OutputChannelsMap) {
+ for (auto& [_, outputChannel] : OutputChannelsMap) {
if (outputChannel.Channel) {
outputChannel.Channel->Terminate();
}
- }
-
- if (RuntimeSettings.TerminateHandler) {
- RuntimeSettings.TerminateHandler(success, message);
- }
-
- this->PassAway();
- }
-
+ }
+
+ if (RuntimeSettings.TerminateHandler) {
+ RuntimeSettings.TerminateHandler(success, message);
+ }
+
+ this->PassAway();
+ }
+
void ReportStateAndMaybeDie(TIssue&& issue) {
ReportStateAndMaybeDie(
State == NDqProto::COMPUTE_STATE_FINISHED ?
Ydb::StatusIds::STATUS_CODE_UNSPECIFIED
: Ydb::StatusIds::ABORTED, TIssues({issue}));
- }
-
- void ReportStateAndDie(NDqProto::EComputeState state, TIssue&& issue) {
- auto execEv = MakeHolder<TEvDqCompute::TEvState>();
- auto& record = execEv->Record;
-
- record.SetState(state);
- if (state != NDqProto::COMPUTE_STATE_FINISHED) {
- record.SetStatus(Ydb::StatusIds::ABORTED);
- } else {
- record.SetStatus(Ydb::StatusIds::STATUS_CODE_UNSPECIFIED);
- }
- record.SetTaskId(Task.GetId());
- if (RuntimeSettings.StatsMode >= NDqProto::DQ_STATS_MODE_BASIC) {
- FillStats(record.MutableStats(), /* last */ true);
- }
-
- if (issue.Message) {
- IssueToMessage(issue, record.MutableIssues()->Add());
- }
-
- this->Send(ExecuterId, execEv.Release());
-
- TerminateSources(issue.Message, state == NDqProto::COMPUTE_STATE_FINISHED);
- Terminate(state == NDqProto::COMPUTE_STATE_FINISHED, issue.Message);
- }
-
+ }
+
+ void ReportStateAndDie(NDqProto::EComputeState state, TIssue&& issue) {
+ auto execEv = MakeHolder<TEvDqCompute::TEvState>();
+ auto& record = execEv->Record;
+
+ record.SetState(state);
+ if (state != NDqProto::COMPUTE_STATE_FINISHED) {
+ record.SetStatus(Ydb::StatusIds::ABORTED);
+ } else {
+ record.SetStatus(Ydb::StatusIds::STATUS_CODE_UNSPECIFIED);
+ }
+ record.SetTaskId(Task.GetId());
+ if (RuntimeSettings.StatsMode >= NDqProto::DQ_STATS_MODE_BASIC) {
+ FillStats(record.MutableStats(), /* last */ true);
+ }
+
+ if (issue.Message) {
+ IssueToMessage(issue, record.MutableIssues()->Add());
+ }
+
+ this->Send(ExecuterId, execEv.Release());
+
+ TerminateSources(issue.Message, state == NDqProto::COMPUTE_STATE_FINISHED);
+ Terminate(state == NDqProto::COMPUTE_STATE_FINISHED, issue.Message);
+ }
+
void ReportStateAndMaybeDie(Ydb::StatusIds::StatusCode status, const TIssues& issues)
- {
- auto execEv = MakeHolder<TEvDqCompute::TEvState>();
- auto& record = execEv->Record;
-
+ {
+ auto execEv = MakeHolder<TEvDqCompute::TEvState>();
+ auto& record = execEv->Record;
+
record.SetState(State);
record.SetStatus(status);
- record.SetTaskId(Task.GetId());
- if (RuntimeSettings.StatsMode >= NDqProto::DQ_STATS_MODE_BASIC) {
- FillStats(record.MutableStats(), /* last */ true);
- }
- IssuesToMessage(issues, record.MutableIssues());
-
- this->Send(ExecuterId, execEv.Release());
-
+ record.SetTaskId(Task.GetId());
+ if (RuntimeSettings.StatsMode >= NDqProto::DQ_STATS_MODE_BASIC) {
+ FillStats(record.MutableStats(), /* last */ true);
+ }
+ IssuesToMessage(issues, record.MutableIssues());
+
+ this->Send(ExecuterId, execEv.Release());
+
if (Checkpoints && State == NDqProto::COMPUTE_STATE_FINISHED) {
// checkpointed CAs must not self-destroy
return;
@@ -487,41 +487,41 @@ protected:
Terminate(State == NDqProto::COMPUTE_STATE_FINISHED, NDqProto::EComputeState_Name(State));
}
- void InternalError(TIssuesIds::EIssueCode issueCode, const TString& message) {
- CA_LOG_E(TIssuesIds::EIssueCode_Name(issueCode) << ": " << message << ".");
- TIssue issue(message);
- SetIssueCode(issueCode, issue);
+ void InternalError(TIssuesIds::EIssueCode issueCode, const TString& message) {
+ CA_LOG_E(TIssuesIds::EIssueCode_Name(issueCode) << ": " << message << ".");
+ TIssue issue(message);
+ SetIssueCode(issueCode, issue);
std::optional<TGuard<NKikimr::NMiniKQL::TScopedAlloc>> guard = MaybeBindAllocator();
State = NDqProto::COMPUTE_STATE_FAILURE;
ReportStateAndMaybeDie(std::move(issue));
}
- void ContinueExecute() {
+ void ContinueExecute() {
if (!ResumeEventScheduled && Running) {
- ResumeEventScheduled = true;
- this->Send(this->SelfId(), new TEvDqCompute::TEvResumeExecution());
- }
- }
-
-public:
- i64 GetInputChannelFreeSpace(ui64 channelId) const override {
- const TInputChannelInfo* inputChannel = InputChannelsMap.FindPtr(channelId);
- YQL_ENSURE(inputChannel, "task: " << Task.GetId() << ", unknown input channelId: " << channelId);
-
- return inputChannel->Channel->GetFreeSpace();
- }
-
+ ResumeEventScheduled = true;
+ this->Send(this->SelfId(), new TEvDqCompute::TEvResumeExecution());
+ }
+ }
+
+public:
+ i64 GetInputChannelFreeSpace(ui64 channelId) const override {
+ const TInputChannelInfo* inputChannel = InputChannelsMap.FindPtr(channelId);
+ YQL_ENSURE(inputChannel, "task: " << Task.GetId() << ", unknown input channelId: " << channelId);
+
+ return inputChannel->Channel->GetFreeSpace();
+ }
+
void TakeInputChannelData(NDqProto::TChannelData&& channelData, bool ack) override {
- TInputChannelInfo* inputChannel = InputChannelsMap.FindPtr(channelData.GetChannelId());
- YQL_ENSURE(inputChannel, "task: " << Task.GetId() << ", unknown input channelId: " << channelData.GetChannelId());
-
- auto channel = inputChannel->Channel;
-
- if (channelData.GetData().GetRows()) {
+ TInputChannelInfo* inputChannel = InputChannelsMap.FindPtr(channelData.GetChannelId());
+ YQL_ENSURE(inputChannel, "task: " << Task.GetId() << ", unknown input channelId: " << channelData.GetChannelId());
+
+ auto channel = inputChannel->Channel;
+
+ if (channelData.GetData().GetRows()) {
auto guard = BindAllocator();
- channel->Push(std::move(*channelData.MutableData()));
- }
-
+ channel->Push(std::move(*channelData.MutableData()));
+ }
+
if (channelData.HasCheckpoint()) {
Y_VERIFY(inputChannel->CheckpointingMode != NDqProto::CHECKPOINTING_MODE_DISABLED);
Y_VERIFY(Checkpoints);
@@ -530,42 +530,42 @@ public:
Checkpoints->RegisterCheckpoint(checkpoint, channelData.GetChannelId());
}
- if (channelData.GetFinished()) {
- channel->Finish();
- }
-
+ if (channelData.GetFinished()) {
+ channel->Finish();
+ }
+
if (ack) {
Channels->SendChannelDataAck(channel->GetChannelId(), channel->GetFreeSpace());
}
ResumeExecution();
- }
-
- void PeerFinished(ui64 channelId) override {
- TOutputChannelInfo* outputChannel = OutputChannelsMap.FindPtr(channelId);
- YQL_ENSURE(outputChannel, "task: " << Task.GetId() << ", output channelId: " << channelId);
-
- outputChannel->Finished = true;
- outputChannel->Channel->Finish();
-
- CA_LOG_D("task: " << Task.GetId() << ", output channelId: " << channelId << " finished prematurely, "
- << " about to clear buffer");
-
- {
+ }
+
+ void PeerFinished(ui64 channelId) override {
+ TOutputChannelInfo* outputChannel = OutputChannelsMap.FindPtr(channelId);
+ YQL_ENSURE(outputChannel, "task: " << Task.GetId() << ", output channelId: " << channelId);
+
+ outputChannel->Finished = true;
+ outputChannel->Channel->Finish();
+
+ CA_LOG_D("task: " << Task.GetId() << ", output channelId: " << channelId << " finished prematurely, "
+ << " about to clear buffer");
+
+ {
auto guard = BindAllocator();
- ui32 dropRows = outputChannel->Channel->Drop();
-
- CA_LOG_I("task: " << Task.GetId() << ", output channelId: " << channelId << " finished prematurely, "
- << "drop " << dropRows << " rows");
- }
-
- DoExecute();
- }
-
- void ResumeExecution() override {
- ContinueExecute();
- }
-
+ ui32 dropRows = outputChannel->Channel->Drop();
+
+ CA_LOG_I("task: " << Task.GetId() << ", output channelId: " << channelId << " finished prematurely, "
+ << "drop " << dropRows << " rows");
+ }
+
+ DoExecute();
+ }
+
+ void ResumeExecution() override {
+ ContinueExecute();
+ }
+
void OnSinkStateSaved(NDqProto::TSinkState&& state, ui64 outputIndex, const NDqProto::TCheckpoint& checkpoint) override {
Y_VERIFY(Checkpoints); // If we are checkpointing, we must have already constructed "checkpoints" object.
Checkpoints->OnSinkStateSaved(std::move(state), outputIndex, checkpoint);
@@ -666,12 +666,12 @@ protected:
State = NDqProto::COMPUTE_STATE_UNKNOWN;
}
-protected:
- struct TInputChannelInfo {
+protected:
+ struct TInputChannelInfo {
ui64 ChannelId;
- IDqInputChannel::TPtr Channel;
+ IDqInputChannel::TPtr Channel;
bool HasPeer = false;
- std::optional<NDqProto::TCheckpoint> PendingCheckpoint;
+ std::optional<NDqProto::TCheckpoint> PendingCheckpoint;
const NDqProto::ECheckpointingMode CheckpointingMode;
ui64 FreeSpace = 0;
@@ -682,10 +682,10 @@ protected:
}
bool IsPaused() const {
- return PendingCheckpoint.has_value();
+ return PendingCheckpoint.has_value();
}
- void Pause(const NDqProto::TCheckpoint& checkpoint) {
+ void Pause(const NDqProto::TCheckpoint& checkpoint) {
YQL_ENSURE(!IsPaused());
YQL_ENSURE(CheckpointingMode != NDqProto::CHECKPOINTING_MODE_DISABLED);
PendingCheckpoint = checkpoint;
@@ -693,11 +693,11 @@ protected:
}
void Resume() {
- PendingCheckpoint.reset();
+ PendingCheckpoint.reset();
Channel->Resume();
}
- };
-
+ };
+
struct TSourceInfo {
ui64 Index;
IDqSource::TPtr Source;
@@ -711,24 +711,24 @@ protected:
TSourceInfo(ui64 index) : Index(index), IssuesBuffer(IssuesBufferSize) {}
};
- struct TOutputChannelInfo {
+ struct TOutputChannelInfo {
ui64 ChannelId;
- IDqOutputChannel::TPtr Channel;
- bool HasPeer = false;
+ IDqOutputChannel::TPtr Channel;
+ bool HasPeer = false;
bool Finished = false; // != Channel->IsFinished() // If channel is in finished state, it sends only checkpoints.
bool PopStarted = false;
-
+
explicit TOutputChannelInfo(ui64 channelId)
: ChannelId(channelId)
{ }
- struct TStats {
- ui64 BlockedByCapacity = 0;
- ui64 NoDstActorId = 0;
- };
- THolder<TStats> Stats;
- };
-
+ struct TStats {
+ ui64 BlockedByCapacity = 0;
+ ui64 NoDstActorId = 0;
+ };
+ THolder<TStats> Stats;
+ };
+
struct TSinkInfo {
IDqSink::TPtr Sink;
IDqSinkActor* SinkActor = nullptr;
@@ -741,19 +741,19 @@ protected:
TSinkInfo() : IssuesBuffer(IssuesBufferSize) {}
};
-protected:
- // virtual methods (TODO: replace with static_cast<TDerived*>(this)->Foo()
+protected:
+ // virtual methods (TODO: replace with static_cast<TDerived*>(this)->Foo()
virtual THolder<NKikimr::IDestructable> GetSourcesState() {
- return nullptr;
+ return nullptr;
}
virtual void PollSources(THolder<NKikimr::IDestructable> /* state */) {
}
- virtual void TerminateSources(const TString& /* message */, bool /* success */) {
- }
-
+ virtual void TerminateSources(const TString& /* message */, bool /* success */) {
+ }
+
virtual TGuard<NKikimr::NMiniKQL::TScopedAlloc> BindAllocator() {
return TaskRunner->BindAllocator();
}
@@ -782,18 +782,18 @@ protected:
return true;
}
-protected:
- void HandleExecuteBase(TEvDqCompute::TEvResumeExecution::TPtr&) {
- ResumeEventScheduled = false;
+protected:
+ void HandleExecuteBase(TEvDqCompute::TEvResumeExecution::TPtr&) {
+ ResumeEventScheduled = false;
if (Running) {
- DoExecute();
+ DoExecute();
}
}
- void HandleExecuteBase(TEvDqCompute::TEvChannelsInfo::TPtr& ev) {
+ void HandleExecuteBase(TEvDqCompute::TEvChannelsInfo::TPtr& ev) {
auto& record = ev->Get()->Record;
- CA_LOG_D("Received channels info: " << record.ShortDebugString());
+ CA_LOG_D("Received channels info: " << record.ShortDebugString());
for (auto& channelUpdate : record.GetUpdate()) {
TInputChannelInfo* inputChannel = InputChannelsMap.FindPtr(channelUpdate.GetId());
@@ -808,72 +808,72 @@ protected:
continue;
}
- TOutputChannelInfo* outputChannel = OutputChannelsMap.FindPtr(channelUpdate.GetId());
+ TOutputChannelInfo* outputChannel = OutputChannelsMap.FindPtr(channelUpdate.GetId());
if (outputChannel && !outputChannel->HasPeer && channelUpdate.GetDstEndpoint().HasActorId()) {
auto peer = NActors::ActorIdFromProto(channelUpdate.GetDstEndpoint().GetActorId());
-
- CA_LOG_D("Update output channelId: " << channelUpdate.GetId() << ", peer: " << peer);
-
- Channels->SetOutputChannelPeer(channelUpdate.GetId(), peer);
- outputChannel->HasPeer = true;
+
+ CA_LOG_D("Update output channelId: " << channelUpdate.GetId() << ", peer: " << peer);
+
+ Channels->SetOutputChannelPeer(channelUpdate.GetId(), peer);
+ outputChannel->HasPeer = true;
continue;
- }
+ }
YQL_ENSURE(inputChannel || outputChannel, "Unknown channelId: " << channelUpdate.GetId() << ", task: " << Task.GetId());
}
- DoExecute();
+ DoExecute();
}
void HandleExecuteBase(NActors::TEvents::TEvWakeup::TPtr& ev) {
auto tag = (EEvWakeupTag) ev->Get()->Tag;
switch (tag) {
- case EEvWakeupTag::TimeoutTag: {
- auto abortEv = MakeHolder<TEvDq::TEvAbortExecution>(Ydb::StatusIds::TIMEOUT, TStringBuilder()
- << "Timeout event from compute actor " << this->SelfId()
- << ", TxId: " << TxId << ", task: " << Task.GetId());
-
- this->Send(ExecuterId, abortEv.Release());
-
- TerminateSources("timeout exceeded", false);
- Terminate(false, "timeout exceeded");
+ case EEvWakeupTag::TimeoutTag: {
+ auto abortEv = MakeHolder<TEvDq::TEvAbortExecution>(Ydb::StatusIds::TIMEOUT, TStringBuilder()
+ << "Timeout event from compute actor " << this->SelfId()
+ << ", TxId: " << TxId << ", task: " << Task.GetId());
+
+ this->Send(ExecuterId, abortEv.Release());
+
+ TerminateSources("timeout exceeded", false);
+ Terminate(false, "timeout exceeded");
break;
- }
+ }
case EEvWakeupTag::PeriodicStatsTag: {
const auto maxInterval = RuntimeSettings.ReportStatsSettings->MaxInterval;
this->Schedule(maxInterval, new NActors::TEvents::TEvWakeup(EEvWakeupTag::PeriodicStatsTag));
- auto now = NActors::TActivationContext::Now();
- if (now - LastSendStatsTime >= maxInterval) {
- ReportStats(now);
+ auto now = NActors::TActivationContext::Now();
+ if (now - LastSendStatsTime >= maxInterval) {
+ ReportStats(now);
}
break;
}
default:
static_cast<TDerived*>(this)->HandleEvWakeup(tag);
- }
- }
-
+ }
+ }
+
void HandleEvWakeup(EEvWakeupTag tag) {
CA_LOG_E("Unhandled wakeup tag " << (ui64)tag);
}
void HandleExecuteBase(NActors::TEvents::TEvUndelivered::TPtr& ev) {
- ui32 lostEventType = ev->Get()->SourceType;
- switch (lostEventType) {
- case TEvDqCompute::TEvState::EventType: {
- CA_LOG_E("Handle undelivered TEvState event, abort execution");
- this->TerminateSources("executer lost", false);
- Terminate(false, "executer lost");
- break;
- }
- default: {
- CA_LOG_C("Handle unexpected event undelivery: " << lostEventType);
- }
- }
- }
-
+ ui32 lostEventType = ev->Get()->SourceType;
+ switch (lostEventType) {
+ case TEvDqCompute::TEvState::EventType: {
+ CA_LOG_E("Handle undelivered TEvState event, abort execution");
+ this->TerminateSources("executer lost", false);
+ Terminate(false, "executer lost");
+ break;
+ }
+ default: {
+ CA_LOG_C("Handle unexpected event undelivery: " << lostEventType);
+ }
+ }
+ }
+
void HandleExecuteBase(TEvDqCompute::TEvRun::TPtr& ev) {
CA_LOG_D("Got TEvRun from actor " << ev->Sender);
Start();
@@ -891,7 +891,7 @@ protected:
evState->Record.SetState(NDqProto::COMPUTE_STATE_EXECUTING);
evState->Record.SetStatus(Ydb::StatusIds::SUCCESS);
evState->Record.SetTaskId(Task.GetId());
- FillStats(evState->Record.MutableStats(), /* last */ false);
+ FillStats(evState->Record.MutableStats(), /* last */ false);
this->Send(ev->Sender, evState.Release(), NActors::IEventHandle::FlagTrackDelivery, ev->Cookie);
}
@@ -902,26 +902,26 @@ protected:
Channels->SetCheckpointsSupport();
}
TAutoPtr<NActors::IEventHandle> handle = new NActors::IEventHandle(Checkpoints->SelfId(), ev->Sender, ev->Release().Release());
- Checkpoints->Receive(handle, NActors::TActivationContext::AsActorContext());
- }
-
- void HandleExecuteBase(TEvDq::TEvAbortExecution::TPtr& ev) {
- TString message = ev->Get()->Record.GetMessage();
- CA_LOG_E("Handle abort execution event from: " << ev->Sender
- << ", status: " << Ydb::StatusIds_StatusCode_Name(ev->Get()->Record.GetStatusCode())
- << ", reason: " << message);
-
- bool success = ev->Get()->Record.GetStatusCode() == Ydb::StatusIds::SUCCESS;
-
- this->TerminateSources(message, success);
-
- if (ev->Sender != ExecuterId) {
- NActors::TActivationContext::Send(ev->Forward(ExecuterId));
- }
-
- Terminate(success, message);
- }
-
+ Checkpoints->Receive(handle, NActors::TActivationContext::AsActorContext());
+ }
+
+ void HandleExecuteBase(TEvDq::TEvAbortExecution::TPtr& ev) {
+ TString message = ev->Get()->Record.GetMessage();
+ CA_LOG_E("Handle abort execution event from: " << ev->Sender
+ << ", status: " << Ydb::StatusIds_StatusCode_Name(ev->Get()->Record.GetStatusCode())
+ << ", reason: " << message);
+
+ bool success = ev->Get()->Record.GetStatusCode() == Ydb::StatusIds::SUCCESS;
+
+ this->TerminateSources(message, success);
+
+ if (ev->Sender != ExecuterId) {
+ NActors::TActivationContext::Send(ev->Forward(ExecuterId));
+ }
+
+ Terminate(success, message);
+ }
+
void HandleExecuteBase(NActors::TEvInterconnect::TEvNodeDisconnected::TPtr& ev) {
TAutoPtr<NActors::IEventHandle> iev(ev.Release());
if (Checkpoints) {
@@ -946,71 +946,71 @@ private:
virtual void DrainOutputChannel(TOutputChannelInfo& outputChannel, const TDqComputeActorChannels::TPeerState& peerState) {
YQL_ENSURE(!outputChannel.Finished || Checkpoints);
-
+
const bool wasFinished = outputChannel.Finished;
- auto channelId = outputChannel.Channel->GetChannelId();
-
+ auto channelId = outputChannel.Channel->GetChannelId();
+
const ui32 allowedOvercommit = AllowedChannelsOvercommit();
-
+
const i64 toSend = peerState.PeerFreeSpace + allowedOvercommit - peerState.InFlightBytes;
-
- CA_LOG_D("About to drain channelId: " << channelId
- << ", hasPeer: " << outputChannel.HasPeer
- << ", peerFreeSpace: " << peerState.PeerFreeSpace
- << ", inFlightBytes: " << peerState.InFlightBytes
- << ", inFlightRows: " << peerState.InFlightRows
- << ", inFlightCount: " << peerState.InFlightCount
- << ", allowedOvercommit: " << allowedOvercommit
- << ", toSend: " << toSend
- << ", finished: " << outputChannel.Channel->IsFinished());
-
+
+ CA_LOG_D("About to drain channelId: " << channelId
+ << ", hasPeer: " << outputChannel.HasPeer
+ << ", peerFreeSpace: " << peerState.PeerFreeSpace
+ << ", inFlightBytes: " << peerState.InFlightBytes
+ << ", inFlightRows: " << peerState.InFlightRows
+ << ", inFlightCount: " << peerState.InFlightCount
+ << ", allowedOvercommit: " << allowedOvercommit
+ << ", toSend: " << toSend
+ << ", finished: " << outputChannel.Channel->IsFinished());
+
ProcessOutputsState.HasDataToSend |= !outputChannel.Finished;
ProcessOutputsState.AllOutputsFinished &= outputChannel.Finished;
- if (toSend <= 0) {
- if (Y_UNLIKELY(outputChannel.Stats)) {
- outputChannel.Stats->BlockedByCapacity++;
- }
- }
-
- i64 remains = toSend;
+ if (toSend <= 0) {
+ if (Y_UNLIKELY(outputChannel.Stats)) {
+ outputChannel.Stats->BlockedByCapacity++;
+ }
+ }
+
+ i64 remains = toSend;
while (remains > 0 && (!outputChannel.Finished || Checkpoints)) {
- ui32 sent = this->SendChannelDataChunk(outputChannel, remains);
- if (sent == 0) {
- break;
- }
- remains -= sent;
- }
-
+ ui32 sent = this->SendChannelDataChunk(outputChannel, remains);
+ if (sent == 0) {
+ break;
+ }
+ remains -= sent;
+ }
+
ProcessOutputsState.HasDataToSend |= !outputChannel.Finished;
ProcessOutputsState.AllOutputsFinished &= outputChannel.Finished;
ProcessOutputsState.DataWasSent |= (!wasFinished && outputChannel.Finished) || remains != toSend;
- }
-
- ui32 SendChannelDataChunk(TOutputChannelInfo& outputChannel, ui64 bytes) {
- auto channel = outputChannel.Channel;
-
- NDqProto::TData data;
- NDqProto::TCheckpoint checkpoint;
+ }
+
+ ui32 SendChannelDataChunk(TOutputChannelInfo& outputChannel, ui64 bytes) {
+ auto channel = outputChannel.Channel;
+
+ NDqProto::TData data;
+ NDqProto::TCheckpoint checkpoint;
bool hasData = channel->Pop(data, bytes);
bool hasCheckpoint = channel->Pop(checkpoint);
if (!hasData && !hasCheckpoint) {
- if (!channel->IsFinished()) {
- CA_LOG_D("output channelId: " << channel->GetChannelId() << ", nothing to send and is not finished");
- return 0; // channel is empty and not finished yet
- }
- }
+ if (!channel->IsFinished()) {
+ CA_LOG_D("output channelId: " << channel->GetChannelId() << ", nothing to send and is not finished");
+ return 0; // channel is empty and not finished yet
+ }
+ }
const bool wasFinished = outputChannel.Finished;
- outputChannel.Finished = channel->IsFinished();
+ outputChannel.Finished = channel->IsFinished();
const bool becameFinished = !wasFinished && outputChannel.Finished;
-
- ui32 dataSize = data.GetRaw().size();
+
+ ui32 dataSize = data.GetRaw().size();
ui32 checkpointSize = checkpoint.ByteSize();
-
- NDqProto::TChannelData channelData;
- channelData.SetChannelId(channel->GetChannelId());
- channelData.SetFinished(outputChannel.Finished);
+
+ NDqProto::TChannelData channelData;
+ channelData.SetChannelId(channel->GetChannelId());
+ channelData.SetFinished(outputChannel.Finished);
if (hasData) {
channelData.MutableData()->Swap(&data);
}
@@ -1019,14 +1019,14 @@ private:
CA_LOG_I("Resume inputs");
ResumeInputs();
}
-
+
if (hasData || hasCheckpoint || becameFinished) {
Channels->SendChannelData(std::move(channelData));
return dataSize + checkpointSize;
}
return 0;
- }
-
+ }
+
virtual void DrainSink(ui64 outputIndex, TSinkInfo& sinkInfo) {
ProcessOutputsState.AllOutputsFinished &= sinkInfo.Finished;
if (sinkInfo.Finished && !Checkpoints) {
@@ -1098,40 +1098,40 @@ private:
return dataSize + checkpointSize;
}
-protected:
+protected:
const TMaybe<NDqProto::TRlPath>& GetRlPath() const {
return RuntimeSettings.RlPath;
}
TTxId GetTxId() const {
- return TxId;
- }
-
- const NDqProto::TDqTask& GetTask() const {
- return Task;
+ return TxId;
}
- NDqProto::EDqStatsMode GetStatsMode() const {
+ const NDqProto::TDqTask& GetTask() const {
+ return Task;
+ }
+
+ NDqProto::EDqStatsMode GetStatsMode() const {
return RuntimeSettings.StatsMode;
- }
-
- bool GetUseLLVM() const {
- return RuntimeSettings.UseLLVM;
- }
-
- const TComputeMemoryLimits& GetMemoryLimits() const {
- return MemoryLimits;
- }
-
-protected:
- void SetTaskRunner(const TIntrusivePtr<IDqTaskRunner>& taskRunner) {
- TaskRunner = taskRunner;
- }
-
- void PrepareTaskRunner(const IDqTaskRunnerExecutionContext& execCtx = TDqTaskRunnerExecutionContext()) {
- YQL_ENSURE(TaskRunner);
-
- auto guard = TaskRunner->BindAllocator(MkqlMemoryLimit);
+ }
+
+ bool GetUseLLVM() const {
+ return RuntimeSettings.UseLLVM;
+ }
+
+ const TComputeMemoryLimits& GetMemoryLimits() const {
+ return MemoryLimits;
+ }
+
+protected:
+ void SetTaskRunner(const TIntrusivePtr<IDqTaskRunner>& taskRunner) {
+ TaskRunner = taskRunner;
+ }
+
+ void PrepareTaskRunner(const IDqTaskRunnerExecutionContext& execCtx = TDqTaskRunnerExecutionContext()) {
+ YQL_ENSURE(TaskRunner);
+
+ auto guard = TaskRunner->BindAllocator(MkqlMemoryLimit);
auto* alloc = guard.GetMutex();
if (CanAllocateExtraMemory) {
@@ -1140,12 +1140,12 @@ protected:
});
}
- TDqTaskRunnerMemoryLimits limits;
- limits.ChannelBufferSize = MemoryLimits.ChannelBufferSize;
- limits.OutputChunkMaxSize = GetDqExecutionSettings().FlowControl.MaxOutputChunkSize;
-
- TaskRunner->Prepare(Task, limits, execCtx);
-
+ TDqTaskRunnerMemoryLimits limits;
+ limits.ChannelBufferSize = MemoryLimits.ChannelBufferSize;
+ limits.OutputChunkMaxSize = GetDqExecutionSettings().FlowControl.MaxOutputChunkSize;
+
+ TaskRunner->Prepare(Task, limits, execCtx);
+
FillChannelMaps(
TaskRunner->GetHolderFactory(),
TaskRunner->GetTypeEnv(),
@@ -1163,7 +1163,7 @@ protected:
for (auto& [channelId, channel] : InputChannelsMap) {
channel.Channel = TaskRunner->GetInputChannel(channelId);
}
- }
+ }
for (auto& [inputIndex, source] : SourcesMap) {
if (TaskRunner) { source.Source = TaskRunner->GetSource(inputIndex); Y_VERIFY(source.Source);}
Y_VERIFY(SourceActorFactory);
@@ -1187,7 +1187,7 @@ protected:
for (auto& [channelId, channel] : OutputChannelsMap) {
channel.Channel = TaskRunner->GetOutputChannel(channelId);
}
- }
+ }
for (auto& [outputIndex, sink] : SinksMap) {
if (TaskRunner) { sink.Sink = TaskRunner->GetSink(outputIndex); }
Y_VERIFY(SinkActorFactory);
@@ -1206,7 +1206,7 @@ protected:
});
this->RegisterWithSameMailbox(sink.Actor);
}
- }
+ }
void PollSourceActors() { // TODO: rename to PollSources()
// Don't produce any input from sources if we're about to save checkpoint.
@@ -1264,30 +1264,30 @@ protected:
InternalError(TIssuesIds::DEFAULT_ERROR, desc);
}
- virtual ui64 CalcMkqlMemoryLimit() {
- auto& opts = Task.GetProgram().GetSettings();
- return opts.GetHasMapJoin()/* || opts.GetHasSort()*/
- ? MemoryLimits.MkqlHeavyProgramMemoryLimit
- : MemoryLimits.MkqlLightProgramMemoryLimit;
- }
-
-private:
- void InitializeTask() {
- for (ui32 i = 0; i < Task.InputsSize(); ++i) {
+ virtual ui64 CalcMkqlMemoryLimit() {
+ auto& opts = Task.GetProgram().GetSettings();
+ return opts.GetHasMapJoin()/* || opts.GetHasSort()*/
+ ? MemoryLimits.MkqlHeavyProgramMemoryLimit
+ : MemoryLimits.MkqlLightProgramMemoryLimit;
+ }
+
+private:
+ void InitializeTask() {
+ for (ui32 i = 0; i < Task.InputsSize(); ++i) {
const auto& inputDesc = Task.GetInputs(i);
Y_VERIFY(!inputDesc.HasSource() || inputDesc.ChannelsSize() == 0); // HasSource => no channels
if (inputDesc.HasSource()) {
auto result = SourcesMap.emplace(i, TSourceInfo(i));
- YQL_ENSURE(result.second);
+ YQL_ENSURE(result.second);
} else {
for (auto& channel : inputDesc.GetChannels()) {
auto result = InputChannelsMap.emplace(channel.GetId(), TInputChannelInfo(channel.GetId(), channel.GetCheckpointingMode()));
YQL_ENSURE(result.second);
}
}
- }
+ }
- for (ui32 i = 0; i < Task.OutputsSize(); ++i) {
+ for (ui32 i = 0; i < Task.OutputsSize(); ++i) {
const auto& outputDesc = Task.GetOutputs(i);
Y_VERIFY(!outputDesc.HasSink() || outputDesc.ChannelsSize() == 0); // HasSink => no channels
Y_VERIFY(outputDesc.HasSink() || outputDesc.ChannelsSize() > 0);
@@ -1298,18 +1298,18 @@ private:
for (auto& channel : outputDesc.GetChannels()) {
TOutputChannelInfo outputChannel(channel.GetId());
outputChannel.HasPeer = channel.GetDstEndpoint().HasActorId();
-
- if (Y_UNLIKELY(RuntimeSettings.StatsMode >= NDqProto::DQ_STATS_MODE_PROFILE)) {
+
+ if (Y_UNLIKELY(RuntimeSettings.StatsMode >= NDqProto::DQ_STATS_MODE_PROFILE)) {
outputChannel.Stats = MakeHolder<typename TOutputChannelInfo::TStats>();
}
auto result = OutputChannelsMap.emplace(channel.GetId(), std::move(outputChannel));
YQL_ENSURE(result.second);
- }
+ }
}
}
-
- MkqlMemoryLimit = CalcMkqlMemoryLimit();
+
+ MkqlMemoryLimit = CalcMkqlMemoryLimit();
}
static ui64 AlignMemorySizeToMbBoundary(ui64 memory) {
@@ -1318,42 +1318,42 @@ private:
return (memory + alignMask) & ~alignMask;
}
- void RequestExtraMemory(ui64 memory, NKikimr::NMiniKQL::TScopedAlloc* alloc) {
+ void RequestExtraMemory(ui64 memory, NKikimr::NMiniKQL::TScopedAlloc* alloc) {
memory = std::max(AlignMemorySizeToMbBoundary(memory), MemoryLimits.MinMemAllocSize);
-
- CA_LOG_I("not enough memory, request +" << memory);
-
- if (MemoryLimits.AllocateMemoryFn(TxId, Task.GetId(), memory)) {
- MkqlMemoryLimit += memory;
- CA_LOG_I("[Mem] memory granted, new limit: " << MkqlMemoryLimit);
- alloc->SetLimit(MkqlMemoryLimit);
- } else {
- CA_LOG_W("[Mem] memory not granted");
-// throw yexception() << "Can not allocate extra memory, limit: " << MkqlMemoryLimit
-// << ", requested: " << memory << ", host: " << HostName();
- }
-
- if (Y_UNLIKELY(ProfileStats)) {
- ProfileStats->MkqlExtraMemoryBytes += memory;
- ProfileStats->MkqlExtraMemoryRequests++;
- }
- }
-
- void FillStats(NDqProto::TDqComputeActorStats* dst, bool last) {
- if (!BasicStats) {
+
+ CA_LOG_I("not enough memory, request +" << memory);
+
+ if (MemoryLimits.AllocateMemoryFn(TxId, Task.GetId(), memory)) {
+ MkqlMemoryLimit += memory;
+ CA_LOG_I("[Mem] memory granted, new limit: " << MkqlMemoryLimit);
+ alloc->SetLimit(MkqlMemoryLimit);
+ } else {
+ CA_LOG_W("[Mem] memory not granted");
+// throw yexception() << "Can not allocate extra memory, limit: " << MkqlMemoryLimit
+// << ", requested: " << memory << ", host: " << HostName();
+ }
+
+ if (Y_UNLIKELY(ProfileStats)) {
+ ProfileStats->MkqlExtraMemoryBytes += memory;
+ ProfileStats->MkqlExtraMemoryRequests++;
+ }
+ }
+
+ void FillStats(NDqProto::TDqComputeActorStats* dst, bool last) {
+ if (!BasicStats) {
return;
}
- if (last) {
- ReportEventElapsedTime();
+ if (last) {
+ ReportEventElapsedTime();
}
- dst->SetCpuTimeUs(BasicStats->CpuTime.MicroSeconds());
-
- if (ProfileStats) {
- dst->SetMkqlMaxMemoryUsage(ProfileStats->MkqlMaxUsedMemory);
- dst->SetMkqlExtraMemoryBytes(ProfileStats->MkqlExtraMemoryBytes);
- dst->SetMkqlExtraMemoryRequests(ProfileStats->MkqlExtraMemoryRequests);
+ dst->SetCpuTimeUs(BasicStats->CpuTime.MicroSeconds());
+
+ if (ProfileStats) {
+ dst->SetMkqlMaxMemoryUsage(ProfileStats->MkqlMaxUsedMemory);
+ dst->SetMkqlExtraMemoryBytes(ProfileStats->MkqlExtraMemoryBytes);
+ dst->SetMkqlExtraMemoryRequests(ProfileStats->MkqlExtraMemoryRequests);
}
if (TaskRunner) {
@@ -1362,42 +1362,42 @@ private:
if (auto* taskStats = TaskRunner->GetStats()) {
auto* protoTask = dst->AddTasks();
FillTaskRunnerStats(Task.GetId(), Task.GetStageId(), *taskStats, protoTask, (bool) ProfileStats);
-
+
for (auto& [outputIndex, sinkInfo] : SinksMap) {
if (auto* sinkStats = sinkInfo.Sink ? sinkInfo.Sink->GetStats() : nullptr) {
protoTask->SetOutputRows(protoTask->GetOutputRows() + sinkStats->RowsIn);
protoTask->SetOutputBytes(protoTask->GetOutputBytes() + sinkStats->Bytes);
-
+
if (ProfileStats) {
auto* protoSink = protoTask->AddSinks();
protoSink->SetOutputIndex(outputIndex);
-
+
protoSink->SetChunks(sinkStats->Chunks);
protoSink->SetBytes(sinkStats->Bytes);
protoSink->SetRowsIn(sinkStats->RowsIn);
protoSink->SetRowsOut(sinkStats->RowsOut);
-
+
protoSink->SetMaxMemoryUsage(sinkStats->MaxMemoryUsage);
protoSink->SetErrorsCount(sinkInfo.IssuesBuffer.GetAllAddedIssuesCount());
}
- }
- }
-
+ }
+ }
+
if (ProfileStats) {
for (auto& protoSource : *protoTask->MutableSources()) {
if (auto* sourceInfo = SourcesMap.FindPtr(protoSource.GetInputIndex())) {
protoSource.SetErrorsCount(sourceInfo->IssuesBuffer.GetAllAddedIssuesCount());
}
- }
-
+ }
+
for (auto& protoInputChannelStats : *protoTask->MutableInputChannels()) {
if (auto* caChannelStats = Channels->GetInputChannelStats(protoInputChannelStats.GetChannelId())) {
protoInputChannelStats.SetPollRequests(caChannelStats->PollRequests);
protoInputChannelStats.SetWaitTimeUs(caChannelStats->WaitTime.MicroSeconds());
protoInputChannelStats.SetResentMessages(caChannelStats->ResentMessages);
}
- }
-
+ }
+
for (auto& protoOutputChannelStats : *protoTask->MutableOutputChannels()) {
if (auto* x = Channels->GetOutputChannelStats(protoOutputChannelStats.GetChannelId())) {
protoOutputChannelStats.SetResentMessages(x->ResentMessages);
@@ -1409,32 +1409,32 @@ private:
protoOutputChannelStats.SetNoDstActorId(x->NoDstActorId);
}
}
- }
- }
- }
- }
+ }
+ }
+ }
+ }
- static_cast<TDerived*>(this)->FillExtraStats(dst, last);
+ static_cast<TDerived*>(this)->FillExtraStats(dst, last);
- if (last) {
- BasicStats.reset();
- ProfileStats.reset();
+ if (last) {
+ BasicStats.reset();
+ ProfileStats.reset();
}
- }
-
- void ReportStats(TInstant now) {
- if (!RuntimeSettings.ReportStatsSettings) {
+ }
+
+ void ReportStats(TInstant now) {
+ if (!RuntimeSettings.ReportStatsSettings) {
return;
- }
+ }
- if (now - LastSendStatsTime < RuntimeSettings.ReportStatsSettings->MinInterval) {
+ if (now - LastSendStatsTime < RuntimeSettings.ReportStatsSettings->MinInterval) {
return;
- }
+ }
- auto evState = std::make_unique<TEvDqCompute::TEvState>();
+ auto evState = std::make_unique<TEvDqCompute::TEvState>();
evState->Record.SetState(NDqProto::COMPUTE_STATE_EXECUTING);
evState->Record.SetTaskId(Task.GetId());
- FillStats(evState->Record.MutableStats(), /* last */ false);
+ FillStats(evState->Record.MutableStats(), /* last */ false);
auto dbgPrintStats = [&]() {
NProtoBuf::TextFormat::Printer printer;
@@ -1450,43 +1450,43 @@ private:
CA_LOG_D("Send stats to executor actor " << ExecuterId << " TaskId: " << Task.GetId()
<< " Stats: " << dbgPrintStats());
- this->Send(ExecuterId, evState.release(), NActors::IEventHandle::FlagTrackDelivery);
+ this->Send(ExecuterId, evState.release(), NActors::IEventHandle::FlagTrackDelivery);
LastSendStatsTime = now;
}
-protected:
- const NActors::TActorId ExecuterId;
+protected:
+ const NActors::TActorId ExecuterId;
const TTxId TxId;
- const NDqProto::TDqTask Task;
- const TComputeRuntimeSettings RuntimeSettings;
- const TComputeMemoryLimits MemoryLimits;
+ const NDqProto::TDqTask Task;
+ const TComputeRuntimeSettings RuntimeSettings;
+ const TComputeMemoryLimits MemoryLimits;
const bool CanAllocateExtraMemory = false;
const IDqSourceActorFactory::TPtr SourceActorFactory;
const IDqSinkActorFactory::TPtr SinkActorFactory;
const NDqProto::ECheckpointingMode CheckpointingMode;
- TIntrusivePtr<IDqTaskRunner> TaskRunner;
- TDqComputeActorChannels* Channels = nullptr;
+ TIntrusivePtr<IDqTaskRunner> TaskRunner;
+ TDqComputeActorChannels* Channels = nullptr;
TDqComputeActorCheckpoints* Checkpoints = nullptr;
THashMap<ui64, TInputChannelInfo> InputChannelsMap; // Channel id -> Channel info
THashMap<ui64, TSourceInfo> SourcesMap; // Input index -> Source info
THashMap<ui64, TOutputChannelInfo> OutputChannelsMap; // Channel id -> Channel info
THashMap<ui64, TSinkInfo> SinksMap; // Output index -> Sink info
- ui64 MkqlMemoryLimit = 0;
- bool ResumeEventScheduled = false;
+ ui64 MkqlMemoryLimit = 0;
+ bool ResumeEventScheduled = false;
NDqProto::EComputeState State;
-
- struct TBasicStats {
- TDuration CpuTime;
- };
- struct TProfileStats {
- ui64 MkqlMaxUsedMemory = 0;
- ui64 MkqlExtraMemoryBytes = 0;
- ui32 MkqlExtraMemoryRequests = 0;
- };
- std::unique_ptr<TBasicStats> BasicStats;
- std::unique_ptr<TProfileStats> ProfileStats;
-
+
+ struct TBasicStats {
+ TDuration CpuTime;
+ };
+ struct TProfileStats {
+ ui64 MkqlMaxUsedMemory = 0;
+ ui64 MkqlExtraMemoryBytes = 0;
+ ui32 MkqlExtraMemoryRequests = 0;
+ };
+ std::unique_ptr<TBasicStats> BasicStats;
+ std::unique_ptr<TProfileStats> ProfileStats;
+
struct TProcessOutputsState {
int Inflight = 0;
bool ChannelsReady = true;
@@ -1502,5 +1502,5 @@ private:
TInstant LastSendStatsTime;
};
-} // namespace NYql
-} // namespace NNq
+} // namespace NYql
+} // namespace NNq
diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h
index 864fe0d301..d2a646cab6 100644
--- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h
+++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h
@@ -46,7 +46,7 @@ struct IDqSourceActor {
virtual ui64 GetInputIndex() const = 0;
// Gets data and returns space used by filled data batch.
- // Method should be called under bound mkql allocator.
+ // Method should be called under bound mkql allocator.
// Could throw YQL errors.
virtual i64 GetSourceData(NKikimr::NMiniKQL::TUnboxedValueVector& batch, bool& finished, i64 freeSpace) = 0;
diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_stats.cpp b/ydb/library/yql/dq/actors/compute/dq_compute_actor_stats.cpp
index 1027a2a97f..cc13ce4d27 100644
--- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_stats.cpp
+++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_stats.cpp
@@ -1,40 +1,40 @@
-#include "dq_compute_actor.h"
-
+#include "dq_compute_actor.h"
+
#include <ydb/library/yql/dq/actors/protos/dq_stats.pb.h>
-
-namespace NYql {
-namespace NDq {
-
-void FillTaskRunnerStats(ui64 taskId, ui32 stageId, const TDqTaskRunnerStats& taskStats,
- NDqProto::TDqTaskStats* protoTask, bool withProfileStats)
-{
- protoTask->SetTaskId(taskId);
- protoTask->SetStageId(stageId);
- protoTask->SetCpuTimeUs(taskStats.ComputeCpuTime.MicroSeconds() + taskStats.BuildCpuTime.MicroSeconds());
- protoTask->SetFinishTimeMs(taskStats.FinishTs.MilliSeconds());
-
- // Cerr << (TStringBuilder() << "FillTaskRunnerStats: " << taskStats.ComputeCpuTime << ", " << taskStats.BuildCpuTime << Endl);
-
- if (Y_UNLIKELY(withProfileStats)) {
- protoTask->SetComputeCpuTimeUs(taskStats.ComputeCpuTime.MicroSeconds());
- protoTask->SetBuildCpuTimeUs(taskStats.BuildCpuTime.MicroSeconds());
- protoTask->SetWaitTimeUs(taskStats.WaitTime.MicroSeconds());
- protoTask->SetWaitOutputTimeUs(taskStats.WaitOutputTime.MicroSeconds());
-
+
+namespace NYql {
+namespace NDq {
+
+void FillTaskRunnerStats(ui64 taskId, ui32 stageId, const TDqTaskRunnerStats& taskStats,
+ NDqProto::TDqTaskStats* protoTask, bool withProfileStats)
+{
+ protoTask->SetTaskId(taskId);
+ protoTask->SetStageId(stageId);
+ protoTask->SetCpuTimeUs(taskStats.ComputeCpuTime.MicroSeconds() + taskStats.BuildCpuTime.MicroSeconds());
+ protoTask->SetFinishTimeMs(taskStats.FinishTs.MilliSeconds());
+
+ // Cerr << (TStringBuilder() << "FillTaskRunnerStats: " << taskStats.ComputeCpuTime << ", " << taskStats.BuildCpuTime << Endl);
+
+ if (Y_UNLIKELY(withProfileStats)) {
+ protoTask->SetComputeCpuTimeUs(taskStats.ComputeCpuTime.MicroSeconds());
+ protoTask->SetBuildCpuTimeUs(taskStats.BuildCpuTime.MicroSeconds());
+ protoTask->SetWaitTimeUs(taskStats.WaitTime.MicroSeconds());
+ protoTask->SetWaitOutputTimeUs(taskStats.WaitOutputTime.MicroSeconds());
+
// All run statuses metrics
protoTask->SetPendingInputTimeUs(taskStats.RunStatusTimeMetrics[ERunStatus::PendingInput].MicroSeconds());
protoTask->SetPendingOutputTimeUs(taskStats.RunStatusTimeMetrics[ERunStatus::PendingOutput].MicroSeconds());
protoTask->SetFinishTimeUs(taskStats.RunStatusTimeMetrics[ERunStatus::Finished].MicroSeconds());
static_assert(TRunStatusTimeMetrics::StatusesCount == 3); // Add all statuses here
- if (taskStats.ComputeCpuTimeByRun) {
- auto snapshot = taskStats.ComputeCpuTimeByRun->Snapshot();
- for (ui32 i = 0; i < snapshot->Count(); i++) {
- auto* protoBucket = protoTask->AddComputeCpuTimeByRun();
- protoBucket->SetBound(snapshot->UpperBound(i));
- protoBucket->SetValue(snapshot->Value(i));
- }
- }
+ if (taskStats.ComputeCpuTimeByRun) {
+ auto snapshot = taskStats.ComputeCpuTimeByRun->Snapshot();
+ for (ui32 i = 0; i < snapshot->Count(); i++) {
+ auto* protoBucket = protoTask->AddComputeCpuTimeByRun();
+ protoBucket->SetBound(snapshot->UpperBound(i));
+ protoBucket->SetValue(snapshot->Value(i));
+ }
+ }
for (const auto& stat : taskStats.MkqlStats) {
auto* s = protoTask->MutableMkqlStats()->Add();
@@ -42,90 +42,90 @@ void FillTaskRunnerStats(ui64 taskId, ui32 stageId, const TDqTaskRunnerStats& ta
s->SetValue(stat.Value);
s->SetDeriv(stat.Key.IsDeriv());
}
- }
-
- ui64 firstRowTs = std::numeric_limits<ui64>::max();
-
- for (auto& [channelId, inputChannelStats] : taskStats.InputChannels) {
- if (!inputChannelStats) {
- continue;
- }
-
- if (inputChannelStats->FirstRowTs) {
- firstRowTs = std::min(firstRowTs, inputChannelStats->FirstRowTs.MilliSeconds());
- }
-
- protoTask->SetInputRows(protoTask->GetInputRows() + inputChannelStats->RowsOut); // Yes, OutputRows of input channel, i.e. rows that were visible by the task runner
- protoTask->SetInputBytes(protoTask->GetInputBytes() + inputChannelStats->Bytes); // should be bytes out...
-
- if (Y_UNLIKELY(withProfileStats)) {
- auto protoChannel = protoTask->AddInputChannels();
- protoChannel->SetChannelId(channelId);
- protoChannel->SetChunks(inputChannelStats->Chunks);
- protoChannel->SetBytes(inputChannelStats->Bytes);
- protoChannel->SetRowsIn(inputChannelStats->RowsIn);
- protoChannel->SetRowsOut(inputChannelStats->RowsOut);
- protoChannel->SetMaxMemoryUsage(inputChannelStats->MaxMemoryUsage);
- protoChannel->SetDeserializationTimeUs(inputChannelStats->DeserializationTime.MicroSeconds());
- }
- }
-
- for (auto& [inputIndex, sourceStats] : taskStats.Sources) {
- if (!sourceStats) {
- continue;
- }
-
- if (sourceStats->FirstRowTs) {
- firstRowTs = std::min(firstRowTs, sourceStats->FirstRowTs.MilliSeconds());
- }
-
- protoTask->SetInputRows(protoTask->GetInputRows() + sourceStats->RowsOut); // the same comment here ... ^^^
- protoTask->SetInputBytes(protoTask->GetInputBytes() + sourceStats->Bytes);
-
- if (Y_UNLIKELY(withProfileStats)) {
- auto* protoSource = protoTask->AddSources();
- protoSource->SetInputIndex(inputIndex);
- protoSource->SetChunks(sourceStats->Chunks);
- protoSource->SetBytes(sourceStats->Bytes);
- protoSource->SetRowsIn(sourceStats->RowsIn);
- protoSource->SetRowsOut(sourceStats->RowsOut);
- protoSource->SetMaxMemoryUsage(sourceStats->MaxMemoryUsage);
- }
- }
-
- for (auto& [channelId, outputChannelStats] : taskStats.OutputChannels) {
- if (!outputChannelStats) {
- continue;
- }
-
- if (outputChannelStats->FirstRowIn) {
- firstRowTs = std::min(firstRowTs, outputChannelStats->FirstRowIn.MilliSeconds());
- }
-
- protoTask->SetOutputRows(protoTask->GetOutputRows() + outputChannelStats->RowsIn);
- protoTask->SetOutputBytes(protoTask->GetOutputBytes() + outputChannelStats->Bytes);
-
- if (Y_UNLIKELY(withProfileStats)) {
- auto* protoChannel = protoTask->AddOutputChannels();
- protoChannel->SetChannelId(channelId);
-
- protoChannel->SetChunks(outputChannelStats->Chunks);
- protoChannel->SetBytes(outputChannelStats->Bytes);
- protoChannel->SetRowsIn(outputChannelStats->RowsIn);
- protoChannel->SetRowsOut(outputChannelStats->RowsOut);
-
- protoChannel->SetMaxMemoryUsage(outputChannelStats->MaxMemoryUsage);
- protoChannel->SetMaxRowsInMemory(outputChannelStats->MaxRowsInMemory);
- protoChannel->SetSerializationTimeUs(outputChannelStats->SerializationTime.MicroSeconds());
-
- protoChannel->SetSpilledBytes(outputChannelStats->SpilledBytes);
- protoChannel->SetSpilledRows(outputChannelStats->SpilledRows);
- protoChannel->SetSpilledBlobs(outputChannelStats->SpilledBlobs);
- }
- }
-
- protoTask->SetFirstRowTimeMs(firstRowTs);
-}
-
-} // namespace NDq
-} // namespace NYql
+ }
+
+ ui64 firstRowTs = std::numeric_limits<ui64>::max();
+
+ for (auto& [channelId, inputChannelStats] : taskStats.InputChannels) {
+ if (!inputChannelStats) {
+ continue;
+ }
+
+ if (inputChannelStats->FirstRowTs) {
+ firstRowTs = std::min(firstRowTs, inputChannelStats->FirstRowTs.MilliSeconds());
+ }
+
+ protoTask->SetInputRows(protoTask->GetInputRows() + inputChannelStats->RowsOut); // Yes, OutputRows of input channel, i.e. rows that were visible by the task runner
+ protoTask->SetInputBytes(protoTask->GetInputBytes() + inputChannelStats->Bytes); // should be bytes out...
+
+ if (Y_UNLIKELY(withProfileStats)) {
+ auto protoChannel = protoTask->AddInputChannels();
+ protoChannel->SetChannelId(channelId);
+ protoChannel->SetChunks(inputChannelStats->Chunks);
+ protoChannel->SetBytes(inputChannelStats->Bytes);
+ protoChannel->SetRowsIn(inputChannelStats->RowsIn);
+ protoChannel->SetRowsOut(inputChannelStats->RowsOut);
+ protoChannel->SetMaxMemoryUsage(inputChannelStats->MaxMemoryUsage);
+ protoChannel->SetDeserializationTimeUs(inputChannelStats->DeserializationTime.MicroSeconds());
+ }
+ }
+
+ for (auto& [inputIndex, sourceStats] : taskStats.Sources) {
+ if (!sourceStats) {
+ continue;
+ }
+
+ if (sourceStats->FirstRowTs) {
+ firstRowTs = std::min(firstRowTs, sourceStats->FirstRowTs.MilliSeconds());
+ }
+
+ protoTask->SetInputRows(protoTask->GetInputRows() + sourceStats->RowsOut); // the same comment here ... ^^^
+ protoTask->SetInputBytes(protoTask->GetInputBytes() + sourceStats->Bytes);
+
+ if (Y_UNLIKELY(withProfileStats)) {
+ auto* protoSource = protoTask->AddSources();
+ protoSource->SetInputIndex(inputIndex);
+ protoSource->SetChunks(sourceStats->Chunks);
+ protoSource->SetBytes(sourceStats->Bytes);
+ protoSource->SetRowsIn(sourceStats->RowsIn);
+ protoSource->SetRowsOut(sourceStats->RowsOut);
+ protoSource->SetMaxMemoryUsage(sourceStats->MaxMemoryUsage);
+ }
+ }
+
+ for (auto& [channelId, outputChannelStats] : taskStats.OutputChannels) {
+ if (!outputChannelStats) {
+ continue;
+ }
+
+ if (outputChannelStats->FirstRowIn) {
+ firstRowTs = std::min(firstRowTs, outputChannelStats->FirstRowIn.MilliSeconds());
+ }
+
+ protoTask->SetOutputRows(protoTask->GetOutputRows() + outputChannelStats->RowsIn);
+ protoTask->SetOutputBytes(protoTask->GetOutputBytes() + outputChannelStats->Bytes);
+
+ if (Y_UNLIKELY(withProfileStats)) {
+ auto* protoChannel = protoTask->AddOutputChannels();
+ protoChannel->SetChannelId(channelId);
+
+ protoChannel->SetChunks(outputChannelStats->Chunks);
+ protoChannel->SetBytes(outputChannelStats->Bytes);
+ protoChannel->SetRowsIn(outputChannelStats->RowsIn);
+ protoChannel->SetRowsOut(outputChannelStats->RowsOut);
+
+ protoChannel->SetMaxMemoryUsage(outputChannelStats->MaxMemoryUsage);
+ protoChannel->SetMaxRowsInMemory(outputChannelStats->MaxRowsInMemory);
+ protoChannel->SetSerializationTimeUs(outputChannelStats->SerializationTime.MicroSeconds());
+
+ protoChannel->SetSpilledBytes(outputChannelStats->SpilledBytes);
+ protoChannel->SetSpilledRows(outputChannelStats->SpilledRows);
+ protoChannel->SetSpilledBlobs(outputChannelStats->SpilledBlobs);
+ }
+ }
+
+ protoTask->SetFirstRowTimeMs(firstRowTs);
+}
+
+} // namespace NDq
+} // namespace NYql
diff --git a/ydb/library/yql/dq/actors/compute/ya.make b/ydb/library/yql/dq/actors/compute/ya.make
index f6bedc92fa..c4b20bd502 100644
--- a/ydb/library/yql/dq/actors/compute/ya.make
+++ b/ydb/library/yql/dq/actors/compute/ya.make
@@ -11,7 +11,7 @@ SRCS(
dq_compute_actor_channels.cpp
dq_compute_actor_checkpoints.cpp
dq_compute_actor_io_actors_factory.cpp
- dq_compute_actor_stats.cpp
+ dq_compute_actor_stats.cpp
dq_compute_issues_buffer.cpp
retry_queue.cpp
)
diff --git a/ydb/library/yql/dq/actors/dq.cpp b/ydb/library/yql/dq/actors/dq.cpp
index 67ff53f8ae..f0b0e5d0e9 100644
--- a/ydb/library/yql/dq/actors/dq.cpp
+++ b/ydb/library/yql/dq/actors/dq.cpp
@@ -1 +1 @@
-#include "dq.h"
+#include "dq.h"
diff --git a/ydb/library/yql/dq/actors/dq.h b/ydb/library/yql/dq/actors/dq.h
index 85545d05a7..025626d05f 100644
--- a/ydb/library/yql/dq/actors/dq.h
+++ b/ydb/library/yql/dq/actors/dq.h
@@ -1,39 +1,39 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/dq/actors/protos/dq_events.pb.h>
#include <ydb/library/yql/dq/actors/dq_events_ids.h>
-
-namespace NYql {
-namespace NDq {
-
-struct TEvDq {
-
- struct TEvAbortExecution : public NActors::TEventPB<TEvAbortExecution, NDqProto::TEvAbortExecution, TDqEvents::EvAbortExecution> {
- static THolder <TEvAbortExecution> Unavailable(const TString& s) {
- return MakeHolder<TEvAbortExecution>(Ydb::StatusIds::UNAVAILABLE, s);
- }
-
- static THolder <TEvAbortExecution> InternalError(const TString& s) {
- return MakeHolder<TEvAbortExecution>(Ydb::StatusIds::INTERNAL_ERROR, s);
- }
-
- static THolder <TEvAbortExecution> Aborted(const TString& s) {
- return MakeHolder<TEvAbortExecution>(Ydb::StatusIds::ABORTED, s);
- }
-
- TEvAbortExecution() = default;
-
- TEvAbortExecution(TEvAbortExecution&&) = default;
-
- TEvAbortExecution(const TEvAbortExecution&) = default;
-
- TEvAbortExecution(Ydb::StatusIds::StatusCode code, const TString& message) {
- Record.SetStatusCode(code);
- Record.SetMessage(message);
- }
- };
-
-};
-
-} // namespace NDq
-} // namespace NYql
+
+namespace NYql {
+namespace NDq {
+
+struct TEvDq {
+
+ struct TEvAbortExecution : public NActors::TEventPB<TEvAbortExecution, NDqProto::TEvAbortExecution, TDqEvents::EvAbortExecution> {
+ static THolder <TEvAbortExecution> Unavailable(const TString& s) {
+ return MakeHolder<TEvAbortExecution>(Ydb::StatusIds::UNAVAILABLE, s);
+ }
+
+ static THolder <TEvAbortExecution> InternalError(const TString& s) {
+ return MakeHolder<TEvAbortExecution>(Ydb::StatusIds::INTERNAL_ERROR, s);
+ }
+
+ static THolder <TEvAbortExecution> Aborted(const TString& s) {
+ return MakeHolder<TEvAbortExecution>(Ydb::StatusIds::ABORTED, s);
+ }
+
+ TEvAbortExecution() = default;
+
+ TEvAbortExecution(TEvAbortExecution&&) = default;
+
+ TEvAbortExecution(const TEvAbortExecution&) = default;
+
+ TEvAbortExecution(Ydb::StatusIds::StatusCode code, const TString& message) {
+ Record.SetStatusCode(code);
+ Record.SetMessage(message);
+ }
+ };
+
+};
+
+} // namespace NDq
+} // namespace NYql
diff --git a/ydb/library/yql/dq/actors/dq_events_ids.h b/ydb/library/yql/dq/actors/dq_events_ids.h
index 4f173a5f49..698377a393 100644
--- a/ydb/library/yql/dq/actors/dq_events_ids.h
+++ b/ydb/library/yql/dq/actors/dq_events_ids.h
@@ -2,41 +2,41 @@
#include <library/cpp/actors/core/events.h>
-namespace NYql {
-namespace NDq {
+namespace NYql {
+namespace NDq {
struct TDqEvents {
enum EEventSpaceDq {
- ES_DQ_COMPUTE_KQP_COMPATIBLE = 4145, // TKikimrEvents::ES_KQP
- ES_DQ_COMPUTE = 4212 //TKikimrEvents::ES_DQ
+ ES_DQ_COMPUTE_KQP_COMPATIBLE = 4145, // TKikimrEvents::ES_KQP
+ ES_DQ_COMPUTE = 4212 //TKikimrEvents::ES_DQ
+ };
+
+ enum EDqEvents {
+ EvAbortExecution = EventSpaceBegin(ES_DQ_COMPUTE_KQP_COMPATIBLE) + 15
};
-
- enum EDqEvents {
- EvAbortExecution = EventSpaceBegin(ES_DQ_COMPUTE_KQP_COMPATIBLE) + 15
- };
};
-struct TDqComputeEvents {
- enum EDqComputeEvents {
- Unused0 = EventSpaceBegin(TDqEvents::ES_DQ_COMPUTE_KQP_COMPATIBLE) + 200,
+struct TDqComputeEvents {
+ enum EDqComputeEvents {
+ Unused0 = EventSpaceBegin(TDqEvents::ES_DQ_COMPUTE_KQP_COMPATIBLE) + 200,
EvState,
EvResumeExecution,
EvChannelData,
- ReservedKqp_EvScanData,
- ReservedKqp_EvScanDataAck,
- EvChannelsInfo,
+ ReservedKqp_EvScanData,
+ ReservedKqp_EvScanDataAck,
+ EvChannelsInfo,
EvChannelDataAck,
- ReservedKqp_EvScanError,
- Unused1,
- EvRetryChannelData,
- EvRetryChannelDataAck,
- ReservedKqp_EvScanInitActor,
- ReservedKqp_EvRemoteScanData,
- ReservedKqp_EvRemoteScanDataAck,
-
- EvRun = EventSpaceBegin(TDqEvents::ES_DQ_COMPUTE),
-
- EvNewCheckpointCoordinator,
+ ReservedKqp_EvScanError,
+ Unused1,
+ EvRetryChannelData,
+ EvRetryChannelDataAck,
+ ReservedKqp_EvScanInitActor,
+ ReservedKqp_EvRemoteScanData,
+ ReservedKqp_EvRemoteScanDataAck,
+
+ EvRun = EventSpaceBegin(TDqEvents::ES_DQ_COMPUTE),
+
+ EvNewCheckpointCoordinator,
EvInjectCheckpoint,
EvSaveTaskState,
EvSaveTaskStateResult,
@@ -45,17 +45,17 @@ struct TDqComputeEvents {
EvRestoreFromCheckpoint,
EvRestoreFromCheckpointResult,
EvGetTaskState,
- EvGetTaskStateResult,
+ EvGetTaskStateResult,
EvStateRequest,
EvNewCheckpointCoordinatorAck,
-
- // place all new events here
-
- EvEnd
+
+ // place all new events here
+
+ EvEnd
};
-
- static_assert(EvEnd < EventSpaceBegin((TDqEvents::ES_DQ_COMPUTE + 1)));
+
+ static_assert(EvEnd < EventSpaceBegin((TDqEvents::ES_DQ_COMPUTE + 1)));
};
-} // namespace NDq
-} // namespace NYql
+} // namespace NDq
+} // namespace NYql
diff --git a/ydb/library/yql/dq/actors/protos/dq_events.proto b/ydb/library/yql/dq/actors/protos/dq_events.proto
index 7ad01b1721..0e38ab3bbc 100644
--- a/ydb/library/yql/dq/actors/protos/dq_events.proto
+++ b/ydb/library/yql/dq/actors/protos/dq_events.proto
@@ -1,4 +1,4 @@
-package NYql.NDqProto;
+package NYql.NDqProto;
option cc_enable_arenas = true;
import "library/cpp/actors/protos/actors.proto";
@@ -42,18 +42,18 @@ enum EComputeState {
COMPUTE_STATE_FINISHED = 3;
};
-message TComputeActorStats {
+message TComputeActorStats {
optional NActorsProto.TActorId ActorId = 1;
- reserved 2; //optional NKqpProto.TKqpStatsRun LegacyStats = 2;
-};
-
+ reserved 2; //optional NKqpProto.TKqpStatsRun LegacyStats = 2;
+};
+
message TEvComputeActorState {
- optional uint32 State = 1; // == EComputeState
- optional TComputeActorStats LegacyStats = 2;
+ optional uint32 State = 1; // == EComputeState
+ optional TComputeActorStats LegacyStats = 2;
optional Ydb.StatusIds.StatusCode Status = 3;
repeated Ydb.Issue.IssueMessage Issues = 4;
- optional uint64 TaskId = 5;
- optional TDqComputeActorStats Stats = 6;
+ optional uint64 TaskId = 5;
+ optional TDqComputeActorStats Stats = 6;
};
message TEvComputeStateRequest {
@@ -61,23 +61,23 @@ message TEvComputeStateRequest {
message TEvComputeChannelData {
optional TChannelData ChannelData = 1;
- optional uint64 SendTime = 2;
- optional uint64 SeqNo = 3;
- optional bool NoAck = 4;
+ optional uint64 SendTime = 2;
+ optional uint64 SeqNo = 3;
+ optional bool NoAck = 4;
};
-
-message TEvComputeChannelDataAck {
- optional uint64 ChannelId = 1;
- optional uint64 SeqNo = 2;
- optional int64 FreeSpace = 3;
- optional bool Finish = 4;
+
+message TEvComputeChannelDataAck {
+ optional uint64 ChannelId = 1;
+ optional uint64 SeqNo = 2;
+ optional int64 FreeSpace = 3;
+ optional bool Finish = 4;
};
-
-message TEvAbortExecution {
- optional Ydb.StatusIds.StatusCode StatusCode = 1;
- optional string Message = 2;
-}
-
+
+message TEvAbortExecution {
+ optional Ydb.StatusIds.StatusCode StatusCode = 1;
+ optional string Message = 2;
+}
+
message TRlPath {
optional string CoordinationNode = 1;
optional string ResourcePath = 2;
@@ -85,23 +85,23 @@ message TRlPath {
optional string Token = 4;
}
-message TComputeRuntimeSettings {
- optional uint32 TimeoutMs = 1;
-
- enum EExecType {
- UNSPECIFIED = 0;
+message TComputeRuntimeSettings {
+ optional uint32 TimeoutMs = 1;
+
+ enum EExecType {
+ UNSPECIFIED = 0;
SCAN = 1;
- DATA = 2;
- }
- optional EExecType ExecType = 2;
-
- reserved 3; //optional NKqpProto.EKqpStatsMode LegacyStatsMode = 3;
- optional EDqStatsMode StatsMode = 8;
- optional bool UseLLVM = 4;
- optional bool UseSpilling = 6;
- optional uint32 TasksOnNodeCount = 5; // approx
+ DATA = 2;
+ }
+ optional EExecType ExecType = 2;
+
+ reserved 3; //optional NKqpProto.EKqpStatsMode LegacyStatsMode = 3;
+ optional EDqStatsMode StatsMode = 8;
+ optional bool UseLLVM = 4;
+ optional bool UseSpilling = 6;
+ optional uint32 TasksOnNodeCount = 5; // approx
optional TRlPath RlPath = 7;
-}
+}
message TEvNewCheckpointCoordinator {
optional uint64 Generation = 1;
diff --git a/ydb/library/yql/dq/actors/protos/dq_legacy_stats.proto b/ydb/library/yql/dq/actors/protos/dq_legacy_stats.proto
index beddc6207b..fe8dd7819b 100644
--- a/ydb/library/yql/dq/actors/protos/dq_legacy_stats.proto
+++ b/ydb/library/yql/dq/actors/protos/dq_legacy_stats.proto
@@ -1,291 +1,291 @@
-syntax = "proto3";
-option cc_enable_arenas = true;
-
-package NKqpProto;
-option java_package = "ru.yandex.kikimr.proto";
-
-enum EKqpStatsMode {
- STATS_MODE_UNSPECIFIED = 0;
- STATS_MODE_NONE = 10;
- STATS_MODE_BASIC = 20;
- STATS_MODE_PROFILE = 30;
-};
-
-message TKqpStatsMinMax {
- uint64 Min = 1;
- uint64 Max = 2;
-}
-
-message TKqpStatsAggr {
- uint64 Min = 1;
- uint64 Max = 2;
- uint64 Sum = 3;
- uint64 Cnt = 4;
-}
-
-message TKqpStatsTable {
- string TableId = 1;
- string TablePath = 2;
-
- uint64 ReadRows = 3;
- uint64 ReadBytes = 4;
- uint64 WriteRows = 5;
- uint64 WriteBytes = 6;
- uint64 EraseRows = 7;
-
- uint64 AffectedPartitions = 8;
-}
-
-message TKqpStatsMemoryUsage {
- uint32 InputChannelsCount = 1;
- uint32 InputChannelsSize = 2;
- uint32 OutputChannelsCount = 3;
- uint32 OutputChannelsSize = 4;
- uint32 ScanBuffersCount = 5;
- uint32 ScanBuffersSize = 6;
- uint32 MkqlSize = 7;
-}
-
-message TKqpStatsTask {
- message TScanStats {
- string TableId = 1;
- string TablePath = 2;
- // total read data
- uint64 Rows = 3;
- uint64 Bytes = 4;
- // IScan stats
- uint64 ScanTotalTime = 5;
- uint64 ScanWaitTime = 6; // wait for row inside IScan object
- uint64 BufferDataTime = 7; // time to store rows taken from IScan to the internal buffer
- uint64 PageFaults = 8;
- // ITableReader stats
- uint64 ReadDataTime = 9; // TableReader read data time
- uint64 WaitDataTime = 10; // TableReader wait data time
- // Network stats
- uint64 Messages = 11;
- uint64 MessagesByPageFault = 12;
- };
-
- message TInputChannelStats {
- uint64 ChannelId = 1;
- uint64 Chunks = 2;
- uint64 Bytes = 3;
- uint64 RowsIn = 4;
- uint64 RowsOut = 12;
- uint64 StartTs = 5;
- uint64 FinishTs = 6;
- uint64 DeserializationTime = 7;
- uint64 PollRequests = 8;
- uint64 WaitTime = 9;
- uint32 ResentAcks = 10;
- uint64 MaxMemoryUsage = 11;
- }
-
- message TSourceStats {
- uint64 ChannelId = 1;
- uint64 Chunks = 2;
- uint64 Bytes = 3;
- uint64 RowsIn = 4;
- uint64 RowsOut = 12;
- uint64 StartTs = 5;
- uint64 FinishTs = 6;
- uint64 InputIndex = 7;
- uint64 PollRequests = 8;
- uint64 WaitTime = 9;
- uint32 ResentAcks = 10;
- uint64 MaxMemoryUsage = 11;
- uint32 ErrorsCount = 13;
- }
-
- message TOutputChannelStats {
- uint64 ChannelId = 1;
- uint64 Chunks = 2;
- uint64 Bytes = 3;
- uint64 RowsIn = 4;
- uint64 RowsOut = 26;
- uint64 StartTs = 5;
- uint64 FinishTs = 6;
- uint64 SerializationTime = 7;
- uint64 BlockedByCapacity = 10;
- uint64 NoDstActorId = 11;
- uint32 Count4KChunks = 14;
- uint32 Count16KChunks = 15;
- uint32 Count64KChunks = 16;
- uint32 Count256KChunks = 17;
- uint32 Count1MChunks = 18;
- uint32 CountBigChunks = 19;
- uint32 ResentMessages = 20;
- uint64 MaxMemoryUsage = 21;
- uint32 MaxRowsInMemory = 22;
- uint64 SpilledBytes = 23;
- uint64 SpilledRows = 24;
- uint64 SpilledBlobs = 25;
- }
-
- message TSinkStats {
- uint64 ChannelId = 1;
- uint64 Chunks = 2;
- uint64 Bytes = 3;
- uint64 RowsIn = 4;
- uint64 RowsOut = 26;
- uint64 StartTs = 5;
- uint64 FinishTs = 6;
- uint32 Count4KChunks = 14;
- uint32 Count16KChunks = 15;
- uint32 Count64KChunks = 16;
- uint32 Count256KChunks = 17;
- uint32 Count1MChunks = 18;
- uint32 CountBigChunks = 19;
- uint64 MaxMemoryUsage = 21;
- uint32 MaxRowsInMemory = 22;
- uint64 SpilledBytes = 23;
- uint64 SpilledRows = 24;
- uint64 SpilledBlobs = 25;
- uint32 ErrorsCount = 27;
- }
-
- message THistBucket {
- double Bound = 1;
- uint64 Value = 2;
- }
-
- uint64 TaskId = 1;
- uint64 StartTs = 2;
- uint64 FinishTs = 3;
- uint64 BuildTaskTime = 4;
- uint64 PrefetchTime = 5;
- uint64 ComputeTime = 6;
- uint64 TotalTime = 7; // ComputeTime + waits
- uint64 WaitTime = 8;
- uint64 WaitInputTime = 9;
- uint64 WaitScanTime = 10;
- uint64 WaitOutputTime = 11;
- repeated THistBucket ComputeTimeByRun = 33;
-
- uint64 InputRows = 12;
- uint64 InputBytes = 13;
- uint64 OutputRows = 14;
- uint64 OutputBytes = 15;
-
- uint64 TotalPageFaults = 16;
-
- uint64 MkqlMaxMemoryUsage = 17;
- uint64 ExtraMemoryBytes = 22;
- uint32 ExtraMemoryRequests = 23;
- uint32 ExtraMemoryWaitTimeMs = 24;
-
- int64 MkqlTotalNodes = 25;
- int64 MkqlCodegenFunctions = 26;
- int64 CodeGenTotalInstructions = 27;
- int64 CodeGenTotalFunctions = 28;
- int64 CodeGenFullTime = 29;
- int64 CodeGenFinalizeTime = 30;
- int64 CodeGenModulePassTime = 31;
-
- repeated TKqpStatsTable Tables = 18;
- repeated TScanStats Scans = 19;
- repeated TInputChannelStats InputChannels = 20;
- repeated TOutputChannelStats OutputChannels = 21;
- repeated TSourceStats Sources = 32;
- repeated TSinkStats Sinks = 34;
-}
-
-message TKqpStatsRun {
- // Basic stats
- uint64 ExecutionTimeUs = 1;
- repeated TKqpStatsTable Tables = 2;
-
- // Profile stats
- TKqpStatsMemoryUsage MemoryUsage = 100;
- repeated TKqpStatsTask Tasks = 101;
-}
-
-message TKqpStatsExecution {
- message TInputChannelStats {
- uint32 SrcStageId = 1;
- TKqpStatsAggr Bytes = 2;
- TKqpStatsAggr Rows = 3;
- TKqpStatsAggr DeserializationTime = 4;
- TKqpStatsAggr PollRequests = 5;
- TKqpStatsAggr WaitTime = 6;
- TKqpStatsAggr ResentAcks = 7;
- };
-
- message TOutputChannelStats {
- uint32 DstStageId = 1;
- TKqpStatsAggr Bytes = 2;
- TKqpStatsAggr Rows = 3;
- TKqpStatsAggr SerializationTime = 4;
- TKqpStatsAggr BlockedByCapacity = 5;
- TKqpStatsAggr NoDstActorId = 6;
- TKqpStatsAggr ResentMessages = 7;
- };
-
- message TScansStats {
- string TablePath = 1;
- TKqpStatsAggr Bytes = 2;
- TKqpStatsAggr Rows = 3;
- TKqpStatsAggr ScanTotalTime = 4;
- TKqpStatsAggr ScanWaitTime = 5;
- TKqpStatsAggr WaitDataTime = 6;
- };
-
- message TStageStats {
- uint32 StageId = 1;
- string Program = 2;
- TKqpStatsMinMax StageStartTs = 3;
- TKqpStatsMinMax StageFinishTs = 4;
- TKqpStatsAggr TasksTotalTime = 5;
- TKqpStatsAggr TasksComputeTime = 6;
- TKqpStatsAggr TasksWaitTime = 7;
- TKqpStatsAggr TasksWaitScanTime = 8;
- TKqpStatsAggr TasksWaitInputTime = 9;
- TKqpStatsAggr TasksWaitOutputTime = 10;
- TKqpStatsAggr TasksScanTotalTime = 11;
- TKqpStatsAggr TasksScanWaitTime = 12;
- TKqpStatsAggr TasksReadBytes = 13;
- TKqpStatsAggr TasksReadRows = 14;
-
- uint32 TasksCount = 15;
-
- repeated TScansStats Scans = 16;
- repeated TInputChannelStats InputChannels = 17;
- repeated TOutputChannelStats OutputChannels = 18;
-
- // raw tasks statistics
- repeated TKqpStatsTask Tasks = 19;
-
- uint64 TxId = 20;
- string StageGuid = 21;
- };
-
- message TNodeResourcesUsage {
- uint32 NodeId = 1;
- uint32 ComputeActorsCount = 2;
- TKqpStatsMemoryUsage MemoryUsage = 3;
- uint64 MaxExecutionTimeUs = 4;
- };
-
- message TTimelineStats {
- uint64 CreateExecutorTs = 1;
- uint64 StartExecuteTs = 2;
- uint64 FinishExecuteTs = 3;
- uint64 FirstResultRowTs = 4;
- };
-
- // Basic stats
- uint32 AffectedShards = 1;
- uint64 ExecuteTimeUs = 2;
- repeated TKqpStatsTable Tables = 3;
- uint64 ResultBytes = 4;
- uint64 ResultRows = 5;
- TKqpStatsAggr ComputeCpuTimeUs = 6;
- TKqpStatsAggr ShardsCpuTimeUs = 7;
- bool LiteralExecution = 8;
-
- // Profile stats
- TTimelineStats Timeline = 100;
- repeated TStageStats Stages = 101;
- repeated TNodeResourcesUsage NodeResourcesUsage = 102;
- repeated string TxPlansWithStats = 103;
-};
+syntax = "proto3";
+option cc_enable_arenas = true;
+
+package NKqpProto;
+option java_package = "ru.yandex.kikimr.proto";
+
+enum EKqpStatsMode {
+ STATS_MODE_UNSPECIFIED = 0;
+ STATS_MODE_NONE = 10;
+ STATS_MODE_BASIC = 20;
+ STATS_MODE_PROFILE = 30;
+};
+
+message TKqpStatsMinMax {
+ uint64 Min = 1;
+ uint64 Max = 2;
+}
+
+message TKqpStatsAggr {
+ uint64 Min = 1;
+ uint64 Max = 2;
+ uint64 Sum = 3;
+ uint64 Cnt = 4;
+}
+
+message TKqpStatsTable {
+ string TableId = 1;
+ string TablePath = 2;
+
+ uint64 ReadRows = 3;
+ uint64 ReadBytes = 4;
+ uint64 WriteRows = 5;
+ uint64 WriteBytes = 6;
+ uint64 EraseRows = 7;
+
+ uint64 AffectedPartitions = 8;
+}
+
+message TKqpStatsMemoryUsage {
+ uint32 InputChannelsCount = 1;
+ uint32 InputChannelsSize = 2;
+ uint32 OutputChannelsCount = 3;
+ uint32 OutputChannelsSize = 4;
+ uint32 ScanBuffersCount = 5;
+ uint32 ScanBuffersSize = 6;
+ uint32 MkqlSize = 7;
+}
+
+message TKqpStatsTask {
+ message TScanStats {
+ string TableId = 1;
+ string TablePath = 2;
+ // total read data
+ uint64 Rows = 3;
+ uint64 Bytes = 4;
+ // IScan stats
+ uint64 ScanTotalTime = 5;
+ uint64 ScanWaitTime = 6; // wait for row inside IScan object
+ uint64 BufferDataTime = 7; // time to store rows taken from IScan to the internal buffer
+ uint64 PageFaults = 8;
+ // ITableReader stats
+ uint64 ReadDataTime = 9; // TableReader read data time
+ uint64 WaitDataTime = 10; // TableReader wait data time
+ // Network stats
+ uint64 Messages = 11;
+ uint64 MessagesByPageFault = 12;
+ };
+
+ message TInputChannelStats {
+ uint64 ChannelId = 1;
+ uint64 Chunks = 2;
+ uint64 Bytes = 3;
+ uint64 RowsIn = 4;
+ uint64 RowsOut = 12;
+ uint64 StartTs = 5;
+ uint64 FinishTs = 6;
+ uint64 DeserializationTime = 7;
+ uint64 PollRequests = 8;
+ uint64 WaitTime = 9;
+ uint32 ResentAcks = 10;
+ uint64 MaxMemoryUsage = 11;
+ }
+
+ message TSourceStats {
+ uint64 ChannelId = 1;
+ uint64 Chunks = 2;
+ uint64 Bytes = 3;
+ uint64 RowsIn = 4;
+ uint64 RowsOut = 12;
+ uint64 StartTs = 5;
+ uint64 FinishTs = 6;
+ uint64 InputIndex = 7;
+ uint64 PollRequests = 8;
+ uint64 WaitTime = 9;
+ uint32 ResentAcks = 10;
+ uint64 MaxMemoryUsage = 11;
+ uint32 ErrorsCount = 13;
+ }
+
+ message TOutputChannelStats {
+ uint64 ChannelId = 1;
+ uint64 Chunks = 2;
+ uint64 Bytes = 3;
+ uint64 RowsIn = 4;
+ uint64 RowsOut = 26;
+ uint64 StartTs = 5;
+ uint64 FinishTs = 6;
+ uint64 SerializationTime = 7;
+ uint64 BlockedByCapacity = 10;
+ uint64 NoDstActorId = 11;
+ uint32 Count4KChunks = 14;
+ uint32 Count16KChunks = 15;
+ uint32 Count64KChunks = 16;
+ uint32 Count256KChunks = 17;
+ uint32 Count1MChunks = 18;
+ uint32 CountBigChunks = 19;
+ uint32 ResentMessages = 20;
+ uint64 MaxMemoryUsage = 21;
+ uint32 MaxRowsInMemory = 22;
+ uint64 SpilledBytes = 23;
+ uint64 SpilledRows = 24;
+ uint64 SpilledBlobs = 25;
+ }
+
+ message TSinkStats {
+ uint64 ChannelId = 1;
+ uint64 Chunks = 2;
+ uint64 Bytes = 3;
+ uint64 RowsIn = 4;
+ uint64 RowsOut = 26;
+ uint64 StartTs = 5;
+ uint64 FinishTs = 6;
+ uint32 Count4KChunks = 14;
+ uint32 Count16KChunks = 15;
+ uint32 Count64KChunks = 16;
+ uint32 Count256KChunks = 17;
+ uint32 Count1MChunks = 18;
+ uint32 CountBigChunks = 19;
+ uint64 MaxMemoryUsage = 21;
+ uint32 MaxRowsInMemory = 22;
+ uint64 SpilledBytes = 23;
+ uint64 SpilledRows = 24;
+ uint64 SpilledBlobs = 25;
+ uint32 ErrorsCount = 27;
+ }
+
+ message THistBucket {
+ double Bound = 1;
+ uint64 Value = 2;
+ }
+
+ uint64 TaskId = 1;
+ uint64 StartTs = 2;
+ uint64 FinishTs = 3;
+ uint64 BuildTaskTime = 4;
+ uint64 PrefetchTime = 5;
+ uint64 ComputeTime = 6;
+ uint64 TotalTime = 7; // ComputeTime + waits
+ uint64 WaitTime = 8;
+ uint64 WaitInputTime = 9;
+ uint64 WaitScanTime = 10;
+ uint64 WaitOutputTime = 11;
+ repeated THistBucket ComputeTimeByRun = 33;
+
+ uint64 InputRows = 12;
+ uint64 InputBytes = 13;
+ uint64 OutputRows = 14;
+ uint64 OutputBytes = 15;
+
+ uint64 TotalPageFaults = 16;
+
+ uint64 MkqlMaxMemoryUsage = 17;
+ uint64 ExtraMemoryBytes = 22;
+ uint32 ExtraMemoryRequests = 23;
+ uint32 ExtraMemoryWaitTimeMs = 24;
+
+ int64 MkqlTotalNodes = 25;
+ int64 MkqlCodegenFunctions = 26;
+ int64 CodeGenTotalInstructions = 27;
+ int64 CodeGenTotalFunctions = 28;
+ int64 CodeGenFullTime = 29;
+ int64 CodeGenFinalizeTime = 30;
+ int64 CodeGenModulePassTime = 31;
+
+ repeated TKqpStatsTable Tables = 18;
+ repeated TScanStats Scans = 19;
+ repeated TInputChannelStats InputChannels = 20;
+ repeated TOutputChannelStats OutputChannels = 21;
+ repeated TSourceStats Sources = 32;
+ repeated TSinkStats Sinks = 34;
+}
+
+message TKqpStatsRun {
+ // Basic stats
+ uint64 ExecutionTimeUs = 1;
+ repeated TKqpStatsTable Tables = 2;
+
+ // Profile stats
+ TKqpStatsMemoryUsage MemoryUsage = 100;
+ repeated TKqpStatsTask Tasks = 101;
+}
+
+message TKqpStatsExecution {
+ message TInputChannelStats {
+ uint32 SrcStageId = 1;
+ TKqpStatsAggr Bytes = 2;
+ TKqpStatsAggr Rows = 3;
+ TKqpStatsAggr DeserializationTime = 4;
+ TKqpStatsAggr PollRequests = 5;
+ TKqpStatsAggr WaitTime = 6;
+ TKqpStatsAggr ResentAcks = 7;
+ };
+
+ message TOutputChannelStats {
+ uint32 DstStageId = 1;
+ TKqpStatsAggr Bytes = 2;
+ TKqpStatsAggr Rows = 3;
+ TKqpStatsAggr SerializationTime = 4;
+ TKqpStatsAggr BlockedByCapacity = 5;
+ TKqpStatsAggr NoDstActorId = 6;
+ TKqpStatsAggr ResentMessages = 7;
+ };
+
+ message TScansStats {
+ string TablePath = 1;
+ TKqpStatsAggr Bytes = 2;
+ TKqpStatsAggr Rows = 3;
+ TKqpStatsAggr ScanTotalTime = 4;
+ TKqpStatsAggr ScanWaitTime = 5;
+ TKqpStatsAggr WaitDataTime = 6;
+ };
+
+ message TStageStats {
+ uint32 StageId = 1;
+ string Program = 2;
+ TKqpStatsMinMax StageStartTs = 3;
+ TKqpStatsMinMax StageFinishTs = 4;
+ TKqpStatsAggr TasksTotalTime = 5;
+ TKqpStatsAggr TasksComputeTime = 6;
+ TKqpStatsAggr TasksWaitTime = 7;
+ TKqpStatsAggr TasksWaitScanTime = 8;
+ TKqpStatsAggr TasksWaitInputTime = 9;
+ TKqpStatsAggr TasksWaitOutputTime = 10;
+ TKqpStatsAggr TasksScanTotalTime = 11;
+ TKqpStatsAggr TasksScanWaitTime = 12;
+ TKqpStatsAggr TasksReadBytes = 13;
+ TKqpStatsAggr TasksReadRows = 14;
+
+ uint32 TasksCount = 15;
+
+ repeated TScansStats Scans = 16;
+ repeated TInputChannelStats InputChannels = 17;
+ repeated TOutputChannelStats OutputChannels = 18;
+
+ // raw tasks statistics
+ repeated TKqpStatsTask Tasks = 19;
+
+ uint64 TxId = 20;
+ string StageGuid = 21;
+ };
+
+ message TNodeResourcesUsage {
+ uint32 NodeId = 1;
+ uint32 ComputeActorsCount = 2;
+ TKqpStatsMemoryUsage MemoryUsage = 3;
+ uint64 MaxExecutionTimeUs = 4;
+ };
+
+ message TTimelineStats {
+ uint64 CreateExecutorTs = 1;
+ uint64 StartExecuteTs = 2;
+ uint64 FinishExecuteTs = 3;
+ uint64 FirstResultRowTs = 4;
+ };
+
+ // Basic stats
+ uint32 AffectedShards = 1;
+ uint64 ExecuteTimeUs = 2;
+ repeated TKqpStatsTable Tables = 3;
+ uint64 ResultBytes = 4;
+ uint64 ResultRows = 5;
+ TKqpStatsAggr ComputeCpuTimeUs = 6;
+ TKqpStatsAggr ShardsCpuTimeUs = 7;
+ bool LiteralExecution = 8;
+
+ // Profile stats
+ TTimelineStats Timeline = 100;
+ repeated TStageStats Stages = 101;
+ repeated TNodeResourcesUsage NodeResourcesUsage = 102;
+ repeated string TxPlansWithStats = 103;
+};
diff --git a/ydb/library/yql/dq/actors/protos/dq_stats.proto b/ydb/library/yql/dq/actors/protos/dq_stats.proto
index 6c1804a264..6b60879030 100644
--- a/ydb/library/yql/dq/actors/protos/dq_stats.proto
+++ b/ydb/library/yql/dq/actors/protos/dq_stats.proto
@@ -1,235 +1,235 @@
-syntax = "proto3";
-
-package NYql.NDqProto;
-
-option cc_enable_arenas = true;
-
-import "google/protobuf/any.proto";
-
-enum EDqStatsMode {
- DQ_STATS_MODE_UNSPECIFIED = 0;
- DQ_STATS_MODE_NONE = 10;
- DQ_STATS_MODE_BASIC = 20;
- DQ_STATS_MODE_PROFILE = 30;
-}
-
-message TDqSourceStats {
- // basic stats
- uint64 InputIndex = 1;
- uint64 Chunks = 2;
- uint64 Bytes = 3;
- uint64 RowsIn = 4;
- uint64 RowsOut = 5;
-
- // profile stats
- uint64 MaxMemoryUsage = 6;
- uint32 ErrorsCount = 7;
-
- google.protobuf.Any Extra = 100;
-}
-
-message TDqInputChannelStats {
- // basic stats
- uint64 ChannelId = 1;
- uint64 Chunks = 2;
- uint64 Bytes = 3;
- uint64 RowsIn = 4;
- uint64 RowsOut = 5;
-
- // profile stats
- uint64 DeserializationTimeUs = 6;
- uint64 MaxMemoryUsage = 7;
-
- uint32 PollRequests = 8;
- uint32 ResentMessages = 9;
- uint64 WaitTimeUs = 10;
-
- google.protobuf.Any Extra = 100;
-}
-
-message TDqSinkStats {
- // basic stats
- uint64 OutputIndex = 1;
- uint64 Chunks = 2;
- uint64 Bytes = 3;
- uint64 RowsIn = 4;
- uint64 RowsOut = 5;
-
- // profile stats
- uint64 MaxMemoryUsage = 6;
- uint32 ErrorsCount = 7;
-
- google.protobuf.Any Extra = 100;
-}
-
-message TDqOutputChannelStats {
- // basic stats
- uint64 ChannelId = 1;
- uint64 Chunks = 2;
- uint64 Bytes = 3;
- uint64 RowsIn = 4;
- uint64 RowsOut = 5;
-
- // profile stats
- uint64 SerializationTimeUs = 6;
- uint32 BlockedByCapacity = 7;
- uint32 ResentMessages = 8;
- uint32 NoDstActorId = 9;
- uint32 MaxRowsInMemory = 10;
- uint64 MaxMemoryUsage = 11;
-
- uint64 SpilledBytes = 12;
- uint64 SpilledRows = 13;
- uint64 SpilledBlobs = 14;
-
- google.protobuf.Any Extra = 100;
-}
-
-message TDqTableStats {
- string TablePath = 1;
-
- uint64 ReadRows = 2;
- uint64 ReadBytes = 3;
- uint64 WriteRows = 4;
- uint64 WriteBytes = 5;
- uint64 EraseRows = 6;
- uint64 EraseBytes = 7;
-
- uint32 AffectedPartitions = 8; // TODO: move it to the Extra
-
- google.protobuf.Any Extra = 100;
-}
-
+syntax = "proto3";
+
+package NYql.NDqProto;
+
+option cc_enable_arenas = true;
+
+import "google/protobuf/any.proto";
+
+enum EDqStatsMode {
+ DQ_STATS_MODE_UNSPECIFIED = 0;
+ DQ_STATS_MODE_NONE = 10;
+ DQ_STATS_MODE_BASIC = 20;
+ DQ_STATS_MODE_PROFILE = 30;
+}
+
+message TDqSourceStats {
+ // basic stats
+ uint64 InputIndex = 1;
+ uint64 Chunks = 2;
+ uint64 Bytes = 3;
+ uint64 RowsIn = 4;
+ uint64 RowsOut = 5;
+
+ // profile stats
+ uint64 MaxMemoryUsage = 6;
+ uint32 ErrorsCount = 7;
+
+ google.protobuf.Any Extra = 100;
+}
+
+message TDqInputChannelStats {
+ // basic stats
+ uint64 ChannelId = 1;
+ uint64 Chunks = 2;
+ uint64 Bytes = 3;
+ uint64 RowsIn = 4;
+ uint64 RowsOut = 5;
+
+ // profile stats
+ uint64 DeserializationTimeUs = 6;
+ uint64 MaxMemoryUsage = 7;
+
+ uint32 PollRequests = 8;
+ uint32 ResentMessages = 9;
+ uint64 WaitTimeUs = 10;
+
+ google.protobuf.Any Extra = 100;
+}
+
+message TDqSinkStats {
+ // basic stats
+ uint64 OutputIndex = 1;
+ uint64 Chunks = 2;
+ uint64 Bytes = 3;
+ uint64 RowsIn = 4;
+ uint64 RowsOut = 5;
+
+ // profile stats
+ uint64 MaxMemoryUsage = 6;
+ uint32 ErrorsCount = 7;
+
+ google.protobuf.Any Extra = 100;
+}
+
+message TDqOutputChannelStats {
+ // basic stats
+ uint64 ChannelId = 1;
+ uint64 Chunks = 2;
+ uint64 Bytes = 3;
+ uint64 RowsIn = 4;
+ uint64 RowsOut = 5;
+
+ // profile stats
+ uint64 SerializationTimeUs = 6;
+ uint32 BlockedByCapacity = 7;
+ uint32 ResentMessages = 8;
+ uint32 NoDstActorId = 9;
+ uint32 MaxRowsInMemory = 10;
+ uint64 MaxMemoryUsage = 11;
+
+ uint64 SpilledBytes = 12;
+ uint64 SpilledRows = 13;
+ uint64 SpilledBlobs = 14;
+
+ google.protobuf.Any Extra = 100;
+}
+
+message TDqTableStats {
+ string TablePath = 1;
+
+ uint64 ReadRows = 2;
+ uint64 ReadBytes = 3;
+ uint64 WriteRows = 4;
+ uint64 WriteBytes = 5;
+ uint64 EraseRows = 6;
+ uint64 EraseBytes = 7;
+
+ uint32 AffectedPartitions = 8; // TODO: move it to the Extra
+
+ google.protobuf.Any Extra = 100;
+}
+
message TDqMkqlStat {
string Name = 1;
int64 Value = 2;
bool Deriv = 3;
}
-message TDqTaskStats {
- // basic stats
- uint64 TaskId = 1;
- uint32 StageId = 2;
- uint64 CpuTimeUs = 3; // total cpu time (build & compute)
- uint64 FirstRowTimeMs = 4; // first row time, timestamp in millis
- uint64 FinishTimeMs = 5; // task finish time, timestamp in millis
- uint64 InputRows = 6;
- uint64 InputBytes = 7;
- uint64 OutputRows = 8;
- uint64 OutputBytes = 9;
-
- repeated TDqTableStats Tables = 10;
-
- // profile stats
- uint64 BuildCpuTimeUs = 103; // prepare task time: build computation graph, prepare parameters, ...
+message TDqTaskStats {
+ // basic stats
+ uint64 TaskId = 1;
+ uint32 StageId = 2;
+ uint64 CpuTimeUs = 3; // total cpu time (build & compute)
+ uint64 FirstRowTimeMs = 4; // first row time, timestamp in millis
+ uint64 FinishTimeMs = 5; // task finish time, timestamp in millis
+ uint64 InputRows = 6;
+ uint64 InputBytes = 7;
+ uint64 OutputRows = 8;
+ uint64 OutputBytes = 9;
+
+ repeated TDqTableStats Tables = 10;
+
+ // profile stats
+ uint64 BuildCpuTimeUs = 103; // prepare task time: build computation graph, prepare parameters, ...
uint64 WaitTimeUs = 104; // total wait (input + output) wall time
- uint64 WaitOutputTimeUs = 105; // wait output wall time (any output: channels, sinks, ...)
+ uint64 WaitOutputTimeUs = 105; // wait output wall time (any output: channels, sinks, ...)
uint64 ComputeCpuTimeUs = 102; // compute time only
uint64 PendingInputTimeUs = 107; // time waiting input data
uint64 PendingOutputTimeUs = 108; // time waiting output data
uint64 FinishTimeUs = 109; // time in finished state // ComputeCpuTimeUs + PendingInputTimeUs + PendingOutputTimeUs + FinishTimeUs == 100% (or == const in aggregated graphs for several stages/tasks)
repeated TDqMkqlStat MkqlStats = 110; // stats from mkql
-
- message THistBucket {
- double Bound = 1;
- uint64 Value = 2;
- }
- repeated THistBucket ComputeCpuTimeByRun = 106;
-
- repeated TDqSourceStats Sources = 150;
- repeated TDqInputChannelStats InputChannels = 151;
- repeated TDqSinkStats Sinks = 152;
- repeated TDqOutputChannelStats OutputChannels = 153;
-
- google.protobuf.Any Extra = 200;
-}
-
-message TDqComputeActorStats {
- // basic stats
- uint64 CpuTimeUs = 1; // total cpu time: tasks cpu time + self cpu time
- uint64 DurationUs = 2; // compute actor duration, wall time (from FirstRowTime to FinishTime)
- repeated TDqTaskStats Tasks = 3; // in the BASIC_MODE only basic fields are used
-
- // profile stats
- uint64 MkqlMaxMemoryUsage = 102; // MKQL allocations stats
- uint64 MkqlExtraMemoryBytes = 103;
- uint32 MkqlExtraMemoryRequests = 104;
-
- google.protobuf.Any Extra = 200;
-}
-
-message TDqStatsAggr {
- uint64 Min = 1;
- uint64 Max = 2;
- uint64 Sum = 3;
- uint64 Cnt = 4;
-}
-
-message TDqStatsMinMax {
- uint64 Min = 1;
- uint64 Max = 2;
-}
-
-message TDqTableAggrStats {
- string TablePath = 1;
-
- TDqStatsAggr ReadRows = 2;
- TDqStatsAggr ReadBytes = 3;
- TDqStatsAggr WriteRows = 4;
- TDqStatsAggr WriteBytes = 5;
- TDqStatsAggr EraseRows = 6;
- TDqStatsAggr EraseBytes = 7;
-
- uint32 AffectedPartitions = 8;
-
- google.protobuf.Any Extra = 100;
-}
-
-// aggregated stats for all tasks of the stage
-message TDqStageStats {
- uint32 StageId = 1;
- string StageGuid = 2;
- string Program = 3;
-
- uint32 TotalTasksCount = 5;
- uint32 FailedTasksCount = 6;
-
- TDqStatsAggr CpuTimeUs = 8;
- TDqStatsAggr InputRows = 9;
- TDqStatsAggr InputBytes = 10;
- TDqStatsAggr OutputRows = 11;
- TDqStatsAggr OutputBytes = 12;
-
- TDqStatsMinMax FirstRowTimeMs = 13;
- TDqStatsMinMax FinishTimeMs = 14;
- uint64 DurationUs = 15; // microseconds from min(task_first_row_time) to max(task_finish_time)
-
- repeated TDqTableAggrStats Tables = 16; // is it required?
-
- repeated TDqComputeActorStats ComputeActors = 17; // more detailed stats
-
- google.protobuf.Any Extra = 100;
-}
-
-// graph execution stats
-message TDqExecutionStats {
- // basic stats
- uint64 CpuTimeUs = 1; // total cpu time, executer + compute actors + ...
- uint64 DurationUs = 2; // execution wall time
-
- uint64 ResultRows = 3;
- uint64 ResultBytes = 4;
-
- repeated TDqTableStats Tables = 6;
-
- // profile stats
- uint64 ExecuterCpuTimeUs = 10; // executer self cpu time, microseconds
- uint64 StartTimeMs = 11; // executer start timestamp, milliseconds
- uint64 FinishTimeMs = 12; // executer finish timestamp, milliseconds
- uint64 FirstRowTimeMs = 13; // first result row timestamp, milliseconds
-
- repeated TDqStageStats Stages = 14;
- repeated string TxPlansWithStats = 15;
-
- google.protobuf.Any Extra = 100;
-}
+
+ message THistBucket {
+ double Bound = 1;
+ uint64 Value = 2;
+ }
+ repeated THistBucket ComputeCpuTimeByRun = 106;
+
+ repeated TDqSourceStats Sources = 150;
+ repeated TDqInputChannelStats InputChannels = 151;
+ repeated TDqSinkStats Sinks = 152;
+ repeated TDqOutputChannelStats OutputChannels = 153;
+
+ google.protobuf.Any Extra = 200;
+}
+
+message TDqComputeActorStats {
+ // basic stats
+ uint64 CpuTimeUs = 1; // total cpu time: tasks cpu time + self cpu time
+ uint64 DurationUs = 2; // compute actor duration, wall time (from FirstRowTime to FinishTime)
+ repeated TDqTaskStats Tasks = 3; // in the BASIC_MODE only basic fields are used
+
+ // profile stats
+ uint64 MkqlMaxMemoryUsage = 102; // MKQL allocations stats
+ uint64 MkqlExtraMemoryBytes = 103;
+ uint32 MkqlExtraMemoryRequests = 104;
+
+ google.protobuf.Any Extra = 200;
+}
+
+message TDqStatsAggr {
+ uint64 Min = 1;
+ uint64 Max = 2;
+ uint64 Sum = 3;
+ uint64 Cnt = 4;
+}
+
+message TDqStatsMinMax {
+ uint64 Min = 1;
+ uint64 Max = 2;
+}
+
+message TDqTableAggrStats {
+ string TablePath = 1;
+
+ TDqStatsAggr ReadRows = 2;
+ TDqStatsAggr ReadBytes = 3;
+ TDqStatsAggr WriteRows = 4;
+ TDqStatsAggr WriteBytes = 5;
+ TDqStatsAggr EraseRows = 6;
+ TDqStatsAggr EraseBytes = 7;
+
+ uint32 AffectedPartitions = 8;
+
+ google.protobuf.Any Extra = 100;
+}
+
+// aggregated stats for all tasks of the stage
+message TDqStageStats {
+ uint32 StageId = 1;
+ string StageGuid = 2;
+ string Program = 3;
+
+ uint32 TotalTasksCount = 5;
+ uint32 FailedTasksCount = 6;
+
+ TDqStatsAggr CpuTimeUs = 8;
+ TDqStatsAggr InputRows = 9;
+ TDqStatsAggr InputBytes = 10;
+ TDqStatsAggr OutputRows = 11;
+ TDqStatsAggr OutputBytes = 12;
+
+ TDqStatsMinMax FirstRowTimeMs = 13;
+ TDqStatsMinMax FinishTimeMs = 14;
+ uint64 DurationUs = 15; // microseconds from min(task_first_row_time) to max(task_finish_time)
+
+ repeated TDqTableAggrStats Tables = 16; // is it required?
+
+ repeated TDqComputeActorStats ComputeActors = 17; // more detailed stats
+
+ google.protobuf.Any Extra = 100;
+}
+
+// graph execution stats
+message TDqExecutionStats {
+ // basic stats
+ uint64 CpuTimeUs = 1; // total cpu time, executer + compute actors + ...
+ uint64 DurationUs = 2; // execution wall time
+
+ uint64 ResultRows = 3;
+ uint64 ResultBytes = 4;
+
+ repeated TDqTableStats Tables = 6;
+
+ // profile stats
+ uint64 ExecuterCpuTimeUs = 10; // executer self cpu time, microseconds
+ uint64 StartTimeMs = 11; // executer start timestamp, milliseconds
+ uint64 FinishTimeMs = 12; // executer finish timestamp, milliseconds
+ uint64 FirstRowTimeMs = 13; // first result row timestamp, milliseconds
+
+ repeated TDqStageStats Stages = 14;
+ repeated string TxPlansWithStats = 15;
+
+ google.protobuf.Any Extra = 100;
+}
diff --git a/ydb/library/yql/dq/actors/protos/ya.make b/ydb/library/yql/dq/actors/protos/ya.make
index e9c7feb27d..81e25d0336 100644
--- a/ydb/library/yql/dq/actors/protos/ya.make
+++ b/ydb/library/yql/dq/actors/protos/ya.make
@@ -4,7 +4,7 @@ OWNER(g:yql)
SRCS(
dq_events.proto
- dq_stats.proto
+ dq_stats.proto
)
PEERDIR(
diff --git a/ydb/library/yql/dq/actors/ya.make b/ydb/library/yql/dq/actors/ya.make
index d005253954..962f15c886 100644
--- a/ydb/library/yql/dq/actors/ya.make
+++ b/ydb/library/yql/dq/actors/ya.make
@@ -4,10 +4,10 @@ GRPC()
OWNER(g:yql)
-SRCS(
- dq.cpp
- dq_events_ids.cpp
-)
+SRCS(
+ dq.cpp
+ dq_events_ids.cpp
+)
PEERDIR(
ydb/library/yql/dq/actors/protos
diff --git a/ydb/library/yql/dq/common/dq_common.cpp b/ydb/library/yql/dq/common/dq_common.cpp
index 623408561f..865ed134a8 100644
--- a/ydb/library/yql/dq/common/dq_common.cpp
+++ b/ydb/library/yql/dq/common/dq_common.cpp
@@ -1,4 +1,4 @@
-#include "dq_common.h"
+#include "dq_common.h"
IOutputStream& operator<<(IOutputStream& stream, const NYql::NDq::TTxId& txId) {
std::visit([&stream](auto arg) {
diff --git a/ydb/library/yql/dq/common/dq_value.cpp b/ydb/library/yql/dq/common/dq_value.cpp
index f6b156239f..5f87331b60 100644
--- a/ydb/library/yql/dq/common/dq_value.cpp
+++ b/ydb/library/yql/dq/common/dq_value.cpp
@@ -1 +1 @@
-#include "dq_value.h"
+#include "dq_value.h"
diff --git a/ydb/library/yql/dq/common/dq_value.h b/ydb/library/yql/dq/common/dq_value.h
index a7b730f856..6699f30411 100644
--- a/ydb/library/yql/dq/common/dq_value.h
+++ b/ydb/library/yql/dq/common/dq_value.h
@@ -6,19 +6,19 @@ namespace NYql::NDq {
class TMkqlValueRef {
public:
- TMkqlValueRef(const NKikimrMiniKQL::TType& type, const NKikimrMiniKQL::TValue& value)
- : Type(&type)
- , Value(&value) {}
+ TMkqlValueRef(const NKikimrMiniKQL::TType& type, const NKikimrMiniKQL::TValue& value)
+ : Type(&type)
+ , Value(&value) {}
- explicit TMkqlValueRef(const NKikimrMiniKQL::TParams& params)
- : TMkqlValueRef(params.GetType(), params.GetValue()) {}
+ explicit TMkqlValueRef(const NKikimrMiniKQL::TParams& params)
+ : TMkqlValueRef(params.GetType(), params.GetValue()) {}
- explicit TMkqlValueRef(const NKikimrMiniKQL::TResult& result)
- : TMkqlValueRef(result.GetType(), result.GetValue()) {}
+ explicit TMkqlValueRef(const NKikimrMiniKQL::TResult& result)
+ : TMkqlValueRef(result.GetType(), result.GetValue()) {}
- const NKikimrMiniKQL::TType& GetType() const { return *Type; }
+ const NKikimrMiniKQL::TType& GetType() const { return *Type; }
- const NKikimrMiniKQL::TValue& GetValue() const { return *Value; }
+ const NKikimrMiniKQL::TValue& GetValue() const { return *Value; }
private:
const NKikimrMiniKQL::TType *Type;
diff --git a/ydb/library/yql/dq/common/ya.make b/ydb/library/yql/dq/common/ya.make
index add67f1d76..a47ae59c37 100644
--- a/ydb/library/yql/dq/common/ya.make
+++ b/ydb/library/yql/dq/common/ya.make
@@ -11,9 +11,9 @@ PEERDIR(
)
SRCS(
- dq_common.cpp
+ dq_common.cpp
dq_resource_quoter.h
- dq_value.cpp
+ dq_value.cpp
)
END()
diff --git a/ydb/library/yql/dq/comp_nodes/yql_common_dq_factory.cpp b/ydb/library/yql/dq/comp_nodes/yql_common_dq_factory.cpp
index fbad5bf0a8..7227430c08 100644
--- a/ydb/library/yql/dq/comp_nodes/yql_common_dq_factory.cpp
+++ b/ydb/library/yql/dq/comp_nodes/yql_common_dq_factory.cpp
@@ -22,7 +22,7 @@ public:
private:
NUdf::TUnboxedValue Run(const NUdf::IValueBuilder*, const NUdf::TUnboxedValuePod*) const override {
ActorSystem->Send(new NActors::IEventHandle(
- CurrentActorId, NActors::TActorId(), new NDq::TEvDqCompute::TEvResumeExecution()));
+ CurrentActorId, NActors::TActorId(), new NDq::TEvDqCompute::TEvResumeExecution()));
return NUdf::TUnboxedValue::Void();
}
diff --git a/ydb/library/yql/dq/connections.md b/ydb/library/yql/dq/connections.md
index 779e4a9402..09da3d1d25 100644
--- a/ydb/library/yql/dq/connections.md
+++ b/ydb/library/yql/dq/connections.md
@@ -1,118 +1,118 @@
-# Intro
-SQL-запрос преобразуется во внутреннее представление в виде DQ-графа.
-
-DQ-граф состоит из стадий (_DqStage_), соединённых рёбрами (коннекшинами типа _DqConnection_).
-
-Каждая стадия исполняется в виде некоторого количества тасков (_DqTask_). Например, читающая таблицу стадия может
+# Intro
+SQL-запрос преобразуется во внутреннее представление в виде DQ-графа.
+
+DQ-граф состоит из стадий (_DqStage_), соединённых рёбрами (коннекшинами типа _DqConnection_).
+
+Каждая стадия исполняется в виде некоторого количества тасков (_DqTask_). Например, читающая таблицу стадия может
исполняться в N тасков, где N - это количество шардов таблицы.
-
-# Виды коннекшинов
-
-## DqCnUnionAll
-* _NodeType_: **UnionAll**
-* _PlanNodeType_: **Connection**
-
-Объединяет результаты всех тасков одной стадии (producer stage) и отдаёт их как единый результат в единственный таск
-другой стадии (consumer stage).
-```
- +--------------------------------------+
- | DqStage_1: DqTask_1, ..., DqTask_N |
- +----------------+--------------+------+
- | ... |
- +--------+-----+ DqCnUnionAll
- |
- V
- +--------------------------------------+
- | DqStage_2: DqTask_1 |
- +--------------------------------------+
-```
-
-## DqCnMerge (In Development)
-* _NodeType_: **Merge**
-* _PlanNodeType_: **Connection**
-
-Частный случай DqUnionAll, когда результаты первой стадии отсортированы по заданному набору колонок. Результат так же
-отсортирован.
-
-
-## DqCnMap
-* _NodeType_: **Map**
-* _PlanNodeType_: **Connection**
-
-Реализует связи тасков стадий 1-к-1. Т.е. producer stage и consumer stage должны иметь одинаковое количество тасков.
-```
- +--------------------------------------+
- | DqStage_1: DqTask_1, ..., DqTask_N |
- +----------------+--------------+------+
- | |
- | ... | DqCnMap
- | |
- V V
- +--------------------------------------+
- | DqStage_2: DqTask_1, ..., DqTask_N |
- +--------------------------------------+
-```
-
-## DqCnBroadcast
-* NodeType: **Broadcast**
-* PlanNodeType: **Connection**
-
-Рассылает результат единственного таска одной стадии (producer stage) на все таски другой (consumer stage).
-```
- +--------------------------------------+
- | DqStage_1: DqTask_1 |
- +-------------------------+------------+
- |
- +--------+-----+ DqCnBroadcast
- | ... |
- V V
- +--------------------------------------+
- | DqStage_2: DqTask_1, ..., DqTask_N |
- +--------------------------------------+
-```
-
-## DqCnHashShuffle
-* _NodeType_: **HashShuffle**
-* _PlanNodeType_: **Connection**
+
+# Виды коннекшинов
+
+## DqCnUnionAll
+* _NodeType_: **UnionAll**
+* _PlanNodeType_: **Connection**
+
+Объединяет результаты всех тасков одной стадии (producer stage) и отдаёт их как единый результат в единственный таск
+другой стадии (consumer stage).
+```
+ +--------------------------------------+
+ | DqStage_1: DqTask_1, ..., DqTask_N |
+ +----------------+--------------+------+
+ | ... |
+ +--------+-----+ DqCnUnionAll
+ |
+ V
+ +--------------------------------------+
+ | DqStage_2: DqTask_1 |
+ +--------------------------------------+
+```
+
+## DqCnMerge (In Development)
+* _NodeType_: **Merge**
+* _PlanNodeType_: **Connection**
+
+Частный случай DqUnionAll, когда результаты первой стадии отсортированы по заданному набору колонок. Результат так же
+отсортирован.
+
+
+## DqCnMap
+* _NodeType_: **Map**
+* _PlanNodeType_: **Connection**
+
+Реализует связи тасков стадий 1-к-1. Т.е. producer stage и consumer stage должны иметь одинаковое количество тасков.
+```
+ +--------------------------------------+
+ | DqStage_1: DqTask_1, ..., DqTask_N |
+ +----------------+--------------+------+
+ | |
+ | ... | DqCnMap
+ | |
+ V V
+ +--------------------------------------+
+ | DqStage_2: DqTask_1, ..., DqTask_N |
+ +--------------------------------------+
+```
+
+## DqCnBroadcast
+* NodeType: **Broadcast**
+* PlanNodeType: **Connection**
+
+Рассылает результат единственного таска одной стадии (producer stage) на все таски другой (consumer stage).
+```
+ +--------------------------------------+
+ | DqStage_1: DqTask_1 |
+ +-------------------------+------------+
+ |
+ +--------+-----+ DqCnBroadcast
+ | ... |
+ V V
+ +--------------------------------------+
+ | DqStage_2: DqTask_1, ..., DqTask_N |
+ +--------------------------------------+
+```
+
+## DqCnHashShuffle
+* _NodeType_: **HashShuffle**
+* _PlanNodeType_: **Connection**
* _KeyColumns_: список колонок, по которым идёт перемешивание
-
-Рассылает результаты с тасков одной стадии (producer stage) на таски другой (consumer stage) по некоторому правилу для
-заданных колонок. Правило жётско описано в коде, но вот список колонок указывается в каждом коннекшине независимо.
-
-Типичным использованием является hash-partitioning для full outer join и для агрегаций.
-```
- +--------------------------------------+
- | DqStage_1: DqTask_1, ..., DqTask_N |
- +----------------+--------------+------+
- |\ /|
+
+Рассылает результаты с тасков одной стадии (producer stage) на таски другой (consumer stage) по некоторому правилу для
+заданных колонок. Правило жётско описано в коде, но вот список колонок указывается в каждом коннекшине независимо.
+
+Типичным использованием является hash-partitioning для full outer join и для агрегаций.
+```
+ +--------------------------------------+
+ | DqStage_1: DqTask_1, ..., DqTask_N |
+ +----------------+--------------+------+
+ |\ /|
| \ ... / | DqCnHashShuffle
- | \ / |
- V V V V
- +--------------------------------------+
- | DqStage_2: DqTask_1, ..., DqTask_M |
- +--------------------------------------+
-```
-
-## DqCnResult
-* _NodeType_: **Result**
-* _PlanNodeType_: --
-
-Вспомогательный вид коннекшина, в который передаётся результат выполнения всего запроса.
-
-## DqCnValue
-Вспомогательный вид коннекшина, который реализует переход из векторного в скалярный контекст. По нему происходит
-разбиение запроса на фазы исполнения.
-
-Не должно попадать в explain.
-
-
-# KQP Specific Connections
-
-## KqpCnMapShard
-* _NodeType_: **Map**
-* _PlanNodeType_: **Connection**
-
-Используется только для inplace-update под прагмой. В этом случае обе стадии выполняются на одном шарде и последовательно.
-
-## KqpCnShuffleShard
+ | \ / |
+ V V V V
+ +--------------------------------------+
+ | DqStage_2: DqTask_1, ..., DqTask_M |
+ +--------------------------------------+
+```
+
+## DqCnResult
+* _NodeType_: **Result**
+* _PlanNodeType_: --
+
+Вспомогательный вид коннекшина, в который передаётся результат выполнения всего запроса.
+
+## DqCnValue
+Вспомогательный вид коннекшина, который реализует переход из векторного в скалярный контекст. По нему происходит
+разбиение запроса на фазы исполнения.
+
+Не должно попадать в explain.
+
+
+# KQP Specific Connections
+
+## KqpCnMapShard
+* _NodeType_: **Map**
+* _PlanNodeType_: **Connection**
+
+Используется только для inplace-update под прагмой. В этом случае обе стадии выполняются на одном шарде и последовательно.
+
+## KqpCnShuffleShard
Задумывалось для межшардового взаимодействия. Нигде не используется.
diff --git a/ydb/library/yql/dq/expr_nodes/dq_expr_nodes.json b/ydb/library/yql/dq/expr_nodes/dq_expr_nodes.json
index 5e0b7a61bf..67627b1b8b 100644
--- a/ydb/library/yql/dq/expr_nodes/dq_expr_nodes.json
+++ b/ydb/library/yql/dq/expr_nodes/dq_expr_nodes.json
@@ -80,10 +80,10 @@
"ListBase": "TDqSink"
},
{
- "Name": "TDqStageBase",
+ "Name": "TDqStageBase",
"Base": "TCallable",
- "Match": {"Type": "CallableBase"},
- "Builder": {"Generate": "None"},
+ "Match": {"Type": "CallableBase"},
+ "Builder": {"Generate": "None"},
"Children": [
{"Index": 0, "Name": "Inputs", "Type": "TExprList"},
{"Index": 1, "Name": "Program", "Type": "TCoLambda"},
@@ -92,29 +92,29 @@
]
},
{
- "Name": "TDqStage",
- "Base": "TDqStageBase",
- "Match": {"Type": "Callable", "Name": "DqStage"}
- },
- {
- "Name": "TDqPhyStage",
- "Base": "TDqStageBase",
+ "Name": "TDqStage",
+ "Base": "TDqStageBase",
+ "Match": {"Type": "Callable", "Name": "DqStage"}
+ },
+ {
+ "Name": "TDqPhyStage",
+ "Base": "TDqStageBase",
"Match": {"Type": "Callable", "Name": "DqPhyStage"}
- },
- {
+ },
+ {
"Name": "TDqStageList",
"ListBase": "TDqStageBase"
},
{
- "Name": "TDqPhyStageList",
- "ListBase": "TDqPhyStage"
- },
- {
+ "Name": "TDqPhyStageList",
+ "ListBase": "TDqPhyStage"
+ },
+ {
"Name": "TDqOutput",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "TDqOutput"},
"Children": [
- {"Index": 0, "Name": "Stage", "Type": "TDqStageBase"},
+ {"Index": 0, "Name": "Stage", "Type": "TDqStageBase"},
{"Index": 1, "Name": "Index", "Type": "TCoAtom"}
]
},
@@ -128,9 +128,9 @@
]
},
{
- "Name": "TDqCnHashShuffle",
+ "Name": "TDqCnHashShuffle",
"Base": "TDqConnection",
- "Match": {"Type": "Callable", "Name": "DqCnHashShuffle"},
+ "Match": {"Type": "Callable", "Name": "DqCnHashShuffle"},
"Children": [
{"Index": 1, "Name": "KeyColumns", "Type": "TCoAtomList"}
]
@@ -153,12 +153,12 @@
{
"Name": "TDqCnResult",
"Base": "TDqConnection",
- "Match": {"Type": "Callable", "Name": "DqCnResult"},
- "Children": [
- {"Index": 1, "Name": "ColumnHints", "Type": "TCoAtomList"}
- ]
- },
- {
+ "Match": {"Type": "Callable", "Name": "DqCnResult"},
+ "Children": [
+ {"Index": 1, "Name": "ColumnHints", "Type": "TCoAtomList"}
+ ]
+ },
+ {
"Name": "TDqCnValue",
"Base": "TDqConnection",
"Match": {"Type": "Callable", "Name": "DqCnValue"}
@@ -185,12 +185,12 @@
]
},
{
- "Name": "TDqReplicate",
- "Base": "TFreeArgCallable",
- "Match": {"Type": "Callable", "Name": "DqReplicate"},
- "Children": [
- {"Index": 0, "Name": "Input", "Type": "TExprBase"}
- ]
+ "Name": "TDqReplicate",
+ "Base": "TFreeArgCallable",
+ "Match": {"Type": "Callable", "Name": "DqReplicate"},
+ "Children": [
+ {"Index": 0, "Name": "Input", "Type": "TExprBase"}
+ ]
},
{
"Name": "TDqQuery",
diff --git a/ydb/library/yql/dq/opt/dq_opt.cpp b/ydb/library/yql/dq/opt/dq_opt.cpp
index 2c16794421..f60f29dd46 100644
--- a/ydb/library/yql/dq/opt/dq_opt.cpp
+++ b/ydb/library/yql/dq/opt/dq_opt.cpp
@@ -11,11 +11,11 @@ namespace NYql::NDq {
TDqStageSettings TDqStageSettings::Parse(const TDqStageBase& node) {
TDqStageSettings settings{};
-
- for (const auto& tuple : node.Settings()) {
+
+ for (const auto& tuple : node.Settings()) {
if (const auto name = tuple.Name().Value(); name == IdSettingName) {
- YQL_ENSURE(tuple.Value().Maybe<TCoAtom>());
- settings.Id = tuple.Value().Cast<TCoAtom>().Value();
+ YQL_ENSURE(tuple.Value().Maybe<TCoAtom>());
+ settings.Id = tuple.Value().Cast<TCoAtom>().Value();
} else if (name == LogicalIdSettingName) {
YQL_ENSURE(tuple.Value().Maybe<TCoAtom>());
settings.LogicalId = FromString<ui64>(tuple.Value().Cast<TCoAtom>().Value());
@@ -35,12 +35,12 @@ TDqStageSettings TDqStageSettings::Parse(const TDqStageBase& node) {
}
} else if (name == TransformConcurrencySetting) {
settings.TransformConcurrency = FromString<ui32>(tuple.Value().Cast<TCoAtom>().Value());
- }
- }
-
- return settings;
-}
-
+ }
+ }
+
+ return settings;
+}
+
TDqStageSettings TDqStageSettings::New(const NNodes::TDqStageBase& node) {
auto settings = Parse(node);
@@ -52,35 +52,35 @@ TDqStageSettings TDqStageSettings::New(const NNodes::TDqStageBase& node) {
}
NNodes::TCoNameValueTupleList TDqStageSettings::BuildNode(TExprContext& ctx, TPositionHandle pos) const {
- TVector<TCoNameValueTuple> settings;
+ TVector<TCoNameValueTuple> settings;
auto logicalId = LogicalId;
if (!logicalId) {
logicalId = ctx.NextUniqueId;
}
-
+
settings.push_back(Build<TCoNameValueTuple>(ctx, pos)
.Name().Build(LogicalIdSettingName)
.Value<TCoAtom>().Build(logicalId)
.Done());
- if (Id) {
- settings.push_back(Build<TCoNameValueTuple>(ctx, pos)
- .Name().Build(IdSettingName)
- .Value<TCoAtom>().Build(Id)
- .Done());
- }
-
+ if (Id) {
+ settings.push_back(Build<TCoNameValueTuple>(ctx, pos)
+ .Name().Build(IdSettingName)
+ .Value<TCoAtom>().Build(Id)
+ .Done());
+ }
+
if (SinglePartition) {
settings.push_back(Build<TCoNameValueTuple>(ctx, pos)
.Name().Build(SinglePartitionSettingName)
.Done());
}
- return Build<TCoNameValueTupleList>(ctx, pos)
- .Add(settings)
- .Done();
-}
-
+ return Build<TCoNameValueTupleList>(ctx, pos)
+ .Add(settings)
+ .Done();
+}
+
ui32 TDqStageSettings::MaxTransformConcurrency() const {
return TransformConcurrency > 0 ? TransformConcurrency : TDqSettings::TDefault::CloudFunctionConcurrency;
}
@@ -109,7 +109,7 @@ TCoLambda BuildIdentityLambda(TPositionHandle pos, TExprContext& ctx) {
bool EnsureDqUnion(const TExprBase& node, TExprContext& ctx) {
if (!node.Maybe<TDqCnUnionAll>()) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()),
- TStringBuilder() << "Expected physical input, got " << node.Ref().Content()));
+ TStringBuilder() << "Expected physical input, got " << node.Ref().Content()));
return false;
}
@@ -134,16 +134,16 @@ ui32 GetConsumersCount(const TExprBase& node, const TParentsMap& parentsMap) {
return GetConsumers(node, parentsMap).size();
}
-bool IsSingleConsumer(const TExprBase& node, const TParentsMap& parentsMap) {
- return GetConsumersCount(node, parentsMap) == 1;
-}
-
+bool IsSingleConsumer(const TExprBase& node, const TParentsMap& parentsMap) {
+ return GetConsumersCount(node, parentsMap) == 1;
+}
+
bool IsSingleConsumerConnection(const TDqConnection& node, const TParentsMap& parentsMap, bool allowStageMultiUsage) {
return IsSingleConsumer(node, parentsMap)
&& IsSingleConsumer(node.Output(), parentsMap)
&& (allowStageMultiUsage || IsSingleConsumer(node.Output().Stage(), parentsMap));
-}
-
+}
+
ui32 GetStageOutputsCount(const TDqStageBase& stage, bool includingSinks) {
auto stageType = stage.Ref().GetTypeAnn();
YQL_ENSURE(stageType);
@@ -161,13 +161,13 @@ TVector<TDqConnection> FindDqConnections(const TExprBase& node) {
VisitExpr(node.Ptr(), [&connections](const TExprNode::TPtr& exprNode) {
TExprBase node(exprNode);
- if (node.Maybe<TDqPhyPrecompute>()) {
- return false;
- }
-
+ if (node.Maybe<TDqPhyPrecompute>()) {
+ return false;
+ }
+
if (auto maybeConnection = node.Maybe<TDqConnection>()) {
- YQL_ENSURE(!maybeConnection.Maybe<TDqCnValue>());
- connections.emplace_back(maybeConnection.Cast());
+ YQL_ENSURE(!maybeConnection.Maybe<TDqCnValue>());
+ connections.emplace_back(maybeConnection.Cast());
return false;
}
@@ -195,8 +195,8 @@ bool IsDqPureExpr(const TExprBase& node, bool isPrecomputePure) {
}
bool IsDqDependsOnStage(const TExprBase& node, const TDqStageBase& stage) {
- return !!FindNode(node.Ptr(), [ptr = stage.Raw()](const TExprNode::TPtr& exprNode) {
- return exprNode.Get() == ptr;
+ return !!FindNode(node.Ptr(), [ptr = stage.Raw()](const TExprNode::TPtr& exprNode) {
+ return exprNode.Get() == ptr;
});
}
diff --git a/ydb/library/yql/dq/opt/dq_opt.h b/ydb/library/yql/dq/opt/dq_opt.h
index aa22ce42f8..5b3146d0f4 100644
--- a/ydb/library/yql/dq/opt/dq_opt.h
+++ b/ydb/library/yql/dq/opt/dq_opt.h
@@ -4,7 +4,7 @@
#include <ydb/library/yql/dq/expr_nodes/dq_expr_nodes.h>
#include <ydb/library/yql/dq/proto/dq_tasks.pb.h>
-#include <util/generic/guid.h>
+#include <util/generic/guid.h>
namespace NYql::NDq {
@@ -18,42 +18,42 @@ struct TDqStageSettings {
static constexpr TStringBuf TransformConcurrencySetting = "concurrency";
ui64 LogicalId = 0;
- TString Id;
+ TString Id;
bool SinglePartition = false;
-
+
bool IsExternalFunction = false;
NDqProto::ETransformType TransformType = NDqProto::TRANSFORM_YANDEX_CLOUD;
TString TransformName;
ui32 TransformConcurrency = 0;
static TDqStageSettings Parse(const NNodes::TDqStageBase& node);
-
+
static TDqStageSettings New(const NNodes::TDqStageBase& node);
static TDqStageSettings New() {
TDqStageSettings s;
- s.Id = CreateGuidAsString();
- return s;
- }
+ s.Id = CreateGuidAsString();
+ return s;
+ }
NNodes::TCoNameValueTupleList BuildNode(TExprContext& ctx, TPositionHandle pos) const;
-
+
ui32 MaxTransformConcurrency() const;
-};
-
+};
+
NNodes::TCoAtom BuildAtom(TStringBuf value, TPositionHandle pos, TExprContext& ctx);
NNodes::TCoAtomList BuildAtomList(TStringBuf value, TPositionHandle pos, TExprContext& ctx);
NNodes::TCoLambda BuildIdentityLambda(TPositionHandle pos, TExprContext& ctx);
bool EnsureDqUnion(const NNodes::TExprBase& node, TExprContext& ctx);
-
+
const TNodeSet& GetConsumers(const NNodes::TExprBase& node, const TParentsMap& parentsMap);
const TNodeMultiSet& GetConsumers(const NNodes::TExprBase& node, const TParentsMultiMap& parentsMap);
ui32 GetConsumersCount(const NNodes::TExprBase& node, const TParentsMap& parentsMap);
-bool IsSingleConsumer(const NNodes::TExprBase& node, const TParentsMap& parentsMap);
-
+bool IsSingleConsumer(const NNodes::TExprBase& node, const TParentsMap& parentsMap);
+
bool IsSingleConsumerConnection(const NNodes::TDqConnection& node, const TParentsMap& parentsMap, bool allowStageMultiUsage = true);
-
+
ui32 GetStageOutputsCount(const NNodes::TDqStageBase& stage, bool includingSinks);
TVector<NNodes::TDqConnection> FindDqConnections(const NNodes::TExprBase& node);
diff --git a/ydb/library/yql/dq/opt/dq_opt_build.cpp b/ydb/library/yql/dq/opt/dq_opt_build.cpp
index f69fda6d59..e57b85a911 100644
--- a/ydb/library/yql/dq/opt/dq_opt_build.cpp
+++ b/ydb/library/yql/dq/opt/dq_opt_build.cpp
@@ -1,35 +1,35 @@
-#include "dq_opt_build.h"
-#include "dq_opt.h"
+#include "dq_opt_build.h"
+#include "dq_opt.h"
#include "dq_opt_phy_finalizing.h"
-
+
#include <ydb/library/yql/ast/yql_expr.h>
#include <ydb/library/yql/core/yql_expr_optimize.h>
#include <ydb/library/yql/dq/expr_nodes/dq_expr_nodes.h>
#include <ydb/library/yql/utils/log/log.h>
#include <ydb/library/yql/providers/common/provider/yql_provider.h>
-
-namespace NYql::NDq {
-
-using namespace NNodes;
-
+
+namespace NYql::NDq {
+
+using namespace NNodes;
+
namespace {
TExprBase RewriteProgramResultToStream(const TExprBase& result, TExprContext& ctx) {
if (result.Ref().GetTypeAnn()->GetKind() != ETypeAnnotationKind::Flow) {
return result;
- }
-
+ }
+
if (auto maybeToFlow = result.Maybe<TCoToFlow>()) {
- auto toFlow = maybeToFlow.Cast();
- if (toFlow.Input().Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Flow) {
+ auto toFlow = maybeToFlow.Cast();
+ if (toFlow.Input().Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Flow) {
return toFlow.Input();
- }
+ }
return Build<TCoToStream>(ctx, result.Pos())
.Input(toFlow.Input())
- .Done();
- }
-
+ .Done();
+ }
+
if (const auto itemType = result.Ref().GetTypeAnn()->Cast<TFlowExprType>()->GetItemType();
ETypeAnnotationKind::Struct == itemType->GetKind() && result.Ref().IsCallable({"PartitionsByKeys", "CombineByKey"})) {
if (const auto structType = itemType->Cast<TStructExprType>(); structType->GetSize() > 0U) {
@@ -75,14 +75,14 @@ TExprBase RewriteProgramResultToStream(const TExprBase& result, TExprContext& ct
return Build<TCoFromFlow>(ctx, result.Pos()) // TODO: TDqOutputReader?
.Input(result)
- .Done();
-}
-
+ .Done();
+}
+
void CollectArgsReplaces(const TDqStage& dqStage, TVector<TCoArgument>& newArgs, TNodeOnNodeOwnedMap& argsMap,
TExprContext& ctx)
{
newArgs.reserve(dqStage.Program().Args().Size());
-
+
for (const auto& arg : dqStage.Program().Args()) {
auto newArg = TCoArgument(ctx.NewArgument(arg.Pos(), arg.Name()));
newArgs.emplace_back(newArg);
@@ -91,18 +91,18 @@ void CollectArgsReplaces(const TDqStage& dqStage, TVector<TCoArgument>& newArgs,
.Input(newArg)
.Done();
argsMap.emplace(arg.Raw(), argReplace.Ptr());
- } else {
+ } else {
argsMap.emplace(arg.Raw(), newArg.Ptr());
- }
- }
+ }
+ }
}
-
+
struct TStageConsumersInfo {
ui32 ConsumersCount = 0;
std::vector<TMaybeNode<TDqOutput>> Consumers;
bool HasDependantConsumers = false;
};
-
+
void MakeConsumerReplaces(
const TDqStage& dqStage,
const std::vector<TDqOutput>& consumers,
@@ -110,13 +110,13 @@ void MakeConsumerReplaces(
TExprContext& ctx)
{
auto replicate = dqStage.Program().Body().Cast<TDqReplicate>();
-
+
TVector<TExprBase> stageResults;
for (const auto& output : consumers) {
auto index = FromString<ui32>(output.Index().Value());
stageResults.push_back(replicate.Args().Get(index + 1));
}
-
+
YQL_ENSURE(!stageResults.empty());
TMaybeNode<TExprBase> stageResult;
if (stageResults.size() == 1) {
@@ -132,12 +132,12 @@ void MakeConsumerReplaces(
.Build()
.Done();
}
-
+
TVector<TCoArgument> newArgs;
newArgs.reserve(dqStage.Inputs().Size());
TNodeOnNodeOwnedMap argsMap;
CollectArgsReplaces(dqStage, newArgs, argsMap, ctx);
-
+
auto newStage = Build<TDqStage>(ctx, dqStage.Pos())
.Inputs(dqStage.Inputs())
.Program<TCoLambda>()
@@ -147,17 +147,17 @@ void MakeConsumerReplaces(
.Settings(TDqStageSettings::New(dqStage).BuildNode(ctx, dqStage.Pos()))
.Sinks(dqStage.Sinks())
.Done();
-
+
for (ui32 i = 0; i < consumers.size(); ++i) {
auto newOutput = Build<TDqOutput>(ctx, dqStage.Pos())
.Stage(newStage)
.Index().Build(ToString(i))
.Done();
-
+
replaces.emplace(consumers[i].Raw(), newOutput.Ptr());
}
}
-
+
void MakeConsumerReplaces(
const TDqStage& dqStage,
const TStageConsumersInfo& info,
@@ -217,7 +217,7 @@ public:
}
} else {
queryRoots.push_back(head);
- }
+ }
}
auto filter = [](const TExprNode::TPtr& exprNode) {
@@ -261,7 +261,7 @@ public:
}
}
}
-
+
TNodeOnNodeOwnedMap replaces;
for (const auto& [stage, info] : consumersMap) {
MakeConsumerReplaces(TDqStage(stage), info, AllowDependantConsumers, replaces, ctx);
@@ -275,7 +275,7 @@ public:
settings.VisitLambdas = false;
return RemapExpr(input, output, replaces, ctx, settings);
}
-
+
private:
const bool AllowDependantConsumers;
};
@@ -283,93 +283,93 @@ private:
class TDqBuildPhysicalStagesTransformer : public TSyncTransformerBase {
public:
explicit TDqBuildPhysicalStagesTransformer() {}
-
+
TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final {
YQL_CLOG(TRACE, CoreDq) << "[DQ/Build/TransformPhysical] " << NCommon::ExprToPrettyString(ctx, *input);
output = input;
-
- TNodeOnNodeOwnedMap replaces;
+
+ TNodeOnNodeOwnedMap replaces;
VisitExpr(input, [&ctx, &replaces](const TExprNode::TPtr& node) {
- if (node->IsLambda()) {
- return false;
- }
-
- if (replaces.contains(node.Get())) {
- return false;
- }
-
- TExprBase expr{node};
-
- if (expr.Maybe<TDqStage>()) {
- auto stage = expr.Cast<TDqStage>();
-
- TVector<TCoArgument> newArgs;
- newArgs.reserve(stage.Inputs().Size());
- TNodeOnNodeOwnedMap argsMap;
- CollectArgsReplaces(stage, newArgs, argsMap, ctx);
-
- auto result = RewriteProgramResultToStream(stage.Program().Body(), ctx).Ptr();
- auto newBody = ctx.ReplaceNodes(std::move(result), argsMap);
-
- auto newStage = Build<TDqPhyStage>(ctx, stage.Pos())
- .Inputs(stage.Inputs())
- .Program()
- .Args(newArgs)
- .Body(newBody)
- .Build()
- .Settings(TDqStageSettings::New(stage).BuildNode(ctx, stage.Pos()))
- .Sinks(stage.Sinks())
- .Done();
-
- replaces.emplace(stage.Raw(), newStage.Ptr());
+ if (node->IsLambda()) {
+ return false;
+ }
+
+ if (replaces.contains(node.Get())) {
+ return false;
+ }
+
+ TExprBase expr{node};
+
+ if (expr.Maybe<TDqStage>()) {
+ auto stage = expr.Cast<TDqStage>();
+
+ TVector<TCoArgument> newArgs;
+ newArgs.reserve(stage.Inputs().Size());
+ TNodeOnNodeOwnedMap argsMap;
+ CollectArgsReplaces(stage, newArgs, argsMap, ctx);
+
+ auto result = RewriteProgramResultToStream(stage.Program().Body(), ctx).Ptr();
+ auto newBody = ctx.ReplaceNodes(std::move(result), argsMap);
+
+ auto newStage = Build<TDqPhyStage>(ctx, stage.Pos())
+ .Inputs(stage.Inputs())
+ .Program()
+ .Args(newArgs)
+ .Body(newBody)
+ .Build()
+ .Settings(TDqStageSettings::New(stage).BuildNode(ctx, stage.Pos()))
+ .Sinks(stage.Sinks())
+ .Done();
+
+ replaces.emplace(stage.Raw(), newStage.Ptr());
YQL_CLOG(TRACE, CoreDq) << " [DQ/Build/TransformPhysical] replace stage #"
<< stage.Ref().UniqueId() << " -> #" << newStage.Ref().UniqueId();
- }
-
- return true;
- });
-
+ }
+
+ return true;
+ });
+
if (replaces.empty()) {
return TStatus::Ok;
}
-
+
TOptimizeExprSettings settings{nullptr};
settings.VisitLambdas = false;
auto status = RemapExpr(input, output, replaces, ctx, settings);
-#if 0
+#if 0
VisitExpr(output, [](const TExprNode::TPtr& node) {
YQL_ENSURE(!TDqStage::Match(node.Get()), "DqStage #" << node->UniqueId());
return true;
});
-#endif
+#endif
return status;
- }
+ }
};
-
+
} // namespace
-
+
TAutoPtr<IGraphTransformer> CreateDqBuildPhyStagesTransformer(bool allowDependantConsumers) {
TVector<TTransformStage> transformers;
-
+
transformers.push_back(TTransformStage(CreateFunctorTransformer(
[](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
return DqReplicateStageMultiOutput(input, output, ctx);
}),
"ReplicateStageMultiOutput",
TIssuesIds::DEFAULT_ERROR));
-
+
transformers.push_back(TTransformStage(
new TDqReplaceStageConsumersTransformer(allowDependantConsumers),
"ReplaceStageConsumers",
TIssuesIds::DEFAULT_ERROR));
-
+
transformers.push_back(TTransformStage(
new TDqBuildPhysicalStagesTransformer(),
"BuildPhysicalStages",
TIssuesIds::DEFAULT_ERROR));
-
+
return CreateCompositeGraphTransformer(transformers, false);
-}
-
-} // namespace NYql::NDq
+}
+
+} // namespace NYql::NDq
diff --git a/ydb/library/yql/dq/opt/dq_opt_build.h b/ydb/library/yql/dq/opt/dq_opt_build.h
index 1393229bac..456b1f0c81 100644
--- a/ydb/library/yql/dq/opt/dq_opt_build.h
+++ b/ydb/library/yql/dq/opt/dq_opt_build.h
@@ -1,10 +1,10 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/core/yql_graph_transformer.h>
#include <ydb/library/yql/dq/common/dq_common.h>
-
-namespace NYql::NDq {
-
+
+namespace NYql::NDq {
+
TAutoPtr<IGraphTransformer> CreateDqBuildPhyStagesTransformer(bool allowDependantConsumers);
-
-} // namespace NYql::NDq
+
+} // namespace NYql::NDq
diff --git a/ydb/library/yql/dq/opt/dq_opt_join.cpp b/ydb/library/yql/dq/opt/dq_opt_join.cpp
index a19de59973..4097b49e24 100644
--- a/ydb/library/yql/dq/opt/dq_opt_join.cpp
+++ b/ydb/library/yql/dq/opt/dq_opt_join.cpp
@@ -32,15 +32,15 @@ void CollectJoinColumns(const TExprBase& joinSettings, THashMap<TStringBuf, TVec
THashSet<TStringBuf>* columnsToDrop)
{
for (const auto& option : joinSettings.Ref().Children()) {
- if (option->Head().IsAtom("rename")) {
- TCoAtom fromName{option->Child(1)};
- YQL_ENSURE(!fromName.Value().Empty());
- TCoAtom toName{option->Child(2)};
- if (!toName.Value().Empty()) {
+ if (option->Head().IsAtom("rename")) {
+ TCoAtom fromName{option->Child(1)};
+ YQL_ENSURE(!fromName.Value().Empty());
+ TCoAtom toName{option->Child(2)};
+ if (!toName.Value().Empty()) {
(*columnsToRename)[fromName.Value()].emplace_back(toName.Value());
- } else {
- columnsToDrop->emplace(fromName.Value());
- }
+ } else {
+ columnsToDrop->emplace(fromName.Value());
+ }
}
}
}
@@ -87,11 +87,11 @@ TMaybe<TJoinInputDesc> BuildDqJoin(const TCoEquiJoinTuple& joinTuple,
}
}
- TStringBuf joinType = joinTuple.Type().Value();
- TSet<std::pair<TStringBuf, TStringBuf>> resultKeys;
+ TStringBuf joinType = joinTuple.Type().Value();
+ TSet<std::pair<TStringBuf, TStringBuf>> resultKeys;
if (joinType != TStringBuf("RightOnly") && joinType != TStringBuf("RightSemi")) {
- resultKeys.insert(left->Keys.begin(), left->Keys.end());
- }
+ resultKeys.insert(left->Keys.begin(), left->Keys.end());
+ }
if (joinType != TStringBuf("LeftOnly") && joinType != TStringBuf("LeftSemi")) {
resultKeys.insert(right->Keys.begin(), right->Keys.end());
}
@@ -154,7 +154,7 @@ TMaybe<TJoinInputDesc> BuildDqJoin(const TCoEquiJoinTuple& joinTuple,
TMaybe<TJoinInputDesc> PrepareJoinInput(const TCoEquiJoinInput& input) {
if (!input.Scope().Maybe<TCoAtom>()) {
- YQL_CLOG(TRACE, CoreDq) << "EquiJoin input scope is not an Atom: " << input.Scope().Ref().Content();
+ YQL_CLOG(TRACE, CoreDq) << "EquiJoin input scope is not an Atom: " << input.Scope().Ref().Content();
return {};
}
auto scope = input.Scope().Cast<TCoAtom>().Value();
@@ -170,40 +170,40 @@ TMaybe<TJoinInputDesc> PrepareJoinInput(const TCoEquiJoinInput& input) {
return TJoinInputDesc(scope, input.List(), std::move(keys));
}
-TStringBuf RotateRightJoinType(TStringBuf joinType) {
- if (joinType == "Right") {
- return "Left";
- }
- if (joinType == "RightOnly") {
- return "LeftOnly";
- }
- if (joinType == "RightSemi") {
- return "LeftSemi";
- }
- YQL_ENSURE(false, "unexpected right join type " << joinType);
-}
-
-std::pair<TVector<TCoAtom>, TVector<TCoAtom>> GetJoinKeys(const TDqJoin& join, TExprContext& ctx) {
+TStringBuf RotateRightJoinType(TStringBuf joinType) {
+ if (joinType == "Right") {
+ return "Left";
+ }
+ if (joinType == "RightOnly") {
+ return "LeftOnly";
+ }
+ if (joinType == "RightSemi") {
+ return "LeftSemi";
+ }
+ YQL_ENSURE(false, "unexpected right join type " << joinType);
+}
+
+std::pair<TVector<TCoAtom>, TVector<TCoAtom>> GetJoinKeys(const TDqJoin& join, TExprContext& ctx) {
TVector<TCoAtom> leftJoinKeys;
TVector<TCoAtom> rightJoinKeys;
- auto size = join.JoinKeys().Size();
- leftJoinKeys.reserve(size);
- rightJoinKeys.reserve(size);
-
+ auto size = join.JoinKeys().Size();
+ leftJoinKeys.reserve(size);
+ rightJoinKeys.reserve(size);
+
for (const auto& keyTuple : join.JoinKeys()) {
auto leftLabel = keyTuple.LeftLabel().Value();
auto rightLabel = keyTuple.RightLabel().Value();
auto leftKey = Build<TCoAtom>(ctx, join.Pos())
.Value(join.LeftLabel().Maybe<TCoAtom>()
- ? keyTuple.LeftColumn().StringValue()
+ ? keyTuple.LeftColumn().StringValue()
: FullColumnName(leftLabel, keyTuple.LeftColumn().Value()))
.Done();
auto rightKey = Build<TCoAtom>(ctx, join.Pos())
.Value(join.RightLabel().Maybe<TCoAtom>()
- ? keyTuple.RightColumn().StringValue()
+ ? keyTuple.RightColumn().StringValue()
: FullColumnName(rightLabel, keyTuple.RightColumn().Value()))
.Done();
@@ -217,17 +217,17 @@ std::pair<TVector<TCoAtom>, TVector<TCoAtom>> GetJoinKeys(const TDqJoin& join, T
TDqPhyMapJoin DqMakePhyMapJoin(const TDqJoin& join, const TExprBase& leftInput, const TExprBase& rightInput,
TExprContext& ctx)
{
- static const std::set<std::string_view> supportedTypes = {"Inner"sv, "Left"sv, "LeftOnly"sv, "LeftSemi"sv};
+ static const std::set<std::string_view> supportedTypes = {"Inner"sv, "Left"sv, "LeftOnly"sv, "LeftSemi"sv};
auto joinType = join.JoinType().Value();
bool supportedJoin = supportedTypes.contains(joinType);
- YQL_ENSURE(supportedJoin, "" << joinType);
+ YQL_ENSURE(supportedJoin, "" << joinType);
auto [leftJoinKeys, rightJoinKeys] = GetJoinKeys(join, ctx);
TVector<TCoAtom> leftFilterKeys;
TVector<TCoAtom> rightFilterKeys;
- if (joinType == "Inner"sv || joinType == "LeftSemi"sv) {
+ if (joinType == "Inner"sv || joinType == "LeftSemi"sv) {
for (const auto& key : leftJoinKeys) {
leftFilterKeys.push_back(key);
}
@@ -258,8 +258,8 @@ bool CheckJoinColumns(const TExprBase& node) {
auto equiJoin = node.Cast<TCoEquiJoin>();
THashMap<TStringBuf, TVector<TStringBuf>> columnsToRename;
THashSet<TStringBuf> columnsToDrop;
- CollectJoinColumns(equiJoin.Arg(equiJoin.ArgCount() - 1), &columnsToRename, &columnsToDrop);
- return true;
+ CollectJoinColumns(equiJoin.Arg(equiJoin.ArgCount() - 1), &columnsToRename, &columnsToDrop);
+ return true;
} catch (...) {
return false;
}
@@ -325,7 +325,7 @@ TExprBase DqRewriteEquiJoin(const TExprBase& node, TExprContext& ctx) {
THashMap<TStringBuf, TVector<TStringBuf>> columnsToRename;
THashSet<TStringBuf> columnsToDrop;
- CollectJoinColumns(equiJoin.Arg(equiJoin.ArgCount() - 1), &columnsToRename, &columnsToDrop);
+ CollectJoinColumns(equiJoin.Arg(equiJoin.ArgCount() - 1), &columnsToRename, &columnsToDrop);
if (columnsToRename.empty() && columnsToDrop.empty()) {
return result->Input;
@@ -378,53 +378,53 @@ TExprBase DqRewriteEquiJoin(const TExprBase& node, TExprContext& ctx) {
return projection;
}
-TExprBase DqRewriteRightJoinToLeft(const TExprBase node, TExprContext& ctx) {
- if (!node.Maybe<TDqJoin>()) {
- return node;
- }
-
- auto dqJoin = node.Cast<TDqJoin>();
- if (!dqJoin.JoinType().Value().StartsWith("Right")) {
- return node;
- }
-
- auto joinKeysBuilder = Build<TDqJoinKeyTupleList>(ctx, dqJoin.Pos());
- for (const auto& keys : dqJoin.JoinKeys()) {
- joinKeysBuilder.Add<TDqJoinKeyTuple>()
- .LeftLabel(keys.RightLabel())
- .LeftColumn(keys.RightColumn())
- .RightLabel(keys.LeftLabel())
- .RightColumn(keys.LeftColumn())
- .Build();
- }
-
- return Build<TDqJoin>(ctx, dqJoin.Pos())
- .LeftInput(dqJoin.RightInput())
- .RightInput(dqJoin.LeftInput())
- .LeftLabel(dqJoin.RightLabel())
- .RightLabel(dqJoin.LeftLabel())
- .JoinType()
- .Value(RotateRightJoinType(dqJoin.JoinType().Value()))
- .Build()
- .JoinKeys(joinKeysBuilder.Done())
- .Done();
-}
-
+TExprBase DqRewriteRightJoinToLeft(const TExprBase node, TExprContext& ctx) {
+ if (!node.Maybe<TDqJoin>()) {
+ return node;
+ }
+
+ auto dqJoin = node.Cast<TDqJoin>();
+ if (!dqJoin.JoinType().Value().StartsWith("Right")) {
+ return node;
+ }
+
+ auto joinKeysBuilder = Build<TDqJoinKeyTupleList>(ctx, dqJoin.Pos());
+ for (const auto& keys : dqJoin.JoinKeys()) {
+ joinKeysBuilder.Add<TDqJoinKeyTuple>()
+ .LeftLabel(keys.RightLabel())
+ .LeftColumn(keys.RightColumn())
+ .RightLabel(keys.LeftLabel())
+ .RightColumn(keys.LeftColumn())
+ .Build();
+ }
+
+ return Build<TDqJoin>(ctx, dqJoin.Pos())
+ .LeftInput(dqJoin.RightInput())
+ .RightInput(dqJoin.LeftInput())
+ .LeftLabel(dqJoin.RightLabel())
+ .RightLabel(dqJoin.LeftLabel())
+ .JoinType()
+ .Value(RotateRightJoinType(dqJoin.JoinType().Value()))
+ .Build()
+ .JoinKeys(joinKeysBuilder.Done())
+ .Done();
+}
+
TExprBase DqPushJoinToStage(const TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
- const TParentsMap& parentsMap, bool allowStageMultiUsage)
+ const TParentsMap& parentsMap, bool allowStageMultiUsage)
{
if (!node.Maybe<TDqJoin>()) {
return node;
- }
-
+ }
+
auto join = node.Cast<TDqJoin>();
- static const std::set<std::string_view> supportedTypes = {"Inner"sv, "Left"sv, "LeftOnly"sv, "LeftSemi"sv};
- auto joinType = join.JoinType().Value();
- if (!supportedTypes.contains(joinType)) {
- return node;
- }
-
+ static const std::set<std::string_view> supportedTypes = {"Inner"sv, "Left"sv, "LeftOnly"sv, "LeftSemi"sv};
+ auto joinType = join.JoinType().Value();
+ if (!supportedTypes.contains(joinType)) {
+ return node;
+ }
+
TMaybeNode<TDqCnUnionAll> connection;
bool pushLeft = false;
if (join.LeftInput().Maybe<TDqCnUnionAll>() && IsDqPureExpr(join.RightInput())) {
@@ -433,23 +433,23 @@ TExprBase DqPushJoinToStage(const TExprBase node, TExprContext& ctx, IOptimizati
}
if (join.RightInput().Maybe<TDqCnUnionAll>() && IsDqPureExpr(join.LeftInput())) {
connection = join.RightInput().Cast<TDqCnUnionAll>();
- pushLeft = false;
+ pushLeft = false;
}
-
+
if (!connection) {
return node;
}
-
+
if (!IsSingleConsumerConnection(connection.Cast(), parentsMap, allowStageMultiUsage)) {
return node;
}
-
+
TCoArgument inputArg{ctx.NewArgument(join.Pos(), "_dq_join_input")};
auto makeFlow = [&ctx](const TExprBase& list) {
return Build<TCoToFlow>(ctx, list.Pos())
.Input(list)
- .Done();
+ .Done();
};
auto phyJoin = DqMakePhyMapJoin(
@@ -463,24 +463,24 @@ TExprBase DqPushJoinToStage(const TExprBase node, TExprContext& ctx, IOptimizati
.Body(phyJoin)
.Done();
- auto result = DqPushLambdaToStageUnionAll(connection.Cast(), lambda, {}, ctx, optCtx);
- if (!result) {
- return node;
- }
-
+ auto result = DqPushLambdaToStageUnionAll(connection.Cast(), lambda, {}, ctx, optCtx);
+ if (!result) {
+ return node;
+ }
+
YQL_ENSURE(result.Maybe<TDqCnUnionAll>());
return result.Cast();
}
-TExprBase DqBuildPhyJoin(const TDqJoin& join, bool pushLeftStage, TExprContext& ctx, IOptimizationContext& optCtx) {
- static const std::set<std::string_view> supportedTypes = {
- "Inner"sv,
- "Left"sv,
- "Cross"sv,
- "LeftOnly"sv,
- "LeftSemi"sv
- };
-
+TExprBase DqBuildPhyJoin(const TDqJoin& join, bool pushLeftStage, TExprContext& ctx, IOptimizationContext& optCtx) {
+ static const std::set<std::string_view> supportedTypes = {
+ "Inner"sv,
+ "Left"sv,
+ "Cross"sv,
+ "LeftOnly"sv,
+ "LeftSemi"sv
+ };
+
auto joinType = join.JoinType().Value();
if (!supportedTypes.contains(joinType)) {
@@ -488,135 +488,135 @@ TExprBase DqBuildPhyJoin(const TDqJoin& join, bool pushLeftStage, TExprContext&
}
YQL_ENSURE(join.LeftInput().Maybe<TDqCnUnionAll>());
- TDqCnUnionAll leftCn = join.LeftInput().Cast<TDqCnUnionAll>();
-
- TMaybeNode<TDqCnUnionAll> rightCn = join.RightInput().Maybe<TDqCnUnionAll>();
- YQL_ENSURE(rightCn || IsDqPureExpr(join.RightInput(), /* isPrecomputePure */ true));
-
- TMaybeNode<TDqCnBroadcast> rightBroadcast;
- TNodeOnNodeOwnedMap rightPrecomputes;
-
- if (rightCn) {
- auto collectRightStage = Build<TDqStage>(ctx, join.Pos())
- .Inputs()
- .Add(rightCn.Cast())
- .Build()
- .Program()
- .Args({"stream"})
- .Body("stream")
- .Build()
- .Settings(TDqStageSettings().BuildNode(ctx, join.Pos()))
- .Done();
-
- rightBroadcast = Build<TDqCnBroadcast>(ctx, join.Pos())
- .Output()
- .Stage(collectRightStage)
- .Index().Build("0")
- .Build()
- .Done();
- } else {
- YQL_CLOG(TRACE, CoreDq) << "-- DqBuildPhyJoin: right input is DqPure expr";
-
- // right input is DqPure expression (may contain precomputes)
- VisitExpr(join.RightInput().Ptr(), [&rightPrecomputes](const TExprNode::TPtr& node) {
- if (TDqPhyPrecompute::Match(node.Get())) {
- rightPrecomputes[node.Get()] = node;
- return false;
- }
- return true;
- },
- [](const TExprNode::TPtr&) { return true; });
-
- if (rightPrecomputes.empty()) {
- // absolutely pure expression
- YQL_CLOG(TRACE, CoreDq) << "-- DqBuildPhyJoin: right input is absolutely pure expr";
- } else {
- YQL_CLOG(TRACE, CoreDq) << "-- DqBuildPhyJoin: right input is DqPure expr with " << rightPrecomputes.size()
- << " precomputes";
-
- if (IsDqDependsOnStage(join.RightInput(), leftCn.Output().Stage())) {
- YQL_CLOG(TRACE, CoreDq) << "-- DqBuildPhyJoin: right input is DqPure expr and depends on left side";
-
- TVector<TCoArgument> args; args.reserve(rightPrecomputes.size());
- TVector<TExprBase> inputs; inputs.reserve(rightPrecomputes.size());
- TNodeOnNodeOwnedMap argsReplaces;
- int i = 0;
- for (auto [raw, ptr] : rightPrecomputes) {
- args.emplace_back(TCoArgument(ctx.NewArgument(raw->Pos(), TStringBuilder() << "precompute_" << (i++))));
- inputs.emplace_back(ptr);
- argsReplaces[raw] = args.back().Ptr();
- }
-
- auto collectRightStage = Build<TDqStage>(ctx, join.Pos())
- .Inputs()
- .Add(inputs)
- .Build()
- .Program()
- .Args(args)
- .Body<TCoToStream>()
- .Input(ctx.ReplaceNodes(join.RightInput().Ptr(), argsReplaces))
- .Build()
- .Build()
- .Settings(TDqStageSettings().BuildNode(ctx, join.Pos()))
- .Done();
-
- rightBroadcast = Build<TDqCnBroadcast>(ctx, join.Pos())
- .Output()
- .Stage(collectRightStage)
- .Index().Build("0")
- .Build()
- .Done();
- } else {
- // do nothing
- YQL_CLOG(TRACE, CoreDq) << "-- right input is DqPure expr and doesn't depend on left side";
- }
- }
- }
-
+ TDqCnUnionAll leftCn = join.LeftInput().Cast<TDqCnUnionAll>();
+
+ TMaybeNode<TDqCnUnionAll> rightCn = join.RightInput().Maybe<TDqCnUnionAll>();
+ YQL_ENSURE(rightCn || IsDqPureExpr(join.RightInput(), /* isPrecomputePure */ true));
+
+ TMaybeNode<TDqCnBroadcast> rightBroadcast;
+ TNodeOnNodeOwnedMap rightPrecomputes;
+
+ if (rightCn) {
+ auto collectRightStage = Build<TDqStage>(ctx, join.Pos())
+ .Inputs()
+ .Add(rightCn.Cast())
+ .Build()
+ .Program()
+ .Args({"stream"})
+ .Body("stream")
+ .Build()
+ .Settings(TDqStageSettings().BuildNode(ctx, join.Pos()))
+ .Done();
+
+ rightBroadcast = Build<TDqCnBroadcast>(ctx, join.Pos())
+ .Output()
+ .Stage(collectRightStage)
+ .Index().Build("0")
+ .Build()
+ .Done();
+ } else {
+ YQL_CLOG(TRACE, CoreDq) << "-- DqBuildPhyJoin: right input is DqPure expr";
+
+ // right input is DqPure expression (may contain precomputes)
+ VisitExpr(join.RightInput().Ptr(), [&rightPrecomputes](const TExprNode::TPtr& node) {
+ if (TDqPhyPrecompute::Match(node.Get())) {
+ rightPrecomputes[node.Get()] = node;
+ return false;
+ }
+ return true;
+ },
+ [](const TExprNode::TPtr&) { return true; });
+
+ if (rightPrecomputes.empty()) {
+ // absolutely pure expression
+ YQL_CLOG(TRACE, CoreDq) << "-- DqBuildPhyJoin: right input is absolutely pure expr";
+ } else {
+ YQL_CLOG(TRACE, CoreDq) << "-- DqBuildPhyJoin: right input is DqPure expr with " << rightPrecomputes.size()
+ << " precomputes";
+
+ if (IsDqDependsOnStage(join.RightInput(), leftCn.Output().Stage())) {
+ YQL_CLOG(TRACE, CoreDq) << "-- DqBuildPhyJoin: right input is DqPure expr and depends on left side";
+
+ TVector<TCoArgument> args; args.reserve(rightPrecomputes.size());
+ TVector<TExprBase> inputs; inputs.reserve(rightPrecomputes.size());
+ TNodeOnNodeOwnedMap argsReplaces;
+ int i = 0;
+ for (auto [raw, ptr] : rightPrecomputes) {
+ args.emplace_back(TCoArgument(ctx.NewArgument(raw->Pos(), TStringBuilder() << "precompute_" << (i++))));
+ inputs.emplace_back(ptr);
+ argsReplaces[raw] = args.back().Ptr();
+ }
+
+ auto collectRightStage = Build<TDqStage>(ctx, join.Pos())
+ .Inputs()
+ .Add(inputs)
+ .Build()
+ .Program()
+ .Args(args)
+ .Body<TCoToStream>()
+ .Input(ctx.ReplaceNodes(join.RightInput().Ptr(), argsReplaces))
+ .Build()
+ .Build()
+ .Settings(TDqStageSettings().BuildNode(ctx, join.Pos()))
+ .Done();
+
+ rightBroadcast = Build<TDqCnBroadcast>(ctx, join.Pos())
+ .Output()
+ .Stage(collectRightStage)
+ .Index().Build("0")
+ .Build()
+ .Done();
+ } else {
+ // do nothing
+ YQL_CLOG(TRACE, CoreDq) << "-- right input is DqPure expr and doesn't depend on left side";
+ }
+ }
+ }
+
TCoArgument leftInputArg{ctx.NewArgument(join.Pos(), "_dq_join_left")};
TCoArgument rightInputArg{ctx.NewArgument(join.Pos(), "_dq_join_right")};
bool buildNewStage = !pushLeftStage;
- if (!rightCn) {
- // right input is DqPure expression, try to push down the join...
- if (rightPrecomputes.empty()) {
- // absolutely pure expression, don't need to create a new stage
- buildNewStage = false;
- } else {
- // right input contains precompute(s), and it may depend on left side (if rightBroadcast is defined)
- buildNewStage = rightBroadcast.IsValid();
- }
- } else if (!buildNewStage) {
- // NOTE: Can't pass data from the stage to itself.
- buildNewStage = IsDqDependsOnStage(join.RightInput(), leftCn.Output().Stage());
- if (!buildNewStage) {
- // NOTE: Do not push join to stage with multiple outputs, reduce memory footprint.
- buildNewStage = GetStageOutputsCount(leftCn.Output().Stage(), true) > 1;
- }
- }
-
- TExprBase joinRightInput = buildNewStage
- ? (TExprBase) rightInputArg
- : (rightBroadcast
- ? (TExprBase) rightBroadcast.Cast()
- : (TExprBase) Build<TCoToFlow>(ctx, join.Pos())
- .Input(join.RightInput())
- .Done());
-
+ if (!rightCn) {
+ // right input is DqPure expression, try to push down the join...
+ if (rightPrecomputes.empty()) {
+ // absolutely pure expression, don't need to create a new stage
+ buildNewStage = false;
+ } else {
+ // right input contains precompute(s), and it may depend on left side (if rightBroadcast is defined)
+ buildNewStage = rightBroadcast.IsValid();
+ }
+ } else if (!buildNewStage) {
+ // NOTE: Can't pass data from the stage to itself.
+ buildNewStage = IsDqDependsOnStage(join.RightInput(), leftCn.Output().Stage());
+ if (!buildNewStage) {
+ // NOTE: Do not push join to stage with multiple outputs, reduce memory footprint.
+ buildNewStage = GetStageOutputsCount(leftCn.Output().Stage(), true) > 1;
+ }
+ }
+
+ TExprBase joinRightInput = buildNewStage
+ ? (TExprBase) rightInputArg
+ : (rightBroadcast
+ ? (TExprBase) rightBroadcast.Cast()
+ : (TExprBase) Build<TCoToFlow>(ctx, join.Pos())
+ .Input(join.RightInput())
+ .Done());
+
TMaybeNode<TExprBase> phyJoin;
- if (join.JoinType().Value() != "Cross"sv) {
- phyJoin = DqMakePhyMapJoin(join, leftInputArg, joinRightInput, ctx);
+ if (join.JoinType().Value() != "Cross"sv) {
+ phyJoin = DqMakePhyMapJoin(join, leftInputArg, joinRightInput, ctx);
} else {
YQL_ENSURE(join.JoinKeys().Empty());
phyJoin = Build<TDqPhyCrossJoin>(ctx, join.Pos())
.LeftInput(leftInputArg)
.LeftLabel(join.LeftLabel())
- .RightInput(joinRightInput)
+ .RightInput(joinRightInput)
.RightLabel(join.RightLabel())
.JoinType(join.JoinType())
.JoinKeys(join.JoinKeys())
- .Done();
+ .Done();
}
TMaybeNode<TDqCnUnionAll> newConnection;
@@ -624,9 +624,9 @@ TExprBase DqBuildPhyJoin(const TDqJoin& join, bool pushLeftStage, TExprContext&
auto newJoinStage = Build<TDqStage>(ctx, join.Pos())
.Inputs()
.Add<TDqCnMap>()
- .Output(leftCn.Output())
+ .Output(leftCn.Output())
.Build()
- .Add(rightBroadcast.Cast())
+ .Add(rightBroadcast.Cast())
.Build()
.Program()
.Args({leftInputArg, rightInputArg})
@@ -647,12 +647,12 @@ TExprBase DqBuildPhyJoin(const TDqJoin& join, bool pushLeftStage, TExprContext&
.Body(phyJoin.Cast())
.Done();
- TVector<TDqConnection> lambdaConnections;
- if (rightBroadcast) {
- lambdaConnections.emplace_back(rightBroadcast.Cast());
- }
-
- auto maybeCn = DqPushLambdaToStageUnionAll(leftCn, lambda, lambdaConnections, ctx, optCtx);
+ TVector<TDqConnection> lambdaConnections;
+ if (rightBroadcast) {
+ lambdaConnections.emplace_back(rightBroadcast.Cast());
+ }
+
+ auto maybeCn = DqPushLambdaToStageUnionAll(leftCn, lambda, lambdaConnections, ctx, optCtx);
YQL_ENSURE(maybeCn);
auto newCn = maybeCn.Cast();
@@ -667,11 +667,11 @@ TExprBase DqBuildPhyJoin(const TDqJoin& join, bool pushLeftStage, TExprContext&
TExprBase DqBuildJoinDict(const TDqJoin& join, TExprContext& ctx) {
auto joinType = join.JoinType().Value();
- if (joinType != "Full"sv && joinType != "Exclusion"sv) {
+ if (joinType != "Full"sv && joinType != "Exclusion"sv) {
return join;
}
- auto buildShuffle = [&ctx, &join](const TExprBase& input, const TVector<TCoAtom>& keys) {
+ auto buildShuffle = [&ctx, &join](const TExprBase& input, const TVector<TCoAtom>& keys) {
auto stage = Build<TDqStage>(ctx, join.Pos())
.Inputs()
.Add(input)
@@ -683,7 +683,7 @@ TExprBase DqBuildJoinDict(const TDqJoin& join, TExprContext& ctx) {
.Settings(TDqStageSettings().BuildNode(ctx, join.Pos()))
.Done();
- return Build<TDqCnHashShuffle>(ctx, join.Pos())
+ return Build<TDqCnHashShuffle>(ctx, join.Pos())
.Output()
.Stage(stage)
.Index().Build("0")
@@ -694,124 +694,124 @@ TExprBase DqBuildJoinDict(const TDqJoin& join, TExprContext& ctx) {
.Done();
};
- bool leftIsUnionAll = join.LeftInput().Maybe<TDqCnUnionAll>().IsValid();
- bool rightIsUnionAll = join.RightInput().Maybe<TDqCnUnionAll>().IsValid();
-
- TMaybeNode<TDqStage> joinStage;
-
- // join streams
- if (leftIsUnionAll && rightIsUnionAll) {
- auto leftCn = join.LeftInput().Cast<TDqCnUnionAll>();
- auto rightCn = join.RightInput().Cast<TDqCnUnionAll>();
-
- auto [leftJoinKeys, rightJoinKeys] = GetJoinKeys(join, ctx);
-
- auto rightShuffle = buildShuffle(rightCn, rightJoinKeys);
- auto leftShuffle = buildShuffle(leftCn, leftJoinKeys);
-
- TCoArgument leftInputArg{ctx.NewArgument(join.Pos(), "_dq_join_left")};
- TCoArgument rightInputArg{ctx.NewArgument(join.Pos(), "_dq_join_right")};
-
- auto phyJoin = Build<TDqPhyJoinDict>(ctx, join.Pos())
- .LeftInput(leftInputArg)
- .LeftLabel(join.LeftLabel())
- .RightInput(rightInputArg)
- .RightLabel(join.RightLabel())
- .JoinType(join.JoinType())
- .JoinKeys(join.JoinKeys())
- .Done();
-
- joinStage = Build<TDqStage>(ctx, join.Pos())
- .Inputs()
- .Add(leftShuffle)
- .Add(rightShuffle)
- .Build()
- .Program()
- .Args({leftInputArg, rightInputArg})
- .Body(phyJoin)
- .Build()
- .Settings(TDqStageSettings().BuildNode(ctx, join.Pos()))
- .Done();
- }
-
- // join stream with pure expr
- else if (leftIsUnionAll && IsDqPureExpr(join.RightInput(), /* isPrecomputePure */ true)) {
- auto leftCn = join.LeftInput().Cast<TDqCnUnionAll>();
-
- auto [leftJoinKeys, _] = GetJoinKeys(join, ctx);
-
- auto leftShuffle = buildShuffle(leftCn, leftJoinKeys);
- TCoArgument leftInputArg{ctx.NewArgument(join.Pos(), "_dq_join_left")};
-
- auto phyJoin = Build<TDqPhyJoinDict>(ctx, join.Pos())
- .LeftInput(leftInputArg)
- .LeftLabel(join.LeftLabel())
- .RightInput<TCoToStream>()
- .Input(join.RightInput())
- .Build()
- .RightLabel(join.RightLabel())
- .JoinType(join.JoinType())
- .JoinKeys(join.JoinKeys())
- .Done();
-
- joinStage = Build<TDqStage>(ctx, join.Pos())
- .Inputs()
- .Add(leftShuffle)
- .Build()
- .Program()
- .Args({leftInputArg})
- .Body(phyJoin)
- .Build()
- .Settings(TDqStageSettings().BuildNode(ctx, join.Pos()))
- .Done();
- }
-
- // join pure expr with stream
- else if (IsDqPureExpr(join.RightInput(), /* isPrecomputePure */ true) && rightIsUnionAll) {
- auto rightCn = join.RightInput().Cast<TDqCnUnionAll>();
-
- auto [_, rightJoinKeys] = GetJoinKeys(join, ctx);
-
- auto rightShuffle = buildShuffle(rightCn, rightJoinKeys);
- TCoArgument rightInputArg{ctx.NewArgument(join.Pos(), "_dq_join_right")};
-
- auto phyJoin = Build<TDqPhyJoinDict>(ctx, join.Pos())
- .LeftInput(join.LeftInput())
- .LeftLabel(join.LeftLabel())
- .RightInput<TCoToStream>()
- .Input(rightInputArg)
- .Build()
- .RightLabel(join.RightLabel())
- .JoinType(join.JoinType())
- .JoinKeys(join.JoinKeys())
- .Done();
-
- joinStage = Build<TDqStage>(ctx, join.Pos())
- .Inputs()
- .Add(rightShuffle)
- .Build()
- .Program()
- .Args({rightInputArg})
- .Body(phyJoin)
- .Build()
- .Settings(TDqStageSettings().BuildNode(ctx, join.Pos()))
- .Done();
- }
-
- else {
- // TODO: pure join, do nothing?
- }
-
- if (joinStage) {
- return Build<TDqCnUnionAll>(ctx, join.Pos())
- .Output()
- .Stage(joinStage.Cast())
- .Index().Build("0")
- .Build()
- .Done();
- }
-
- return join;
+ bool leftIsUnionAll = join.LeftInput().Maybe<TDqCnUnionAll>().IsValid();
+ bool rightIsUnionAll = join.RightInput().Maybe<TDqCnUnionAll>().IsValid();
+
+ TMaybeNode<TDqStage> joinStage;
+
+ // join streams
+ if (leftIsUnionAll && rightIsUnionAll) {
+ auto leftCn = join.LeftInput().Cast<TDqCnUnionAll>();
+ auto rightCn = join.RightInput().Cast<TDqCnUnionAll>();
+
+ auto [leftJoinKeys, rightJoinKeys] = GetJoinKeys(join, ctx);
+
+ auto rightShuffle = buildShuffle(rightCn, rightJoinKeys);
+ auto leftShuffle = buildShuffle(leftCn, leftJoinKeys);
+
+ TCoArgument leftInputArg{ctx.NewArgument(join.Pos(), "_dq_join_left")};
+ TCoArgument rightInputArg{ctx.NewArgument(join.Pos(), "_dq_join_right")};
+
+ auto phyJoin = Build<TDqPhyJoinDict>(ctx, join.Pos())
+ .LeftInput(leftInputArg)
+ .LeftLabel(join.LeftLabel())
+ .RightInput(rightInputArg)
+ .RightLabel(join.RightLabel())
+ .JoinType(join.JoinType())
+ .JoinKeys(join.JoinKeys())
+ .Done();
+
+ joinStage = Build<TDqStage>(ctx, join.Pos())
+ .Inputs()
+ .Add(leftShuffle)
+ .Add(rightShuffle)
+ .Build()
+ .Program()
+ .Args({leftInputArg, rightInputArg})
+ .Body(phyJoin)
+ .Build()
+ .Settings(TDqStageSettings().BuildNode(ctx, join.Pos()))
+ .Done();
+ }
+
+ // join stream with pure expr
+ else if (leftIsUnionAll && IsDqPureExpr(join.RightInput(), /* isPrecomputePure */ true)) {
+ auto leftCn = join.LeftInput().Cast<TDqCnUnionAll>();
+
+ auto [leftJoinKeys, _] = GetJoinKeys(join, ctx);
+
+ auto leftShuffle = buildShuffle(leftCn, leftJoinKeys);
+ TCoArgument leftInputArg{ctx.NewArgument(join.Pos(), "_dq_join_left")};
+
+ auto phyJoin = Build<TDqPhyJoinDict>(ctx, join.Pos())
+ .LeftInput(leftInputArg)
+ .LeftLabel(join.LeftLabel())
+ .RightInput<TCoToStream>()
+ .Input(join.RightInput())
+ .Build()
+ .RightLabel(join.RightLabel())
+ .JoinType(join.JoinType())
+ .JoinKeys(join.JoinKeys())
+ .Done();
+
+ joinStage = Build<TDqStage>(ctx, join.Pos())
+ .Inputs()
+ .Add(leftShuffle)
+ .Build()
+ .Program()
+ .Args({leftInputArg})
+ .Body(phyJoin)
+ .Build()
+ .Settings(TDqStageSettings().BuildNode(ctx, join.Pos()))
+ .Done();
+ }
+
+ // join pure expr with stream
+ else if (IsDqPureExpr(join.RightInput(), /* isPrecomputePure */ true) && rightIsUnionAll) {
+ auto rightCn = join.RightInput().Cast<TDqCnUnionAll>();
+
+ auto [_, rightJoinKeys] = GetJoinKeys(join, ctx);
+
+ auto rightShuffle = buildShuffle(rightCn, rightJoinKeys);
+ TCoArgument rightInputArg{ctx.NewArgument(join.Pos(), "_dq_join_right")};
+
+ auto phyJoin = Build<TDqPhyJoinDict>(ctx, join.Pos())
+ .LeftInput(join.LeftInput())
+ .LeftLabel(join.LeftLabel())
+ .RightInput<TCoToStream>()
+ .Input(rightInputArg)
+ .Build()
+ .RightLabel(join.RightLabel())
+ .JoinType(join.JoinType())
+ .JoinKeys(join.JoinKeys())
+ .Done();
+
+ joinStage = Build<TDqStage>(ctx, join.Pos())
+ .Inputs()
+ .Add(rightShuffle)
+ .Build()
+ .Program()
+ .Args({rightInputArg})
+ .Body(phyJoin)
+ .Build()
+ .Settings(TDqStageSettings().BuildNode(ctx, join.Pos()))
+ .Done();
+ }
+
+ else {
+ // TODO: pure join, do nothing?
+ }
+
+ if (joinStage) {
+ return Build<TDqCnUnionAll>(ctx, join.Pos())
+ .Output()
+ .Stage(joinStage.Cast())
+ .Index().Build("0")
+ .Build()
+ .Done();
+ }
+
+ return join;
}
-
+
} // namespace NYql::NDq
diff --git a/ydb/library/yql/dq/opt/dq_opt_log.cpp b/ydb/library/yql/dq/opt/dq_opt_log.cpp
index 28b802cb22..fed0f09f0f 100644
--- a/ydb/library/yql/dq/opt/dq_opt_log.cpp
+++ b/ydb/library/yql/dq/opt/dq_opt_log.cpp
@@ -24,23 +24,23 @@ TExprBase DqRewriteAggregate(TExprBase node, TExprContext& ctx) {
return TExprBase(result);
}
-// Take . Sort -> TopSort
-// Take . Skip . Sort -> Take . Skip . TopSort
+// Take . Sort -> TopSort
+// Take . Skip . Sort -> Take . Skip . TopSort
TExprBase DqRewriteTakeSortToTopSort(TExprBase node, TExprContext& ctx, const TParentsMap& parents) {
- if (!node.Maybe<TCoTake>()) {
+ if (!node.Maybe<TCoTake>()) {
return node;
}
auto take = node.Cast<TCoTake>();
- auto maybeSkip = take.Input().Maybe<TCoSkip>();
- auto maybeSort = maybeSkip ? maybeSkip.Cast().Input().Maybe<TCoSort>() : take.Input().Maybe<TCoSort>();
-
- if (!maybeSort) {
- return node;
- }
- auto sort = maybeSort.Cast();
-
- if (!IsSingleConsumer(sort, parents)) {
+ auto maybeSkip = take.Input().Maybe<TCoSkip>();
+ auto maybeSort = maybeSkip ? maybeSkip.Cast().Input().Maybe<TCoSort>() : take.Input().Maybe<TCoSort>();
+
+ if (!maybeSort) {
+ return node;
+ }
+ auto sort = maybeSort.Cast();
+
+ if (!IsSingleConsumer(sort, parents)) {
return node;
}
@@ -48,34 +48,34 @@ TExprBase DqRewriteTakeSortToTopSort(TExprBase node, TExprContext& ctx, const TP
return node;
}
- if (maybeSkip) {
- auto skip = maybeSkip.Cast();
-
- if (!IsSingleConsumer(skip, parents)) {
- return node;
- }
-
- if (!IsDqPureExpr(skip)) {
- return node;
- }
-
- return Build<TCoTake>(ctx, node.Pos())
- .Input<TCoSkip>()
- .Input<TCoTopSort>()
- .Input(sort.Input())
- .KeySelectorLambda(sort.KeySelectorLambda())
- .SortDirections(sort.SortDirections())
- .Count<TCoPlus>()
- .Left(take.Count())
- .Right(skip.Count())
- .Build()
- .Build()
- .Count(skip.Count())
- .Build()
- .Count(take.Count())
- .Done();
- }
-
+ if (maybeSkip) {
+ auto skip = maybeSkip.Cast();
+
+ if (!IsSingleConsumer(skip, parents)) {
+ return node;
+ }
+
+ if (!IsDqPureExpr(skip)) {
+ return node;
+ }
+
+ return Build<TCoTake>(ctx, node.Pos())
+ .Input<TCoSkip>()
+ .Input<TCoTopSort>()
+ .Input(sort.Input())
+ .KeySelectorLambda(sort.KeySelectorLambda())
+ .SortDirections(sort.SortDirections())
+ .Count<TCoPlus>()
+ .Left(take.Count())
+ .Right(skip.Count())
+ .Build()
+ .Build()
+ .Count(skip.Count())
+ .Build()
+ .Count(take.Count())
+ .Done();
+ }
+
return Build<TCoTopSort>(ctx, node.Pos())
.Input(sort.Input())
.KeySelectorLambda(sort.KeySelectorLambda())
diff --git a/ydb/library/yql/dq/opt/dq_opt_peephole.cpp b/ydb/library/yql/dq/opt/dq_opt_peephole.cpp
index e8bedc8034..87ac170eea 100644
--- a/ydb/library/yql/dq/opt/dq_opt_peephole.cpp
+++ b/ydb/library/yql/dq/opt/dq_opt_peephole.cpp
@@ -5,7 +5,7 @@
#include <ydb/library/yql/core/yql_expr_type_annotation.h>
#include <ydb/library/yql/utils/log/log.h>
-
+
#include <util/generic/size_literals.h>
namespace NYql::NDq {
@@ -14,8 +14,8 @@ using namespace NYql::NNodes;
namespace {
-inline std::string_view GetTableLabel(const TExprBase& node) {
- static const std::string_view empty;
+inline std::string_view GetTableLabel(const TExprBase& node) {
+ static const std::string_view empty;
if (node.Maybe<TCoAtom>()) {
return node.Cast<TCoAtom>().Value();
@@ -24,8 +24,8 @@ inline std::string_view GetTableLabel(const TExprBase& node) {
return empty;
}
-inline TString GetColumnName(std::string_view label, const TItemExprType *key) {
- if (!label.empty()) {
+inline TString GetColumnName(std::string_view label, const TItemExprType *key) {
+ if (!label.empty()) {
return FullColumnName(label, key->GetName());
}
@@ -33,7 +33,7 @@ inline TString GetColumnName(std::string_view label, const TItemExprType *key) {
}
std::pair<TExprNode::TListType, TExprNode::TListType> JoinKeysToAtoms(TExprContext& ctx, const TDqJoinBase& join,
- std::string_view leftTableLabel, std::string_view rightTableLabel)
+ std::string_view leftTableLabel, std::string_view rightTableLabel)
{
TExprNode::TListType leftNodes;
TExprNode::TListType rightNodes;
@@ -63,63 +63,63 @@ std::pair<TExprNode::TListType, TExprNode::TListType> JoinKeysToAtoms(TExprConte
rightNodes.emplace_back(rightValue);
}
- return {std::move(leftNodes), std::move(rightNodes)};
+ return {std::move(leftNodes), std::move(rightNodes)};
+}
+
+TExprNode::TPtr BuildDictKeySelector(TExprContext& ctx, TPositionHandle pos, const TExprNode::TListType& keyAtoms,
+ const TTypeAnnotationNode::TListType& keyDryTypes, bool optional)
+{
+ YQL_ENSURE(keyAtoms.size() == keyDryTypes.size());
+
+ TExprNode::TListType keysTuple;
+
+ auto keySelectorArg = Build<TCoArgument>(ctx, pos)
+ .Name("keyArg")
+ .Done();
+
+ for (const auto& atom: keyAtoms) {
+ auto member = Build<TCoMember>(ctx, pos)
+ .Struct(keySelectorArg)
+ .Name(atom)
+ .Done();
+
+ keysTuple.emplace_back(member.Ptr());
+ }
+
+ if (keysTuple.size() == 1) {
+ return optional
+ ? Build<TCoLambda>(ctx, pos)
+ .Args({keySelectorArg})
+ .Body<TCoStrictCast>()
+ .Value(keysTuple[0])
+ .Type(ExpandType(pos, *keyDryTypes[0], ctx))
+ .Build()
+ .Done().Ptr()
+ : Build<TCoLambda>(ctx, pos)
+ .Args({keySelectorArg})
+ .Body<TCoJust>()
+ .Input(keysTuple[0])
+ .Build()
+ .Done().Ptr();
+ }
+
+ auto type = ctx.MakeType<TOptionalExprType>(ctx.MakeType<TTupleExprType>(keyDryTypes));
+ return optional
+ ? Build<TCoLambda>(ctx, pos)
+ .Args({keySelectorArg})
+ .Body<TCoStrictCast>()
+ .Value(ctx.NewList(pos, std::move(keysTuple)))
+ .Type(ExpandType(pos, *type, ctx))
+ .Build()
+ .Done().Ptr()
+ : Build<TCoLambda>(ctx, pos)
+ .Args({keySelectorArg})
+ .Body<TCoJust>()
+ .Input(ctx.NewList(pos, std::move(keysTuple)))
+ .Build()
+ .Done().Ptr();
}
-TExprNode::TPtr BuildDictKeySelector(TExprContext& ctx, TPositionHandle pos, const TExprNode::TListType& keyAtoms,
- const TTypeAnnotationNode::TListType& keyDryTypes, bool optional)
-{
- YQL_ENSURE(keyAtoms.size() == keyDryTypes.size());
-
- TExprNode::TListType keysTuple;
-
- auto keySelectorArg = Build<TCoArgument>(ctx, pos)
- .Name("keyArg")
- .Done();
-
- for (const auto& atom: keyAtoms) {
- auto member = Build<TCoMember>(ctx, pos)
- .Struct(keySelectorArg)
- .Name(atom)
- .Done();
-
- keysTuple.emplace_back(member.Ptr());
- }
-
- if (keysTuple.size() == 1) {
- return optional
- ? Build<TCoLambda>(ctx, pos)
- .Args({keySelectorArg})
- .Body<TCoStrictCast>()
- .Value(keysTuple[0])
- .Type(ExpandType(pos, *keyDryTypes[0], ctx))
- .Build()
- .Done().Ptr()
- : Build<TCoLambda>(ctx, pos)
- .Args({keySelectorArg})
- .Body<TCoJust>()
- .Input(keysTuple[0])
- .Build()
- .Done().Ptr();
- }
-
- auto type = ctx.MakeType<TOptionalExprType>(ctx.MakeType<TTupleExprType>(keyDryTypes));
- return optional
- ? Build<TCoLambda>(ctx, pos)
- .Args({keySelectorArg})
- .Body<TCoStrictCast>()
- .Value(ctx.NewList(pos, std::move(keysTuple)))
- .Type(ExpandType(pos, *type, ctx))
- .Build()
- .Done().Ptr()
- : Build<TCoLambda>(ctx, pos)
- .Args({keySelectorArg})
- .Body<TCoJust>()
- .Input(ctx.NewList(pos, std::move(keysTuple)))
- .Build()
- .Done().Ptr();
-}
-
} // anonymous namespace end
/**
@@ -128,8 +128,8 @@ TExprNode::TPtr BuildDictKeySelector(TExprContext& ctx, TPositionHandle pos, con
* Restrictions:
* - Don't select join strategy, always use `MapJoin`
* - Explicitly convert right input to the dict
- * - Use quite pretty trick: do `MapJoinCore` in `FlatMap`-lambda
- * (rely on the fact that there will be only one element in the `FlatMap`-stream)
+ * - Use quite pretty trick: do `MapJoinCore` in `FlatMap`-lambda
+ * (rely on the fact that there will be only one element in the `FlatMap`-stream)
*/
TExprBase DqPeepholeRewriteMapJoin(const TExprBase& node, TExprContext& ctx) {
if (!node.Maybe<TDqPhyMapJoin>()) {
@@ -175,7 +175,7 @@ TExprBase DqPeepholeRewriteMapJoin(const TExprBase& node, TExprContext& ctx) {
if (!keyTypes[i])
keyTypes.clear();
}
-
+
auto leftInput = ctx.NewCallable(mapJoin.LeftInput().Pos(), "ToFlow", {mapJoin.LeftInput().Ptr()});
auto rightInput = ctx.NewCallable(mapJoin.RightInput().Pos(), "ToFlow", {mapJoin.RightInput().Ptr()});
@@ -233,7 +233,7 @@ TExprBase DqPeepholeRewriteMapJoin(const TExprBase& node, TExprContext& ctx) {
.LeftKeysColumns(ctx.NewList(pos, std::move(leftKeyColumnNodes)))
.LeftRenames(ctx.NewList(pos, std::move(leftRenames)))
.RightRenames(ctx.NewList(pos, std::move(rightRenames)))
- .Build()
+ .Build()
.Build()
.Done();
}
@@ -271,9 +271,9 @@ TExprBase DqPeepholeRewriteCrossJoin(const TExprBase& node, TExprContext& ctx) {
// because stream supports single iteration only
auto itemArg = Build<TCoArgument>(ctx, crossJoin.Pos()).Name("item").Done();
auto rightAsStreamOfLists = Build<TCoCondense1>(ctx, crossJoin.Pos())
- .Input<TCoToFlow>()
- .Input(crossJoin.RightInput())
- .Build()
+ .Input<TCoToFlow>()
+ .Input(crossJoin.RightInput())
+ .Build()
.InitHandler()
.Args({itemArg})
.Body<TCoAsList>()
@@ -321,22 +321,22 @@ TExprBase DqPeepholeRewriteCrossJoin(const TExprBase& node, TExprContext& ctx) {
namespace {
-TExprNode::TPtr UnpackJoinedData(const TStructExprType* leftRowType, const TStructExprType* rightRowType,
- std::string_view leftLabel, std::string_view rightLabel, TPositionHandle pos, TExprContext& ctx)
+TExprNode::TPtr UnpackJoinedData(const TStructExprType* leftRowType, const TStructExprType* rightRowType,
+ std::string_view leftLabel, std::string_view rightLabel, TPositionHandle pos, TExprContext& ctx)
{
- auto arg = Build<TCoArgument>(ctx, pos)
+ auto arg = Build<TCoArgument>(ctx, pos)
.Name("packedItem")
.Done();
- const auto& leftScheme = leftRowType->GetItems();
- const auto& rightScheme = rightRowType->GetItems();
+ const auto& leftScheme = leftRowType->GetItems();
+ const auto& rightScheme = rightRowType->GetItems();
TExprNode::TListType outValueItems;
- outValueItems.reserve(leftScheme.size() + rightScheme.size());
+ outValueItems.reserve(leftScheme.size() + rightScheme.size());
for (int tableIndex = 0; tableIndex < 2; tableIndex++) {
- const auto& scheme = tableIndex ? rightScheme : leftScheme;
- const auto label = tableIndex ? rightLabel : leftLabel;
+ const auto& scheme = tableIndex ? rightScheme : leftScheme;
+ const auto label = tableIndex ? rightLabel : leftLabel;
for (const auto& item : scheme) {
auto nameAtom = ctx.NewAtom(pos, item->GetName());
@@ -373,72 +373,72 @@ NNodes::TExprBase DqPeepholeRewriteJoinDict(const NNodes::TExprBase& node, TExpr
return node;
}
- const auto joinDict = node.Cast<TDqPhyJoinDict>();
- const auto joinKind = joinDict.JoinType().Value();
-
- YQL_ENSURE(joinKind != "Cross"sv);
-
- const auto leftTableLabel = GetTableLabel(joinDict.LeftLabel());
- const auto rightTableLabel = GetTableLabel(joinDict.RightLabel());
-
- auto [leftKeys, rightKeys] = JoinKeysToAtoms(ctx, joinDict, leftTableLabel, rightTableLabel);
-
- YQL_CLOG(TRACE, CoreDq) << "[DqPeepholeRewriteJoinDict] join types"
- << ", left: " << *joinDict.LeftInput().Ref().GetTypeAnn()
- << ", right: " << *joinDict.RightInput().Ref().GetTypeAnn();
-
- const auto* leftRowType = GetSeqItemType(joinDict.LeftInput().Ref().GetTypeAnn())->Cast<TStructExprType>();
- const auto* rightRowType = GetSeqItemType(joinDict.RightInput().Ref().GetTypeAnn())->Cast<TStructExprType>();
-
- bool optKeyLeft = false, optKeyRight = false, badKey = false;
- TTypeAnnotationNode::TListType keyTypeItems;
- keyTypeItems.reserve(leftKeys.size());
- for (auto i = 0U; i < leftKeys.size(); ++i) {
- auto leftKeyType = leftRowType->FindItemType(leftKeys[i]->Content());
- auto rightKeyType = rightRowType->FindItemType(rightKeys[i]->Content());
+ const auto joinDict = node.Cast<TDqPhyJoinDict>();
+ const auto joinKind = joinDict.JoinType().Value();
+
+ YQL_ENSURE(joinKind != "Cross"sv);
+
+ const auto leftTableLabel = GetTableLabel(joinDict.LeftLabel());
+ const auto rightTableLabel = GetTableLabel(joinDict.RightLabel());
+
+ auto [leftKeys, rightKeys] = JoinKeysToAtoms(ctx, joinDict, leftTableLabel, rightTableLabel);
+
+ YQL_CLOG(TRACE, CoreDq) << "[DqPeepholeRewriteJoinDict] join types"
+ << ", left: " << *joinDict.LeftInput().Ref().GetTypeAnn()
+ << ", right: " << *joinDict.RightInput().Ref().GetTypeAnn();
+
+ const auto* leftRowType = GetSeqItemType(joinDict.LeftInput().Ref().GetTypeAnn())->Cast<TStructExprType>();
+ const auto* rightRowType = GetSeqItemType(joinDict.RightInput().Ref().GetTypeAnn())->Cast<TStructExprType>();
+
+ bool optKeyLeft = false, optKeyRight = false, badKey = false;
+ TTypeAnnotationNode::TListType keyTypeItems;
+ keyTypeItems.reserve(leftKeys.size());
+ for (auto i = 0U; i < leftKeys.size(); ++i) {
+ auto leftKeyType = leftRowType->FindItemType(leftKeys[i]->Content());
+ auto rightKeyType = rightRowType->FindItemType(rightKeys[i]->Content());
keyTypeItems.emplace_back(CommonType<true>(node.Pos(), DryType(leftKeyType, optKeyLeft, ctx), DryType(rightKeyType, optKeyRight, ctx), ctx));
- badKey = !keyTypeItems.back();
- if (badKey) {
- YQL_CLOG(DEBUG, CoreDq) << "Not comparable keys in join: " << leftKeys[i]->Content()
- << "(" << *leftKeyType << ") vs " << rightKeys[i]->Content() << "(" << *rightKeyType << ")";
- break;
- }
- }
-
- TExprNode::TPtr leftKeySelector;
- TExprNode::TPtr rightKeySelector;
-
- if (badKey) {
- leftKeySelector = Build<TCoLambda>(ctx, node.Pos())
- .Args({"row"})
- .Body<TCoBool>()
- .Literal().Build("true")
- .Build()
- .Done().Ptr();
-
- rightKeySelector = Build<TCoLambda>(ctx, node.Pos())
- .Args({"item"})
- .Body<TCoBool>()
- .Literal().Build("false")
- .Build()
- .Done().Ptr();
- } else {
- leftKeySelector = BuildDictKeySelector(ctx, joinDict.Pos(), leftKeys, keyTypeItems, optKeyLeft);
- rightKeySelector = BuildDictKeySelector(ctx, joinDict.Pos(), rightKeys, keyTypeItems, optKeyRight);
- }
-
- auto streamToDict = [&ctx](const TExprBase& input, const TExprNode::TPtr& keySelector) {
- return Build<TCoSqueezeToDict>(ctx, input.Pos())
+ badKey = !keyTypeItems.back();
+ if (badKey) {
+ YQL_CLOG(DEBUG, CoreDq) << "Not comparable keys in join: " << leftKeys[i]->Content()
+ << "(" << *leftKeyType << ") vs " << rightKeys[i]->Content() << "(" << *rightKeyType << ")";
+ break;
+ }
+ }
+
+ TExprNode::TPtr leftKeySelector;
+ TExprNode::TPtr rightKeySelector;
+
+ if (badKey) {
+ leftKeySelector = Build<TCoLambda>(ctx, node.Pos())
+ .Args({"row"})
+ .Body<TCoBool>()
+ .Literal().Build("true")
+ .Build()
+ .Done().Ptr();
+
+ rightKeySelector = Build<TCoLambda>(ctx, node.Pos())
+ .Args({"item"})
+ .Body<TCoBool>()
+ .Literal().Build("false")
+ .Build()
+ .Done().Ptr();
+ } else {
+ leftKeySelector = BuildDictKeySelector(ctx, joinDict.Pos(), leftKeys, keyTypeItems, optKeyLeft);
+ rightKeySelector = BuildDictKeySelector(ctx, joinDict.Pos(), rightKeys, keyTypeItems, optKeyRight);
+ }
+
+ auto streamToDict = [&ctx](const TExprBase& input, const TExprNode::TPtr& keySelector) {
+ return Build<TCoSqueezeToDict>(ctx, input.Pos())
.Stream(input)
.KeySelector(keySelector)
.PayloadSelector()
- .Args({"item"})
- .Body("item")
+ .Args({"item"})
+ .Body("item")
.Build()
.Settings()
- .Add<TCoAtom>().Build("Hashed")
- .Add<TCoAtom>().Build("Many")
- .Add<TCoAtom>().Build("Compact")
+ .Add<TCoAtom>().Build("Hashed")
+ .Add<TCoAtom>().Build("Many")
+ .Add<TCoAtom>().Build("Compact")
.Build()
.Done();
};
@@ -447,16 +447,16 @@ NNodes::TExprBase DqPeepholeRewriteJoinDict(const NNodes::TExprBase& node, TExpr
auto rightDict = streamToDict(joinDict.RightInput(), rightKeySelector);
auto join = Build<TCoFlatMap>(ctx, joinDict.Pos())
- .Input(leftDict) // only 1 element with dict
+ .Input(leftDict) // only 1 element with dict
.Lambda()
- .Args({"left"})
+ .Args({"left"})
.Body<TCoFlatMap>()
- .Input(rightDict) // only 1 element with dict
+ .Input(rightDict) // only 1 element with dict
.Lambda()
- .Args({"right"})
+ .Args({"right"})
.Body<TCoJoinDict>()
- .LeftInput("left")
- .RightInput("right")
+ .LeftInput("left")
+ .RightInput("right")
.JoinKind(joinDict.JoinType())
.Build()
.Build()
@@ -467,7 +467,7 @@ NNodes::TExprBase DqPeepholeRewriteJoinDict(const NNodes::TExprBase& node, TExpr
// Join return list of tuple of structs. I.e. if you have tables t1 and t2 with values t1.a, t1.b and t2.c, t2.d,
// you will receive List<Tuple<Struct<t1.a, t1.b>, Struct<t2.c, t2.d>>> and this data should be unpacked to
// List<Struct<t1.a, t1.b, t2.c, t2.d>>
- auto unpackData = UnpackJoinedData(leftRowType, rightRowType, leftTableLabel, rightTableLabel, join.Pos(), ctx);
+ auto unpackData = UnpackJoinedData(leftRowType, rightRowType, leftTableLabel, rightTableLabel, join.Pos(), ctx);
return Build<TCoMap>(ctx, joinDict.Pos())
.Input(join)
diff --git a/ydb/library/yql/dq/opt/dq_opt_phy.cpp b/ydb/library/yql/dq/opt/dq_opt_phy.cpp
index a7102fa9e4..cbe871059d 100644
--- a/ydb/library/yql/dq/opt/dq_opt_phy.cpp
+++ b/ydb/library/yql/dq/opt/dq_opt_phy.cpp
@@ -12,9 +12,9 @@ using namespace NYql::NNodes;
namespace {
-TVector<TCoArgument> PrepareArgumentsReplacement(const TCoArguments& args, const TVector<TDqConnection>& newInputs,
+TVector<TCoArgument> PrepareArgumentsReplacement(const TCoArguments& args, const TVector<TDqConnection>& newInputs,
TExprContext& ctx, TNodeOnNodeOwnedMap& replaceMap)
-{
+{
TVector<TCoArgument> newArgs;
newArgs.reserve(args.Size() + newInputs.size());
replaceMap.clear();
@@ -43,15 +43,15 @@ TExprBase DqBuildPartitionsStageStub(TExprBase node, TExprContext& ctx, const TP
}
auto partition = node.Cast<TPartition>();
- if (!IsDqPureExpr(partition.KeySelectorLambda()) ||
- !IsDqPureExpr(partition.ListHandlerLambda()) ||
- !IsDqPureExpr(partition.SortKeySelectorLambda()))
- {
+ if (!IsDqPureExpr(partition.KeySelectorLambda()) ||
+ !IsDqPureExpr(partition.ListHandlerLambda()) ||
+ !IsDqPureExpr(partition.SortKeySelectorLambda()))
+ {
return node;
}
auto dqUnion = partition.Input().template Cast<TDqCnUnionAll>();
- if (!IsSingleConsumerConnection(dqUnion, parentsMap)) {
+ if (!IsSingleConsumerConnection(dqUnion, parentsMap)) {
return node;
}
@@ -87,7 +87,7 @@ TExprBase DqBuildPartitionsStageStub(TExprBase node, TExprContext& ctx, const TP
TExprNode::TPtr newConnection;
if (!keyColumns.empty() && allKeysAreMembers) {
- newConnection = Build<TDqCnHashShuffle>(ctx, node.Pos())
+ newConnection = Build<TDqCnHashShuffle>(ctx, node.Pos())
.Output()
.Stage(dqUnion.Output().Stage())
.Index(dqUnion.Output().Index())
@@ -132,9 +132,9 @@ TExprBase DqBuildPartitionsStageStub(TExprBase node, TExprContext& ctx, const TP
.Input("rows")
.KeySelectorLambda(ctx.DeepCopyLambda(partition.KeySelectorLambda().Ref()))
.SortDirections(partition.SortDirections())
- .SortKeySelectorLambda(partition.SortKeySelectorLambda().template Maybe<TCoLambda>()
- ? ctx.DeepCopyLambda(partition.SortKeySelectorLambda().Ref())
- : partition.SortKeySelectorLambda().Ptr())
+ .SortKeySelectorLambda(partition.SortKeySelectorLambda().template Maybe<TCoLambda>()
+ ? ctx.DeepCopyLambda(partition.SortKeySelectorLambda().Ref())
+ : partition.SortKeySelectorLambda().Ptr())
.ListHandlerLambda()
.Args({"list"})
.template Body<TExprApplier>()
@@ -155,147 +155,147 @@ TExprBase DqBuildPartitionsStageStub(TExprBase node, TExprContext& ctx, const TP
.Done();
}
-template <typename TMembersFilter>
-TExprBase DqPushMembersFilterToStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
- const TParentsMap& parentsMap, bool allowStageMultiUsage)
-{
- if (!node.Maybe<TMembersFilter>().Input().template Maybe<TDqCnUnionAll>()) {
+template <typename TMembersFilter>
+TExprBase DqPushMembersFilterToStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
+ const TParentsMap& parentsMap, bool allowStageMultiUsage)
+{
+ if (!node.Maybe<TMembersFilter>().Input().template Maybe<TDqCnUnionAll>()) {
return node;
}
- auto filter = node.Cast<TMembersFilter>();
- auto dqUnion = filter.Input().template Cast<TDqCnUnionAll>();
+ auto filter = node.Cast<TMembersFilter>();
+ auto dqUnion = filter.Input().template Cast<TDqCnUnionAll>();
if (!IsSingleConsumerConnection(dqUnion, parentsMap, allowStageMultiUsage)) {
return node;
}
- auto lambda = Build<TCoLambda>(ctx, filter.Pos())
+ auto lambda = Build<TCoLambda>(ctx, filter.Pos())
.Args({"stream"})
- .template Body<TMembersFilter>()
+ .template Body<TMembersFilter>()
.Input("stream")
- .Members(filter.Members())
+ .Members(filter.Members())
.Build()
.Done();
- auto result = DqPushLambdaToStageUnionAll(dqUnion, lambda, {}, ctx, optCtx);
+ auto result = DqPushLambdaToStageUnionAll(dqUnion, lambda, {}, ctx, optCtx);
if (!result) {
return node;
}
return result.Cast();
-}
+}
-} // namespace
-
-TMaybeNode<TDqStage> DqPushLambdaToStage(const TDqStage& stage, const TCoAtom& outputIndex, TCoLambda& lambda,
- const TVector<TDqConnection>& lambdaInputs, TExprContext& ctx, IOptimizationContext& optCtx)
+} // namespace
+
+TMaybeNode<TDqStage> DqPushLambdaToStage(const TDqStage& stage, const TCoAtom& outputIndex, TCoLambda& lambda,
+ const TVector<TDqConnection>& lambdaInputs, TExprContext& ctx, IOptimizationContext& optCtx)
{
- YQL_CLOG(TRACE, CoreDq) << "stage #" << stage.Ref().UniqueId() << ": " << PrintDqStageOnly(stage, ctx)
- << ", add lambda to output #" << outputIndex.Value();
+ YQL_CLOG(TRACE, CoreDq) << "stage #" << stage.Ref().UniqueId() << ": " << PrintDqStageOnly(stage, ctx)
+ << ", add lambda to output #" << outputIndex.Value();
if (IsDqDependsOnStage(lambda, stage)) {
- YQL_CLOG(TRACE, CoreDq) << "Lambda " << lambda.Ref().Dump() << " depends on stage: " << PrintDqStageOnly(stage, ctx);
+ YQL_CLOG(TRACE, CoreDq) << "Lambda " << lambda.Ref().Dump() << " depends on stage: " << PrintDqStageOnly(stage, ctx);
return {};
}
- auto program = TCoLambda(DuplicateIndependentStreams(stage.Program().Ptr(),
- [](const TExprNode* node) {
- return TDqPhyPrecompute::Match(node) || TDqConnection::Match(node);
- }, ctx));
+ auto program = TCoLambda(DuplicateIndependentStreams(stage.Program().Ptr(),
+ [](const TExprNode* node) {
+ return TDqPhyPrecompute::Match(node) || TDqConnection::Match(node);
+ }, ctx));
ui32 index = FromString<ui32>(outputIndex.Value());
ui32 branchesCount = GetStageOutputsCount(stage, true);
- TExprNode::TPtr newProgram;
+ TExprNode::TPtr newProgram;
if (branchesCount == 1) {
- newProgram = ctx.FuseLambdas(lambda.Ref(), program.Ref());
- } else {
- auto dqReplicate = program.Body().Cast<TDqReplicate>();
-
- TNodeOnNodeOwnedMap argReplaces;
- auto newArgs = PrepareArgumentsReplacement(program.Args(), {}, ctx, argReplaces);
-
-// for (auto& x : argReplaces) {
-// YQL_CLOG(TRACE, CoreDq) << "-- replace arg #" << x.first->UniqueId() << " -> #" << x.second->UniqueId();
-// }
-
- TVector<TExprNode::TPtr> newBranchLambdas;
- newBranchLambdas.reserve(branchesCount);
-
- for (size_t i = 0; i < branchesCount; ++i) {
- auto branchLambda = dqReplicate.Arg(/* input */ 1 + i).Cast<TCoLambda>();
+ newProgram = ctx.FuseLambdas(lambda.Ref(), program.Ref());
+ } else {
+ auto dqReplicate = program.Body().Cast<TDqReplicate>();
+
+ TNodeOnNodeOwnedMap argReplaces;
+ auto newArgs = PrepareArgumentsReplacement(program.Args(), {}, ctx, argReplaces);
+
+// for (auto& x : argReplaces) {
+// YQL_CLOG(TRACE, CoreDq) << "-- replace arg #" << x.first->UniqueId() << " -> #" << x.second->UniqueId();
+// }
+
+ TVector<TExprNode::TPtr> newBranchLambdas;
+ newBranchLambdas.reserve(branchesCount);
+
+ for (size_t i = 0; i < branchesCount; ++i) {
+ auto branchLambda = dqReplicate.Arg(/* input */ 1 + i).Cast<TCoLambda>();
YQL_ENSURE(branchLambda.Args().Size() == 1);
-
- TExprNode::TPtr newBranchProgram;
- if (index == i) {
- newBranchProgram = ctx.FuseLambdas(lambda.Ref(), branchLambda.Ref());
+
+ TExprNode::TPtr newBranchProgram;
+ if (index == i) {
+ newBranchProgram = ctx.FuseLambdas(lambda.Ref(), branchLambda.Ref());
} else {
- newBranchProgram = ctx.DeepCopyLambda(branchLambda.Ref());
+ newBranchProgram = ctx.DeepCopyLambda(branchLambda.Ref());
}
-
- newBranchLambdas.emplace_back(ctx.ReplaceNodes(std::move(newBranchProgram), argReplaces));
- }
-
- newProgram = Build<TCoLambda>(ctx, dqReplicate.Pos())
- .Args(newArgs)
- .Body<TDqReplicate>()
- .Input(ctx.ReplaceNodes(dqReplicate.Input().Ptr(), argReplaces))
- .FreeArgs()
- .Add(newBranchLambdas)
+
+ newBranchLambdas.emplace_back(ctx.ReplaceNodes(std::move(newBranchProgram), argReplaces));
+ }
+
+ newProgram = Build<TCoLambda>(ctx, dqReplicate.Pos())
+ .Args(newArgs)
+ .Body<TDqReplicate>()
+ .Input(ctx.ReplaceNodes(dqReplicate.Input().Ptr(), argReplaces))
+ .FreeArgs()
+ .Add(newBranchLambdas)
.Build()
.Build()
- .Done().Ptr();
-
- // YQL_CLOG(TRACE, CoreDq) << "-- newProgram: " << newProgram->Dump();
- }
+ .Done().Ptr();
+
+ // YQL_CLOG(TRACE, CoreDq) << "-- newProgram: " << newProgram->Dump();
+ }
TNodeOnNodeOwnedMap inputArgReplaces;
- TVector<TCoArgument> newArgs = PrepareArgumentsReplacement(TCoLambda(newProgram).Args(), lambdaInputs, ctx, inputArgReplaces);
- TVector<TExprBase> inputNodes;
-
- // if lambda contains precomputes -> move them to the stage inputs
- {
- TNodeOnNodeOwnedMap precomputesInsideLambda;
- VisitExpr(newProgram, [&precomputesInsideLambda](const TExprNode::TPtr& node) {
- if (TDqPhyPrecompute::Match(node.Get())) {
- precomputesInsideLambda[node.Get()] = node;
- return false;
- }
- return true;
- });
-
- YQL_CLOG(TRACE, CoreDq) << "lambda with " << precomputesInsideLambda.size() << " precomputes and "
- << lambdaInputs.size() << " inputs";
-
- auto prevInputs = stage.Inputs();
-
- inputNodes.reserve(newArgs.size() + precomputesInsideLambda.size());
- inputNodes.insert(inputNodes.end(), prevInputs.begin(), prevInputs.end());
- inputNodes.insert(inputNodes.end(), lambdaInputs.begin(), lambdaInputs.end());
-
- for (auto [raw, ptr]: precomputesInsideLambda) {
- auto it = std::find_if(prevInputs.begin(), prevInputs.end(), [raw=raw](auto x) { return x.Raw() == raw; });
- if (it != prevInputs.end()) {
- ui64 inputIndex = std::distance(prevInputs.begin(), it);
- inputArgReplaces[raw] = newArgs[inputIndex].Ptr();
- } else {
- inputNodes.emplace_back(TExprBase(ptr));
- newArgs.emplace_back(TCoArgument(ctx.NewArgument(raw->Pos(), "precompute")));
- inputArgReplaces[raw] = newArgs.back().Ptr();
- }
- }
- }
-
- YQL_ENSURE(newArgs.size() == inputNodes.size(), "" << newArgs.size() << " != " << inputNodes.size());
-
+ TVector<TCoArgument> newArgs = PrepareArgumentsReplacement(TCoLambda(newProgram).Args(), lambdaInputs, ctx, inputArgReplaces);
+ TVector<TExprBase> inputNodes;
+
+ // if lambda contains precomputes -> move them to the stage inputs
+ {
+ TNodeOnNodeOwnedMap precomputesInsideLambda;
+ VisitExpr(newProgram, [&precomputesInsideLambda](const TExprNode::TPtr& node) {
+ if (TDqPhyPrecompute::Match(node.Get())) {
+ precomputesInsideLambda[node.Get()] = node;
+ return false;
+ }
+ return true;
+ });
+
+ YQL_CLOG(TRACE, CoreDq) << "lambda with " << precomputesInsideLambda.size() << " precomputes and "
+ << lambdaInputs.size() << " inputs";
+
+ auto prevInputs = stage.Inputs();
+
+ inputNodes.reserve(newArgs.size() + precomputesInsideLambda.size());
+ inputNodes.insert(inputNodes.end(), prevInputs.begin(), prevInputs.end());
+ inputNodes.insert(inputNodes.end(), lambdaInputs.begin(), lambdaInputs.end());
+
+ for (auto [raw, ptr]: precomputesInsideLambda) {
+ auto it = std::find_if(prevInputs.begin(), prevInputs.end(), [raw=raw](auto x) { return x.Raw() == raw; });
+ if (it != prevInputs.end()) {
+ ui64 inputIndex = std::distance(prevInputs.begin(), it);
+ inputArgReplaces[raw] = newArgs[inputIndex].Ptr();
+ } else {
+ inputNodes.emplace_back(TExprBase(ptr));
+ newArgs.emplace_back(TCoArgument(ctx.NewArgument(raw->Pos(), "precompute")));
+ inputArgReplaces[raw] = newArgs.back().Ptr();
+ }
+ }
+ }
+
+ YQL_ENSURE(newArgs.size() == inputNodes.size(), "" << newArgs.size() << " != " << inputNodes.size());
+
auto newStage = Build<TDqStage>(ctx, stage.Pos())
- .Inputs()
- .Add(inputNodes)
- .Build()
- .Program()
- .Args(newArgs)
- .Body(ctx.ReplaceNodes(newProgram->TailPtr(), inputArgReplaces))
- .Build()
+ .Inputs()
+ .Add(inputNodes)
+ .Build()
+ .Program()
+ .Args(newArgs)
+ .Body(ctx.ReplaceNodes(newProgram->TailPtr(), inputArgReplaces))
+ .Build()
.Settings(TDqStageSettings().BuildNode(ctx, stage.Pos()))
.Done();
@@ -305,10 +305,10 @@ TMaybeNode<TDqStage> DqPushLambdaToStage(const TDqStage& stage, const TCoAtom& o
}
TMaybeNode<TDqConnection> DqPushLambdaToStageUnionAll(const TDqConnection& connection, TCoLambda& lambda,
- const TVector<TDqConnection>& lambdaInputs, TExprContext& ctx, IOptimizationContext& optCtx)
+ const TVector<TDqConnection>& lambdaInputs, TExprContext& ctx, IOptimizationContext& optCtx)
{
auto stage = connection.Output().Stage().Cast<TDqStage>();
- auto newStage = DqPushLambdaToStage(stage, connection.Output().Index(), lambda, lambdaInputs, ctx, optCtx);
+ auto newStage = DqPushLambdaToStage(stage, connection.Output().Index(), lambda, lambdaInputs, ctx, optCtx);
if (!newStage) {
return {};
}
@@ -318,24 +318,24 @@ TMaybeNode<TDqConnection> DqPushLambdaToStageUnionAll(const TDqConnection& conne
.Index(connection.Output().Index())
.Done();
- return TDqConnection(ctx.ChangeChild(connection.Ref(), TDqConnection::idx_Output, output.Ptr()));
+ return TDqConnection(ctx.ChangeChild(connection.Ref(), TDqConnection::idx_Output, output.Ptr()));
}
-TExprBase DqPushSkipNullMembersToStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
- const TParentsMap& parentsMap, bool allowStageMultiUsage)
-{
- return DqPushMembersFilterToStage<TCoSkipNullMembers>(node, ctx, optCtx, parentsMap, allowStageMultiUsage);
+TExprBase DqPushSkipNullMembersToStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
+ const TParentsMap& parentsMap, bool allowStageMultiUsage)
+{
+ return DqPushMembersFilterToStage<TCoSkipNullMembers>(node, ctx, optCtx, parentsMap, allowStageMultiUsage);
+}
+
+TExprBase DqPushExtractMembersToStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
+ const TParentsMap& parentsMap, bool allowStageMultiUsage)
+{
+ return DqPushMembersFilterToStage<TCoExtractMembers>(node, ctx, optCtx, parentsMap, allowStageMultiUsage);
}
-TExprBase DqPushExtractMembersToStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
- const TParentsMap& parentsMap, bool allowStageMultiUsage)
-{
- return DqPushMembersFilterToStage<TCoExtractMembers>(node, ctx, optCtx, parentsMap, allowStageMultiUsage);
-}
-
TExprBase DqBuildFlatmapStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
- const TParentsMap& parentsMap, bool allowStageMultiUsage)
-{
+ const TParentsMap& parentsMap, bool allowStageMultiUsage)
+{
if (!node.Maybe<TCoFlatMapBase>().Input().Maybe<TDqCnUnionAll>()) {
return node;
}
@@ -347,9 +347,9 @@ TExprBase DqBuildFlatmapStage(TExprBase node, TExprContext& ctx, IOptimizationCo
}
if (!IsDqPureExpr(flatmap.Lambda())) {
- return node;
- }
-
+ return node;
+ }
+
auto lambda = TCoLambda(ctx.Builder(flatmap.Lambda().Pos())
.Lambda()
.Param("stream")
@@ -359,7 +359,7 @@ TExprBase DqBuildFlatmapStage(TExprBase node, TExprContext& ctx, IOptimizationCo
.Seal()
.Seal().Build());
- auto pushResult = DqPushLambdaToStageUnionAll(dqUnion, lambda, {}, ctx, optCtx);
+ auto pushResult = DqPushLambdaToStageUnionAll(dqUnion, lambda, {}, ctx, optCtx);
if (pushResult) {
return pushResult.Cast();
}
@@ -384,7 +384,7 @@ TExprBase DqBuildFlatmapStage(TExprBase node, TExprContext& ctx, IOptimizationCo
template <typename BaseLMap>
TExprBase DqPushBaseLMapToStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
- const TParentsMap& parentsMap, bool allowStageMultiUsage = true)
+ const TParentsMap& parentsMap, bool allowStageMultiUsage = true)
{
if (!node.Maybe<BaseLMap>().Input().template Maybe<TDqCnUnionAll>()) {
return node;
@@ -410,7 +410,7 @@ TExprBase DqPushBaseLMapToStage(TExprBase node, TExprContext& ctx, IOptimization
.Build()
.Done();
- auto result = DqPushLambdaToStageUnionAll(dqUnion, lambda, {}, ctx, optCtx);
+ auto result = DqPushLambdaToStageUnionAll(dqUnion, lambda, {}, ctx, optCtx);
if (!result) {
return node;
}
@@ -419,15 +419,15 @@ TExprBase DqPushBaseLMapToStage(TExprBase node, TExprContext& ctx, IOptimization
}
TExprBase DqPushOrderedLMapToStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
- const TParentsMap& parentsMap, bool allowStageMultiUsage)
+ const TParentsMap& parentsMap, bool allowStageMultiUsage)
{
- return DqPushBaseLMapToStage<TCoOrderedLMap>(node, ctx, optCtx, parentsMap, allowStageMultiUsage);
+ return DqPushBaseLMapToStage<TCoOrderedLMap>(node, ctx, optCtx, parentsMap, allowStageMultiUsage);
}
TExprBase DqPushLMapToStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
- const TParentsMap& parentsMap, bool allowStageMultiUsage)
+ const TParentsMap& parentsMap, bool allowStageMultiUsage)
{
- return DqPushBaseLMapToStage<TCoLMap>(node, ctx, optCtx, parentsMap, allowStageMultiUsage);
+ return DqPushBaseLMapToStage<TCoLMap>(node, ctx, optCtx, parentsMap, allowStageMultiUsage);
}
TExprBase DqBuildExtFunctionStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
@@ -547,9 +547,9 @@ TExprBase DqBuildExtFunctionStage(TExprBase node, TExprContext& ctx, IOptimizati
return externalStage;
}
-TExprBase DqPushCombineToStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
- const TParentsMap& parentsMap, bool allowStageMultiUsage)
-{
+TExprBase DqPushCombineToStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
+ const TParentsMap& parentsMap, bool allowStageMultiUsage)
+{
if (!node.Maybe<TCoCombineByKey>().Input().Maybe<TDqCnUnionAll>()) {
return node;
}
@@ -566,9 +566,9 @@ TExprBase DqPushCombineToStage(TExprBase node, TExprContext& ctx, IOptimizationC
!CanPushDqExpr(combine.UpdateHandlerLambda(), dqUnion) ||
!CanPushDqExpr(combine.FinishHandlerLambda(), dqUnion))
{
- return node;
- }
-
+ return node;
+ }
+
auto lambda = Build<TCoLambda>(ctx, combine.Pos())
.Args({"stream"})
.Body<TCoCombineByKey>()
@@ -581,7 +581,7 @@ TExprBase DqPushCombineToStage(TExprBase node, TExprContext& ctx, IOptimizationC
.Build()
.Done();
- auto result = DqPushLambdaToStageUnionAll(dqUnion, lambda, {}, ctx, optCtx);
+ auto result = DqPushLambdaToStageUnionAll(dqUnion, lambda, {}, ctx, optCtx);
if (!result) {
return node;
}
@@ -589,11 +589,11 @@ TExprBase DqPushCombineToStage(TExprBase node, TExprContext& ctx, IOptimizationC
return result.Cast();
}
-TExprBase DqBuildPartitionsStage(TExprBase node, TExprContext& ctx, const TParentsMap& parentsMap) {
+TExprBase DqBuildPartitionsStage(TExprBase node, TExprContext& ctx, const TParentsMap& parentsMap) {
return DqBuildPartitionsStageStub<TCoPartitionsByKeys>(std::move(node), ctx, parentsMap);
}
-TExprBase DqBuildPartitionStage(TExprBase node, TExprContext& ctx, const TParentsMap& parentsMap) {
+TExprBase DqBuildPartitionStage(TExprBase node, TExprContext& ctx, const TParentsMap& parentsMap) {
return DqBuildPartitionsStageStub<TCoPartitionByKey>(std::move(node), ctx, parentsMap);
}
@@ -607,13 +607,13 @@ TExprBase DqBuildPartitionStage(TExprBase node, TExprContext& ctx, const TParent
* StructMember_i := function_i(ToOptional (TDqCnUnionAll ...))
* @endcode
*
- * Each list item (AsStruct) represents a single aggregation result.
+ * Each list item (AsStruct) represents a single aggregation result.
* If for all `i` function `function_i` depends on single connection via `ToOptional` callable
* then we do the following stuff:
*
* For each list item create a new stage with `Condense` callable:
*
- * 1. with `initState` (it is the _default_ _value_ in case of empty input stream)
+ * 1. with `initState` (it is the _default_ _value_ in case of empty input stream)
* @code
* emptyList := EmptyList of type `AsStruct` from the snippet above
* initState := Apply `AsStruct` emptyList
@@ -621,20 +621,20 @@ TExprBase DqBuildPartitionStage(TExprBase node, TExprContext& ctx, const TParent
*
* 2. with `updateHandler`
* @code
- * updateHandler := `AsStruct` callable with all `ToOptional (DqCnUnionAll))` replaced with input state argument
+ * updateHandler := `AsStruct` callable with all `ToOptional (DqCnUnionAll))` replaced with input state argument
* @endocde
*
* If there are more than a single list item, create a separate stage to merge all the results from previous stages.
*/
// TODO: Creating a separate stage for each AsList element is redundant, it's better to use
-// a single stage with something like MultiCondense here.
+// a single stage with something like MultiCondense here.
// TODO: The way such context switch is presented in the expression graph is very implicit, so it is
-// hard to work with. We should consider making it more explicit, something like ProcessScalar on the
-// top level of expression graph.
+// hard to work with. We should consider making it more explicit, something like ProcessScalar on the
+// top level of expression graph.
-TExprBase DqBuildAggregationResultStage(TExprBase node, TExprContext& ctx, IOptimizationContext&) {
+TExprBase DqBuildAggregationResultStage(TExprBase node, TExprContext& ctx, IOptimizationContext&) {
if (!node.Maybe<TCoAsList>()) {
return node;
}
@@ -839,9 +839,9 @@ TExprBase GetSortDirection(TExprBase& sortDirections, size_t index) {
}
} // End of anonymous namespace
-TExprBase DqBuildTopSortStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
- const TParentsMap& parentsMap, bool allowStageMultiUsage)
-{
+TExprBase DqBuildTopSortStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
+ const TParentsMap& parentsMap, bool allowStageMultiUsage)
+{
if (!node.Maybe<TCoTopSort>().Input().Maybe<TDqCnUnionAll>()) {
return node;
}
@@ -853,9 +853,9 @@ TExprBase DqBuildTopSortStage(TExprBase node, TExprContext& ctx, IOptimizationCo
}
if (!CanPushDqExpr(topSort.Count(), dqUnion) || !CanPushDqExpr(topSort.KeySelectorLambda(), dqUnion)) {
- return node;
- }
-
+ return node;
+ }
+
auto result = dqUnion.Output().Stage().Program().Body();
auto sortKeySelector = topSort.KeySelectorLambda();
@@ -871,7 +871,7 @@ TExprBase DqBuildTopSortStage(TExprBase node, TExprContext& ctx, IOptimizationCo
.Done();
auto stage = dqUnion.Output().Stage().Cast<TDqStage>();
- auto newStage = DqPushLambdaToStage(stage, dqUnion.Output().Index(), lambda, {}, ctx, optCtx);
+ auto newStage = DqPushLambdaToStage(stage, dqUnion.Output().Index(), lambda, {}, ctx, optCtx);
if (!newStage) {
return node;
}
@@ -955,20 +955,20 @@ TExprBase DqBuildTopSortStage(TExprBase node, TExprContext& ctx, IOptimizationCo
.Done();
}
-TExprBase DqBuildSortStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
- const TParentsMap& parentsMap, bool allowStageMultiUsage)
-{
+TExprBase DqBuildSortStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
+ const TParentsMap& parentsMap, bool allowStageMultiUsage)
+{
if (!node.Maybe<TCoSortBase>().Input().Maybe<TDqCnUnionAll>()) {
return node;
}
auto sort = node.Cast<TCoSortBase>();
auto dqUnion = sort.Input().Cast<TDqCnUnionAll>();
-
+
if (!IsSingleConsumerConnection(dqUnion, parentsMap, allowStageMultiUsage)) {
- return node;
- }
-
+ return node;
+ }
+
auto result = dqUnion.Output().Stage().Program().Body();
auto sortKeySelector = sort.KeySelectorLambda();
@@ -1012,7 +1012,7 @@ TExprBase DqBuildSortStage(TExprBase node, TExprContext& ctx, IOptimizationConte
.Done();
auto stage = dqUnion.Output().Stage().Cast<TDqStage>();
- auto newStage = DqPushLambdaToStage(stage, dqUnion.Output().Index(), lambda, {}, ctx, optCtx);
+ auto newStage = DqPushLambdaToStage(stage, dqUnion.Output().Index(), lambda, {}, ctx, optCtx);
if (!newStage) {
return node;
}
@@ -1066,8 +1066,8 @@ TExprBase DqBuildSortStage(TExprBase node, TExprContext& ctx, IOptimizationConte
// (let $7 (DqPhyStage '((DqCnUnionAll (TDqOutput $5 '"0"))) (lambda '($13) (FromFlow (Take (Skip (ToFlow $13) (Uint64 '1)) $6))) '()))
// (let $8 (DqPhyStage '((DqCnUnionAll (TDqOutput $7 '"0"))) (lambda '($14) (FromFlow (Take (ToFlow $14) $6))) '()))
// maybe optimize (Take (Skip ...) ...) ?
-TExprBase DqBuildSkipStage(TExprBase node, TExprContext& ctx, IOptimizationContext& /* optCtx */,
- const TParentsMap& parentsMap, bool allowStageMultiUsage)
+TExprBase DqBuildSkipStage(TExprBase node, TExprContext& ctx, IOptimizationContext& /* optCtx */,
+ const TParentsMap& parentsMap, bool allowStageMultiUsage)
{
if (!node.Maybe<TCoSkip>().Input().Maybe<TDqCnUnionAll>()) {
return node;
@@ -1105,9 +1105,9 @@ TExprBase DqBuildSkipStage(TExprBase node, TExprContext& ctx, IOptimizationConte
.Done();
}
-TExprBase DqBuildTakeStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
- const TParentsMap& parentsMap, bool allowStageMultiUsage)
-{
+TExprBase DqBuildTakeStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
+ const TParentsMap& parentsMap, bool allowStageMultiUsage)
+{
if (!node.Maybe<TCoTake>().Input().Maybe<TDqCnUnionAll>()) {
return node;
}
@@ -1119,9 +1119,9 @@ TExprBase DqBuildTakeStage(TExprBase node, TExprContext& ctx, IOptimizationConte
}
if (!CanPushDqExpr(take.Count(), dqUnion)) {
- return node;
- }
-
+ return node;
+ }
+
auto result = dqUnion.Output().Stage().Program().Body();
auto stage = dqUnion.Output().Stage();
@@ -1133,7 +1133,7 @@ TExprBase DqBuildTakeStage(TExprBase node, TExprContext& ctx, IOptimizationConte
.Build()
.Done();
- auto newDqUnion = DqPushLambdaToStageUnionAll(dqUnion, lambda, {}, ctx, optCtx);
+ auto newDqUnion = DqPushLambdaToStageUnionAll(dqUnion, lambda, {}, ctx, optCtx);
if (!newDqUnion) {
return node;
}
@@ -1161,71 +1161,71 @@ TExprBase DqBuildTakeStage(TExprBase node, TExprContext& ctx, IOptimizationConte
.Done();
}
-TExprBase DqBuildTakeSkipStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
- const TParentsMap& parentsMap, bool allowStageMultiUsage)
-{
- if (!node.Maybe<TCoTake>().Input().Maybe<TCoSkip>().Input().Maybe<TDqCnUnionAll>()) {
- return node;
- }
-
- auto take = node.Cast<TCoTake>();
+TExprBase DqBuildTakeSkipStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
+ const TParentsMap& parentsMap, bool allowStageMultiUsage)
+{
+ if (!node.Maybe<TCoTake>().Input().Maybe<TCoSkip>().Input().Maybe<TDqCnUnionAll>()) {
+ return node;
+ }
+
+ auto take = node.Cast<TCoTake>();
auto skip = take.Input().Cast<TCoSkip>();
auto dqUnion = skip.Input().Cast<TDqCnUnionAll>();
if (!IsSingleConsumerConnection(dqUnion, parentsMap, allowStageMultiUsage)) {
- return node;
- }
-
+ return node;
+ }
+
if (!CanPushDqExpr(take.Count(), dqUnion)) {
- return node;
- }
-
+ return node;
+ }
+
if (!CanPushDqExpr(skip.Count(), dqUnion)) {
- return node;
- }
-
- auto lambda = Build<TCoLambda>(ctx, node.Pos())
- .Args({"stream"})
- .Body<TCoTake>()
- .Input("stream")
- .Count<TCoPlus>()
- .Left(take.Count())
- .Right(skip.Count())
- .Build()
- .Build()
- .Done();
-
- auto newDqUnion = DqPushLambdaToStageUnionAll(dqUnion, lambda, {}, ctx, optCtx);
- if (!newDqUnion) {
- return node;
- }
-
- auto outerTakeSkipStage = Build<TDqStage>(ctx, node.Pos())
- .Inputs()
- .Add(newDqUnion.Cast())
- .Build()
- .Program()
- .Args({"stream"})
- .Body<TCoTake>()
- .Input<TCoSkip>()
- .Input("stream")
- .Count(skip.Count())
- .Build()
- .Count(take.Count())
- .Build()
- .Build()
+ return node;
+ }
+
+ auto lambda = Build<TCoLambda>(ctx, node.Pos())
+ .Args({"stream"})
+ .Body<TCoTake>()
+ .Input("stream")
+ .Count<TCoPlus>()
+ .Left(take.Count())
+ .Right(skip.Count())
+ .Build()
+ .Build()
+ .Done();
+
+ auto newDqUnion = DqPushLambdaToStageUnionAll(dqUnion, lambda, {}, ctx, optCtx);
+ if (!newDqUnion) {
+ return node;
+ }
+
+ auto outerTakeSkipStage = Build<TDqStage>(ctx, node.Pos())
+ .Inputs()
+ .Add(newDqUnion.Cast())
+ .Build()
+ .Program()
+ .Args({"stream"})
+ .Body<TCoTake>()
+ .Input<TCoSkip>()
+ .Input("stream")
+ .Count(skip.Count())
+ .Build()
+ .Count(take.Count())
+ .Build()
+ .Build()
.Settings(TDqStageSettings().BuildNode(ctx, node.Pos()))
- .Done();
-
- return Build<TDqCnUnionAll>(ctx, node.Pos())
- .Output()
- .Stage(outerTakeSkipStage)
- .Index().Build("0")
- .Build()
- .Done();
-}
-
-TExprBase DqRewriteLengthOfStageOutput(TExprBase node, TExprContext& ctx, IOptimizationContext&) {
+ .Done();
+
+ return Build<TDqCnUnionAll>(ctx, node.Pos())
+ .Output()
+ .Stage(outerTakeSkipStage)
+ .Index().Build("0")
+ .Build()
+ .Done();
+}
+
+TExprBase DqRewriteLengthOfStageOutput(TExprBase node, TExprContext& ctx, IOptimizationContext&) {
if (!node.Maybe<TCoLength>().List().Maybe<TDqCnUnionAll>()) {
return node;
}
@@ -1445,9 +1445,9 @@ TExprBase DqBuildPrecomputeStage(TExprBase node, TExprContext& ctx) {
connection = input.Ptr();
value = true;
} else if (IsDqPureExpr(input)) {
- if (input.Ref().GetTypeAnn()->GetKind() != ETypeAnnotationKind::List &&
- input.Ref().GetTypeAnn()->GetKind() != ETypeAnnotationKind::Data)
- {
+ if (input.Ref().GetTypeAnn()->GetKind() != ETypeAnnotationKind::List &&
+ input.Ref().GetTypeAnn()->GetKind() != ETypeAnnotationKind::Data)
+ {
return node;
}
@@ -1456,23 +1456,23 @@ TExprBase DqBuildPrecomputeStage(TExprBase node, TExprContext& ctx) {
.Build()
.Program()
.Args({})
- .Body<TCoToStream>()
- .Input<TCoJust>()
- .Input(input)
- .Build()
- .Build()
+ .Body<TCoToStream>()
+ .Input<TCoJust>()
+ .Input(input)
+ .Build()
+ .Build()
.Build()
.Settings().Build()
.Done();
- connection = Build<TDqCnValue>(ctx, node.Pos())
- .Output()
- .Stage(dataStage)
- .Index().Build("0")
- .Build()
- .Done().Ptr();
-
- value = true;
+ connection = Build<TDqCnValue>(ctx, node.Pos())
+ .Output()
+ .Stage(dataStage)
+ .Index().Build("0")
+ .Build()
+ .Done().Ptr();
+
+ value = true;
} else {
return node;
}
@@ -1490,10 +1490,10 @@ TExprBase DqBuildPrecomputeStage(TExprBase node, TExprContext& ctx) {
.Add(phyPrecompute)
.Build()
.Program()
- .Args({"zzz"})
- .Body<TCoIterator>()
- .List("zzz")
- .Build()
+ .Args({"zzz"})
+ .Body<TCoIterator>()
+ .List("zzz")
+ .Build()
.Build()
.Settings().Build()
.Done();
@@ -1506,7 +1506,7 @@ TExprBase DqBuildPrecomputeStage(TExprBase node, TExprContext& ctx) {
.Done();
}
-TExprBase DqBuildHasItems(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx) {
+TExprBase DqBuildHasItems(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx) {
if (!node.Maybe<TCoHasItems>()) {
return node;
}
@@ -1533,7 +1533,7 @@ TExprBase DqBuildHasItems(TExprBase node, TExprContext& ctx, IOptimizationContex
.Build()
.Done();
- auto newUnion = DqPushLambdaToStageUnionAll(unionAll, takeProgram, {}, ctx, optCtx);
+ auto newUnion = DqPushLambdaToStageUnionAll(unionAll, takeProgram, {}, ctx, optCtx);
if (!newUnion.IsValid()) {
return node;
@@ -1582,7 +1582,7 @@ TExprBase DqBuildHasItems(TExprBase node, TExprContext& ctx, IOptimizationContex
return precompute;
}
-TExprBase DqBuildScalarPrecompute(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx) {
+TExprBase DqBuildScalarPrecompute(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx) {
if (!node.Maybe<TCoToOptional>()) {
return node;
}
@@ -1650,7 +1650,7 @@ TExprBase DqBuildScalarPrecompute(TExprBase node, TExprContext& ctx, IOptimizati
.Build()
.Done();
- auto newUnion = DqPushLambdaToStageUnionAll(unionAll, newProgram, {}, ctx, optCtx);
+ auto newUnion = DqPushLambdaToStageUnionAll(unionAll, newProgram, {}, ctx, optCtx);
if (!newUnion.IsValid()) {
return node;
diff --git a/ydb/library/yql/dq/opt/dq_opt_phy.h b/ydb/library/yql/dq/opt/dq_opt_phy.h
index 3bca561b9e..70a9d70082 100644
--- a/ydb/library/yql/dq/opt/dq_opt_phy.h
+++ b/ydb/library/yql/dq/opt/dq_opt_phy.h
@@ -12,31 +12,31 @@ namespace NYql::NDq {
NNodes::TMaybeNode<NNodes::TDqStage> DqPushLambdaToStage(const NNodes::TDqStage &stage,
const NNodes::TCoAtom& outputIndex, NNodes::TCoLambda& lambda,
- const TVector<NNodes::TDqConnection>& lambdaInputs, TExprContext& ctx, IOptimizationContext& optCtx);
+ const TVector<NNodes::TDqConnection>& lambdaInputs, TExprContext& ctx, IOptimizationContext& optCtx);
NNodes::TMaybeNode<NNodes::TDqConnection> DqPushLambdaToStageUnionAll(const NNodes::TDqConnection& connection, NNodes::TCoLambda& lambda,
- const TVector<NNodes::TDqConnection>& lambdaInputs, TExprContext& ctx, IOptimizationContext& optCtx);
+ const TVector<NNodes::TDqConnection>& lambdaInputs, TExprContext& ctx, IOptimizationContext& optCtx);
NNodes::TExprBase DqPushSkipNullMembersToStage(NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
- const TParentsMap& parentsMap, bool allowStageMultiUsage = true);
+ const TParentsMap& parentsMap, bool allowStageMultiUsage = true);
NNodes::TExprBase DqPushExtractMembersToStage(NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
- const TParentsMap& parentsMap, bool allowStageMultiUsage = true);
+ const TParentsMap& parentsMap, bool allowStageMultiUsage = true);
NNodes::TExprBase DqPushOrderedLMapToStage(NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
- const TParentsMap& parentsMap, bool allowStageMultiUsage = true);
+ const TParentsMap& parentsMap, bool allowStageMultiUsage = true);
NNodes::TExprBase DqPushLMapToStage(NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
- const TParentsMap& parentsMap, bool allowStageMultiUsage = true);
+ const TParentsMap& parentsMap, bool allowStageMultiUsage = true);
NNodes::TExprBase DqBuildExtFunctionStage(NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
const TParentsMap& parentsMap, bool allowStageMultiUsage = true);
NNodes::TExprBase DqBuildFlatmapStage(NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
- const TParentsMap& parentsMap, bool allowStageMultiUsage = true);
+ const TParentsMap& parentsMap, bool allowStageMultiUsage = true);
NNodes::TExprBase DqPushCombineToStage(NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
- const TParentsMap& parentsMap, bool allowStageMultiUsage = true);
+ const TParentsMap& parentsMap, bool allowStageMultiUsage = true);
NNodes::TExprBase DqBuildPartitionsStage(NNodes::TExprBase node, TExprContext& ctx, const TParentsMap& parentsMap);
@@ -46,34 +46,34 @@ NNodes::TExprBase DqBuildAggregationResultStage(NNodes::TExprBase node, TExprCon
IOptimizationContext& optCtx);
NNodes::TExprBase DqBuildTopSortStage(NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
- const TParentsMap& parentsMap, bool allowStageMultiUsage = true);
+ const TParentsMap& parentsMap, bool allowStageMultiUsage = true);
NNodes::TExprBase DqBuildSortStage(NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
- const TParentsMap& parentsMap, bool allowStageMultiUsage = true);
+ const TParentsMap& parentsMap, bool allowStageMultiUsage = true);
NNodes::TExprBase DqBuildSkipStage(NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
- const TParentsMap& parentsMap, bool allowStageMultiUsage = true);
+ const TParentsMap& parentsMap, bool allowStageMultiUsage = true);
NNodes::TExprBase DqBuildTakeStage(NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
- const TParentsMap& parentsMap, bool allowStageMultiUsage = true);
+ const TParentsMap& parentsMap, bool allowStageMultiUsage = true);
+
+NNodes::TExprBase DqBuildTakeSkipStage(NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
+ const TParentsMap& parentsMap, bool allowStageMultiUsage = true);
-NNodes::TExprBase DqBuildTakeSkipStage(NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
- const TParentsMap& parentsMap, bool allowStageMultiUsage = true);
-
NNodes::TExprBase DqRewriteLengthOfStageOutput(NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx);
-NNodes::TExprBase DqRewriteRightJoinToLeft(const NNodes::TExprBase node, TExprContext& ctx);
-
+NNodes::TExprBase DqRewriteRightJoinToLeft(const NNodes::TExprBase node, TExprContext& ctx);
+
NNodes::TExprBase DqPushJoinToStage(const NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx,
- const TParentsMap& parentsMap, bool allowStageMultiUsage = true);
+ const TParentsMap& parentsMap, bool allowStageMultiUsage = true);
NNodes::TExprBase DqBuildPhyJoin(const NNodes::TDqJoin& join, bool pushLeftStage, TExprContext& ctx,
- IOptimizationContext& optCtx);
+ IOptimizationContext& optCtx);
NNodes::TExprBase DqBuildJoinDict(const NNodes::TDqJoin& join, TExprContext& ctx);
TMaybe<std::pair<NNodes::TExprBase, NNodes::TDqConnection>> ExtractPureExprStage(TExprNode::TPtr input,
- TExprContext& ctx);
+ TExprContext& ctx);
NNodes::TExprBase DqBuildPureExprStage(NNodes::TExprBase node, TExprContext& ctx);
@@ -82,9 +82,9 @@ NNodes::TExprBase DqBuildExtendStage(NNodes::TExprBase node, TExprContext& ctx);
NNodes::TExprBase DqBuildPrecomputeStage(NNodes::TExprBase node, TExprContext& ctx);
NYql::NNodes::TExprBase DqBuildHasItems(NYql::NNodes::TExprBase node, NYql::TExprContext& ctx,
- NYql::IOptimizationContext& optCtx);
+ NYql::IOptimizationContext& optCtx);
NYql::NNodes::TExprBase DqBuildScalarPrecompute(NYql::NNodes::TExprBase node, NYql::TExprContext& ctx,
- NYql::IOptimizationContext& optCtx);
+ NYql::IOptimizationContext& optCtx);
} // namespace NYql::NDq
diff --git a/ydb/library/yql/dq/opt/dq_opt_phy_finalizing.cpp b/ydb/library/yql/dq/opt/dq_opt_phy_finalizing.cpp
index 4a74050845..a1c8cf10c2 100644
--- a/ydb/library/yql/dq/opt/dq_opt_phy_finalizing.cpp
+++ b/ydb/library/yql/dq/opt/dq_opt_phy_finalizing.cpp
@@ -1,206 +1,206 @@
#include "dq_opt_phy_finalizing.h"
-
+
#include <ydb/library/yql/dq/type_ann/dq_type_ann.h>
#include <ydb/library/yql/utils/log/log.h>
#include <ydb/library/yql/providers/common/provider/yql_provider.h>
-
+
#include <algorithm>
-namespace NYql::NDq {
-
-using namespace NNodes;
-
-namespace {
-
-// returns new DqStage and list of added output indexes
-std::pair<TDqStage, TVector<TCoAtom>> ReplicateStageOutput(const TDqStage& stage, const TCoAtom& indexAtom,
- const TVector<TCoLambda>& lambdas, TExprContext& ctx)
-{
+namespace NYql::NDq {
+
+using namespace NNodes;
+
+namespace {
+
+// returns new DqStage and list of added output indexes
+std::pair<TDqStage, TVector<TCoAtom>> ReplicateStageOutput(const TDqStage& stage, const TCoAtom& indexAtom,
+ const TVector<TCoLambda>& lambdas, TExprContext& ctx)
+{
auto result = stage.Program().Body();
- auto resultType = result.Ref().GetTypeAnn();
-
- const TTypeAnnotationNode* resultItemType = nullptr;
+ auto resultType = result.Ref().GetTypeAnn();
+
+ const TTypeAnnotationNode* resultItemType = nullptr;
if (!EnsureNewSeqType<false, false>(result.Pos(), *resultType, ctx, &resultItemType)) {
- YQL_ENSURE(false, "got " << FormatType(resultType));
- }
-
- ui32 index = FromString<ui32>(indexAtom.Value());
- ui32 outputsCount = 0;
- if (index > 0 || resultItemType->GetKind() == ETypeAnnotationKind::Variant) {
- YQL_ENSURE(resultItemType->GetKind() == ETypeAnnotationKind::Variant);
- auto variantUnderlyingType = resultItemType->Cast<TVariantExprType>()->GetUnderlyingType();
- YQL_ENSURE(variantUnderlyingType->GetKind() == ETypeAnnotationKind::Tuple);
- YQL_ENSURE(variantUnderlyingType->Cast<TTupleExprType>()->GetSize() > index);
- outputsCount = variantUnderlyingType->Cast<TTupleExprType>()->GetSize();
- } else {
- outputsCount = 1;
- }
-
- YQL_CLOG(TRACE, CoreDq) << "replicate stage (#" << stage.Ref().UniqueId() << ", " << index << "), outputs: "
- << outputsCount << ", about to add " << lambdas.size() << " copies." << Endl << PrintDqStageOnly(stage, ctx);
-
- TExprNode::TPtr newResult;
- ui32 newOutputIndexStart;
-
- if (outputsCount > 1) {
- YQL_ENSURE(result.Maybe<TDqReplicate>(), "got: " << NCommon::ExprToPrettyString(ctx, result.Ref()));
- auto outputTypesTuple = resultItemType->Cast<TVariantExprType>()->GetUnderlyingType()->Cast<TTupleExprType>();
- newOutputIndexStart = outputTypesTuple->GetSize();
- } else {
- newOutputIndexStart = 1;
- }
-
- ui32 newOutputIndexEnd = newOutputIndexStart + lambdas.size(); // exclusive
-
- YQL_CLOG(TRACE, CoreDq) << "add output indexes: [" << newOutputIndexStart << ".." << newOutputIndexEnd << ")";
-
- TVector<TExprBase> variants;
- variants.reserve(newOutputIndexEnd);
-
- TExprNode::TPtr input;
-
- if (outputsCount > 1) {
- auto dqReplicate = result.Cast<TDqReplicate>();
- for (ui32 i = 0; i < newOutputIndexStart; ++i) {
- variants.emplace_back(dqReplicate.Args().Get(1 + i));
- }
- input = dqReplicate.Input().Ptr();
- } else {
- variants.emplace_back(BuildIdentityLambda(stage.Pos(), ctx));
- input = result.Ptr();
- }
-
- if (input->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Stream) {
- input = Build<TCoToFlow>(ctx, input->Pos())
- .Input(input)
- .Done().Ptr();
- } else {
- YQL_ENSURE(input->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Flow, "" << *input->GetTypeAnn());
- }
-
- for (ui32 i = newOutputIndexStart, j = 0; i < newOutputIndexEnd; ++i, ++j) {
+ YQL_ENSURE(false, "got " << FormatType(resultType));
+ }
+
+ ui32 index = FromString<ui32>(indexAtom.Value());
+ ui32 outputsCount = 0;
+ if (index > 0 || resultItemType->GetKind() == ETypeAnnotationKind::Variant) {
+ YQL_ENSURE(resultItemType->GetKind() == ETypeAnnotationKind::Variant);
+ auto variantUnderlyingType = resultItemType->Cast<TVariantExprType>()->GetUnderlyingType();
+ YQL_ENSURE(variantUnderlyingType->GetKind() == ETypeAnnotationKind::Tuple);
+ YQL_ENSURE(variantUnderlyingType->Cast<TTupleExprType>()->GetSize() > index);
+ outputsCount = variantUnderlyingType->Cast<TTupleExprType>()->GetSize();
+ } else {
+ outputsCount = 1;
+ }
+
+ YQL_CLOG(TRACE, CoreDq) << "replicate stage (#" << stage.Ref().UniqueId() << ", " << index << "), outputs: "
+ << outputsCount << ", about to add " << lambdas.size() << " copies." << Endl << PrintDqStageOnly(stage, ctx);
+
+ TExprNode::TPtr newResult;
+ ui32 newOutputIndexStart;
+
+ if (outputsCount > 1) {
+ YQL_ENSURE(result.Maybe<TDqReplicate>(), "got: " << NCommon::ExprToPrettyString(ctx, result.Ref()));
+ auto outputTypesTuple = resultItemType->Cast<TVariantExprType>()->GetUnderlyingType()->Cast<TTupleExprType>();
+ newOutputIndexStart = outputTypesTuple->GetSize();
+ } else {
+ newOutputIndexStart = 1;
+ }
+
+ ui32 newOutputIndexEnd = newOutputIndexStart + lambdas.size(); // exclusive
+
+ YQL_CLOG(TRACE, CoreDq) << "add output indexes: [" << newOutputIndexStart << ".." << newOutputIndexEnd << ")";
+
+ TVector<TExprBase> variants;
+ variants.reserve(newOutputIndexEnd);
+
+ TExprNode::TPtr input;
+
+ if (outputsCount > 1) {
+ auto dqReplicate = result.Cast<TDqReplicate>();
+ for (ui32 i = 0; i < newOutputIndexStart; ++i) {
+ variants.emplace_back(dqReplicate.Args().Get(1 + i));
+ }
+ input = dqReplicate.Input().Ptr();
+ } else {
+ variants.emplace_back(BuildIdentityLambda(stage.Pos(), ctx));
+ input = result.Ptr();
+ }
+
+ if (input->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Stream) {
+ input = Build<TCoToFlow>(ctx, input->Pos())
+ .Input(input)
+ .Done().Ptr();
+ } else {
+ YQL_ENSURE(input->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Flow, "" << *input->GetTypeAnn());
+ }
+
+ for (ui32 i = newOutputIndexStart, j = 0; i < newOutputIndexEnd; ++i, ++j) {
if (outputsCount > 1) {
- variants.emplace_back(ctx.FuseLambdas(lambdas[j].Ref(), variants[index].Ref()));
+ variants.emplace_back(ctx.FuseLambdas(lambdas[j].Ref(), variants[index].Ref()));
} else {
- variants.emplace_back(ctx.DeepCopyLambda(lambdas[j].Ref()));
+ variants.emplace_back(ctx.DeepCopyLambda(lambdas[j].Ref()));
}
- }
-
- newResult = Build<TDqReplicate>(ctx, stage.Pos())
- .Input(input)
- .FreeArgs()
- .Add(variants)
- .Build()
- .Done().Ptr();
-
- std::vector<TExprNode::TPtr> newStageArgs;
- newStageArgs.reserve(stage.Program().Args().Size());
- TNodeOnNodeOwnedMap stageArgsReplaces;
- for (size_t i = 0; i < stage.Program().Args().Size(); ++i) {
- auto oldArg = stage.Program().Args().Arg(i);
- newStageArgs.emplace_back(ctx.NewArgument(oldArg.Pos(), TStringBuilder() << "arg_" << i));
- stageArgsReplaces[oldArg.Raw()] = newStageArgs.back();
- }
-
- auto newStage = Build<TDqStage>(ctx, stage.Pos())
- .Inputs(stage.Inputs())
- .Program()
- .Args(newStageArgs)
- .Body(ctx.ReplaceNodes(std::move(newResult), stageArgsReplaces))
- .Build()
+ }
+
+ newResult = Build<TDqReplicate>(ctx, stage.Pos())
+ .Input(input)
+ .FreeArgs()
+ .Add(variants)
+ .Build()
+ .Done().Ptr();
+
+ std::vector<TExprNode::TPtr> newStageArgs;
+ newStageArgs.reserve(stage.Program().Args().Size());
+ TNodeOnNodeOwnedMap stageArgsReplaces;
+ for (size_t i = 0; i < stage.Program().Args().Size(); ++i) {
+ auto oldArg = stage.Program().Args().Arg(i);
+ newStageArgs.emplace_back(ctx.NewArgument(oldArg.Pos(), TStringBuilder() << "arg_" << i));
+ stageArgsReplaces[oldArg.Raw()] = newStageArgs.back();
+ }
+
+ auto newStage = Build<TDqStage>(ctx, stage.Pos())
+ .Inputs(stage.Inputs())
+ .Program()
+ .Args(newStageArgs)
+ .Body(ctx.ReplaceNodes(std::move(newResult), stageArgsReplaces))
+ .Build()
.Settings(TDqStageSettings().BuildNode(ctx, stage.Pos()))
- .Done();
-
- YQL_CLOG(TRACE, CoreDq) << "new stage #" << newStage.Ref().UniqueId();
-
- TVector<TCoAtom> indexes;
- indexes.reserve(lambdas.size());
- for (ui32 i = newOutputIndexStart; i < newOutputIndexEnd; ++i) {
- indexes.emplace_back(BuildAtom(ToString(i), stage.Pos(), ctx));
- }
- return {newStage, std::move(indexes)};
-}
-
-struct TMultiUsedOutput {
- TDqOutput Output;
+ .Done();
+
+ YQL_CLOG(TRACE, CoreDq) << "new stage #" << newStage.Ref().UniqueId();
+
+ TVector<TCoAtom> indexes;
+ indexes.reserve(lambdas.size());
+ for (ui32 i = newOutputIndexStart; i < newOutputIndexEnd; ++i) {
+ indexes.emplace_back(BuildAtom(ToString(i), stage.Pos(), ctx));
+ }
+ return {newStage, std::move(indexes)};
+}
+
+struct TMultiUsedOutput {
+ TDqOutput Output;
const TNodeMultiSet& Consumers;
-
+
TMultiUsedOutput(const TDqOutput& output, const TNodeMultiSet& consumers)
- : Output(output)
- , Consumers(consumers) {}
-};
-
-struct TMultiUsedConnection {
- TDqConnection Connection;
+ : Output(output)
+ , Consumers(consumers) {}
+};
+
+struct TMultiUsedConnection {
+ TDqConnection Connection;
const TNodeMultiSet& Consumers;
// set if Connection has single consumer, but it's Output has many consumers
- // in that case, Connection - is on of connections, references to this Output
- TMaybe<TMultiUsedOutput> Output;
-
+ // in that case, Connection - is on of connections, references to this Output
+ TMaybe<TMultiUsedOutput> Output;
+
TMultiUsedConnection(const TDqConnection& connection, const TNodeMultiSet& consumers)
- : Connection(connection)
- , Consumers(consumers) {}
-
- TString Print(TExprContext& ctx) const {
- TStringBuilder sb;
- if (!Output.Defined()) {
- auto output = Connection.Output();
- sb << "multiused connection: " << Connection.Ref().Content() << ", #" << Connection.Ref().UniqueId()
- << ", index: " << output.Index().Value() << ", stage: " << PrintDqStageOnly(output.Stage(), ctx)
- << ", consumers: " << Consumers.size() << Endl;
- for (const auto& consumer : Consumers) {
- sb << "consumer: " << PrintConsumer(consumer, output.Stage(), ctx) << Endl;
- }
- } else {
+ : Connection(connection)
+ , Consumers(consumers) {}
+
+ TString Print(TExprContext& ctx) const {
+ TStringBuilder sb;
+ if (!Output.Defined()) {
+ auto output = Connection.Output();
+ sb << "multiused connection: " << Connection.Ref().Content() << ", #" << Connection.Ref().UniqueId()
+ << ", index: " << output.Index().Value() << ", stage: " << PrintDqStageOnly(output.Stage(), ctx)
+ << ", consumers: " << Consumers.size() << Endl;
+ for (const auto& consumer : Consumers) {
+ sb << "consumer: " << PrintConsumer(consumer, output.Stage(), ctx) << Endl;
+ }
+ } else {
sb << "multiused output, index: " << Output->Output.Index().Value()
<< ", #" << Output->Output.Ref().UniqueId()
- << ", stage: " << PrintDqStageOnly(Output->Output.Stage(), ctx)
- << ", consumers: " << Output->Consumers.size() << Endl;
- for (const auto& consumer : Output->Consumers) {
- sb << "consumer: " << PrintConsumer(consumer, Output->Output.Stage(), ctx) << Endl;
- }
- }
- return sb;
- }
-
- TString PrintConsumer(const TExprNode* consumer, const TDqStageBase& stage, TExprContext& ctx) const {
- auto stageReplacement = Build<TCoParameter>(ctx, consumer->Pos())
- .Name().Build("<__stage__>")
- .Type(ExpandType(stage.Pos(), *stage.Ref().GetTypeAnn(), ctx))
- .Done();
-
- TNodeOnNodeOwnedMap clones;
- auto x = ctx.DeepCopy(*consumer, ctx, clones, true, true, false,
- [&](const TExprNode& node, TExprNode::TListType& newChildren) -> bool {
- if (TDqOutput::Match(&node)) {
- auto output = TDqOutput(&node);
- if (output.Stage().Raw() == stage.Raw()) {
- newChildren.emplace_back(stageReplacement.Ptr());
- newChildren.emplace_back(output.Index().Ptr());
- return true;
- }
- }
- return false;
- });
-
- try {
- return NCommon::ExprToPrettyString(ctx, *x);
- } catch (...) {
- return x->Dump();
- }
- }
-};
-
-TString IndexesToString(const TCoAtom& head, const TVector<TCoAtom>& tail) {
- TStringBuilder sb;
- sb << head.Value();
- for (auto& idx : tail) {
- sb << "," << idx.Value();
- }
- return sb;
-}
-
+ << ", stage: " << PrintDqStageOnly(Output->Output.Stage(), ctx)
+ << ", consumers: " << Output->Consumers.size() << Endl;
+ for (const auto& consumer : Output->Consumers) {
+ sb << "consumer: " << PrintConsumer(consumer, Output->Output.Stage(), ctx) << Endl;
+ }
+ }
+ return sb;
+ }
+
+ TString PrintConsumer(const TExprNode* consumer, const TDqStageBase& stage, TExprContext& ctx) const {
+ auto stageReplacement = Build<TCoParameter>(ctx, consumer->Pos())
+ .Name().Build("<__stage__>")
+ .Type(ExpandType(stage.Pos(), *stage.Ref().GetTypeAnn(), ctx))
+ .Done();
+
+ TNodeOnNodeOwnedMap clones;
+ auto x = ctx.DeepCopy(*consumer, ctx, clones, true, true, false,
+ [&](const TExprNode& node, TExprNode::TListType& newChildren) -> bool {
+ if (TDqOutput::Match(&node)) {
+ auto output = TDqOutput(&node);
+ if (output.Stage().Raw() == stage.Raw()) {
+ newChildren.emplace_back(stageReplacement.Ptr());
+ newChildren.emplace_back(output.Index().Ptr());
+ return true;
+ }
+ }
+ return false;
+ });
+
+ try {
+ return NCommon::ExprToPrettyString(ctx, *x);
+ } catch (...) {
+ return x->Dump();
+ }
+ }
+};
+
+TString IndexesToString(const TCoAtom& head, const TVector<TCoAtom>& tail) {
+ TStringBuilder sb;
+ sb << head.Value();
+ for (auto& idx : tail) {
+ sb << "," << idx.Value();
+ }
+ return sb;
+}
+
void BuildIdentityLambdas(TVector<TCoLambda>& lambdas, ui32 count, TExprContext& ctx, TPositionHandle pos) {
lambdas.reserve(count);
for (size_t i = 0; i < count; ++i) {
@@ -208,7 +208,7 @@ void BuildIdentityLambdas(TVector<TCoLambda>& lambdas, ui32 count, TExprContext&
}
}
-TExprNode::TPtr ReplicateDqOutput(TExprNode::TPtr&& input, const TMultiUsedConnection& muConnection, TExprContext& ctx)
+TExprNode::TPtr ReplicateDqOutput(TExprNode::TPtr&& input, const TMultiUsedConnection& muConnection, TExprContext& ctx)
{
YQL_ENSURE(muConnection.Output.Defined());
auto dqOutput = muConnection.Output->Output;
@@ -220,13 +220,13 @@ TExprNode::TPtr ReplicateDqOutput(TExprNode::TPtr&& input, const TMultiUsedConne
TVector<TCoLambda> lambdas;
BuildIdentityLambdas(lambdas, consumersCount - 1, ctx, dqOutput.Pos());
- auto ret = ReplicateStageOutput(dqStage, outputIndex, lambdas, ctx);
+ auto ret = ReplicateStageOutput(dqStage, outputIndex, lambdas, ctx);
auto newStage = ret.first;
auto newAdditionalIndexes = ret.second;
- YQL_CLOG(TRACE, CoreDq) << "Replicate DQ output (stage #" << dqStage.Ref().UniqueId() << ", "
- << outputIndex.Value() << ") -> (#" << newStage.Ref().UniqueId() << ", ["
- << IndexesToString(outputIndex, newAdditionalIndexes) << "])." << Endl << PrintDqStageOnly(newStage, ctx);
+ YQL_CLOG(TRACE, CoreDq) << "Replicate DQ output (stage #" << dqStage.Ref().UniqueId() << ", "
+ << outputIndex.Value() << ") -> (#" << newStage.Ref().UniqueId() << ", ["
+ << IndexesToString(outputIndex, newAdditionalIndexes) << "])." << Endl << PrintDqStageOnly(newStage, ctx);
TNodeOnNodeOwnedMap replaces;
replaces[dqStage.Raw()] = newStage.Ptr();
@@ -256,10 +256,10 @@ TExprNode::TPtr ReplicateDqOutput(TExprNode::TPtr&& input, const TMultiUsedConne
}
TExprNode::TPtr ReplicateDqConnection(TExprNode::TPtr&& input, const TMultiUsedConnection& muConnection,
- TExprContext& ctx)
+ TExprContext& ctx)
{
- YQL_CLOG(TRACE, CoreDq) << "-- ReplicateDqConnection: " << NCommon::ExprToPrettyString(ctx, *input);
-
+ YQL_CLOG(TRACE, CoreDq) << "-- ReplicateDqConnection: " << NCommon::ExprToPrettyString(ctx, *input);
+
YQL_ENSURE(!muConnection.Output.Defined());
auto dqOutput = muConnection.Connection.Output();
auto dqStage = dqOutput.Stage().Cast<TDqStage>();
@@ -272,10 +272,10 @@ TExprNode::TPtr ReplicateDqConnection(TExprNode::TPtr&& input, const TMultiUsedC
// Ensure stable order by processing connection with minimal ID
auto& consumer = *std::min_element(consumers.begin(), consumers.end(),
[](auto l, auto r) { return l->UniqueId() < r->UniqueId(); });
-
+
auto usagesCount = consumers.count(consumer);
bool isLastConsumer = consumers.size() == usagesCount;
-
+
YQL_CLOG(TRACE, CoreDq) << "-- usagesCount: " << usagesCount << ", isLastConsumer: " << isLastConsumer;
if (isLastConsumer) {
@@ -286,8 +286,8 @@ TExprNode::TPtr ReplicateDqConnection(TExprNode::TPtr&& input, const TMultiUsedC
? usagesCount - 1
: usagesCount;
- YQL_CLOG(TRACE, CoreDq) << "-- lambdas count: " << lambdasCount;
-
+ YQL_CLOG(TRACE, CoreDq) << "-- lambdas count: " << lambdasCount;
+
TVector<TCoLambda> lambdas;
BuildIdentityLambdas(lambdas, lambdasCount, ctx, dqOutput.Pos());
@@ -416,113 +416,113 @@ bool GatherConsumers(const TExprNode& root, TNodeMap<TNodeMultiSet>& consumers)
return GatherConsumersImpl(root, consumers, visited);
}
-} // anonymous namespace
-
-IGraphTransformer::TStatus DqReplicateStageMultiOutput(TExprNode::TPtr input, TExprNode::TPtr& output,
- TExprContext& ctx)
-{
+} // anonymous namespace
+
+IGraphTransformer::TStatus DqReplicateStageMultiOutput(TExprNode::TPtr input, TExprNode::TPtr& output,
+ TExprContext& ctx)
+{
output = input;
-
+
// YQL_CLOG(TRACE, CoreDq) << "-- replicate query: " << NCommon::ExprToPrettyString(ctx, *input);
// YQL_CLOG(TRACE, CoreDq) << "-- replicate query: " << input->Dump();
-
+
TNodeMap<TNodeMultiSet> consumersMap;
if (!GatherConsumers(*input, consumersMap)) {
return IGraphTransformer::TStatus::Ok;
}
- // rewrite only 1 (any of) multi-used connection at a time
- std::optional<TMultiUsedConnection> multiUsedConnection;
- TDeque<TExprNode::TPtr> precomputes;
- TNodeSet visitedNodes;
-
- precomputes.emplace_back(input);
-
- while (!precomputes.empty()) {
- auto head = precomputes.front();
- precomputes.pop_front();
-
+ // rewrite only 1 (any of) multi-used connection at a time
+ std::optional<TMultiUsedConnection> multiUsedConnection;
+ TDeque<TExprNode::TPtr> precomputes;
+ TNodeSet visitedNodes;
+
+ precomputes.emplace_back(input);
+
+ while (!precomputes.empty()) {
+ auto head = precomputes.front();
+ precomputes.pop_front();
+
YQL_CLOG(TRACE, CoreDq) << "DqReplicateStageMultiOutput: start traverse node #" << head->UniqueId()
<< ", " << head->Content();
-
- visitedNodes.erase(head.Get());
-
- VisitExpr(
- head,
- [&](const TExprNode::TPtr& ptr) {
- if (ptr == head) {
- return true;
- }
- if (multiUsedConnection.has_value()) {
- YQL_CLOG(TRACE, CoreDq) << "DqReplicateStageMultiOutput: already have multiused connection";
- return false;
- }
- if (ptr->IsLambda()) {
- return false;
- }
-
- TExprBase expr{ptr};
-
- if (auto precompute = expr.Maybe<TDqPhyPrecompute>()) {
+
+ visitedNodes.erase(head.Get());
+
+ VisitExpr(
+ head,
+ [&](const TExprNode::TPtr& ptr) {
+ if (ptr == head) {
+ return true;
+ }
+ if (multiUsedConnection.has_value()) {
+ YQL_CLOG(TRACE, CoreDq) << "DqReplicateStageMultiOutput: already have multiused connection";
+ return false;
+ }
+ if (ptr->IsLambda()) {
+ return false;
+ }
+
+ TExprBase expr{ptr};
+
+ if (auto precompute = expr.Maybe<TDqPhyPrecompute>()) {
YQL_CLOG(TRACE, CoreDq) << "DqReplicateStageMultiOutput: got precompute (#"
<< ptr->UniqueId() << "),"
- << " stop iteration. Child: " << precompute.Connection().Ref().Content()
- << ", #" << precompute.Connection().Ref().UniqueId();
- precomputes.emplace_back(ptr);
- return false;
- }
-
+ << " stop iteration. Child: " << precompute.Connection().Ref().Content()
+ << ", #" << precompute.Connection().Ref().UniqueId();
+ precomputes.emplace_back(ptr);
+ return false;
+ }
+
if (expr.Maybe<TDqConnection>() && consumersMap.find(expr.Raw()) != consumersMap.end()) {
- auto connection = expr.Cast<TDqConnection>();
- YQL_CLOG(TRACE, CoreDq) << "DqReplicateStageMultiOutput: test connection "
- << connection.Ref().Content() << " (#" << ptr->UniqueId() << ")";
+ auto connection = expr.Cast<TDqConnection>();
+ YQL_CLOG(TRACE, CoreDq) << "DqReplicateStageMultiOutput: test connection "
+ << connection.Ref().Content() << " (#" << ptr->UniqueId() << ")";
const auto& consumers = GetConsumers(connection, consumersMap);
- if (consumers.size() > 1) {
- // if connection has multiple consumers - stop traversing and return this connection
- multiUsedConnection.emplace(connection, consumers);
- return false;
- }
- auto output = connection.Output();
+ if (consumers.size() > 1) {
+ // if connection has multiple consumers - stop traversing and return this connection
+ multiUsedConnection.emplace(connection, consumers);
+ return false;
+ }
+ auto output = connection.Output();
const auto& outputConsumers = GetConsumers(output, consumersMap);
- if (outputConsumers.size() > 1) {
- // connection has single consumer, but it's output has multiple ones
- // in that case we check, that this output is used by single-client connections only
- bool allConnectionsWithSingleConsumer = true;
- for (auto conn : outputConsumers) {
+ if (outputConsumers.size() > 1) {
+ // connection has single consumer, but it's output has multiple ones
+ // in that case we check, that this output is used by single-client connections only
+ bool allConnectionsWithSingleConsumer = true;
+ for (auto conn : outputConsumers) {
bool singleConsumer = GetConsumers(TExprBase(conn), consumersMap).size() == 1;
allConnectionsWithSingleConsumer &= singleConsumer;
- }
- if (allConnectionsWithSingleConsumer) {
- multiUsedConnection.emplace(connection, consumers);
- multiUsedConnection->Output.ConstructInPlace(output, outputConsumers);
- } else {
- YQL_ENSURE(false);
- }
- }
- }
-
- return true;
- },
- visitedNodes);
- }
-
- if (!multiUsedConnection.has_value()) {
- return IGraphTransformer::TStatus::Ok;
- }
-
- YQL_CLOG(TRACE, CoreDq) << "DqReplicateStageMultiOutput: " << multiUsedConnection->Print(ctx);
-
+ }
+ if (allConnectionsWithSingleConsumer) {
+ multiUsedConnection.emplace(connection, consumers);
+ multiUsedConnection->Output.ConstructInPlace(output, outputConsumers);
+ } else {
+ YQL_ENSURE(false);
+ }
+ }
+ }
+
+ return true;
+ },
+ visitedNodes);
+ }
+
+ if (!multiUsedConnection.has_value()) {
+ return IGraphTransformer::TStatus::Ok;
+ }
+
+ YQL_CLOG(TRACE, CoreDq) << "DqReplicateStageMultiOutput: " << multiUsedConnection->Print(ctx);
+
if (multiUsedConnection->Output.Defined()) {
- output = ReplicateDqOutput(std::move(input), *multiUsedConnection, ctx);
- } else {
- output = ReplicateDqConnection(std::move(input), *multiUsedConnection, ctx);
- }
-
- return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Repeat, true);
-}
-
+ output = ReplicateDqOutput(std::move(input), *multiUsedConnection, ctx);
+ } else {
+ output = ReplicateDqConnection(std::move(input), *multiUsedConnection, ctx);
+ }
+
+ return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Repeat, true);
+}
+
IGraphTransformer::TStatus DqExtractPrecomputeToStageInput(const TExprNode::TPtr& input, TExprNode::TPtr& output,
- TExprContext& ctx)
+ TExprContext& ctx)
{
auto stages = CollectNodes<TDqStage>(input);
@@ -533,8 +533,8 @@ IGraphTransformer::TStatus DqExtractPrecomputeToStageInput(const TExprNode::TPtr
continue;
}
- YQL_CLOG(TRACE, CoreDq) << "DqExtractPrecomputeToStageInput: stage: " << PrintDqStageOnly(stage, ctx)
- << ", DqPhyPrecompute: " << dqPrecomputes.size();
+ YQL_CLOG(TRACE, CoreDq) << "DqExtractPrecomputeToStageInput: stage: " << PrintDqStageOnly(stage, ctx)
+ << ", DqPhyPrecompute: " << dqPrecomputes.size();
TVector<TExprNode::TPtr> inputs;
TVector<TExprNode::TPtr> args;
@@ -577,4 +577,4 @@ IGraphTransformer::TStatus DqExtractPrecomputeToStageInput(const TExprNode::TPtr
return RemapExpr(input, output, replaces, ctx, TOptimizeExprSettings(nullptr));
}
-} // NKikimr::NKqp
+} // NKikimr::NKqp
diff --git a/ydb/library/yql/dq/opt/dq_opt_phy_finalizing.h b/ydb/library/yql/dq/opt/dq_opt_phy_finalizing.h
index a95a0c0634..2f4ac58f36 100644
--- a/ydb/library/yql/dq/opt/dq_opt_phy_finalizing.h
+++ b/ydb/library/yql/dq/opt/dq_opt_phy_finalizing.h
@@ -11,9 +11,9 @@
namespace NYql::NDq {
IGraphTransformer::TStatus DqReplicateStageMultiOutput(TExprNode::TPtr input, TExprNode::TPtr& output,
- TExprContext& ctx);
+ TExprContext& ctx);
IGraphTransformer::TStatus DqExtractPrecomputeToStageInput(const TExprNode::TPtr& input, TExprNode::TPtr& output,
- TExprContext& ctx);
+ TExprContext& ctx);
} // namespace NYql::NDq
diff --git a/ydb/library/yql/dq/opt/ya.make b/ydb/library/yql/dq/opt/ya.make
index 9e86933f6b..be88e1e1f6 100644
--- a/ydb/library/yql/dq/opt/ya.make
+++ b/ydb/library/yql/dq/opt/ya.make
@@ -16,7 +16,7 @@ PEERDIR(
SRCS(
dq_opt.cpp
- dq_opt_build.cpp
+ dq_opt_build.cpp
dq_opt_join.cpp
dq_opt_log.cpp
dq_opt_peephole.cpp
diff --git a/ydb/library/yql/dq/proto/dq_tasks.proto b/ydb/library/yql/dq/proto/dq_tasks.proto
index 8720b63178..e29e64ef2c 100644
--- a/ydb/library/yql/dq/proto/dq_tasks.proto
+++ b/ydb/library/yql/dq/proto/dq_tasks.proto
@@ -3,7 +3,7 @@ option cc_enable_arenas = true;
package NYql.NDqProto;
-import "google/protobuf/any.proto";
+import "google/protobuf/any.proto";
import "library/cpp/actors/protos/actors.proto";
import "ydb/library/yql/dq/proto/dq_transport.proto";
@@ -21,15 +21,15 @@ enum ECheckpointingMode {
}
message TProgram {
- message TSettings {
- bool HasMapJoin = 1;
- bool HasSort = 2;
- bool HasUdf = 3;
- }
+ message TSettings {
+ bool HasMapJoin = 1;
+ bool HasSort = 2;
+ bool HasUdf = 3;
+ }
uint32 RuntimeVersion = 1;
bytes Raw = 2;
- TSettings Settings = 3;
+ TSettings Settings = 3;
}
message TDqUri {
@@ -56,7 +56,7 @@ message TChannel {
TEndpoint DstEndpoint = 6;
bool IsPersistent = 7;
- bool InMemory = 8;
+ bool InMemory = 8;
ECheckpointingMode CheckpointingMode = 9;
}
@@ -141,13 +141,13 @@ message TDqTransform {
message TDqTask {
uint64 Id = 1;
- uint32 StageId = 9;
+ uint32 StageId = 9;
TEndpoint Executer = 2;
TProgram Program = 3;
map<string, TData> Parameters = 4;
repeated TTaskInput Inputs = 5;
repeated TTaskOutput Outputs = 6;
- google.protobuf.Any Meta = 7;
+ google.protobuf.Any Meta = 7;
bool CreateSuspended = 8;
optional TDqTransform OutputTransform = 12;
}
diff --git a/ydb/library/yql/dq/proto/dq_transport.proto b/ydb/library/yql/dq/proto/dq_transport.proto
index 04bc18d1c7..d78fa7806c 100644
--- a/ydb/library/yql/dq/proto/dq_transport.proto
+++ b/ydb/library/yql/dq/proto/dq_transport.proto
@@ -13,5 +13,5 @@ enum EDataTransportVersion {
message TData {
uint32 TransportVersion = 1;
bytes Raw = 2;
- uint32 Rows = 3;
+ uint32 Rows = 3;
}
diff --git a/ydb/library/yql/dq/runtime/dq_channel_storage.h b/ydb/library/yql/dq/runtime/dq_channel_storage.h
index d80bddce21..d42238f683 100644
--- a/ydb/library/yql/dq/runtime/dq_channel_storage.h
+++ b/ydb/library/yql/dq/runtime/dq_channel_storage.h
@@ -1,29 +1,29 @@
-#pragma once
-
-#include <util/generic/buffer.h>
-#include <util/generic/yexception.h>
-
-
-namespace NYql::NDq {
-
-class TDqChannelStorageException : public yexception {
-};
-
-class IDqChannelStorage : public TSimpleRefCount<IDqChannelStorage> {
-public:
- using TPtr = TIntrusivePtr<IDqChannelStorage>;
-
- using TWakeUpCallback = std::function<void()>;
-
-public:
- virtual ~IDqChannelStorage() = default;
-
- virtual bool IsEmpty() const = 0;
- virtual bool IsFull() const = 0;
-
- // these methods can throw `TDqChannelStorageException`
- virtual void Put(ui64 blobId, TBuffer&& blob) = 0;
- virtual bool Get(ui64 blobId, TBuffer& data) = 0;
-};
-
-} // namespace NYql::NDq
+#pragma once
+
+#include <util/generic/buffer.h>
+#include <util/generic/yexception.h>
+
+
+namespace NYql::NDq {
+
+class TDqChannelStorageException : public yexception {
+};
+
+class IDqChannelStorage : public TSimpleRefCount<IDqChannelStorage> {
+public:
+ using TPtr = TIntrusivePtr<IDqChannelStorage>;
+
+ using TWakeUpCallback = std::function<void()>;
+
+public:
+ virtual ~IDqChannelStorage() = default;
+
+ virtual bool IsEmpty() const = 0;
+ virtual bool IsFull() const = 0;
+
+ // these methods can throw `TDqChannelStorageException`
+ virtual void Put(ui64 blobId, TBuffer&& blob) = 0;
+ virtual bool Get(ui64 blobId, TBuffer& data) = 0;
+};
+
+} // namespace NYql::NDq
diff --git a/ydb/library/yql/dq/runtime/dq_columns_resolve.cpp b/ydb/library/yql/dq/runtime/dq_columns_resolve.cpp
index 593799a0cc..850f6108d9 100644
--- a/ydb/library/yql/dq/runtime/dq_columns_resolve.cpp
+++ b/ydb/library/yql/dq/runtime/dq_columns_resolve.cpp
@@ -8,7 +8,7 @@ using namespace NKikimr::NMiniKQL;
TMaybe<TColumnInfo> FindColumnInfo(const NKikimr::NMiniKQL::TType* type, TStringBuf columnName) {
YQL_ENSURE(type->GetKind() == TType::EKind::Struct);
- const auto& structType = static_cast<const TStructType&>(*type);
+ const auto& structType = static_cast<const TStructType&>(*type);
auto columnIndex = structType.FindMemberIndex(columnName);
if (!columnIndex) {
diff --git a/ydb/library/yql/dq/runtime/dq_columns_resolve.h b/ydb/library/yql/dq/runtime/dq_columns_resolve.h
index d9082bfb25..b61d14f477 100644
--- a/ydb/library/yql/dq/runtime/dq_columns_resolve.h
+++ b/ydb/library/yql/dq/runtime/dq_columns_resolve.h
@@ -22,18 +22,18 @@ struct TSortColumnInfo : public TColumnInfo {
};
TMaybe<TColumnInfo> FindColumnInfo(const NKikimr::NMiniKQL::TType* type, TStringBuf column);
-TColumnInfo GetColumnInfo(const NKikimr::NMiniKQL::TType* type, TStringBuf column);
+TColumnInfo GetColumnInfo(const NKikimr::NMiniKQL::TType* type, TStringBuf column);
template<typename TList>
-void GetColumnsInfo(const NKikimr::NMiniKQL::TType* type, const TList& columns,
+void GetColumnsInfo(const NKikimr::NMiniKQL::TType* type, const TList& columns,
TVector<NUdf::TDataTypeId>& columnTypes, TVector<ui32>& columnIndices)
{
columnTypes.clear();
columnIndices.clear();
- columnTypes.reserve(columns.size());
- columnIndices.reserve(columns.size());
-
+ columnTypes.reserve(columns.size());
+ columnIndices.reserve(columns.size());
+
for (auto& column : columns) {
auto columnInfo = GetColumnInfo(type, column);
columnTypes.push_back(columnInfo.TypeId);
diff --git a/ydb/library/yql/dq/runtime/dq_compute.h b/ydb/library/yql/dq/runtime/dq_compute.h
index 946e0586a4..6aca63b8c4 100644
--- a/ydb/library/yql/dq/runtime/dq_compute.h
+++ b/ydb/library/yql/dq/runtime/dq_compute.h
@@ -4,10 +4,10 @@
namespace NYql::NDq {
-class TDqComputeContextBase : private TNonCopyable {
-public:
- virtual ~TDqComputeContextBase() = default;
-};
+class TDqComputeContextBase : private TNonCopyable {
+public:
+ virtual ~TDqComputeContextBase() = default;
+};
NKikimr::NMiniKQL::TComputationNodeFactory GetDqBaseComputeFactory(const TDqComputeContextBase* computeCtx);
diff --git a/ydb/library/yql/dq/runtime/dq_input.h b/ydb/library/yql/dq/runtime/dq_input.h
index ee5f80b018..4067549a82 100644
--- a/ydb/library/yql/dq/runtime/dq_input.h
+++ b/ydb/library/yql/dq/runtime/dq_input.h
@@ -6,14 +6,14 @@
namespace NYql::NDq {
struct TDqInputStats {
- // basic stats
+ // basic stats
ui64 Chunks = 0;
ui64 Bytes = 0;
ui64 RowsIn = 0;
ui64 RowsOut = 0;
- TInstant FirstRowTs;
-
- // profile stats
+ TInstant FirstRowTs;
+
+ // profile stats
ui64 RowsInMemory = 0;
ui64 MaxMemoryUsage = 0;
};
diff --git a/ydb/library/yql/dq/runtime/dq_input_channel.cpp b/ydb/library/yql/dq/runtime/dq_input_channel.cpp
index dc204869e8..bd0a541e37 100644
--- a/ydb/library/yql/dq/runtime/dq_input_channel.cpp
+++ b/ydb/library/yql/dq/runtime/dq_input_channel.cpp
@@ -1,4 +1,4 @@
-#include "dq_input_channel.h"
+#include "dq_input_channel.h"
#include "dq_input_impl.h"
namespace NYql::NDq {
@@ -7,14 +7,14 @@ class TDqInputChannel : public TDqInputImpl<TDqInputChannel, IDqInputChannel> {
using TBaseImpl = TDqInputImpl<TDqInputChannel, IDqInputChannel>;
friend TBaseImpl;
public:
- TDqInputChannel(ui64 channelId, NKikimr::NMiniKQL::TType* inputType, ui64 maxBufferBytes, bool collectProfileStats,
+ TDqInputChannel(ui64 channelId, NKikimr::NMiniKQL::TType* inputType, ui64 maxBufferBytes, bool collectProfileStats,
const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv, const NKikimr::NMiniKQL::THolderFactory& holderFactory,
NDqProto::EDataTransportVersion transportVersion)
: TBaseImpl(inputType, maxBufferBytes)
, ChannelId(channelId)
- , BasicStats(ChannelId)
- , ProfileStats(collectProfileStats ? &BasicStats : nullptr)
- , DataSerializer(typeEnv, holderFactory, transportVersion) {}
+ , BasicStats(ChannelId)
+ , ProfileStats(collectProfileStats ? &BasicStats : nullptr)
+ , DataSerializer(typeEnv, holderFactory, transportVersion) {}
ui64 GetChannelId() const override {
return ChannelId;
@@ -23,43 +23,43 @@ public:
void Push(NDqProto::TData&& data) override {
YQL_ENSURE(!Finished, "input channel " << ChannelId << " already finished");
- if (Y_UNLIKELY(data.GetRows() == 0)) {
- return;
- }
-
+ if (Y_UNLIKELY(data.GetRows() == 0)) {
+ return;
+ }
+
const i64 space = data.GetRaw().size();
- NKikimr::NMiniKQL::TUnboxedValueVector buffer;
- buffer.reserve(data.GetRows());
-
- if (Y_UNLIKELY(ProfileStats)) {
- auto startTime = TInstant::Now();
- DataSerializer.Deserialize(data, InputType, buffer);
- ProfileStats->DeserializationTime += (TInstant::Now() - startTime);
- } else {
- DataSerializer.Deserialize(data, InputType, buffer);
+ NKikimr::NMiniKQL::TUnboxedValueVector buffer;
+ buffer.reserve(data.GetRows());
+
+ if (Y_UNLIKELY(ProfileStats)) {
+ auto startTime = TInstant::Now();
+ DataSerializer.Deserialize(data, InputType, buffer);
+ ProfileStats->DeserializationTime += (TInstant::Now() - startTime);
+ } else {
+ DataSerializer.Deserialize(data, InputType, buffer);
}
AddBatch(std::move(buffer), space);
}
const TDqInputChannelStats* GetStats() const override {
- return &BasicStats;
+ return &BasicStats;
}
private:
const ui64 ChannelId;
- TDqInputChannelStats BasicStats;
- TDqInputChannelStats* ProfileStats = nullptr;
+ TDqInputChannelStats BasicStats;
+ TDqInputChannelStats* ProfileStats = nullptr;
TDqDataSerializer DataSerializer;
};
-IDqInputChannel::TPtr CreateDqInputChannel(ui64 channelId, NKikimr::NMiniKQL::TType* inputType, ui64 maxBufferBytes,
- bool collectProfileStats, const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv,
- const NKikimr::NMiniKQL::THolderFactory& holderFactory, NDqProto::EDataTransportVersion transportVersion)
+IDqInputChannel::TPtr CreateDqInputChannel(ui64 channelId, NKikimr::NMiniKQL::TType* inputType, ui64 maxBufferBytes,
+ bool collectProfileStats, const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv,
+ const NKikimr::NMiniKQL::THolderFactory& holderFactory, NDqProto::EDataTransportVersion transportVersion)
{
- return new TDqInputChannel(channelId, inputType, maxBufferBytes, collectProfileStats, typeEnv, holderFactory,
- transportVersion);
+ return new TDqInputChannel(channelId, inputType, maxBufferBytes, collectProfileStats, typeEnv, holderFactory,
+ transportVersion);
}
} // namespace NYql::NDq
diff --git a/ydb/library/yql/dq/runtime/dq_input_channel.h b/ydb/library/yql/dq/runtime/dq_input_channel.h
index 374fd8653c..27020103cc 100644
--- a/ydb/library/yql/dq/runtime/dq_input_channel.h
+++ b/ydb/library/yql/dq/runtime/dq_input_channel.h
@@ -1,38 +1,38 @@
-#pragma once
+#pragma once
#include "dq_input.h"
-#include "dq_transport.h"
-
+#include "dq_transport.h"
+
#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
#include <ydb/library/yql/minikql/mkql_node.h>
#include <ydb/library/yql/utils/yql_panic.h>
-
-namespace NYql::NDq {
-
+
+namespace NYql::NDq {
+
struct TDqInputChannelStats : TDqInputStats {
- ui64 ChannelId = 0;
-
- // profile stats
- TDuration DeserializationTime;
-
- explicit TDqInputChannelStats(ui64 channelId)
- : ChannelId(channelId) {}
-};
-
+ ui64 ChannelId = 0;
+
+ // profile stats
+ TDuration DeserializationTime;
+
+ explicit TDqInputChannelStats(ui64 channelId)
+ : ChannelId(channelId) {}
+};
+
class IDqInputChannel : public IDqInput {
-public:
+public:
using TPtr = TIntrusivePtr<IDqInputChannel>;
-
+
virtual ui64 GetChannelId() const = 0;
-
+
virtual void Push(NDqProto::TData&& data) = 0;
-
+
virtual void Finish() = 0;
-
+
virtual const TDqInputChannelStats* GetStats() const = 0;
};
-
-IDqInputChannel::TPtr CreateDqInputChannel(ui64 channelId, NKikimr::NMiniKQL::TType* inputType, ui64 maxBufferBytes,
- bool collectProfileStats, const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv,
- const NKikimr::NMiniKQL::THolderFactory& holderFactory, NDqProto::EDataTransportVersion transportVersion);
-
-} // namespace NYql::NDq
+
+IDqInputChannel::TPtr CreateDqInputChannel(ui64 channelId, NKikimr::NMiniKQL::TType* inputType, ui64 maxBufferBytes,
+ bool collectProfileStats, const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv,
+ const NKikimr::NMiniKQL::THolderFactory& holderFactory, NDqProto::EDataTransportVersion transportVersion);
+
+} // namespace NYql::NDq
diff --git a/ydb/library/yql/dq/runtime/dq_input_impl.h b/ydb/library/yql/dq/runtime/dq_input_impl.h
index eafb53ccc3..dca47f85dd 100644
--- a/ydb/library/yql/dq/runtime/dq_input_impl.h
+++ b/ydb/library/yql/dq/runtime/dq_input_impl.h
@@ -28,18 +28,18 @@ public:
StoredBytes += space;
StoredRows += batch.size();
- auto& stats = MutableBasicStats();
- stats.Chunks++;
- stats.Bytes += space;
- stats.RowsIn += batch.size();
- if (!stats.FirstRowTs) {
- stats.FirstRowTs = TInstant::Now();
+ auto& stats = MutableBasicStats();
+ stats.Chunks++;
+ stats.Bytes += space;
+ stats.RowsIn += batch.size();
+ if (!stats.FirstRowTs) {
+ stats.FirstRowTs = TInstant::Now();
+ }
+
+ if (auto* profile = MutableProfileStats()) {
+ profile->MaxMemoryUsage = std::max(profile->MaxMemoryUsage, StoredBytes);
}
- if (auto* profile = MutableProfileStats()) {
- profile->MaxMemoryUsage = std::max(profile->MaxMemoryUsage, StoredBytes);
- }
-
Batches.emplace_back(std::move(batch));
}
@@ -81,12 +81,12 @@ public:
Batches.clear();
}
- MutableBasicStats().RowsOut += batch.size();
+ MutableBasicStats().RowsOut += batch.size();
return true;
}
void Finish() override {
- Finished = true;
+ Finished = true;
}
bool IsFinished() const override {
@@ -97,14 +97,14 @@ public:
return InputType;
}
- auto& MutableBasicStats() {
- return static_cast<TDerived*>(this)->BasicStats;
+ auto& MutableBasicStats() {
+ return static_cast<TDerived*>(this)->BasicStats;
+ }
+
+ auto* MutableProfileStats() {
+ return static_cast<TDerived*>(this)->ProfileStats;
}
- auto* MutableProfileStats() {
- return static_cast<TDerived*>(this)->ProfileStats;
- }
-
void Pause() override {
Y_VERIFY(!IsPaused());
if (!Finished) {
diff --git a/ydb/library/yql/dq/runtime/dq_input_producer.cpp b/ydb/library/yql/dq/runtime/dq_input_producer.cpp
index 3e6c7a50e1..3fe4830a08 100644
--- a/ydb/library/yql/dq/runtime/dq_input_producer.cpp
+++ b/ydb/library/yql/dq/runtime/dq_input_producer.cpp
@@ -1,63 +1,63 @@
-#include "dq_input_producer.h"
-
+#include "dq_input_producer.h"
+
#include "dq_columns_resolve.h"
#include <ydb/library/yql/dq/type_ann/dq_type_ann.h>
-namespace NYql::NDq {
-
-using namespace NKikimr;
-using namespace NMiniKQL;
+namespace NYql::NDq {
+
+using namespace NKikimr;
+using namespace NMiniKQL;
using namespace NUdf;
-
-namespace {
-
-class TDqInputUnionStreamValue : public TComputationValue<TDqInputUnionStreamValue> {
-public:
+
+namespace {
+
+class TDqInputUnionStreamValue : public TComputationValue<TDqInputUnionStreamValue> {
+public:
TDqInputUnionStreamValue(TMemoryUsageInfo* memInfo, TVector<IDqInput::TPtr>&& inputs)
- : TComputationValue<TDqInputUnionStreamValue>(memInfo)
+ : TComputationValue<TDqInputUnionStreamValue>(memInfo)
, Inputs(std::move(inputs))
- , CurrentItemIndex(0) {}
-
-private:
- NUdf::EFetchStatus Fetch(NKikimr::NUdf::TUnboxedValue& result) final {
- if (CurrentItemIndex >= CurrentBuffer.size()) {
- auto status = FindBuffer();
- switch (status) {
- case NUdf::EFetchStatus::Ok:
- break;
- case NUdf::EFetchStatus::Finish:
- case NUdf::EFetchStatus::Yield:
- return status;
- }
- }
-
- result = std::move(CurrentBuffer[CurrentItemIndex]);
- ++CurrentItemIndex;
- return NUdf::EFetchStatus::Ok;
- }
-
- NUdf::EFetchStatus FindBuffer() {
- bool allFinished = true;
- CurrentBuffer.clear();
-
+ , CurrentItemIndex(0) {}
+
+private:
+ NUdf::EFetchStatus Fetch(NKikimr::NUdf::TUnboxedValue& result) final {
+ if (CurrentItemIndex >= CurrentBuffer.size()) {
+ auto status = FindBuffer();
+ switch (status) {
+ case NUdf::EFetchStatus::Ok:
+ break;
+ case NUdf::EFetchStatus::Finish:
+ case NUdf::EFetchStatus::Yield:
+ return status;
+ }
+ }
+
+ result = std::move(CurrentBuffer[CurrentItemIndex]);
+ ++CurrentItemIndex;
+ return NUdf::EFetchStatus::Ok;
+ }
+
+ NUdf::EFetchStatus FindBuffer() {
+ bool allFinished = true;
+ CurrentBuffer.clear();
+
for (auto& input : Inputs) {
if (input->Pop(CurrentBuffer)) {
- CurrentItemIndex = 0;
- return NUdf::EFetchStatus::Ok;
- }
+ CurrentItemIndex = 0;
+ return NUdf::EFetchStatus::Ok;
+ }
allFinished &= input->IsFinished();
- }
-
- return allFinished ? NUdf::EFetchStatus::Finish : NUdf::EFetchStatus::Yield;
- }
-
-private:
+ }
+
+ return allFinished ? NUdf::EFetchStatus::Finish : NUdf::EFetchStatus::Yield;
+ }
+
+private:
TVector<IDqInput::TPtr> Inputs;
- TUnboxedValueVector CurrentBuffer;
- ui64 CurrentItemIndex;
-};
-
+ TUnboxedValueVector CurrentBuffer;
+ ui64 CurrentItemIndex;
+};
+
class TDqInputMergeStreamValue : public TComputationValue<TDqInputMergeStreamValue> {
public:
TDqInputMergeStreamValue(TMemoryUsageInfo* memInfo, TVector<IDqInput::TPtr>&& inputs,
@@ -169,18 +169,18 @@ private:
TMap<ui32, EDataSlot> SortColTypes;
};
-} // namespace
-
+} // namespace
+
NUdf::TUnboxedValue CreateInputUnionValue(TVector<IDqInput::TPtr>&& inputs,
- const NMiniKQL::THolderFactory& factory)
-{
+ const NMiniKQL::THolderFactory& factory)
+{
return factory.Create<TDqInputUnionStreamValue>(std::move(inputs));
-}
-
+}
+
NKikimr::NUdf::TUnboxedValue CreateInputMergeValue(TVector<IDqInput::TPtr>&& inputs,
TVector<TSortColumnInfo>&& sortCols, const NKikimr::NMiniKQL::THolderFactory& factory)
{
return factory.Create<TDqInputMergeStreamValue>(std::move(inputs), std::move(sortCols));
}
-} // namespace NYql::NDq
+} // namespace NYql::NDq
diff --git a/ydb/library/yql/dq/runtime/dq_input_producer.h b/ydb/library/yql/dq/runtime/dq_input_producer.h
index 685521f917..7a590b23be 100644
--- a/ydb/library/yql/dq/runtime/dq_input_producer.h
+++ b/ydb/library/yql/dq/runtime/dq_input_producer.h
@@ -1,14 +1,14 @@
-#pragma once
-
-#include "dq_input_channel.h"
+#pragma once
+
+#include "dq_input_channel.h"
#include "dq_columns_resolve.h"
-
-namespace NYql::NDq {
-
+
+namespace NYql::NDq {
+
NKikimr::NUdf::TUnboxedValue CreateInputUnionValue(TVector<IDqInput::TPtr>&& inputs,
- const NKikimr::NMiniKQL::THolderFactory& holderFactory);
-
+ const NKikimr::NMiniKQL::THolderFactory& holderFactory);
+
NKikimr::NUdf::TUnboxedValue CreateInputMergeValue(TVector<IDqInput::TPtr>&& inputs,
TVector<TSortColumnInfo>&& sortCols, const NKikimr::NMiniKQL::THolderFactory& factory);
-} // namespace NYql::NDq
+} // namespace NYql::NDq
diff --git a/ydb/library/yql/dq/runtime/dq_output.h b/ydb/library/yql/dq/runtime/dq_output.h
index 2e8eb85bb9..e793ff5231 100644
--- a/ydb/library/yql/dq/runtime/dq_output.h
+++ b/ydb/library/yql/dq/runtime/dq_output.h
@@ -19,14 +19,14 @@ class TUnboxedValue;
namespace NDq {
struct TDqOutputStats {
- // basic stats
+ // basic stats
ui64 Chunks = 0;
ui64 Bytes = 0;
ui64 RowsIn = 0;
ui64 RowsOut = 0;
- TInstant FirstRowIn;
-
- // profile stats
+ TInstant FirstRowIn;
+
+ // profile stats
ui64 MaxMemoryUsage = 0;
ui64 MaxRowsInMemory = 0;
};
@@ -42,7 +42,7 @@ public:
virtual bool IsFull() const = 0;
// can throw TDqChannelStorageException
virtual void Push(NUdf::TUnboxedValue&& value) = 0;
- // Push checkpoint. Checkpoints may be pushed to channel even after it is finished.
+ // Push checkpoint. Checkpoints may be pushed to channel even after it is finished.
virtual void Push(NDqProto::TCheckpoint&& checkpoint) = 0;
virtual void Finish() = 0;
diff --git a/ydb/library/yql/dq/runtime/dq_output_channel.cpp b/ydb/library/yql/dq/runtime/dq_output_channel.cpp
index 71ba447afc..211d4ac39a 100644
--- a/ydb/library/yql/dq/runtime/dq_output_channel.cpp
+++ b/ydb/library/yql/dq/runtime/dq_output_channel.cpp
@@ -1,675 +1,675 @@
-#include "dq_output_channel.h"
-#include "dq_transport.h"
-
+#include "dq_output_channel.h"
+#include "dq_transport.h"
+
#include <ydb/library/yql/utils/yql_panic.h>
-
-#include <util/generic/buffer.h>
-#include <util/generic/size_literals.h>
-#include <util/stream/buffer.h>
-
-
-namespace NYql::NDq {
-
-#define LOG(s) do { if (Y_UNLIKELY(LogFunc)) { LogFunc(TStringBuilder() << "channelId: " << ChannelId << ". " << s); } } while (0)
-
-#ifndef NDEBUG
-#define DLOG(s) LOG(s)
-#else
-#define DLOG(s)
-#endif
-
-namespace {
-
-using namespace NKikimr;
-
-const ui32 ChunkSizeHardLimit = 48 * 1024 * 1024; // 48 MB
-
-bool IsSafeToEstimateValueSize(const NMiniKQL::TType* type) {
- if (type->GetKind() == NMiniKQL::TType::EKind::Data) {
- return true;
- }
- if (type->GetKind() == NMiniKQL::TType::EKind::Optional) {
- const auto* optionalType = static_cast<const NMiniKQL::TOptionalType*>(type);
- return IsSafeToEstimateValueSize(optionalType->GetItemType());
- }
- if (type->GetKind() == NMiniKQL::TType::EKind::Struct) {
- const auto* structType = static_cast<const NMiniKQL::TStructType*>(type);
- for (ui32 i = 0; i < structType->GetMembersCount(); ++i) {
- if (!IsSafeToEstimateValueSize(structType->GetMemberType(i))) {
- return false;
- }
- }
- return true;
- }
- if (type->GetKind() == NMiniKQL::TType::EKind::Tuple) {
- const auto* tupleType = static_cast<const NMiniKQL::TTupleType*>(type);
- for (ui32 i = 0; i < tupleType->GetElementsCount(); ++i) {
- if (!IsSafeToEstimateValueSize(tupleType->GetElementType(i))) {
- return false;
- }
- }
- return true;
- }
- if (type->GetKind() == NMiniKQL::TType::EKind::Variant) {
- const auto* variantType = static_cast<const NMiniKQL::TVariantType*>(type);
- for (ui32 i = 0; i < variantType->GetAlternativesCount(); ++i) {
- if (!IsSafeToEstimateValueSize(variantType->GetAlternativeType(i))) {
- return false;
- }
- }
- return true;
- }
- return false;
-}
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-class TDqOutputChannelOld : public IDqOutputChannel {
-public:
- TDqOutputChannelOld(ui64 channelId, NMiniKQL::TType* outputType, bool collectProfileStats,
- const NMiniKQL::TTypeEnvironment& typeEnv, const NMiniKQL::THolderFactory& holderFactory, ui64 maxStoredBytes,
- ui64 maxChunkBytes, NDqProto::EDataTransportVersion transportVersion, const TLogFunc& logFunc)
- : ChannelId(channelId)
- , OutputType(outputType)
- , BasicStats(ChannelId)
- , ProfileStats(collectProfileStats ? &BasicStats : nullptr)
+
+#include <util/generic/buffer.h>
+#include <util/generic/size_literals.h>
+#include <util/stream/buffer.h>
+
+
+namespace NYql::NDq {
+
+#define LOG(s) do { if (Y_UNLIKELY(LogFunc)) { LogFunc(TStringBuilder() << "channelId: " << ChannelId << ". " << s); } } while (0)
+
+#ifndef NDEBUG
+#define DLOG(s) LOG(s)
+#else
+#define DLOG(s)
+#endif
+
+namespace {
+
+using namespace NKikimr;
+
+const ui32 ChunkSizeHardLimit = 48 * 1024 * 1024; // 48 MB
+
+bool IsSafeToEstimateValueSize(const NMiniKQL::TType* type) {
+ if (type->GetKind() == NMiniKQL::TType::EKind::Data) {
+ return true;
+ }
+ if (type->GetKind() == NMiniKQL::TType::EKind::Optional) {
+ const auto* optionalType = static_cast<const NMiniKQL::TOptionalType*>(type);
+ return IsSafeToEstimateValueSize(optionalType->GetItemType());
+ }
+ if (type->GetKind() == NMiniKQL::TType::EKind::Struct) {
+ const auto* structType = static_cast<const NMiniKQL::TStructType*>(type);
+ for (ui32 i = 0; i < structType->GetMembersCount(); ++i) {
+ if (!IsSafeToEstimateValueSize(structType->GetMemberType(i))) {
+ return false;
+ }
+ }
+ return true;
+ }
+ if (type->GetKind() == NMiniKQL::TType::EKind::Tuple) {
+ const auto* tupleType = static_cast<const NMiniKQL::TTupleType*>(type);
+ for (ui32 i = 0; i < tupleType->GetElementsCount(); ++i) {
+ if (!IsSafeToEstimateValueSize(tupleType->GetElementType(i))) {
+ return false;
+ }
+ }
+ return true;
+ }
+ if (type->GetKind() == NMiniKQL::TType::EKind::Variant) {
+ const auto* variantType = static_cast<const NMiniKQL::TVariantType*>(type);
+ for (ui32 i = 0; i < variantType->GetAlternativesCount(); ++i) {
+ if (!IsSafeToEstimateValueSize(variantType->GetAlternativeType(i))) {
+ return false;
+ }
+ }
+ return true;
+ }
+ return false;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+class TDqOutputChannelOld : public IDqOutputChannel {
+public:
+ TDqOutputChannelOld(ui64 channelId, NMiniKQL::TType* outputType, bool collectProfileStats,
+ const NMiniKQL::TTypeEnvironment& typeEnv, const NMiniKQL::THolderFactory& holderFactory, ui64 maxStoredBytes,
+ ui64 maxChunkBytes, NDqProto::EDataTransportVersion transportVersion, const TLogFunc& logFunc)
+ : ChannelId(channelId)
+ , OutputType(outputType)
+ , BasicStats(ChannelId)
+ , ProfileStats(collectProfileStats ? &BasicStats : nullptr)
, DataSerializer(typeEnv, holderFactory, transportVersion)
- , MaxStoredBytes(maxStoredBytes)
- , MaxChunkBytes(maxChunkBytes)
- , LogFunc(logFunc) {}
-
- ui64 GetChannelId() const override {
- return ChannelId;
- }
-
- ui64 GetValuesCount(bool /* inMemoryOnly */) const override {
- return Data.size();
- }
-
- [[nodiscard]]
- bool IsFull() const override {
- return Data.size() * EstimatedRowBytes >= MaxStoredBytes;
- }
-
- void Push(NUdf::TUnboxedValue&& value) override {
- if (!BasicStats.FirstRowIn) {
- BasicStats.FirstRowIn = TInstant::Now();
- }
-
- ui64 rowsInMemory = Data.size();
-
- LOG("Push request, rows in memory: " << rowsInMemory << ", bytesInMemory: " << (rowsInMemory * EstimatedRowBytes)
- << ", finished: " << Finished);
- YQL_ENSURE(!IsFull());
-
- if (Finished) {
- return;
- }
-
- if (!EstimatedRowBytes) {
- EstimatedRowBytes = IsSafeToEstimateValueSize(OutputType)
- ? TDqDataSerializer::EstimateSize(value, OutputType)
- : DataSerializer.CalcSerializedSize(value, OutputType);
- LOG("EstimatedRowSize: " << EstimatedRowBytes << ", type: " << *OutputType
- << ", safe: " << IsSafeToEstimateValueSize(OutputType));
- }
-
- Data.emplace_back(std::move(value));
- rowsInMemory++;
-
- BasicStats.Bytes += EstimatedRowBytes;
- BasicStats.RowsIn++;
-
- if (Y_UNLIKELY(ProfileStats)) {
- ProfileStats->MaxMemoryUsage = std::max(ProfileStats->MaxMemoryUsage, rowsInMemory * EstimatedRowBytes);
- ProfileStats->MaxRowsInMemory = std::max(ProfileStats->MaxRowsInMemory, rowsInMemory);
- }
- }
-
- void Push(NDqProto::TCheckpoint&& checkpoint) override {
- YQL_ENSURE(!Checkpoint);
- Checkpoint.ConstructInPlace(std::move(checkpoint));
- }
-
- [[nodiscard]]
- bool Pop(NDqProto::TData& data, ui64 bytes) override {
- LOG("Pop request, rows in memory: " << Data.size() << ", finished: " << Finished << ", requested: " << bytes
- << ", estimatedRowSize: " << EstimatedRowBytes);
-
- if (!HasData()) {
- if (Finished) {
- data.SetTransportVersion(DataSerializer.GetTransportVersion());
- data.SetRows(0);
- data.ClearRaw();
- }
- return false;
- }
-
- bytes = std::min(bytes, MaxChunkBytes);
- ui64 takeRows = std::max<ui64>(bytes / EstimatedRowBytes, 1);
- takeRows = std::min<ui64>(takeRows, Data.size());
-
- DLOG("About to take " << takeRows << " rows");
-
- auto last = std::next(Data.begin(), takeRows);
-
- if (Y_UNLIKELY(ProfileStats)) {
- TInstant startTime = TInstant::Now();
- data = DataSerializer.Serialize(Data.begin(), last, OutputType);
- ProfileStats->SerializationTime += (TInstant::Now() - startTime);
- } else {
- data = DataSerializer.Serialize(Data.begin(), last, OutputType);
- }
-
- EstimatedRowBytes = EstimatedRowBytes * 0.6 + std::max<ui64>(data.GetRaw().Size() / takeRows, 1) * 0.4;
- DLOG("Recalc estimated row size: " << EstimatedRowBytes);
-
- while (data.GetRaw().size() >= ChunkSizeHardLimit && takeRows > 1) {
- ui64 newTakeRows = std::max<ui64>(bytes / EstimatedRowBytes, 1);
- newTakeRows = std::min<ui64>(newTakeRows, Data.size());
-
- takeRows = newTakeRows < takeRows ? newTakeRows : takeRows / 2;
-
- DLOG("Too big data, split it. Try to take " << takeRows);
-
- NMiniKQL::TUnboxedValueVector values;
- values.reserve(data.GetRows());
-
- if (Y_UNLIKELY(ProfileStats)) {
- TInstant startTime = TInstant::Now();
- DataSerializer.Deserialize(data, OutputType, values);
- ProfileStats->SerializationTime += (TInstant::Now() - startTime);
- } else {
- DataSerializer.Deserialize(data, OutputType, values);
- }
-
- for (ui32 i = 0; i < data.GetRows(); ++i) {
- Data[i] = std::move(values[i]);
- }
-
- last = std::next(Data.begin(), takeRows);
-
- if (Y_UNLIKELY(ProfileStats)) {
- TInstant startTime = TInstant::Now();
- data = DataSerializer.Serialize(Data.begin(), last, OutputType);
- ProfileStats->SerializationTime += (TInstant::Now() - startTime);
- } else {
- data = DataSerializer.Serialize(Data.begin(), last, OutputType);
- }
- }
-
- YQL_ENSURE(data.GetRaw().size() < ChunkSizeHardLimit);
-
- BasicStats.Chunks++;
- BasicStats.RowsOut += takeRows;
-
- Data.erase(Data.begin(), last);
- return true;
- }
-
- [[nodiscard]]
- bool Pop(NDqProto::TCheckpoint& checkpoint) override {
- if (!HasData() && Checkpoint) {
- checkpoint = std::move(*Checkpoint);
- Checkpoint = Nothing();
- return true;
- }
- return false;
- }
-
- [[nodiscard]]
- bool PopAll(NDqProto::TData& data) override {
- if (Data.empty()) {
- if (Finished) {
- data.SetTransportVersion(DataSerializer.GetTransportVersion());
- data.SetRows(0);
- data.ClearRaw();
- }
- return false;
- }
-
- if (Y_UNLIKELY(ProfileStats)) {
- TInstant startTime = TInstant::Now();
- data = DataSerializer.Serialize(Data.begin(), Data.end(), OutputType);
- ProfileStats->SerializationTime += (TInstant::Now() - startTime);
- } else {
- data = DataSerializer.Serialize(Data.begin(), Data.end(), OutputType);
- }
-
- BasicStats.Chunks++;
- BasicStats.RowsOut += data.GetRows();
-
- Data.clear();
- return true;
- }
-
- bool PopAll(NKikimr::NMiniKQL::TUnboxedValueVector& data) override {
- if (Data.empty()) {
- return false;
- }
-
- data.reserve(data.size() + Data.size());
- for (auto&& v : Data) {
- data.emplace_back(std::move(v));
- }
-
- BasicStats.Chunks++;
- BasicStats.RowsOut += data.size();
-
- Data.clear();
- return true;
- }
-
- void Finish() override {
- LOG("Finish request");
- Finished = true;
-
- if (!BasicStats.FirstRowIn) {
- BasicStats.FirstRowIn = TInstant::Now();
- }
- }
-
- bool HasData() const override {
- return !Data.empty();
- }
-
- bool IsFinished() const override {
- return Finished && !HasData();
- }
-
- ui64 Drop() override { // Drop channel data because channel was finished. Leave checkpoint because checkpoints keep going through channel after finishing channel data transfer.
- ui64 rows = Data.size();
- Data.clear();
- return rows;
- }
-
+ , MaxStoredBytes(maxStoredBytes)
+ , MaxChunkBytes(maxChunkBytes)
+ , LogFunc(logFunc) {}
+
+ ui64 GetChannelId() const override {
+ return ChannelId;
+ }
+
+ ui64 GetValuesCount(bool /* inMemoryOnly */) const override {
+ return Data.size();
+ }
+
+ [[nodiscard]]
+ bool IsFull() const override {
+ return Data.size() * EstimatedRowBytes >= MaxStoredBytes;
+ }
+
+ void Push(NUdf::TUnboxedValue&& value) override {
+ if (!BasicStats.FirstRowIn) {
+ BasicStats.FirstRowIn = TInstant::Now();
+ }
+
+ ui64 rowsInMemory = Data.size();
+
+ LOG("Push request, rows in memory: " << rowsInMemory << ", bytesInMemory: " << (rowsInMemory * EstimatedRowBytes)
+ << ", finished: " << Finished);
+ YQL_ENSURE(!IsFull());
+
+ if (Finished) {
+ return;
+ }
+
+ if (!EstimatedRowBytes) {
+ EstimatedRowBytes = IsSafeToEstimateValueSize(OutputType)
+ ? TDqDataSerializer::EstimateSize(value, OutputType)
+ : DataSerializer.CalcSerializedSize(value, OutputType);
+ LOG("EstimatedRowSize: " << EstimatedRowBytes << ", type: " << *OutputType
+ << ", safe: " << IsSafeToEstimateValueSize(OutputType));
+ }
+
+ Data.emplace_back(std::move(value));
+ rowsInMemory++;
+
+ BasicStats.Bytes += EstimatedRowBytes;
+ BasicStats.RowsIn++;
+
+ if (Y_UNLIKELY(ProfileStats)) {
+ ProfileStats->MaxMemoryUsage = std::max(ProfileStats->MaxMemoryUsage, rowsInMemory * EstimatedRowBytes);
+ ProfileStats->MaxRowsInMemory = std::max(ProfileStats->MaxRowsInMemory, rowsInMemory);
+ }
+ }
+
+ void Push(NDqProto::TCheckpoint&& checkpoint) override {
+ YQL_ENSURE(!Checkpoint);
+ Checkpoint.ConstructInPlace(std::move(checkpoint));
+ }
+
+ [[nodiscard]]
+ bool Pop(NDqProto::TData& data, ui64 bytes) override {
+ LOG("Pop request, rows in memory: " << Data.size() << ", finished: " << Finished << ", requested: " << bytes
+ << ", estimatedRowSize: " << EstimatedRowBytes);
+
+ if (!HasData()) {
+ if (Finished) {
+ data.SetTransportVersion(DataSerializer.GetTransportVersion());
+ data.SetRows(0);
+ data.ClearRaw();
+ }
+ return false;
+ }
+
+ bytes = std::min(bytes, MaxChunkBytes);
+ ui64 takeRows = std::max<ui64>(bytes / EstimatedRowBytes, 1);
+ takeRows = std::min<ui64>(takeRows, Data.size());
+
+ DLOG("About to take " << takeRows << " rows");
+
+ auto last = std::next(Data.begin(), takeRows);
+
+ if (Y_UNLIKELY(ProfileStats)) {
+ TInstant startTime = TInstant::Now();
+ data = DataSerializer.Serialize(Data.begin(), last, OutputType);
+ ProfileStats->SerializationTime += (TInstant::Now() - startTime);
+ } else {
+ data = DataSerializer.Serialize(Data.begin(), last, OutputType);
+ }
+
+ EstimatedRowBytes = EstimatedRowBytes * 0.6 + std::max<ui64>(data.GetRaw().Size() / takeRows, 1) * 0.4;
+ DLOG("Recalc estimated row size: " << EstimatedRowBytes);
+
+ while (data.GetRaw().size() >= ChunkSizeHardLimit && takeRows > 1) {
+ ui64 newTakeRows = std::max<ui64>(bytes / EstimatedRowBytes, 1);
+ newTakeRows = std::min<ui64>(newTakeRows, Data.size());
+
+ takeRows = newTakeRows < takeRows ? newTakeRows : takeRows / 2;
+
+ DLOG("Too big data, split it. Try to take " << takeRows);
+
+ NMiniKQL::TUnboxedValueVector values;
+ values.reserve(data.GetRows());
+
+ if (Y_UNLIKELY(ProfileStats)) {
+ TInstant startTime = TInstant::Now();
+ DataSerializer.Deserialize(data, OutputType, values);
+ ProfileStats->SerializationTime += (TInstant::Now() - startTime);
+ } else {
+ DataSerializer.Deserialize(data, OutputType, values);
+ }
+
+ for (ui32 i = 0; i < data.GetRows(); ++i) {
+ Data[i] = std::move(values[i]);
+ }
+
+ last = std::next(Data.begin(), takeRows);
+
+ if (Y_UNLIKELY(ProfileStats)) {
+ TInstant startTime = TInstant::Now();
+ data = DataSerializer.Serialize(Data.begin(), last, OutputType);
+ ProfileStats->SerializationTime += (TInstant::Now() - startTime);
+ } else {
+ data = DataSerializer.Serialize(Data.begin(), last, OutputType);
+ }
+ }
+
+ YQL_ENSURE(data.GetRaw().size() < ChunkSizeHardLimit);
+
+ BasicStats.Chunks++;
+ BasicStats.RowsOut += takeRows;
+
+ Data.erase(Data.begin(), last);
+ return true;
+ }
+
+ [[nodiscard]]
+ bool Pop(NDqProto::TCheckpoint& checkpoint) override {
+ if (!HasData() && Checkpoint) {
+ checkpoint = std::move(*Checkpoint);
+ Checkpoint = Nothing();
+ return true;
+ }
+ return false;
+ }
+
+ [[nodiscard]]
+ bool PopAll(NDqProto::TData& data) override {
+ if (Data.empty()) {
+ if (Finished) {
+ data.SetTransportVersion(DataSerializer.GetTransportVersion());
+ data.SetRows(0);
+ data.ClearRaw();
+ }
+ return false;
+ }
+
+ if (Y_UNLIKELY(ProfileStats)) {
+ TInstant startTime = TInstant::Now();
+ data = DataSerializer.Serialize(Data.begin(), Data.end(), OutputType);
+ ProfileStats->SerializationTime += (TInstant::Now() - startTime);
+ } else {
+ data = DataSerializer.Serialize(Data.begin(), Data.end(), OutputType);
+ }
+
+ BasicStats.Chunks++;
+ BasicStats.RowsOut += data.GetRows();
+
+ Data.clear();
+ return true;
+ }
+
+ bool PopAll(NKikimr::NMiniKQL::TUnboxedValueVector& data) override {
+ if (Data.empty()) {
+ return false;
+ }
+
+ data.reserve(data.size() + Data.size());
+ for (auto&& v : Data) {
+ data.emplace_back(std::move(v));
+ }
+
+ BasicStats.Chunks++;
+ BasicStats.RowsOut += data.size();
+
+ Data.clear();
+ return true;
+ }
+
+ void Finish() override {
+ LOG("Finish request");
+ Finished = true;
+
+ if (!BasicStats.FirstRowIn) {
+ BasicStats.FirstRowIn = TInstant::Now();
+ }
+ }
+
+ bool HasData() const override {
+ return !Data.empty();
+ }
+
+ bool IsFinished() const override {
+ return Finished && !HasData();
+ }
+
+ ui64 Drop() override { // Drop channel data because channel was finished. Leave checkpoint because checkpoints keep going through channel after finishing channel data transfer.
+ ui64 rows = Data.size();
+ Data.clear();
+ return rows;
+ }
+
NKikimr::NMiniKQL::TType* GetOutputType() const override {
return OutputType;
}
- const TDqOutputChannelStats* GetStats() const override {
- return &BasicStats;
- }
-
- void Terminate() override {
- }
-
-private:
- const ui64 ChannelId;
- NKikimr::NMiniKQL::TType* OutputType;
- TDqOutputChannelStats BasicStats;
- TDqOutputChannelStats* ProfileStats = nullptr;
- TDqDataSerializer DataSerializer;
- const ui64 MaxStoredBytes;
- ui64 MaxChunkBytes;
- TLogFunc LogFunc;
-
- using TDataType = TDeque<NUdf::TUnboxedValue, NKikimr::NMiniKQL::TMKQLAllocator<NUdf::TUnboxedValue>>;
- TDataType Data;
-
- ui32 EstimatedRowBytes = 0;
- bool Finished = false;
-
- TMaybe<NDqProto::TCheckpoint> Checkpoint;
-};
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-class TDqOutputChannelNew : public IDqOutputChannel {
- struct TSpilledBlob;
-
-public:
- TDqOutputChannelNew(ui64 channelId, NMiniKQL::TType* outputType, const NMiniKQL::TTypeEnvironment& typeEnv,
- const NMiniKQL::THolderFactory& holderFactory, const TDqOutputChannelSettings& settings, const TLogFunc& log)
- : ChannelId(channelId)
- , OutputType(outputType)
- , BasicStats(ChannelId)
- , ProfileStats(settings.CollectProfileStats ? &BasicStats : nullptr)
- , DataSerializer(typeEnv, holderFactory, settings.TransportVersion)
- , Storage(settings.ChannelStorage)
- , MaxStoredBytes(settings.MaxStoredBytes)
- , MaxChunkBytes(settings.MaxChunkBytes)
- , LogFunc(log)
- {
- if (Storage && 3 * MaxChunkBytes > MaxStoredBytes) {
- MaxChunkBytes = Max(MaxStoredBytes / 3, 1_MB);
- }
- }
-
- ui64 GetChannelId() const override {
- return ChannelId;
- }
-
- ui64 GetValuesCount(bool inMemoryOnly) const override {
- if (inMemoryOnly) {
- return DataHead.size() + DataTail.size();
- }
- return DataHead.size() + DataTail.size() + SpilledRows;
- }
-
- [[nodiscard]]
- bool IsFull() const override {
- if (!Storage) {
- if (MemoryUsed >= MaxStoredBytes) {
- YQL_ENSURE(HasData());
- return true;
- }
- return false;
- }
-
- return Storage->IsFull();
- }
-
- void Push(NUdf::TUnboxedValue&& value) override {
- if (!BasicStats.FirstRowIn) {
- BasicStats.FirstRowIn = TInstant::Now();
- }
-
- ui64 rowsInMemory = DataHead.size() + DataTail.size();
-
- LOG("Push request, rows in memory: " << rowsInMemory << ", bytesInMemory: " << MemoryUsed
- << ", spilled rows: " << SpilledRows << ", spilled blobs: " << SpilledBlobs.size()
- << ", finished: " << Finished);
- YQL_ENSURE(!IsFull());
-
- if (Finished) {
- return;
- }
-
- ui64 estimatedRowSize;
-
- if (RowFixedSize.has_value()) {
- estimatedRowSize = *RowFixedSize;
- } else {
- bool fixed;
- estimatedRowSize = TDqDataSerializer::EstimateSize(value, OutputType, &fixed);
- if (fixed) {
- RowFixedSize.emplace(estimatedRowSize);
- LOG("Raw has fixed size: " << estimatedRowSize);
- }
- }
-
- DLOG("Row size: " << estimatedRowSize);
-
- if (SpilledRows == 0) {
- YQL_ENSURE(DataTail.empty() && SizeTail.empty());
- DataHead.emplace_back(std::move(value));
- SizeHead.emplace_back(estimatedRowSize);
- } else {
- DataTail.emplace_back(std::move(value));
- SizeTail.emplace_back(estimatedRowSize);
- }
- rowsInMemory++;
- MemoryUsed += estimatedRowSize;
-
- ui64 spilledRows = 0;
- ui64 spilledBytes = 0;
- ui64 spilledBlobs = 0;
-
- // TODO: add PushAndFinish call
- // if processing is finished we don't have to spill data
-
- if (MemoryUsed > MaxStoredBytes && Storage) {
- LOG("Not enough memory to store rows in memory only, start spilling, rowsInMemory: " << rowsInMemory
- << ", memoryUsed: " << MemoryUsed << ", MaxStoredBytes: " << MaxStoredBytes);
-
- TDataType* data = nullptr;
- TDeque<ui64>* size = nullptr;
-
- ui32 startIndex = std::numeric_limits<ui32>::max();
-
- if (SpilledRows) {
- // we have spilled rows already, so we can spill all DataTail
- data = &DataTail;
- size = &SizeTail;
- startIndex = 0;
- } else {
- // spill from head
- // but only if we have rows for at least 2 full chunks
- ui32 chunks = 1;
- ui64 chunkSize = 0;
- startIndex = 0;
- for (ui64 i = 0; i < SizeHead.size(); ++i) {
- chunkSize += SizeHead.at(i);
- if (chunkSize >= MaxChunkBytes && (i - startIndex) > 0) {
- // LOG("one more chunk, size: " << chunkSize << ", start next chunk from " << i);
- chunks++; // this row goes to the next chunk
- chunkSize = 0;
-
- if (chunks >= 3) {
- data = &DataHead;
- size = &SizeHead;
- break;
- }
-
- startIndex = i;
- }
- }
- }
-
- // LOG("about to spill from index " << startIndex);
-
- if (data) {
- while (true) {
- ui64 chunkSize = 0;
- ui32 rowsInChunk = 0;
- ui32 idx = startIndex + spilledRows;
-
- while (idx < size->size()) {
- ui64 rowSize = size->at(idx);
- if (chunkSize == 0 || chunkSize + rowSize <= MaxChunkBytes) {
- chunkSize += rowSize;
- ++rowsInChunk;
- ++idx;
- } else {
- break;
- }
- }
-
- if (rowsInChunk > 0) {
- // LOG("about to spill from " << (startIndex + spilledRows) << " to " << (startIndex + spilledRows + rowsInChunk));
-
- TDataType::iterator first = std::next(data->begin(), startIndex + spilledRows);
- TDataType::iterator last = std::next(data->begin(), startIndex + spilledRows + rowsInChunk);
-
- NDqProto::TData protoBlob = DataSerializer.Serialize(first, last, OutputType);
- TBuffer blob;
- TBufferOutput out{blob};
- protoBlob.SerializeToArcadiaStream(&out);
-
- ui64 blobSize = blob.size();
- ui64 blobId = NextBlobId++;
-
- LOG("Spill blobId: " << blobId << ", size: " << blobSize << ", rows: " << rowsInChunk);
- Storage->Put(blobId, std::move(blob));
-
- SpilledBlobs.emplace_back(TSpilledBlob(blobId, chunkSize, blobSize, rowsInChunk));
- SpilledRows += rowsInChunk;
- MemoryUsed -= chunkSize;
-
- spilledRows += rowsInChunk;
- spilledBytes += blobSize;
- spilledBlobs++;
- } else {
- break;
- }
- }
-
- if (data == &DataHead) {
- // move rows after spilled ones to the DataTail
- YQL_ENSURE(DataTail.empty());
- for (ui32 i = startIndex + spilledRows; i < data->size(); ++i) {
- DataTail.emplace_back(std::move(DataHead[i]));
- SizeTail.emplace_back(SizeHead[i]);
- }
- data->erase(std::next(data->begin(), startIndex), data->end());
- size->erase(std::next(size->begin(), startIndex), size->end());
- } else {
- YQL_ENSURE(startIndex == 0);
- data->erase(data->begin(), std::next(data->begin(), spilledRows));
- size->erase(size->begin(), std::next(size->begin(), spilledRows));
- }
- } else {
- // LOG("nothing to spill");
- }
- }
-
- BasicStats.Bytes += estimatedRowSize;
- BasicStats.RowsIn++;
-
- if (Y_UNLIKELY(ProfileStats)) {
- ProfileStats->MaxMemoryUsage = std::max(ProfileStats->MaxMemoryUsage, MemoryUsed);
- ProfileStats->MaxRowsInMemory = std::max(ProfileStats->MaxRowsInMemory, DataHead.size() + DataTail.size());
- if (spilledRows) {
- ProfileStats->SpilledRows += spilledRows;
- ProfileStats->SpilledBytes += spilledBytes;
- ProfileStats->SpilledBlobs += spilledBlobs;
- }
- }
-
- ValidateUsedMemory();
-
-#if 0
- TStringStream ss;
- ss << "-- DUMP --" << Endl;
- ss << " Head:" << Endl;
- for (auto& v : DataHead) {
- ss << " "; v.Dump(ss); ss << Endl;
- }
- ss << " Spilled: " << SpilledRows << Endl;
- ss << " Tail:" << Endl;
- for (auto& v : DataTail) {
- ss << " "; v.Dump(ss); ss << Endl;
- }
- LOG(ss.Str());
-#endif
- }
-
- void Push(NDqProto::TCheckpoint&& checkpoint) override {
+ const TDqOutputChannelStats* GetStats() const override {
+ return &BasicStats;
+ }
+
+ void Terminate() override {
+ }
+
+private:
+ const ui64 ChannelId;
+ NKikimr::NMiniKQL::TType* OutputType;
+ TDqOutputChannelStats BasicStats;
+ TDqOutputChannelStats* ProfileStats = nullptr;
+ TDqDataSerializer DataSerializer;
+ const ui64 MaxStoredBytes;
+ ui64 MaxChunkBytes;
+ TLogFunc LogFunc;
+
+ using TDataType = TDeque<NUdf::TUnboxedValue, NKikimr::NMiniKQL::TMKQLAllocator<NUdf::TUnboxedValue>>;
+ TDataType Data;
+
+ ui32 EstimatedRowBytes = 0;
+ bool Finished = false;
+
+ TMaybe<NDqProto::TCheckpoint> Checkpoint;
+};
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+class TDqOutputChannelNew : public IDqOutputChannel {
+ struct TSpilledBlob;
+
+public:
+ TDqOutputChannelNew(ui64 channelId, NMiniKQL::TType* outputType, const NMiniKQL::TTypeEnvironment& typeEnv,
+ const NMiniKQL::THolderFactory& holderFactory, const TDqOutputChannelSettings& settings, const TLogFunc& log)
+ : ChannelId(channelId)
+ , OutputType(outputType)
+ , BasicStats(ChannelId)
+ , ProfileStats(settings.CollectProfileStats ? &BasicStats : nullptr)
+ , DataSerializer(typeEnv, holderFactory, settings.TransportVersion)
+ , Storage(settings.ChannelStorage)
+ , MaxStoredBytes(settings.MaxStoredBytes)
+ , MaxChunkBytes(settings.MaxChunkBytes)
+ , LogFunc(log)
+ {
+ if (Storage && 3 * MaxChunkBytes > MaxStoredBytes) {
+ MaxChunkBytes = Max(MaxStoredBytes / 3, 1_MB);
+ }
+ }
+
+ ui64 GetChannelId() const override {
+ return ChannelId;
+ }
+
+ ui64 GetValuesCount(bool inMemoryOnly) const override {
+ if (inMemoryOnly) {
+ return DataHead.size() + DataTail.size();
+ }
+ return DataHead.size() + DataTail.size() + SpilledRows;
+ }
+
+ [[nodiscard]]
+ bool IsFull() const override {
+ if (!Storage) {
+ if (MemoryUsed >= MaxStoredBytes) {
+ YQL_ENSURE(HasData());
+ return true;
+ }
+ return false;
+ }
+
+ return Storage->IsFull();
+ }
+
+ void Push(NUdf::TUnboxedValue&& value) override {
+ if (!BasicStats.FirstRowIn) {
+ BasicStats.FirstRowIn = TInstant::Now();
+ }
+
+ ui64 rowsInMemory = DataHead.size() + DataTail.size();
+
+ LOG("Push request, rows in memory: " << rowsInMemory << ", bytesInMemory: " << MemoryUsed
+ << ", spilled rows: " << SpilledRows << ", spilled blobs: " << SpilledBlobs.size()
+ << ", finished: " << Finished);
+ YQL_ENSURE(!IsFull());
+
+ if (Finished) {
+ return;
+ }
+
+ ui64 estimatedRowSize;
+
+ if (RowFixedSize.has_value()) {
+ estimatedRowSize = *RowFixedSize;
+ } else {
+ bool fixed;
+ estimatedRowSize = TDqDataSerializer::EstimateSize(value, OutputType, &fixed);
+ if (fixed) {
+ RowFixedSize.emplace(estimatedRowSize);
+ LOG("Raw has fixed size: " << estimatedRowSize);
+ }
+ }
+
+ DLOG("Row size: " << estimatedRowSize);
+
+ if (SpilledRows == 0) {
+ YQL_ENSURE(DataTail.empty() && SizeTail.empty());
+ DataHead.emplace_back(std::move(value));
+ SizeHead.emplace_back(estimatedRowSize);
+ } else {
+ DataTail.emplace_back(std::move(value));
+ SizeTail.emplace_back(estimatedRowSize);
+ }
+ rowsInMemory++;
+ MemoryUsed += estimatedRowSize;
+
+ ui64 spilledRows = 0;
+ ui64 spilledBytes = 0;
+ ui64 spilledBlobs = 0;
+
+ // TODO: add PushAndFinish call
+ // if processing is finished we don't have to spill data
+
+ if (MemoryUsed > MaxStoredBytes && Storage) {
+ LOG("Not enough memory to store rows in memory only, start spilling, rowsInMemory: " << rowsInMemory
+ << ", memoryUsed: " << MemoryUsed << ", MaxStoredBytes: " << MaxStoredBytes);
+
+ TDataType* data = nullptr;
+ TDeque<ui64>* size = nullptr;
+
+ ui32 startIndex = std::numeric_limits<ui32>::max();
+
+ if (SpilledRows) {
+ // we have spilled rows already, so we can spill all DataTail
+ data = &DataTail;
+ size = &SizeTail;
+ startIndex = 0;
+ } else {
+ // spill from head
+ // but only if we have rows for at least 2 full chunks
+ ui32 chunks = 1;
+ ui64 chunkSize = 0;
+ startIndex = 0;
+ for (ui64 i = 0; i < SizeHead.size(); ++i) {
+ chunkSize += SizeHead.at(i);
+ if (chunkSize >= MaxChunkBytes && (i - startIndex) > 0) {
+ // LOG("one more chunk, size: " << chunkSize << ", start next chunk from " << i);
+ chunks++; // this row goes to the next chunk
+ chunkSize = 0;
+
+ if (chunks >= 3) {
+ data = &DataHead;
+ size = &SizeHead;
+ break;
+ }
+
+ startIndex = i;
+ }
+ }
+ }
+
+ // LOG("about to spill from index " << startIndex);
+
+ if (data) {
+ while (true) {
+ ui64 chunkSize = 0;
+ ui32 rowsInChunk = 0;
+ ui32 idx = startIndex + spilledRows;
+
+ while (idx < size->size()) {
+ ui64 rowSize = size->at(idx);
+ if (chunkSize == 0 || chunkSize + rowSize <= MaxChunkBytes) {
+ chunkSize += rowSize;
+ ++rowsInChunk;
+ ++idx;
+ } else {
+ break;
+ }
+ }
+
+ if (rowsInChunk > 0) {
+ // LOG("about to spill from " << (startIndex + spilledRows) << " to " << (startIndex + spilledRows + rowsInChunk));
+
+ TDataType::iterator first = std::next(data->begin(), startIndex + spilledRows);
+ TDataType::iterator last = std::next(data->begin(), startIndex + spilledRows + rowsInChunk);
+
+ NDqProto::TData protoBlob = DataSerializer.Serialize(first, last, OutputType);
+ TBuffer blob;
+ TBufferOutput out{blob};
+ protoBlob.SerializeToArcadiaStream(&out);
+
+ ui64 blobSize = blob.size();
+ ui64 blobId = NextBlobId++;
+
+ LOG("Spill blobId: " << blobId << ", size: " << blobSize << ", rows: " << rowsInChunk);
+ Storage->Put(blobId, std::move(blob));
+
+ SpilledBlobs.emplace_back(TSpilledBlob(blobId, chunkSize, blobSize, rowsInChunk));
+ SpilledRows += rowsInChunk;
+ MemoryUsed -= chunkSize;
+
+ spilledRows += rowsInChunk;
+ spilledBytes += blobSize;
+ spilledBlobs++;
+ } else {
+ break;
+ }
+ }
+
+ if (data == &DataHead) {
+ // move rows after spilled ones to the DataTail
+ YQL_ENSURE(DataTail.empty());
+ for (ui32 i = startIndex + spilledRows; i < data->size(); ++i) {
+ DataTail.emplace_back(std::move(DataHead[i]));
+ SizeTail.emplace_back(SizeHead[i]);
+ }
+ data->erase(std::next(data->begin(), startIndex), data->end());
+ size->erase(std::next(size->begin(), startIndex), size->end());
+ } else {
+ YQL_ENSURE(startIndex == 0);
+ data->erase(data->begin(), std::next(data->begin(), spilledRows));
+ size->erase(size->begin(), std::next(size->begin(), spilledRows));
+ }
+ } else {
+ // LOG("nothing to spill");
+ }
+ }
+
+ BasicStats.Bytes += estimatedRowSize;
+ BasicStats.RowsIn++;
+
+ if (Y_UNLIKELY(ProfileStats)) {
+ ProfileStats->MaxMemoryUsage = std::max(ProfileStats->MaxMemoryUsage, MemoryUsed);
+ ProfileStats->MaxRowsInMemory = std::max(ProfileStats->MaxRowsInMemory, DataHead.size() + DataTail.size());
+ if (spilledRows) {
+ ProfileStats->SpilledRows += spilledRows;
+ ProfileStats->SpilledBytes += spilledBytes;
+ ProfileStats->SpilledBlobs += spilledBlobs;
+ }
+ }
+
+ ValidateUsedMemory();
+
+#if 0
+ TStringStream ss;
+ ss << "-- DUMP --" << Endl;
+ ss << " Head:" << Endl;
+ for (auto& v : DataHead) {
+ ss << " "; v.Dump(ss); ss << Endl;
+ }
+ ss << " Spilled: " << SpilledRows << Endl;
+ ss << " Tail:" << Endl;
+ for (auto& v : DataTail) {
+ ss << " "; v.Dump(ss); ss << Endl;
+ }
+ LOG(ss.Str());
+#endif
+ }
+
+ void Push(NDqProto::TCheckpoint&& checkpoint) override {
YQL_ENSURE(!Checkpoint);
Checkpoint.ConstructInPlace(std::move(checkpoint));
}
- [[nodiscard]]
- bool Pop(NDqProto::TData& data, ui64 bytes) override {
- LOG("Pop request, rows in memory: " << DataHead.size() << "/" << DataTail.size()
- << ", spilled rows: " << SpilledRows << ", spilled blobs: " << SpilledBlobs.size()
- << ", finished: " << Finished << ", requested: " << bytes << ", maxChunkBytes: " << MaxChunkBytes);
-
+ [[nodiscard]]
+ bool Pop(NDqProto::TData& data, ui64 bytes) override {
+ LOG("Pop request, rows in memory: " << DataHead.size() << "/" << DataTail.size()
+ << ", spilled rows: " << SpilledRows << ", spilled blobs: " << SpilledBlobs.size()
+ << ", finished: " << Finished << ", requested: " << bytes << ", maxChunkBytes: " << MaxChunkBytes);
+
if (!HasData()) {
- if (Finished) {
- data.SetTransportVersion(DataSerializer.GetTransportVersion());
- data.SetRows(0);
- data.ClearRaw();
- }
- return false;
- }
-
- bytes = std::min(bytes, MaxChunkBytes);
-
- if (DataHead.empty()) {
- YQL_ENSURE(SpilledRows);
-
- TSpilledBlob spilledBlob = SpilledBlobs.front();
-
- LOG("Loading spilled blob. BlobId: " << spilledBlob.BlobId << ", rows: " << spilledBlob.Rows
- << ", bytes: " << spilledBlob.SerializedSize);
- TBuffer blob;
- if (!Storage->Get(spilledBlob.BlobId, blob)) {
- LOG("BlobId " << spilledBlob.BlobId << " not ready yet");
- // not loaded yet
- return false;
- }
-
- YQL_ENSURE(blob.size() == spilledBlob.SerializedSize, "" << blob.size() << " != " << spilledBlob.SerializedSize);
-
- NDqProto::TData protoBlob;
- YQL_ENSURE(protoBlob.ParseFromArray(blob.Data(), blob.size()));
- blob.Reset();
-
- YQL_ENSURE(protoBlob.GetRows() == spilledBlob.Rows);
- bool hasResult = false;
-
- // LOG("bytes: " << bytes << ", loaded blob: " << spilledBlob.SerializedSize);
-
- if (spilledBlob.SerializedSize <= bytes * 2) {
- data.Swap(&protoBlob);
- YQL_ENSURE(data.ByteSizeLong() <= ChunkSizeHardLimit);
- hasResult = true;
- // LOG("return loaded blob as-is");
- } else {
- NKikimr::NMiniKQL::TUnboxedValueVector buffer;
- DataSerializer.Deserialize(protoBlob, OutputType, buffer);
-
- for (ui32 i = 0; i < buffer.size(); ++i) {
- SizeHead.emplace_back(RowFixedSize
- ? *RowFixedSize
- : TDqDataSerializer::EstimateSize(buffer[i], OutputType));
- DataHead.emplace_back(std::move(buffer[i]));
-
- MemoryUsed += SizeHead.back();
- }
- }
-
- SpilledRows -= spilledBlob.Rows;
-
- SpilledBlobs.pop_front();
- if (SpilledBlobs.empty()) {
- // LOG("no more spilled blobs, move " << DataTail.size() << " tail rows to the head");
- DataHead.insert(DataHead.end(), std::make_move_iterator(DataTail.begin()), std::make_move_iterator(DataTail.end()));
- SizeHead.insert(SizeHead.end(), std::make_move_iterator(SizeTail.begin()), std::make_move_iterator(SizeTail.end()));
- DataTail.clear();
- SizeTail.clear();
- }
-
- if (hasResult) {
- BasicStats.Chunks++;
- BasicStats.RowsOut += data.GetRows();
- ValidateUsedMemory();
- return true;
- }
- }
-
- ui32 takeRows = 0;
- ui64 chunkSize = 0;
-
- while (takeRows == 0 || (takeRows < SizeHead.size() && chunkSize + SizeHead[takeRows] <= bytes)) {
- chunkSize += SizeHead[takeRows];
- ++takeRows;
- }
-
- DLOG("return rows: " << takeRows << ", size: " << chunkSize);
-
- auto firstDataIt = DataHead.begin();
- auto lastDataIt = std::next(firstDataIt, takeRows);
-
- auto firstSizeIt = SizeHead.begin();
- auto lastSizeIt = std::next(firstSizeIt, takeRows);
-
- if (Y_UNLIKELY(ProfileStats)) {
- TInstant startTime = TInstant::Now();
- data = DataSerializer.Serialize(firstDataIt, lastDataIt, OutputType);
- ProfileStats->SerializationTime += (TInstant::Now() - startTime);
- } else {
- data = DataSerializer.Serialize(firstDataIt, lastDataIt, OutputType);
- }
-
- YQL_ENSURE(data.ByteSizeLong() <= ChunkSizeHardLimit);
-
- BasicStats.Chunks++;
- BasicStats.RowsOut += takeRows;
-
- DataHead.erase(firstDataIt, lastDataIt);
- SizeHead.erase(firstSizeIt, lastSizeIt);
- MemoryUsed -= chunkSize;
-
- ValidateUsedMemory();
-
- return true;
- }
-
- [[nodiscard]]
- bool Pop(NDqProto::TCheckpoint& checkpoint) override {
+ if (Finished) {
+ data.SetTransportVersion(DataSerializer.GetTransportVersion());
+ data.SetRows(0);
+ data.ClearRaw();
+ }
+ return false;
+ }
+
+ bytes = std::min(bytes, MaxChunkBytes);
+
+ if (DataHead.empty()) {
+ YQL_ENSURE(SpilledRows);
+
+ TSpilledBlob spilledBlob = SpilledBlobs.front();
+
+ LOG("Loading spilled blob. BlobId: " << spilledBlob.BlobId << ", rows: " << spilledBlob.Rows
+ << ", bytes: " << spilledBlob.SerializedSize);
+ TBuffer blob;
+ if (!Storage->Get(spilledBlob.BlobId, blob)) {
+ LOG("BlobId " << spilledBlob.BlobId << " not ready yet");
+ // not loaded yet
+ return false;
+ }
+
+ YQL_ENSURE(blob.size() == spilledBlob.SerializedSize, "" << blob.size() << " != " << spilledBlob.SerializedSize);
+
+ NDqProto::TData protoBlob;
+ YQL_ENSURE(protoBlob.ParseFromArray(blob.Data(), blob.size()));
+ blob.Reset();
+
+ YQL_ENSURE(protoBlob.GetRows() == spilledBlob.Rows);
+ bool hasResult = false;
+
+ // LOG("bytes: " << bytes << ", loaded blob: " << spilledBlob.SerializedSize);
+
+ if (spilledBlob.SerializedSize <= bytes * 2) {
+ data.Swap(&protoBlob);
+ YQL_ENSURE(data.ByteSizeLong() <= ChunkSizeHardLimit);
+ hasResult = true;
+ // LOG("return loaded blob as-is");
+ } else {
+ NKikimr::NMiniKQL::TUnboxedValueVector buffer;
+ DataSerializer.Deserialize(protoBlob, OutputType, buffer);
+
+ for (ui32 i = 0; i < buffer.size(); ++i) {
+ SizeHead.emplace_back(RowFixedSize
+ ? *RowFixedSize
+ : TDqDataSerializer::EstimateSize(buffer[i], OutputType));
+ DataHead.emplace_back(std::move(buffer[i]));
+
+ MemoryUsed += SizeHead.back();
+ }
+ }
+
+ SpilledRows -= spilledBlob.Rows;
+
+ SpilledBlobs.pop_front();
+ if (SpilledBlobs.empty()) {
+ // LOG("no more spilled blobs, move " << DataTail.size() << " tail rows to the head");
+ DataHead.insert(DataHead.end(), std::make_move_iterator(DataTail.begin()), std::make_move_iterator(DataTail.end()));
+ SizeHead.insert(SizeHead.end(), std::make_move_iterator(SizeTail.begin()), std::make_move_iterator(SizeTail.end()));
+ DataTail.clear();
+ SizeTail.clear();
+ }
+
+ if (hasResult) {
+ BasicStats.Chunks++;
+ BasicStats.RowsOut += data.GetRows();
+ ValidateUsedMemory();
+ return true;
+ }
+ }
+
+ ui32 takeRows = 0;
+ ui64 chunkSize = 0;
+
+ while (takeRows == 0 || (takeRows < SizeHead.size() && chunkSize + SizeHead[takeRows] <= bytes)) {
+ chunkSize += SizeHead[takeRows];
+ ++takeRows;
+ }
+
+ DLOG("return rows: " << takeRows << ", size: " << chunkSize);
+
+ auto firstDataIt = DataHead.begin();
+ auto lastDataIt = std::next(firstDataIt, takeRows);
+
+ auto firstSizeIt = SizeHead.begin();
+ auto lastSizeIt = std::next(firstSizeIt, takeRows);
+
+ if (Y_UNLIKELY(ProfileStats)) {
+ TInstant startTime = TInstant::Now();
+ data = DataSerializer.Serialize(firstDataIt, lastDataIt, OutputType);
+ ProfileStats->SerializationTime += (TInstant::Now() - startTime);
+ } else {
+ data = DataSerializer.Serialize(firstDataIt, lastDataIt, OutputType);
+ }
+
+ YQL_ENSURE(data.ByteSizeLong() <= ChunkSizeHardLimit);
+
+ BasicStats.Chunks++;
+ BasicStats.RowsOut += takeRows;
+
+ DataHead.erase(firstDataIt, lastDataIt);
+ SizeHead.erase(firstSizeIt, lastSizeIt);
+ MemoryUsed -= chunkSize;
+
+ ValidateUsedMemory();
+
+ return true;
+ }
+
+ [[nodiscard]]
+ bool Pop(NDqProto::TCheckpoint& checkpoint) override {
if (!HasData() && Checkpoint) {
checkpoint = std::move(*Checkpoint);
Checkpoint = Nothing();
@@ -679,167 +679,167 @@ public:
}
[[nodiscard]]
- bool PopAll(NDqProto::TData& data) override {
- YQL_ENSURE(!SpilledRows);
- YQL_ENSURE(DataTail.empty());
-
- if (DataHead.empty()) {
- if (Finished) {
- data.SetTransportVersion(DataSerializer.GetTransportVersion());
- data.SetRows(0);
- data.ClearRaw();
- }
- return false;
- }
-
- if (Y_UNLIKELY(ProfileStats)) {
- TInstant startTime = TInstant::Now();
- data = DataSerializer.Serialize(DataHead.begin(), DataHead.end(), OutputType);
- ProfileStats->SerializationTime += (TInstant::Now() - startTime);
- } else {
- data = DataSerializer.Serialize(DataHead.begin(), DataHead.end(), OutputType);
- }
-
- BasicStats.Chunks++;
- BasicStats.RowsOut += data.GetRows();
-
- DataHead.clear();
- SizeHead.clear();
- MemoryUsed = 0;
-
- return true;
- }
-
- bool PopAll(NKikimr::NMiniKQL::TUnboxedValueVector& data) override {
- YQL_ENSURE(!SpilledRows);
- YQL_ENSURE(DataTail.empty());
-
- if (DataHead.empty()) {
- return false;
- }
-
- data.reserve(data.size() + DataHead.size());
- for (auto&& v : DataHead) {
- data.emplace_back(std::move(v));
- }
-
- BasicStats.Chunks++;
- BasicStats.RowsOut += data.size();
-
- DataHead.clear();
- SizeHead.clear();
- MemoryUsed = 0;
-
- return true;
- }
-
- void Finish() override {
- DLOG("Finish request");
- Finished = true;
-
- if (!BasicStats.FirstRowIn) {
- BasicStats.FirstRowIn = TInstant::Now();
- }
- }
-
+ bool PopAll(NDqProto::TData& data) override {
+ YQL_ENSURE(!SpilledRows);
+ YQL_ENSURE(DataTail.empty());
+
+ if (DataHead.empty()) {
+ if (Finished) {
+ data.SetTransportVersion(DataSerializer.GetTransportVersion());
+ data.SetRows(0);
+ data.ClearRaw();
+ }
+ return false;
+ }
+
+ if (Y_UNLIKELY(ProfileStats)) {
+ TInstant startTime = TInstant::Now();
+ data = DataSerializer.Serialize(DataHead.begin(), DataHead.end(), OutputType);
+ ProfileStats->SerializationTime += (TInstant::Now() - startTime);
+ } else {
+ data = DataSerializer.Serialize(DataHead.begin(), DataHead.end(), OutputType);
+ }
+
+ BasicStats.Chunks++;
+ BasicStats.RowsOut += data.GetRows();
+
+ DataHead.clear();
+ SizeHead.clear();
+ MemoryUsed = 0;
+
+ return true;
+ }
+
+ bool PopAll(NKikimr::NMiniKQL::TUnboxedValueVector& data) override {
+ YQL_ENSURE(!SpilledRows);
+ YQL_ENSURE(DataTail.empty());
+
+ if (DataHead.empty()) {
+ return false;
+ }
+
+ data.reserve(data.size() + DataHead.size());
+ for (auto&& v : DataHead) {
+ data.emplace_back(std::move(v));
+ }
+
+ BasicStats.Chunks++;
+ BasicStats.RowsOut += data.size();
+
+ DataHead.clear();
+ SizeHead.clear();
+ MemoryUsed = 0;
+
+ return true;
+ }
+
+ void Finish() override {
+ DLOG("Finish request");
+ Finished = true;
+
+ if (!BasicStats.FirstRowIn) {
+ BasicStats.FirstRowIn = TInstant::Now();
+ }
+ }
+
bool HasData() const override {
return !DataHead.empty() || SpilledRows != 0 || !DataTail.empty();
}
- bool IsFinished() const override {
+ bool IsFinished() const override {
return Finished && !HasData();
- }
-
+ }
+
ui64 Drop() override { // Drop channel data because channel was finished. Leave checkpoint because checkpoints keep going through channel after finishing channel data transfer.
- ui64 rows = DataHead.size() + SpilledRows + DataTail.size();
- DataHead.clear();
- SizeHead.clear();
- DataTail.clear();
- SizeTail.clear();
- MemoryUsed = 0;
- // todo: send remove request
- SpilledRows = 0;
- SpilledBlobs.clear();
- return rows;
- }
-
+ ui64 rows = DataHead.size() + SpilledRows + DataTail.size();
+ DataHead.clear();
+ SizeHead.clear();
+ DataTail.clear();
+ SizeTail.clear();
+ MemoryUsed = 0;
+ // todo: send remove request
+ SpilledRows = 0;
+ SpilledBlobs.clear();
+ return rows;
+ }
+
NKikimr::NMiniKQL::TType* GetOutputType() const override {
return OutputType;
}
- const TDqOutputChannelStats* GetStats() const override {
- return &BasicStats;
- }
-
- void Terminate() override {
- if (Storage) {
- Storage.Reset();
- }
- }
-
- void ValidateUsedMemory() {
-#ifndef NDEBUG
- ui64 x = 0;
- for (auto z : SizeHead) x += z;
- for (auto z : SizeTail) x += z;
- YQL_ENSURE(x == MemoryUsed, "" << x << " != " << MemoryUsed);
-#endif
- }
-
-private:
- const ui64 ChannelId;
- NKikimr::NMiniKQL::TType* OutputType;
- TDqOutputChannelStats BasicStats;
- TDqOutputChannelStats* ProfileStats = nullptr;
- TDqDataSerializer DataSerializer;
- IDqChannelStorage::TPtr Storage;
- const ui64 MaxStoredBytes;
- ui64 MaxChunkBytes;
- TLogFunc LogFunc;
- std::optional<ui32> RowFixedSize;
-
- struct TSpilledBlob {
- ui64 BlobId;
- ui64 InMemorySize;
- ui64 SerializedSize;
- ui32 Rows;
-
- TSpilledBlob(ui64 blobId, ui64 inMemorySize, ui64 serializedSize, ui32 rows)
- : BlobId(blobId), InMemorySize(inMemorySize), SerializedSize(serializedSize), Rows(rows) {}
- };
-
- // DataHead ( . SpilledBlobs (. DataTail)? )?
- using TDataType = TDeque<NUdf::TUnboxedValue, NKikimr::NMiniKQL::TMKQLAllocator<NUdf::TUnboxedValue>>;
- TDataType DataHead;
- TDeque<ui64> SizeHead;
- TDeque<TSpilledBlob> SpilledBlobs;
- TDataType DataTail;
- TDeque<ui64> SizeTail;
-
- ui64 MemoryUsed = 0; // approx memory usage
-
- ui64 NextBlobId = 1;
- ui64 SpilledRows = 0;
-
- bool Finished = false;
-
- TMaybe<NDqProto::TCheckpoint> Checkpoint;
-};
-
-} // anonymous namespace
-
-
-IDqOutputChannel::TPtr CreateDqOutputChannel(ui64 channelId, NKikimr::NMiniKQL::TType* outputType,
- const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv, const NKikimr::NMiniKQL::THolderFactory& holderFactory,
- const TDqOutputChannelSettings& settings, const TLogFunc& logFunc)
-{
- if (settings.AllowGeneratorsInUnboxedValues) {
- YQL_ENSURE(!settings.ChannelStorage);
- return new TDqOutputChannelOld(channelId, outputType, settings.CollectProfileStats, typeEnv, holderFactory,
- settings.MaxStoredBytes, settings.MaxChunkBytes, settings.TransportVersion, logFunc);
- } else {
- return new TDqOutputChannelNew(channelId, outputType, typeEnv, holderFactory, settings, logFunc);
- }
-}
-
-} // namespace NYql::NDq
+ const TDqOutputChannelStats* GetStats() const override {
+ return &BasicStats;
+ }
+
+ void Terminate() override {
+ if (Storage) {
+ Storage.Reset();
+ }
+ }
+
+ void ValidateUsedMemory() {
+#ifndef NDEBUG
+ ui64 x = 0;
+ for (auto z : SizeHead) x += z;
+ for (auto z : SizeTail) x += z;
+ YQL_ENSURE(x == MemoryUsed, "" << x << " != " << MemoryUsed);
+#endif
+ }
+
+private:
+ const ui64 ChannelId;
+ NKikimr::NMiniKQL::TType* OutputType;
+ TDqOutputChannelStats BasicStats;
+ TDqOutputChannelStats* ProfileStats = nullptr;
+ TDqDataSerializer DataSerializer;
+ IDqChannelStorage::TPtr Storage;
+ const ui64 MaxStoredBytes;
+ ui64 MaxChunkBytes;
+ TLogFunc LogFunc;
+ std::optional<ui32> RowFixedSize;
+
+ struct TSpilledBlob {
+ ui64 BlobId;
+ ui64 InMemorySize;
+ ui64 SerializedSize;
+ ui32 Rows;
+
+ TSpilledBlob(ui64 blobId, ui64 inMemorySize, ui64 serializedSize, ui32 rows)
+ : BlobId(blobId), InMemorySize(inMemorySize), SerializedSize(serializedSize), Rows(rows) {}
+ };
+
+ // DataHead ( . SpilledBlobs (. DataTail)? )?
+ using TDataType = TDeque<NUdf::TUnboxedValue, NKikimr::NMiniKQL::TMKQLAllocator<NUdf::TUnboxedValue>>;
+ TDataType DataHead;
+ TDeque<ui64> SizeHead;
+ TDeque<TSpilledBlob> SpilledBlobs;
+ TDataType DataTail;
+ TDeque<ui64> SizeTail;
+
+ ui64 MemoryUsed = 0; // approx memory usage
+
+ ui64 NextBlobId = 1;
+ ui64 SpilledRows = 0;
+
+ bool Finished = false;
+
+ TMaybe<NDqProto::TCheckpoint> Checkpoint;
+};
+
+} // anonymous namespace
+
+
+IDqOutputChannel::TPtr CreateDqOutputChannel(ui64 channelId, NKikimr::NMiniKQL::TType* outputType,
+ const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv, const NKikimr::NMiniKQL::THolderFactory& holderFactory,
+ const TDqOutputChannelSettings& settings, const TLogFunc& logFunc)
+{
+ if (settings.AllowGeneratorsInUnboxedValues) {
+ YQL_ENSURE(!settings.ChannelStorage);
+ return new TDqOutputChannelOld(channelId, outputType, settings.CollectProfileStats, typeEnv, holderFactory,
+ settings.MaxStoredBytes, settings.MaxChunkBytes, settings.TransportVersion, logFunc);
+ } else {
+ return new TDqOutputChannelNew(channelId, outputType, typeEnv, holderFactory, settings, logFunc);
+ }
+}
+
+} // namespace NYql::NDq
diff --git a/ydb/library/yql/dq/runtime/dq_output_channel.h b/ydb/library/yql/dq/runtime/dq_output_channel.h
index 951081b10d..f7a4887d2d 100644
--- a/ydb/library/yql/dq/runtime/dq_output_channel.h
+++ b/ydb/library/yql/dq/runtime/dq_output_channel.h
@@ -1,74 +1,74 @@
-#pragma once
+#pragma once
#include "dq_output.h"
#include "dq_channel_storage.h"
-
+
#include <ydb/library/yql/dq/common/dq_common.h>
#include <ydb/library/yql/dq/proto/dq_transport.pb.h>
#include <ydb/library/yql/dq/actors/protos/dq_events.pb.h>
-
+
#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
#include <ydb/library/yql/minikql/mkql_node.h>
-
-#include <util/generic/size_literals.h>
-
-
-namespace NYql::NDq {
-
+
+#include <util/generic/size_literals.h>
+
+
+namespace NYql::NDq {
+
struct TDqOutputChannelStats : TDqOutputStats {
- ui64 ChannelId = 0;
-
- // profile stats
- TDuration SerializationTime;
-
- ui64 SpilledBytes = 0;
- ui64 SpilledRows = 0;
- ui64 SpilledBlobs = 0;
-
- explicit TDqOutputChannelStats(ui64 channelId)
- : ChannelId(channelId) {}
-};
-
+ ui64 ChannelId = 0;
+
+ // profile stats
+ TDuration SerializationTime;
+
+ ui64 SpilledBytes = 0;
+ ui64 SpilledRows = 0;
+ ui64 SpilledBlobs = 0;
+
+ explicit TDqOutputChannelStats(ui64 channelId)
+ : ChannelId(channelId) {}
+};
+
class IDqOutputChannel : public IDqOutput {
-public:
- using TPtr = TIntrusivePtr<IDqOutputChannel>;
-
- virtual ui64 GetChannelId() const = 0;
- virtual ui64 GetValuesCount(bool inMemoryOnly = true) const = 0;
-
- // <| consumer methods
+public:
+ using TPtr = TIntrusivePtr<IDqOutputChannel>;
+
+ virtual ui64 GetChannelId() const = 0;
+ virtual ui64 GetValuesCount(bool inMemoryOnly = true) const = 0;
+
+ // <| consumer methods
// can throw TDqChannelStorageException
- [[nodiscard]]
- virtual bool Pop(NDqProto::TData& data, ui64 bytes) = 0;
+ [[nodiscard]]
+ virtual bool Pop(NDqProto::TData& data, ui64 bytes) = 0;
// Pop chechpoint. Checkpoints may be taken from channel even after it is finished.
[[nodiscard]]
- virtual bool Pop(NDqProto::TCheckpoint& checkpoint) = 0;
- // Only for data-queries
- // TODO: remove this method and create independent Data- and Stream-query implementations.
- // Stream-query implementation should be without PopAll method.
- // Data-query implementation should be one-shot for Pop (a-la PopAll) call and without ChannelStorage.
+ virtual bool Pop(NDqProto::TCheckpoint& checkpoint) = 0;
+ // Only for data-queries
+ // TODO: remove this method and create independent Data- and Stream-query implementations.
+ // Stream-query implementation should be without PopAll method.
+ // Data-query implementation should be one-shot for Pop (a-la PopAll) call and without ChannelStorage.
// can throw TDqChannelStorageException
- [[nodiscard]]
- virtual bool PopAll(NDqProto::TData& data) = 0;
- virtual bool PopAll(NKikimr::NMiniKQL::TUnboxedValueVector& rows) = 0;
- // |>
-
- virtual ui64 Drop() = 0;
-
- virtual const TDqOutputChannelStats* GetStats() const = 0;
- virtual void Terminate() = 0;
-};
-
-struct TDqOutputChannelSettings {
- ui64 MaxStoredBytes = 8_MB;
- ui64 MaxChunkBytes = 2_MB;
- NDqProto::EDataTransportVersion TransportVersion = NDqProto::EDataTransportVersion::DATA_TRANSPORT_UV_PICKLE_1_0;
- IDqChannelStorage::TPtr ChannelStorage;
- bool CollectProfileStats = false;
- bool AllowGeneratorsInUnboxedValues = true;
-};
-
-IDqOutputChannel::TPtr CreateDqOutputChannel(ui64 channelId, NKikimr::NMiniKQL::TType* outputType,
- const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv, const NKikimr::NMiniKQL::THolderFactory& holderFactory,
- const TDqOutputChannelSettings& settings, const TLogFunc& logFunc = {});
-
-} // namespace NYql::NDq
+ [[nodiscard]]
+ virtual bool PopAll(NDqProto::TData& data) = 0;
+ virtual bool PopAll(NKikimr::NMiniKQL::TUnboxedValueVector& rows) = 0;
+ // |>
+
+ virtual ui64 Drop() = 0;
+
+ virtual const TDqOutputChannelStats* GetStats() const = 0;
+ virtual void Terminate() = 0;
+};
+
+struct TDqOutputChannelSettings {
+ ui64 MaxStoredBytes = 8_MB;
+ ui64 MaxChunkBytes = 2_MB;
+ NDqProto::EDataTransportVersion TransportVersion = NDqProto::EDataTransportVersion::DATA_TRANSPORT_UV_PICKLE_1_0;
+ IDqChannelStorage::TPtr ChannelStorage;
+ bool CollectProfileStats = false;
+ bool AllowGeneratorsInUnboxedValues = true;
+};
+
+IDqOutputChannel::TPtr CreateDqOutputChannel(ui64 channelId, NKikimr::NMiniKQL::TType* outputType,
+ const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv, const NKikimr::NMiniKQL::THolderFactory& holderFactory,
+ const TDqOutputChannelSettings& settings, const TLogFunc& logFunc = {});
+
+} // namespace NYql::NDq
diff --git a/ydb/library/yql/dq/runtime/dq_output_channel_ut.cpp b/ydb/library/yql/dq/runtime/dq_output_channel_ut.cpp
index fe4b92ac74..2c3c1ad100 100644
--- a/ydb/library/yql/dq/runtime/dq_output_channel_ut.cpp
+++ b/ydb/library/yql/dq/runtime/dq_output_channel_ut.cpp
@@ -1,548 +1,548 @@
#include <ydb/library/yql/dq/runtime/dq_output_channel.h>
#include <ydb/library/yql/dq/runtime/dq_transport.h>
#include <ydb/library/yql/dq/runtime/ut/ut_helper.h>
-
+
#include <ydb/library/yql/public/udf/udf_value.h>
#include <ydb/library/yql/minikql/computation/mkql_value_builder.h>
#include <ydb/library/yql/minikql/mkql_string_util.h>
-
+
#include <library/cpp/testing/unittest/registar.h>
-
-using namespace NActors;
-using namespace NKikimr;
-using namespace NKikimr::NMiniKQL;
-using namespace NYql;
-using namespace NYql::NDq;
-
-namespace {
-
-// #define DEBUG_LOGS
-
-void Log(TStringBuf msg) {
-#ifdef DEBUG_LOGS
- Cerr << msg << Endl;
-#else
- Y_UNUSED(msg);
-#endif
-}
-
-struct TTestContext {
- TScopedAlloc Alloc;
- TTypeEnvironment TypeEnv;
- TMemoryUsageInfo MemInfo;
- THolderFactory HolderFactory;
- TDefaultValueBuilder Vb;
+
+using namespace NActors;
+using namespace NKikimr;
+using namespace NKikimr::NMiniKQL;
+using namespace NYql;
+using namespace NYql::NDq;
+
+namespace {
+
+// #define DEBUG_LOGS
+
+void Log(TStringBuf msg) {
+#ifdef DEBUG_LOGS
+ Cerr << msg << Endl;
+#else
+ Y_UNUSED(msg);
+#endif
+}
+
+struct TTestContext {
+ TScopedAlloc Alloc;
+ TTypeEnvironment TypeEnv;
+ TMemoryUsageInfo MemInfo;
+ THolderFactory HolderFactory;
+ TDefaultValueBuilder Vb;
NDqProto::EDataTransportVersion TransportVersion;
- TDqDataSerializer Ds;
- TStructType* OutputType = nullptr;
-
- TTestContext(NDqProto::EDataTransportVersion transportVersion = NDqProto::DATA_TRANSPORT_UV_PICKLE_1_0, bool bigRows = false)
- : Alloc()
- , TypeEnv(Alloc)
- , MemInfo("Mem")
- , HolderFactory(Alloc.Ref(), MemInfo)
- , Vb(HolderFactory)
+ TDqDataSerializer Ds;
+ TStructType* OutputType = nullptr;
+
+ TTestContext(NDqProto::EDataTransportVersion transportVersion = NDqProto::DATA_TRANSPORT_UV_PICKLE_1_0, bool bigRows = false)
+ : Alloc()
+ , TypeEnv(Alloc)
+ , MemInfo("Mem")
+ , HolderFactory(Alloc.Ref(), MemInfo)
+ , Vb(HolderFactory)
, TransportVersion(transportVersion)
, Ds(TypeEnv, HolderFactory, TransportVersion)
- {
- if (bigRows) {
- TStructMember members[3] = {
- {"x", TDataType::Create(NUdf::TDataType<i32>::Id, TypeEnv)},
- {"y", TDataType::Create(NUdf::TDataType<ui64>::Id, TypeEnv)},
- {"z", TDataType::Create(NUdf::TDataType<char*>::Id, TypeEnv)}
- };
- OutputType = TStructType::Create(3, members, TypeEnv);
- } else {
- TStructMember members[2] = {
- {"x", TDataType::Create(NUdf::TDataType<i32>::Id, TypeEnv)},
- {"y", TDataType::Create(NUdf::TDataType<ui64>::Id, TypeEnv)}
- };
- OutputType = TStructType::Create(2, members, TypeEnv);
- }
- }
-
- NUdf::TUnboxedValue CreateRow(ui32 value) {
- NUdf::TUnboxedValue* items;
- auto row = Vb.NewArray(OutputType->GetMembersCount(), items);
- items[0] = NUdf::TUnboxedValuePod(value);
- items[1] = NUdf::TUnboxedValuePod((ui64) (value * value));
- if (OutputType->GetMembersCount() == 3) {
- items[2] = NMiniKQL::MakeString("***");
- }
- return row;
- }
-
- NUdf::TUnboxedValue CreateBigRow(ui32 value, ui32 size) {
- NUdf::TUnboxedValue* items;
- auto row = Vb.NewArray(OutputType->GetMembersCount(), items);
- items[0] = NUdf::TUnboxedValuePod(value);
- items[1] = NUdf::TUnboxedValuePod((ui64) (value * value));
- if (OutputType->GetMembersCount() == 3) {
- items[2] = NMiniKQL::MakeString(std::string(size, '*'));
- }
- return row;
- }
-};
-
-void TestSingleRead(TTestContext& ctx, bool quantum) {
- TDqOutputChannelSettings settings;
- settings.MaxStoredBytes = 1000;
- settings.MaxChunkBytes = 200;
- settings.CollectProfileStats = true;
- settings.TransportVersion = ctx.TransportVersion;
- settings.AllowGeneratorsInUnboxedValues = quantum;
-
- auto ch = CreateDqOutputChannel(1, ctx.OutputType, ctx.TypeEnv, ctx.HolderFactory, settings, Log);
-
- for (i32 i = 0; i < 10; ++i) {
- auto row = ctx.CreateRow(i);
- UNIT_ASSERT(!ch->IsFull());
- ch->Push(std::move(row));
- }
-
- UNIT_ASSERT_VALUES_EQUAL(0, ch->GetStats()->Chunks);
- UNIT_ASSERT_VALUES_EQUAL(10, ch->GetStats()->RowsIn);
- UNIT_ASSERT_VALUES_EQUAL(0, ch->GetStats()->RowsOut);
-
- NDqProto::TData data;
- UNIT_ASSERT(ch->Pop(data, 1000));
-
- UNIT_ASSERT_VALUES_EQUAL(10, data.GetRows());
- UNIT_ASSERT_VALUES_EQUAL(1, ch->GetStats()->Chunks);
- UNIT_ASSERT_VALUES_EQUAL(10, ch->GetStats()->RowsIn);
- UNIT_ASSERT_VALUES_EQUAL(10, ch->GetStats()->RowsOut);
-
- TUnboxedValueVector buffer;
- ctx.Ds.Deserialize(data, ctx.OutputType, buffer);
-
- UNIT_ASSERT_VALUES_EQUAL(10, buffer.size());
- for (i32 i = 0; i < 10; ++i) {
- UNIT_ASSERT_VALUES_EQUAL(i, buffer[i].GetElement(0).Get<i32>());
- UNIT_ASSERT_VALUES_EQUAL(i * i, buffer[i].GetElement(1).Get<ui64>());
- }
-
- data.Clear();
- UNIT_ASSERT(!ch->Pop(data, 1000));
-}
-
-void TestPartialRead(TTestContext& ctx, bool quantum) {
- TDqOutputChannelSettings settings;
- settings.MaxStoredBytes = 1000;
- settings.MaxChunkBytes = 100;
- settings.CollectProfileStats = true;
- settings.TransportVersion = ctx.TransportVersion;
- settings.AllowGeneratorsInUnboxedValues = quantum;
-
- auto ch = CreateDqOutputChannel(1, ctx.OutputType, ctx.TypeEnv, ctx.HolderFactory, settings, Log);
-
- for (i32 i = 0; i < 9; ++i) {
- auto row = ctx.CreateRow(i);
- UNIT_ASSERT(!ch->IsFull());
- ch->Push(std::move(row));
- }
-
- UNIT_ASSERT_VALUES_EQUAL(0, ch->GetStats()->Chunks);
- UNIT_ASSERT_VALUES_EQUAL(9, ch->GetStats()->RowsIn);
- UNIT_ASSERT_VALUES_EQUAL(0, ch->GetStats()->RowsOut);
-
- int req = 0;
- ui32 expected[] = {3, 3, 3};
- ui32 expectedQ[] = {3, 5, 1};
-
- ui32 readChunks = 0;
- ui32 readRows = 0;
- while (readRows < 9) {
- NDqProto::TData data;
- UNIT_ASSERT(ch->Pop(data, 50));
-
- ui32 v = quantum ? expectedQ[req] : expected[req];
- ++req;
-
- UNIT_ASSERT_VALUES_EQUAL(v, data.GetRows());
- UNIT_ASSERT_VALUES_EQUAL(++readChunks, ch->GetStats()->Chunks);
- UNIT_ASSERT_VALUES_EQUAL(9, ch->GetStats()->RowsIn);
- UNIT_ASSERT_VALUES_EQUAL(readRows + data.GetRows(), ch->GetStats()->RowsOut);
-
- TUnboxedValueVector buffer;
- ctx.Ds.Deserialize(data, ctx.OutputType, buffer);
-
- UNIT_ASSERT_VALUES_EQUAL(data.GetRows(), buffer.size());
- for (ui32 i = 0; i < data.GetRows(); ++i) {
- ui32 j = readRows + i;
- UNIT_ASSERT_VALUES_EQUAL(j, buffer[i].GetElement(0).Get<i32>());
- UNIT_ASSERT_VALUES_EQUAL(j * j, buffer[i].GetElement(1).Get<ui64>());
- }
-
- readRows += data.GetRows();
- }
-
- NDqProto::TData data;
- UNIT_ASSERT(!ch->Pop(data, 1000));
-}
-
-void TestOverflow(TTestContext& ctx, bool quantum) {
- TDqOutputChannelSettings settings;
- settings.MaxStoredBytes = 100;
- settings.MaxChunkBytes = 10;
- settings.CollectProfileStats = true;
- settings.TransportVersion = ctx.TransportVersion;
- settings.AllowGeneratorsInUnboxedValues = quantum;
-
- auto ch = CreateDqOutputChannel(1, ctx.OutputType, ctx.TypeEnv, ctx.HolderFactory, settings, Log);
-
- for (i32 i = 0; i < 8; ++i) {
- auto row = ctx.CreateRow(i);
- UNIT_ASSERT(!ch->IsFull());
- ch->Push(std::move(row));
- }
-
- UNIT_ASSERT_VALUES_EQUAL(8, ch->GetStats()->RowsIn);
- UNIT_ASSERT_VALUES_EQUAL(0, ch->GetStats()->RowsOut);
-
- UNIT_ASSERT(ch->IsFull());
- try {
- auto row = ctx.CreateRow(100'500);
- ch->Push(std::move(row));
- UNIT_FAIL("");
- } catch (yexception& e) {
- UNIT_ASSERT(TString(e.what()).Contains("requirement !IsFull() failed"));
- }
-}
-
-void TestPopAll(TTestContext& ctx, bool quantum) {
- TDqOutputChannelSettings settings;
- settings.MaxStoredBytes = 1000;
- settings.MaxChunkBytes = 10;
- settings.CollectProfileStats = true;
- settings.TransportVersion = ctx.TransportVersion;
- settings.AllowGeneratorsInUnboxedValues = quantum;
-
- auto ch = CreateDqOutputChannel(1, ctx.OutputType, ctx.TypeEnv, ctx.HolderFactory, settings, Log);
-
- for (i32 i = 0; i < 50; ++i) {
- auto row = ctx.CreateRow(i);
- UNIT_ASSERT(!ch->IsFull());
- ch->Push(std::move(row));
- }
-
- UNIT_ASSERT_VALUES_EQUAL(50, ch->GetStats()->RowsIn);
- UNIT_ASSERT_VALUES_EQUAL(0, ch->GetStats()->RowsOut);
-
- NDqProto::TData data;
- TUnboxedValueVector buffer;
-
- UNIT_ASSERT(ch->PopAll(data));
-
- UNIT_ASSERT_VALUES_EQUAL(50, data.GetRows());
-
- ctx.Ds.Deserialize(data, ctx.OutputType, buffer);
- UNIT_ASSERT_VALUES_EQUAL(50, buffer.size());
-
- for (i32 i = 0; i < 50; ++i) {
- UNIT_ASSERT_VALUES_EQUAL(i, buffer[i].GetElement(0).Get<i32>());
- UNIT_ASSERT_VALUES_EQUAL(i * i, buffer[i].GetElement(1).Get<ui64>());
- }
-
- data.Clear();
- UNIT_ASSERT(!ch->Pop(data, 100'500));
-}
-
-void TestBigRow(TTestContext& ctx, bool quantum) {
- TDqOutputChannelSettings settings;
- settings.MaxStoredBytes = std::numeric_limits<ui32>::max();
- settings.MaxChunkBytes = 2_MB;
- settings.CollectProfileStats = true;
- settings.TransportVersion = ctx.TransportVersion;
- settings.AllowGeneratorsInUnboxedValues = quantum;
-
- auto ch = CreateDqOutputChannel(1, ctx.OutputType, ctx.TypeEnv, ctx.HolderFactory, settings, Log);
-
- {
- auto row = ctx.CreateRow(1);
- UNIT_ASSERT(!ch->IsFull());
- ch->Push(std::move(row));
- }
- {
- for (ui32 i = 2; i < 10; ++i) {
- auto row = ctx.CreateBigRow(i, 10_MB);
- UNIT_ASSERT(!ch->IsFull());
- ch->Push(std::move(row));
- }
- }
-
- UNIT_ASSERT_VALUES_EQUAL(0, ch->GetStats()->Chunks);
- UNIT_ASSERT_VALUES_EQUAL(9, ch->GetStats()->RowsIn);
- UNIT_ASSERT_VALUES_EQUAL(0, ch->GetStats()->RowsOut);
-
- for (ui32 i = 1; i < 10; ++i) {
- NDqProto::TData data;
- UNIT_ASSERT(ch->Pop(data, 1_MB));
-
- UNIT_ASSERT_VALUES_EQUAL(1, data.GetRows());
- UNIT_ASSERT_VALUES_EQUAL(i, ch->GetStats()->Chunks);
- UNIT_ASSERT_VALUES_EQUAL(9, ch->GetStats()->RowsIn);
- UNIT_ASSERT_VALUES_EQUAL(i, ch->GetStats()->RowsOut);
-
- TUnboxedValueVector buffer;
- ctx.Ds.Deserialize(data, ctx.OutputType, buffer);
-
- UNIT_ASSERT_VALUES_EQUAL(1, buffer.size());
- UNIT_ASSERT_VALUES_EQUAL(i, buffer[0].GetElement(0).Get<i32>());
- UNIT_ASSERT_VALUES_EQUAL(i * i, buffer[0].GetElement(1).Get<ui64>());
- }
-
- NDqProto::TData data;
- UNIT_ASSERT(!ch->Pop(data, 10_MB));
-}
-
-
+ {
+ if (bigRows) {
+ TStructMember members[3] = {
+ {"x", TDataType::Create(NUdf::TDataType<i32>::Id, TypeEnv)},
+ {"y", TDataType::Create(NUdf::TDataType<ui64>::Id, TypeEnv)},
+ {"z", TDataType::Create(NUdf::TDataType<char*>::Id, TypeEnv)}
+ };
+ OutputType = TStructType::Create(3, members, TypeEnv);
+ } else {
+ TStructMember members[2] = {
+ {"x", TDataType::Create(NUdf::TDataType<i32>::Id, TypeEnv)},
+ {"y", TDataType::Create(NUdf::TDataType<ui64>::Id, TypeEnv)}
+ };
+ OutputType = TStructType::Create(2, members, TypeEnv);
+ }
+ }
+
+ NUdf::TUnboxedValue CreateRow(ui32 value) {
+ NUdf::TUnboxedValue* items;
+ auto row = Vb.NewArray(OutputType->GetMembersCount(), items);
+ items[0] = NUdf::TUnboxedValuePod(value);
+ items[1] = NUdf::TUnboxedValuePod((ui64) (value * value));
+ if (OutputType->GetMembersCount() == 3) {
+ items[2] = NMiniKQL::MakeString("***");
+ }
+ return row;
+ }
+
+ NUdf::TUnboxedValue CreateBigRow(ui32 value, ui32 size) {
+ NUdf::TUnboxedValue* items;
+ auto row = Vb.NewArray(OutputType->GetMembersCount(), items);
+ items[0] = NUdf::TUnboxedValuePod(value);
+ items[1] = NUdf::TUnboxedValuePod((ui64) (value * value));
+ if (OutputType->GetMembersCount() == 3) {
+ items[2] = NMiniKQL::MakeString(std::string(size, '*'));
+ }
+ return row;
+ }
+};
+
+void TestSingleRead(TTestContext& ctx, bool quantum) {
+ TDqOutputChannelSettings settings;
+ settings.MaxStoredBytes = 1000;
+ settings.MaxChunkBytes = 200;
+ settings.CollectProfileStats = true;
+ settings.TransportVersion = ctx.TransportVersion;
+ settings.AllowGeneratorsInUnboxedValues = quantum;
+
+ auto ch = CreateDqOutputChannel(1, ctx.OutputType, ctx.TypeEnv, ctx.HolderFactory, settings, Log);
+
+ for (i32 i = 0; i < 10; ++i) {
+ auto row = ctx.CreateRow(i);
+ UNIT_ASSERT(!ch->IsFull());
+ ch->Push(std::move(row));
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(0, ch->GetStats()->Chunks);
+ UNIT_ASSERT_VALUES_EQUAL(10, ch->GetStats()->RowsIn);
+ UNIT_ASSERT_VALUES_EQUAL(0, ch->GetStats()->RowsOut);
+
+ NDqProto::TData data;
+ UNIT_ASSERT(ch->Pop(data, 1000));
+
+ UNIT_ASSERT_VALUES_EQUAL(10, data.GetRows());
+ UNIT_ASSERT_VALUES_EQUAL(1, ch->GetStats()->Chunks);
+ UNIT_ASSERT_VALUES_EQUAL(10, ch->GetStats()->RowsIn);
+ UNIT_ASSERT_VALUES_EQUAL(10, ch->GetStats()->RowsOut);
+
+ TUnboxedValueVector buffer;
+ ctx.Ds.Deserialize(data, ctx.OutputType, buffer);
+
+ UNIT_ASSERT_VALUES_EQUAL(10, buffer.size());
+ for (i32 i = 0; i < 10; ++i) {
+ UNIT_ASSERT_VALUES_EQUAL(i, buffer[i].GetElement(0).Get<i32>());
+ UNIT_ASSERT_VALUES_EQUAL(i * i, buffer[i].GetElement(1).Get<ui64>());
+ }
+
+ data.Clear();
+ UNIT_ASSERT(!ch->Pop(data, 1000));
+}
+
+void TestPartialRead(TTestContext& ctx, bool quantum) {
+ TDqOutputChannelSettings settings;
+ settings.MaxStoredBytes = 1000;
+ settings.MaxChunkBytes = 100;
+ settings.CollectProfileStats = true;
+ settings.TransportVersion = ctx.TransportVersion;
+ settings.AllowGeneratorsInUnboxedValues = quantum;
+
+ auto ch = CreateDqOutputChannel(1, ctx.OutputType, ctx.TypeEnv, ctx.HolderFactory, settings, Log);
+
+ for (i32 i = 0; i < 9; ++i) {
+ auto row = ctx.CreateRow(i);
+ UNIT_ASSERT(!ch->IsFull());
+ ch->Push(std::move(row));
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(0, ch->GetStats()->Chunks);
+ UNIT_ASSERT_VALUES_EQUAL(9, ch->GetStats()->RowsIn);
+ UNIT_ASSERT_VALUES_EQUAL(0, ch->GetStats()->RowsOut);
+
+ int req = 0;
+ ui32 expected[] = {3, 3, 3};
+ ui32 expectedQ[] = {3, 5, 1};
+
+ ui32 readChunks = 0;
+ ui32 readRows = 0;
+ while (readRows < 9) {
+ NDqProto::TData data;
+ UNIT_ASSERT(ch->Pop(data, 50));
+
+ ui32 v = quantum ? expectedQ[req] : expected[req];
+ ++req;
+
+ UNIT_ASSERT_VALUES_EQUAL(v, data.GetRows());
+ UNIT_ASSERT_VALUES_EQUAL(++readChunks, ch->GetStats()->Chunks);
+ UNIT_ASSERT_VALUES_EQUAL(9, ch->GetStats()->RowsIn);
+ UNIT_ASSERT_VALUES_EQUAL(readRows + data.GetRows(), ch->GetStats()->RowsOut);
+
+ TUnboxedValueVector buffer;
+ ctx.Ds.Deserialize(data, ctx.OutputType, buffer);
+
+ UNIT_ASSERT_VALUES_EQUAL(data.GetRows(), buffer.size());
+ for (ui32 i = 0; i < data.GetRows(); ++i) {
+ ui32 j = readRows + i;
+ UNIT_ASSERT_VALUES_EQUAL(j, buffer[i].GetElement(0).Get<i32>());
+ UNIT_ASSERT_VALUES_EQUAL(j * j, buffer[i].GetElement(1).Get<ui64>());
+ }
+
+ readRows += data.GetRows();
+ }
+
+ NDqProto::TData data;
+ UNIT_ASSERT(!ch->Pop(data, 1000));
+}
+
+void TestOverflow(TTestContext& ctx, bool quantum) {
+ TDqOutputChannelSettings settings;
+ settings.MaxStoredBytes = 100;
+ settings.MaxChunkBytes = 10;
+ settings.CollectProfileStats = true;
+ settings.TransportVersion = ctx.TransportVersion;
+ settings.AllowGeneratorsInUnboxedValues = quantum;
+
+ auto ch = CreateDqOutputChannel(1, ctx.OutputType, ctx.TypeEnv, ctx.HolderFactory, settings, Log);
+
+ for (i32 i = 0; i < 8; ++i) {
+ auto row = ctx.CreateRow(i);
+ UNIT_ASSERT(!ch->IsFull());
+ ch->Push(std::move(row));
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(8, ch->GetStats()->RowsIn);
+ UNIT_ASSERT_VALUES_EQUAL(0, ch->GetStats()->RowsOut);
+
+ UNIT_ASSERT(ch->IsFull());
+ try {
+ auto row = ctx.CreateRow(100'500);
+ ch->Push(std::move(row));
+ UNIT_FAIL("");
+ } catch (yexception& e) {
+ UNIT_ASSERT(TString(e.what()).Contains("requirement !IsFull() failed"));
+ }
+}
+
+void TestPopAll(TTestContext& ctx, bool quantum) {
+ TDqOutputChannelSettings settings;
+ settings.MaxStoredBytes = 1000;
+ settings.MaxChunkBytes = 10;
+ settings.CollectProfileStats = true;
+ settings.TransportVersion = ctx.TransportVersion;
+ settings.AllowGeneratorsInUnboxedValues = quantum;
+
+ auto ch = CreateDqOutputChannel(1, ctx.OutputType, ctx.TypeEnv, ctx.HolderFactory, settings, Log);
+
+ for (i32 i = 0; i < 50; ++i) {
+ auto row = ctx.CreateRow(i);
+ UNIT_ASSERT(!ch->IsFull());
+ ch->Push(std::move(row));
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(50, ch->GetStats()->RowsIn);
+ UNIT_ASSERT_VALUES_EQUAL(0, ch->GetStats()->RowsOut);
+
+ NDqProto::TData data;
+ TUnboxedValueVector buffer;
+
+ UNIT_ASSERT(ch->PopAll(data));
+
+ UNIT_ASSERT_VALUES_EQUAL(50, data.GetRows());
+
+ ctx.Ds.Deserialize(data, ctx.OutputType, buffer);
+ UNIT_ASSERT_VALUES_EQUAL(50, buffer.size());
+
+ for (i32 i = 0; i < 50; ++i) {
+ UNIT_ASSERT_VALUES_EQUAL(i, buffer[i].GetElement(0).Get<i32>());
+ UNIT_ASSERT_VALUES_EQUAL(i * i, buffer[i].GetElement(1).Get<ui64>());
+ }
+
+ data.Clear();
+ UNIT_ASSERT(!ch->Pop(data, 100'500));
+}
+
+void TestBigRow(TTestContext& ctx, bool quantum) {
+ TDqOutputChannelSettings settings;
+ settings.MaxStoredBytes = std::numeric_limits<ui32>::max();
+ settings.MaxChunkBytes = 2_MB;
+ settings.CollectProfileStats = true;
+ settings.TransportVersion = ctx.TransportVersion;
+ settings.AllowGeneratorsInUnboxedValues = quantum;
+
+ auto ch = CreateDqOutputChannel(1, ctx.OutputType, ctx.TypeEnv, ctx.HolderFactory, settings, Log);
+
+ {
+ auto row = ctx.CreateRow(1);
+ UNIT_ASSERT(!ch->IsFull());
+ ch->Push(std::move(row));
+ }
+ {
+ for (ui32 i = 2; i < 10; ++i) {
+ auto row = ctx.CreateBigRow(i, 10_MB);
+ UNIT_ASSERT(!ch->IsFull());
+ ch->Push(std::move(row));
+ }
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(0, ch->GetStats()->Chunks);
+ UNIT_ASSERT_VALUES_EQUAL(9, ch->GetStats()->RowsIn);
+ UNIT_ASSERT_VALUES_EQUAL(0, ch->GetStats()->RowsOut);
+
+ for (ui32 i = 1; i < 10; ++i) {
+ NDqProto::TData data;
+ UNIT_ASSERT(ch->Pop(data, 1_MB));
+
+ UNIT_ASSERT_VALUES_EQUAL(1, data.GetRows());
+ UNIT_ASSERT_VALUES_EQUAL(i, ch->GetStats()->Chunks);
+ UNIT_ASSERT_VALUES_EQUAL(9, ch->GetStats()->RowsIn);
+ UNIT_ASSERT_VALUES_EQUAL(i, ch->GetStats()->RowsOut);
+
+ TUnboxedValueVector buffer;
+ ctx.Ds.Deserialize(data, ctx.OutputType, buffer);
+
+ UNIT_ASSERT_VALUES_EQUAL(1, buffer.size());
+ UNIT_ASSERT_VALUES_EQUAL(i, buffer[0].GetElement(0).Get<i32>());
+ UNIT_ASSERT_VALUES_EQUAL(i * i, buffer[0].GetElement(1).Get<ui64>());
+ }
+
+ NDqProto::TData data;
+ UNIT_ASSERT(!ch->Pop(data, 10_MB));
+}
+
+
void TestSpillWithMockStorage(TTestContext& ctx) {
- TDqOutputChannelSettings settings;
- settings.MaxStoredBytes = 100;
- settings.MaxChunkBytes = 10;
- settings.CollectProfileStats = true;
- settings.TransportVersion = ctx.TransportVersion;
- settings.AllowGeneratorsInUnboxedValues = false;
-
- auto storage = MakeIntrusive<TMockChannelStorage>(100'500ul);
- settings.ChannelStorage = storage;
-
- auto ch = CreateDqOutputChannel(1, ctx.OutputType, ctx.TypeEnv, ctx.HolderFactory, settings, Log);
-
- for (i32 i = 0; i < 35; ++i) {
- auto row = ctx.CreateRow(i);
- UNIT_ASSERT(!ch->IsFull());
- ch->Push(std::move(row));
- }
-
- UNIT_ASSERT_VALUES_EQUAL(7, ch->GetValuesCount(/* inMemoryOnly */ true));
- UNIT_ASSERT_VALUES_EQUAL(35, ch->GetValuesCount(/* inMemoryOnly */ false));
-
- UNIT_ASSERT_VALUES_EQUAL(35, ch->GetStats()->RowsIn);
- UNIT_ASSERT_VALUES_EQUAL(0, ch->GetStats()->RowsOut);
- UNIT_ASSERT_VALUES_EQUAL(35 - 7, ch->GetStats()->SpilledRows);
- UNIT_ASSERT_VALUES_EQUAL(35 - 7, ch->GetStats()->SpilledBlobs);
- UNIT_ASSERT(ch->GetStats()->SpilledBytes > 200);
-
- ui32 loadedRows = 0;
- storage->SetBlankGetRequests(2);
-
- {
- Cerr << "-- pop rows before spilled ones\n";
- NDqProto::TData data;
- while (ch->Pop(data, 1000)) {
- TUnboxedValueVector buffer;
- ctx.Ds.Deserialize(data, ctx.OutputType, buffer);
-
- UNIT_ASSERT_VALUES_EQUAL(data.GetRows(), buffer.size());
- for (ui32 i = 0; i < data.GetRows(); ++i) {
- auto j = loadedRows + i;
- UNIT_ASSERT_VALUES_EQUAL(j, buffer[i].GetElement(0).Get<i32>());
- UNIT_ASSERT_VALUES_EQUAL(j * j, buffer[i].GetElement(1).Get<ui64>());
- }
-
- loadedRows += data.GetRows();
- }
- }
-
- UNIT_ASSERT_VALUES_EQUAL(7 - loadedRows, ch->GetValuesCount(/* inMemoryOnly */ true));
- UNIT_ASSERT_VALUES_EQUAL(35 - loadedRows, ch->GetValuesCount(/* inMemoryOnly */ false));
-
- // just blank request
- {
- NDqProto::TData data;
- UNIT_ASSERT(!ch->Pop(data, 1000));
-
- UNIT_ASSERT_VALUES_EQUAL(7 - loadedRows, ch->GetValuesCount(/* inMemoryOnly */ true));
- UNIT_ASSERT_VALUES_EQUAL(35 - loadedRows, ch->GetValuesCount(/* inMemoryOnly */ false));
- }
-
- while (loadedRows < 35) {
- NDqProto::TData data;
- while (ch->Pop(data, 10)) {
- storage->SetBlankGetRequests(1);
-
- TUnboxedValueVector buffer;
- ctx.Ds.Deserialize(data, ctx.OutputType, buffer);
-
- UNIT_ASSERT_VALUES_EQUAL(data.GetRows(), buffer.size());
- for (ui32 i = 0; i < data.GetRows(); ++i) {
- auto j = loadedRows + i;
- UNIT_ASSERT_VALUES_EQUAL(j, buffer[i].GetElement(0).Get<i32>());
- UNIT_ASSERT_VALUES_EQUAL(j * j, buffer[i].GetElement(1).Get<ui64>());
- }
-
- loadedRows += data.GetRows();
-
- UNIT_ASSERT_VALUES_EQUAL(35 - loadedRows, ch->GetValuesCount(/* inMemoryOnly */ false));
- }
- }
-
- UNIT_ASSERT_VALUES_EQUAL(35, loadedRows);
-
- // in memory only
- {
- storage->SetBlankGetRequests(0);
- loadedRows = 0;
-
- for (i32 i = 100; i < 105; ++i) {
- auto row = ctx.CreateRow(i);
- UNIT_ASSERT(!ch->IsFull());
- ch->Push(std::move(row));
- }
-
- UNIT_ASSERT_VALUES_EQUAL(5, ch->GetValuesCount(/* inMemoryOnly */ true));
- UNIT_ASSERT_VALUES_EQUAL(5, ch->GetValuesCount(/* inMemoryOnly */ false));
-
- NDqProto::TData data;
- while (ch->Pop(data, 1000)) {
- TUnboxedValueVector buffer;
- ctx.Ds.Deserialize(data, ctx.OutputType, buffer);
-
- UNIT_ASSERT_VALUES_EQUAL(data.GetRows(), buffer.size());
- for (ui32 i = 0; i < data.GetRows(); ++i) {
- auto j = 100 + loadedRows + i;
- UNIT_ASSERT_VALUES_EQUAL(j, buffer[i].GetElement(0).Get<i32>());
- UNIT_ASSERT_VALUES_EQUAL(j * j, buffer[i].GetElement(1).Get<ui64>());
- }
-
- loadedRows += data.GetRows();
- }
- }
-
- UNIT_ASSERT_VALUES_EQUAL(0, ch->GetValuesCount(/* inMemoryOnly */ true));
- UNIT_ASSERT_VALUES_EQUAL(0, ch->GetValuesCount(/* inMemoryOnly */ false));
-}
-
+ TDqOutputChannelSettings settings;
+ settings.MaxStoredBytes = 100;
+ settings.MaxChunkBytes = 10;
+ settings.CollectProfileStats = true;
+ settings.TransportVersion = ctx.TransportVersion;
+ settings.AllowGeneratorsInUnboxedValues = false;
+
+ auto storage = MakeIntrusive<TMockChannelStorage>(100'500ul);
+ settings.ChannelStorage = storage;
+
+ auto ch = CreateDqOutputChannel(1, ctx.OutputType, ctx.TypeEnv, ctx.HolderFactory, settings, Log);
+
+ for (i32 i = 0; i < 35; ++i) {
+ auto row = ctx.CreateRow(i);
+ UNIT_ASSERT(!ch->IsFull());
+ ch->Push(std::move(row));
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(7, ch->GetValuesCount(/* inMemoryOnly */ true));
+ UNIT_ASSERT_VALUES_EQUAL(35, ch->GetValuesCount(/* inMemoryOnly */ false));
+
+ UNIT_ASSERT_VALUES_EQUAL(35, ch->GetStats()->RowsIn);
+ UNIT_ASSERT_VALUES_EQUAL(0, ch->GetStats()->RowsOut);
+ UNIT_ASSERT_VALUES_EQUAL(35 - 7, ch->GetStats()->SpilledRows);
+ UNIT_ASSERT_VALUES_EQUAL(35 - 7, ch->GetStats()->SpilledBlobs);
+ UNIT_ASSERT(ch->GetStats()->SpilledBytes > 200);
+
+ ui32 loadedRows = 0;
+ storage->SetBlankGetRequests(2);
+
+ {
+ Cerr << "-- pop rows before spilled ones\n";
+ NDqProto::TData data;
+ while (ch->Pop(data, 1000)) {
+ TUnboxedValueVector buffer;
+ ctx.Ds.Deserialize(data, ctx.OutputType, buffer);
+
+ UNIT_ASSERT_VALUES_EQUAL(data.GetRows(), buffer.size());
+ for (ui32 i = 0; i < data.GetRows(); ++i) {
+ auto j = loadedRows + i;
+ UNIT_ASSERT_VALUES_EQUAL(j, buffer[i].GetElement(0).Get<i32>());
+ UNIT_ASSERT_VALUES_EQUAL(j * j, buffer[i].GetElement(1).Get<ui64>());
+ }
+
+ loadedRows += data.GetRows();
+ }
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(7 - loadedRows, ch->GetValuesCount(/* inMemoryOnly */ true));
+ UNIT_ASSERT_VALUES_EQUAL(35 - loadedRows, ch->GetValuesCount(/* inMemoryOnly */ false));
+
+ // just blank request
+ {
+ NDqProto::TData data;
+ UNIT_ASSERT(!ch->Pop(data, 1000));
+
+ UNIT_ASSERT_VALUES_EQUAL(7 - loadedRows, ch->GetValuesCount(/* inMemoryOnly */ true));
+ UNIT_ASSERT_VALUES_EQUAL(35 - loadedRows, ch->GetValuesCount(/* inMemoryOnly */ false));
+ }
+
+ while (loadedRows < 35) {
+ NDqProto::TData data;
+ while (ch->Pop(data, 10)) {
+ storage->SetBlankGetRequests(1);
+
+ TUnboxedValueVector buffer;
+ ctx.Ds.Deserialize(data, ctx.OutputType, buffer);
+
+ UNIT_ASSERT_VALUES_EQUAL(data.GetRows(), buffer.size());
+ for (ui32 i = 0; i < data.GetRows(); ++i) {
+ auto j = loadedRows + i;
+ UNIT_ASSERT_VALUES_EQUAL(j, buffer[i].GetElement(0).Get<i32>());
+ UNIT_ASSERT_VALUES_EQUAL(j * j, buffer[i].GetElement(1).Get<ui64>());
+ }
+
+ loadedRows += data.GetRows();
+
+ UNIT_ASSERT_VALUES_EQUAL(35 - loadedRows, ch->GetValuesCount(/* inMemoryOnly */ false));
+ }
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(35, loadedRows);
+
+ // in memory only
+ {
+ storage->SetBlankGetRequests(0);
+ loadedRows = 0;
+
+ for (i32 i = 100; i < 105; ++i) {
+ auto row = ctx.CreateRow(i);
+ UNIT_ASSERT(!ch->IsFull());
+ ch->Push(std::move(row));
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(5, ch->GetValuesCount(/* inMemoryOnly */ true));
+ UNIT_ASSERT_VALUES_EQUAL(5, ch->GetValuesCount(/* inMemoryOnly */ false));
+
+ NDqProto::TData data;
+ while (ch->Pop(data, 1000)) {
+ TUnboxedValueVector buffer;
+ ctx.Ds.Deserialize(data, ctx.OutputType, buffer);
+
+ UNIT_ASSERT_VALUES_EQUAL(data.GetRows(), buffer.size());
+ for (ui32 i = 0; i < data.GetRows(); ++i) {
+ auto j = 100 + loadedRows + i;
+ UNIT_ASSERT_VALUES_EQUAL(j, buffer[i].GetElement(0).Get<i32>());
+ UNIT_ASSERT_VALUES_EQUAL(j * j, buffer[i].GetElement(1).Get<ui64>());
+ }
+
+ loadedRows += data.GetRows();
+ }
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(0, ch->GetValuesCount(/* inMemoryOnly */ true));
+ UNIT_ASSERT_VALUES_EQUAL(0, ch->GetValuesCount(/* inMemoryOnly */ false));
+}
+
void TestOverflowWithMockStorage(TTestContext& ctx) {
- TDqOutputChannelSettings settings;
- settings.MaxStoredBytes = 100;
- settings.MaxChunkBytes = 10;
- settings.CollectProfileStats = true;
- settings.TransportVersion = ctx.TransportVersion;
- settings.AllowGeneratorsInUnboxedValues = false;
-
- auto storage = MakeIntrusive<TMockChannelStorage>(500ul);
- settings.ChannelStorage = storage;
-
- auto ch = CreateDqOutputChannel(1, ctx.OutputType, ctx.TypeEnv, ctx.HolderFactory, settings, Log);
-
- for (i32 i = 0; i < 42; ++i) {
- auto row = ctx.CreateRow(i);
- UNIT_ASSERT(!ch->IsFull());
- ch->Push(std::move(row));
- }
-
- UNIT_ASSERT_VALUES_EQUAL(42, ch->GetStats()->RowsIn);
- UNIT_ASSERT_VALUES_EQUAL(0, ch->GetStats()->RowsOut);
-
- // UNIT_ASSERT(ch->IsFull()); it can be false-negative with storage enabled
- try {
- auto row = ctx.CreateRow(100'500);
- ch->Push(std::move(row));
- UNIT_FAIL("");
- } catch (yexception& e) {
- UNIT_ASSERT(TString(e.what()).Contains("Space limit exceeded"));
- }
-}
-
+ TDqOutputChannelSettings settings;
+ settings.MaxStoredBytes = 100;
+ settings.MaxChunkBytes = 10;
+ settings.CollectProfileStats = true;
+ settings.TransportVersion = ctx.TransportVersion;
+ settings.AllowGeneratorsInUnboxedValues = false;
+
+ auto storage = MakeIntrusive<TMockChannelStorage>(500ul);
+ settings.ChannelStorage = storage;
+
+ auto ch = CreateDqOutputChannel(1, ctx.OutputType, ctx.TypeEnv, ctx.HolderFactory, settings, Log);
+
+ for (i32 i = 0; i < 42; ++i) {
+ auto row = ctx.CreateRow(i);
+ UNIT_ASSERT(!ch->IsFull());
+ ch->Push(std::move(row));
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(42, ch->GetStats()->RowsIn);
+ UNIT_ASSERT_VALUES_EQUAL(0, ch->GetStats()->RowsOut);
+
+ // UNIT_ASSERT(ch->IsFull()); it can be false-negative with storage enabled
+ try {
+ auto row = ctx.CreateRow(100'500);
+ ch->Push(std::move(row));
+ UNIT_FAIL("");
+ } catch (yexception& e) {
+ UNIT_ASSERT(TString(e.what()).Contains("Space limit exceeded"));
+ }
+}
+
} // anonymous namespace
Y_UNIT_TEST_SUITE(DqOutputChannelNoStorageTests) {
Y_UNIT_TEST(SingleRead) {
TTestContext ctx;
- TestSingleRead(ctx, false);
-}
-
-Y_UNIT_TEST(SingleReadQ) {
- TTestContext ctx;
- TestSingleRead(ctx, true);
-}
-
+ TestSingleRead(ctx, false);
+}
+
+Y_UNIT_TEST(SingleReadQ) {
+ TTestContext ctx;
+ TestSingleRead(ctx, true);
+}
+
Y_UNIT_TEST(SingleReadWithArrow) {
TTestContext ctx(NDqProto::DATA_TRANSPORT_ARROW_1_0);
- TestSingleRead(ctx, false);
+ TestSingleRead(ctx, false);
+}
+
+Y_UNIT_TEST(SingleReadWithArrowQ) {
+ TTestContext ctx(NDqProto::DATA_TRANSPORT_ARROW_1_0);
+ TestSingleRead(ctx, true);
}
-Y_UNIT_TEST(SingleReadWithArrowQ) {
- TTestContext ctx(NDqProto::DATA_TRANSPORT_ARROW_1_0);
- TestSingleRead(ctx, true);
-}
-
Y_UNIT_TEST(PartialRead) {
TTestContext ctx;
- TestPartialRead(ctx, false);
+ TestPartialRead(ctx, false);
}
-Y_UNIT_TEST(PartialReadQ) {
- TTestContext ctx;
- TestPartialRead(ctx, true);
+Y_UNIT_TEST(PartialReadQ) {
+ TTestContext ctx;
+ TestPartialRead(ctx, true);
}
-// too heavy messages...
-//Y_UNIT_TEST(PartialReadWithArrow) {
-// TTestContext ctx(NDqProto::DATA_TRANSPORT_ARROW_1_0);
-// TestPartialRead(ctx, false);
-//}
-//
-//Y_UNIT_TEST(PartialReadWithArrowQ) {
-// TTestContext ctx(NDqProto::DATA_TRANSPORT_ARROW_1_0);
-// TestPartialRead(ctx, true);
-//}
-
+// too heavy messages...
+//Y_UNIT_TEST(PartialReadWithArrow) {
+// TTestContext ctx(NDqProto::DATA_TRANSPORT_ARROW_1_0);
+// TestPartialRead(ctx, false);
+//}
+//
+//Y_UNIT_TEST(PartialReadWithArrowQ) {
+// TTestContext ctx(NDqProto::DATA_TRANSPORT_ARROW_1_0);
+// TestPartialRead(ctx, true);
+//}
+
Y_UNIT_TEST(Overflow) {
TTestContext ctx;
- TestOverflow(ctx, false);
+ TestOverflow(ctx, false);
+}
+
+Y_UNIT_TEST(OverflowQ) {
+ TTestContext ctx;
+ TestOverflow(ctx, true);
}
-Y_UNIT_TEST(OverflowQ) {
- TTestContext ctx;
- TestOverflow(ctx, true);
-}
-
Y_UNIT_TEST(OverflowWithArrow) {
TTestContext ctx(NDqProto::DATA_TRANSPORT_ARROW_1_0);
- TestOverflow(ctx, false);
+ TestOverflow(ctx, false);
+}
+
+Y_UNIT_TEST(OverflowWithArrowQ) {
+ TTestContext ctx(NDqProto::DATA_TRANSPORT_ARROW_1_0);
+ TestOverflow(ctx, true);
}
-Y_UNIT_TEST(OverflowWithArrowQ) {
- TTestContext ctx(NDqProto::DATA_TRANSPORT_ARROW_1_0);
- TestOverflow(ctx, true);
-}
-
Y_UNIT_TEST(PopAll) {
TTestContext ctx;
- TestPopAll(ctx, false);
+ TestPopAll(ctx, false);
+}
+
+Y_UNIT_TEST(PopAllQ) {
+ TTestContext ctx;
+ TestPopAll(ctx, true);
}
-Y_UNIT_TEST(PopAllQ) {
- TTestContext ctx;
- TestPopAll(ctx, true);
-}
-
Y_UNIT_TEST(PopAllWithArrow) {
TTestContext ctx(NDqProto::DATA_TRANSPORT_ARROW_1_0);
- TestPopAll(ctx, false);
+ TestPopAll(ctx, false);
}
-
-Y_UNIT_TEST(PopAllWithArrowQ) {
- TTestContext ctx(NDqProto::DATA_TRANSPORT_ARROW_1_0);
- TestPopAll(ctx, true);
+
+Y_UNIT_TEST(PopAllWithArrowQ) {
+ TTestContext ctx(NDqProto::DATA_TRANSPORT_ARROW_1_0);
+ TestPopAll(ctx, true);
+}
+
+Y_UNIT_TEST(BigRow) {
+ TTestContext ctx(NDqProto::DATA_TRANSPORT_UV_PICKLE_1_0, true);
+ TestBigRow(ctx, false);
+}
+
+Y_UNIT_TEST(BigRowQ) {
+ TTestContext ctx(NDqProto::DATA_TRANSPORT_UV_PICKLE_1_0, true);
+ TestBigRow(ctx, true);
+}
+
}
-Y_UNIT_TEST(BigRow) {
- TTestContext ctx(NDqProto::DATA_TRANSPORT_UV_PICKLE_1_0, true);
- TestBigRow(ctx, false);
-}
-
-Y_UNIT_TEST(BigRowQ) {
- TTestContext ctx(NDqProto::DATA_TRANSPORT_UV_PICKLE_1_0, true);
- TestBigRow(ctx, true);
-}
-
-}
-
Y_UNIT_TEST_SUITE(DqOutputChannelWithStorageTests) {
Y_UNIT_TEST(Spill) {
diff --git a/ydb/library/yql/dq/runtime/dq_output_consumer.cpp b/ydb/library/yql/dq/runtime/dq_output_consumer.cpp
index 8da7ab5fdf..74bd5ba0ce 100644
--- a/ydb/library/yql/dq/runtime/dq_output_consumer.cpp
+++ b/ydb/library/yql/dq/runtime/dq_output_consumer.cpp
@@ -1,203 +1,203 @@
-#include "dq_output_consumer.h"
-
+#include "dq_output_consumer.h"
+
#include <ydb/library/yql/public/udf/udf_value.h>
#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
#include <ydb/library/yql/minikql/mkql_node.h>
#include <ydb/library/yql/utils/yql_panic.h>
-
-namespace NYql::NDq {
-
-namespace {
-
-using namespace NKikimr;
-using namespace NMiniKQL;
-using namespace NUdf;
-
-
-class TDqOutputMultiConsumer : public IDqOutputConsumer {
-public:
- explicit TDqOutputMultiConsumer(TVector<IDqOutputConsumer::TPtr>&& consumers)
- : Consumers(std::move(consumers))
- {
- YQL_ENSURE(!Consumers.empty());
- }
-
- bool IsFull() const override {
- return AnyOf(Consumers, [](const auto& consumer) { return consumer->IsFull(); });
- }
-
- void Consume(TUnboxedValue&& value) override {
- if (Consumers.size() == 1) {
- Consumers[0]->Consume(std::move(value));
- return;
- }
-
- auto index = value.GetVariantIndex();
- YQL_ENSURE(index < Consumers.size());
- auto variantItem = value.GetVariantItem();
- Consumers[index]->Consume(std::move(variantItem));
- }
-
- void Finish() override {
- for (auto& consumer : Consumers) {
- consumer->Finish();
- }
- }
-
-private:
- TVector<IDqOutputConsumer::TPtr> Consumers;
-};
-
-class TDqOutputMapConsumer : public IDqOutputConsumer {
-public:
+
+namespace NYql::NDq {
+
+namespace {
+
+using namespace NKikimr;
+using namespace NMiniKQL;
+using namespace NUdf;
+
+
+class TDqOutputMultiConsumer : public IDqOutputConsumer {
+public:
+ explicit TDqOutputMultiConsumer(TVector<IDqOutputConsumer::TPtr>&& consumers)
+ : Consumers(std::move(consumers))
+ {
+ YQL_ENSURE(!Consumers.empty());
+ }
+
+ bool IsFull() const override {
+ return AnyOf(Consumers, [](const auto& consumer) { return consumer->IsFull(); });
+ }
+
+ void Consume(TUnboxedValue&& value) override {
+ if (Consumers.size() == 1) {
+ Consumers[0]->Consume(std::move(value));
+ return;
+ }
+
+ auto index = value.GetVariantIndex();
+ YQL_ENSURE(index < Consumers.size());
+ auto variantItem = value.GetVariantItem();
+ Consumers[index]->Consume(std::move(variantItem));
+ }
+
+ void Finish() override {
+ for (auto& consumer : Consumers) {
+ consumer->Finish();
+ }
+ }
+
+private:
+ TVector<IDqOutputConsumer::TPtr> Consumers;
+};
+
+class TDqOutputMapConsumer : public IDqOutputConsumer {
+public:
TDqOutputMapConsumer(IDqOutput::TPtr output)
: Output(output) {}
-
- bool IsFull() const override {
+
+ bool IsFull() const override {
return Output->IsFull();
- }
-
- void Consume(TUnboxedValue&& value) override {
+ }
+
+ void Consume(TUnboxedValue&& value) override {
Output->Push(std::move(value));
- }
-
- void Finish() override {
+ }
+
+ void Finish() override {
Output->Finish();
- }
-
-private:
+ }
+
+private:
IDqOutput::TPtr Output;
-};
-
-class TDqOutputHashPartitionConsumer : public IDqOutputConsumer {
-public:
+};
+
+class TDqOutputHashPartitionConsumer : public IDqOutputConsumer {
+public:
TDqOutputHashPartitionConsumer(const TTypeEnvironment& typeEnv, TVector<IDqOutput::TPtr>&& outputs,
- TVector<TDataTypeId>&& keyColumnTypes, TVector<ui32>&& keyColumnIndices)
- : TypeEnv(typeEnv)
+ TVector<TDataTypeId>&& keyColumnTypes, TVector<ui32>&& keyColumnIndices)
+ : TypeEnv(typeEnv)
, Outputs(std::move(outputs))
- , KeyColumnTypes(std::move(keyColumnTypes))
- , KeyColumnIndices(std::move(keyColumnIndices))
- {
- MKQL_ENSURE_S(KeyColumnTypes.size() == KeyColumnIndices.size());
-
- for (size_t keyId = 0; keyId < KeyColumnTypes.size(); keyId++) {
- NMiniKQL::TType* type;
- if (KeyColumnTypes[keyId] == NUdf::TDataType<NUdf::TDecimal>::Id) {
- type = NMiniKQL::TDataDecimalType::Create(22, 9, TypeEnv);
- } else {
- type = NMiniKQL::TDataType::Create(KeyColumnTypes[keyId], TypeEnv);
- }
- bool isTuple;
- bool encoded;
- GetDictionaryKeyTypes(type, KeyTypes, isTuple, encoded);
-
- ValueHashers.emplace_back(KeyTypes, isTuple);
- }
- }
-
- bool IsFull() const override {
+ , KeyColumnTypes(std::move(keyColumnTypes))
+ , KeyColumnIndices(std::move(keyColumnIndices))
+ {
+ MKQL_ENSURE_S(KeyColumnTypes.size() == KeyColumnIndices.size());
+
+ for (size_t keyId = 0; keyId < KeyColumnTypes.size(); keyId++) {
+ NMiniKQL::TType* type;
+ if (KeyColumnTypes[keyId] == NUdf::TDataType<NUdf::TDecimal>::Id) {
+ type = NMiniKQL::TDataDecimalType::Create(22, 9, TypeEnv);
+ } else {
+ type = NMiniKQL::TDataType::Create(KeyColumnTypes[keyId], TypeEnv);
+ }
+ bool isTuple;
+ bool encoded;
+ GetDictionaryKeyTypes(type, KeyTypes, isTuple, encoded);
+
+ ValueHashers.emplace_back(KeyTypes, isTuple);
+ }
+ }
+
+ bool IsFull() const override {
return AnyOf(Outputs, [](const auto& output) { return output->IsFull(); });
- }
-
- void Consume(TUnboxedValue&& value) final {
- ui32 partitionIndex = GetHashPartitionIndex(value);
+ }
+
+ void Consume(TUnboxedValue&& value) final {
+ ui32 partitionIndex = GetHashPartitionIndex(value);
Outputs[partitionIndex]->Push(std::move(value));
- }
-
- void Finish() final {
+ }
+
+ void Finish() final {
for (auto& output : Outputs) {
output->Finish();
- }
- }
-
-private:
- size_t GetHashPartitionIndex(const TUnboxedValue& value) {
- ui64 hash = 0;
-
- for (size_t keyId = 0; keyId < KeyColumnIndices.size(); keyId++) {
- auto columnValue = value.GetElement(KeyColumnIndices[keyId]);
- hash = CombineHashes(hash, HashColumn(keyId, columnValue));
- }
-
+ }
+ }
+
+private:
+ size_t GetHashPartitionIndex(const TUnboxedValue& value) {
+ ui64 hash = 0;
+
+ for (size_t keyId = 0; keyId < KeyColumnIndices.size(); keyId++) {
+ auto columnValue = value.GetElement(KeyColumnIndices[keyId]);
+ hash = CombineHashes(hash, HashColumn(keyId, columnValue));
+ }
+
return hash % Outputs.size();
- }
-
- ui64 HashColumn(size_t keyId, const TUnboxedValue& value) const {
- if (!value.HasValue()) {
- return 0;
- }
-
- #define APPLY_HASHER(type, layout) \
- case TDataType<type>::Id: return hasher(value);
-
- auto& hasher = ValueHashers[keyId];
- switch (KeyColumnTypes[keyId]) {
- KNOWN_FIXED_VALUE_TYPES(APPLY_HASHER)
- case TDataType<TDecimal>::Id:
- return GetValueHash<EDataSlot::Decimal>(value);
- }
-
- #undef APPLY_HASHER
-
- return GetValueHash<EDataSlot::String>(value);
- }
-
-private:
- const TTypeEnvironment& TypeEnv;
+ }
+
+ ui64 HashColumn(size_t keyId, const TUnboxedValue& value) const {
+ if (!value.HasValue()) {
+ return 0;
+ }
+
+ #define APPLY_HASHER(type, layout) \
+ case TDataType<type>::Id: return hasher(value);
+
+ auto& hasher = ValueHashers[keyId];
+ switch (KeyColumnTypes[keyId]) {
+ KNOWN_FIXED_VALUE_TYPES(APPLY_HASHER)
+ case TDataType<TDecimal>::Id:
+ return GetValueHash<EDataSlot::Decimal>(value);
+ }
+
+ #undef APPLY_HASHER
+
+ return GetValueHash<EDataSlot::String>(value);
+ }
+
+private:
+ const TTypeEnvironment& TypeEnv;
TVector<IDqOutput::TPtr> Outputs;
- TVector<TDataTypeId> KeyColumnTypes;
- TVector<ui32> KeyColumnIndices;
- TVector<TValueHasher> ValueHashers;
- TKeyTypes KeyTypes;
-};
-
-class TDqOutputBroadcastConsumer : public IDqOutputConsumer {
-public:
+ TVector<TDataTypeId> KeyColumnTypes;
+ TVector<ui32> KeyColumnIndices;
+ TVector<TValueHasher> ValueHashers;
+ TKeyTypes KeyTypes;
+};
+
+class TDqOutputBroadcastConsumer : public IDqOutputConsumer {
+public:
TDqOutputBroadcastConsumer(TVector<IDqOutput::TPtr>&& outputs)
: Outputs(std::move(outputs)) {}
-
- bool IsFull() const override {
+
+ bool IsFull() const override {
return AnyOf(Outputs, [](const auto& output) { return output->IsFull(); });
- }
-
- void Consume(TUnboxedValue&& value) final {
+ }
+
+ void Consume(TUnboxedValue&& value) final {
for (auto& output : Outputs) {
- TUnboxedValue copy{ value };
+ TUnboxedValue copy{ value };
output->Push(std::move(copy));
- }
- }
-
- void Finish() override {
+ }
+ }
+
+ void Finish() override {
for (auto& output : Outputs) {
output->Finish();
- }
- }
-
-private:
+ }
+ }
+
+private:
TVector<IDqOutput::TPtr> Outputs;
-};
-
-} // namespace
-
-IDqOutputConsumer::TPtr CreateOutputMultiConsumer(TVector<IDqOutputConsumer::TPtr>&& consumers) {
- return MakeIntrusive<TDqOutputMultiConsumer>(std::move(consumers));
-}
-
+};
+
+} // namespace
+
+IDqOutputConsumer::TPtr CreateOutputMultiConsumer(TVector<IDqOutputConsumer::TPtr>&& consumers) {
+ return MakeIntrusive<TDqOutputMultiConsumer>(std::move(consumers));
+}
+
IDqOutputConsumer::TPtr CreateOutputMapConsumer(IDqOutput::TPtr output) {
return MakeIntrusive<TDqOutputMapConsumer>(output);
-}
-
-IDqOutputConsumer::TPtr CreateOutputHashPartitionConsumer(
+}
+
+IDqOutputConsumer::TPtr CreateOutputHashPartitionConsumer(
TVector<IDqOutput::TPtr>&& outputs,
- TVector<NUdf::TDataTypeId>&& keyColumnTypes, TVector<ui32>&& keyColumnIndices,
- const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv)
-{
+ TVector<NUdf::TDataTypeId>&& keyColumnTypes, TVector<ui32>&& keyColumnIndices,
+ const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv)
+{
return MakeIntrusive<TDqOutputHashPartitionConsumer>(typeEnv, std::move(outputs), std::move(keyColumnTypes),
- std::move(keyColumnIndices));
-}
-
+ std::move(keyColumnIndices));
+}
+
IDqOutputConsumer::TPtr CreateOutputBroadcastConsumer(TVector<IDqOutput::TPtr>&& outputs) {
return MakeIntrusive<TDqOutputBroadcastConsumer>(std::move(outputs));
-}
-
-} // namespace NYql::NDq
+}
+
+} // namespace NYql::NDq
diff --git a/ydb/library/yql/dq/runtime/dq_output_consumer.h b/ydb/library/yql/dq/runtime/dq_output_consumer.h
index 7817a60224..80e69ae065 100644
--- a/ydb/library/yql/dq/runtime/dq_output_consumer.h
+++ b/ydb/library/yql/dq/runtime/dq_output_consumer.h
@@ -1,37 +1,37 @@
-#pragma once
-
+#pragma once
+
#include "dq_output.h"
-
+
#include <ydb/library/yql/minikql/mkql_alloc.h>
-
+
namespace NKikimr::NMiniKQL {
class TTypeEnvironment;
} // namespace NKikimr::NMiniKQL
-
-namespace NYql::NDq {
-
-class IDqOutputConsumer : public TSimpleRefCount<IDqOutputConsumer>, public NKikimr::NMiniKQL::TWithMiniKQLAlloc {
-public:
- using TPtr = TIntrusivePtr<IDqOutputConsumer>;
-
-public:
- virtual ~IDqOutputConsumer() = default;
-
- virtual bool IsFull() const = 0;
- virtual void Consume(NKikimr::NUdf::TUnboxedValue&& value) = 0;
- virtual void Finish() = 0;
-};
-
-IDqOutputConsumer::TPtr CreateOutputMultiConsumer(TVector<IDqOutputConsumer::TPtr>&& consumers);
-
+
+namespace NYql::NDq {
+
+class IDqOutputConsumer : public TSimpleRefCount<IDqOutputConsumer>, public NKikimr::NMiniKQL::TWithMiniKQLAlloc {
+public:
+ using TPtr = TIntrusivePtr<IDqOutputConsumer>;
+
+public:
+ virtual ~IDqOutputConsumer() = default;
+
+ virtual bool IsFull() const = 0;
+ virtual void Consume(NKikimr::NUdf::TUnboxedValue&& value) = 0;
+ virtual void Finish() = 0;
+};
+
+IDqOutputConsumer::TPtr CreateOutputMultiConsumer(TVector<IDqOutputConsumer::TPtr>&& consumers);
+
IDqOutputConsumer::TPtr CreateOutputMapConsumer(IDqOutput::TPtr output);
-
-IDqOutputConsumer::TPtr CreateOutputHashPartitionConsumer(
+
+IDqOutputConsumer::TPtr CreateOutputHashPartitionConsumer(
TVector<IDqOutput::TPtr>&& outputs,
- TVector<NUdf::TDataTypeId>&& keyColumnTypes, TVector<ui32>&& keyColumnIndices,
- const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv);
-
+ TVector<NUdf::TDataTypeId>&& keyColumnTypes, TVector<ui32>&& keyColumnIndices,
+ const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv);
+
IDqOutputConsumer::TPtr CreateOutputBroadcastConsumer(TVector<IDqOutput::TPtr>&& outputs);
-
-
-} // namespace NYql::NDq
+
+
+} // namespace NYql::NDq
diff --git a/ydb/library/yql/dq/runtime/dq_sink.cpp b/ydb/library/yql/dq/runtime/dq_sink.cpp
index 0b4458c977..ff86523ef4 100644
--- a/ydb/library/yql/dq/runtime/dq_sink.cpp
+++ b/ydb/library/yql/dq/runtime/dq_sink.cpp
@@ -1,5 +1,5 @@
#include "dq_sink.h"
-#include "dq_transport.h"
+#include "dq_transport.h"
#include <ydb/library/yql/utils/yql_panic.h>
@@ -31,12 +31,12 @@ class TDqSink : public IDqSink {
};
public:
- TDqSink(ui64 outputIndex, NKikimr::NMiniKQL::TType* outputType, ui64 maxStoredBytes, bool collectProfileStats)
+ TDqSink(ui64 outputIndex, NKikimr::NMiniKQL::TType* outputType, ui64 maxStoredBytes, bool collectProfileStats)
: OutputIndex(outputIndex)
, MaxStoredBytes(maxStoredBytes)
, OutputType(outputType)
- , BasicStats(OutputIndex)
- , ProfileStats(collectProfileStats ? &BasicStats : nullptr) {}
+ , BasicStats(OutputIndex)
+ , ProfileStats(collectProfileStats ? &BasicStats : nullptr) {}
ui64 GetOutputIndex() const override {
return OutputIndex;
@@ -47,8 +47,8 @@ public:
}
void Push(NUdf::TUnboxedValue&& value) override {
- if (!BasicStats.FirstRowIn) {
- BasicStats.FirstRowIn = TInstant::Now();
+ if (!BasicStats.FirstRowIn) {
+ BasicStats.FirstRowIn = TInstant::Now();
}
if (ValuesPushed++ % 1000 == 0) {
@@ -72,8 +72,8 @@ public:
void Finish() override {
Finished = true;
- if (!BasicStats.FirstRowIn) {
- BasicStats.FirstRowIn = TInstant::Now();
+ if (!BasicStats.FirstRowIn) {
+ BasicStats.FirstRowIn = TInstant::Now();
}
}
@@ -142,12 +142,12 @@ public:
}
const TDqSinkStats* GetStats() const override {
- return &BasicStats;
+ return &BasicStats;
}
private:
void ReestimateRowBytes(const NUdf::TUnboxedValue& value) {
- const ui64 valueSize = TDqDataSerializer::EstimateSize(value, OutputType);
+ const ui64 valueSize = TDqDataSerializer::EstimateSize(value, OutputType);
if (EstimatedRowBytes) {
EstimatedRowBytes = static_cast<ui64>(0.6 * valueSize + 0.4 * EstimatedRowBytes);
} else {
@@ -159,17 +159,17 @@ private:
}
void ReportChunkIn() {
- BasicStats.Bytes += EstimatedRowBytes;
- BasicStats.RowsIn++;
- if (ProfileStats) {
- ProfileStats->MaxMemoryUsage = std::max(ProfileStats->MaxMemoryUsage, EstimatedStoredBytes);
- ProfileStats->MaxRowsInMemory = std::max(ProfileStats->MaxRowsInMemory, Values.size());
+ BasicStats.Bytes += EstimatedRowBytes;
+ BasicStats.RowsIn++;
+ if (ProfileStats) {
+ ProfileStats->MaxMemoryUsage = std::max(ProfileStats->MaxMemoryUsage, EstimatedStoredBytes);
+ ProfileStats->MaxRowsInMemory = std::max(ProfileStats->MaxRowsInMemory, Values.size());
}
}
- void ReportChunkOut(ui64 rowsCount, ui64 /* usedBytes */) {
- BasicStats.Chunks++;
- BasicStats.RowsOut += rowsCount;
+ void ReportChunkOut(ui64 rowsCount, ui64 /* usedBytes */) {
+ BasicStats.Chunks++;
+ BasicStats.RowsOut += rowsCount;
}
private:
@@ -181,16 +181,16 @@ private:
bool Finished = false;
std::deque<TValueDesc> Values;
ui64 EstimatedRowBytes = 0;
- TDqSinkStats BasicStats;
- TDqSinkStats* ProfileStats = nullptr;
+ TDqSinkStats BasicStats;
+ TDqSinkStats* ProfileStats = nullptr;
};
} // namespace
-IDqSink::TPtr CreateDqSink(ui64 outputIndex, NKikimr::NMiniKQL::TType* outputType, ui64 maxStoredBytes,
- bool collectProfileStats)
+IDqSink::TPtr CreateDqSink(ui64 outputIndex, NKikimr::NMiniKQL::TType* outputType, ui64 maxStoredBytes,
+ bool collectProfileStats)
{
- return MakeIntrusive<TDqSink>(outputIndex, outputType, maxStoredBytes, collectProfileStats);
+ return MakeIntrusive<TDqSink>(outputIndex, outputType, maxStoredBytes, collectProfileStats);
}
} // namespace NYql::NDq
diff --git a/ydb/library/yql/dq/runtime/dq_sink.h b/ydb/library/yql/dq/runtime/dq_sink.h
index b707266bff..63c91c9660 100644
--- a/ydb/library/yql/dq/runtime/dq_sink.h
+++ b/ydb/library/yql/dq/runtime/dq_sink.h
@@ -8,7 +8,7 @@
namespace NYql::NDq {
struct TDqSinkStats : TDqOutputStats {
- const ui64 OutputIndex;
+ const ui64 OutputIndex;
explicit TDqSinkStats(ui64 outputIndex)
: OutputIndex(outputIndex) {}
@@ -30,7 +30,7 @@ public:
virtual const TDqSinkStats* GetStats() const = 0;
};
-IDqSink::TPtr CreateDqSink(ui64 outputIndex, NKikimr::NMiniKQL::TType* outputType, ui64 maxStoredBytes,
- bool collectProfileStats);
+IDqSink::TPtr CreateDqSink(ui64 outputIndex, NKikimr::NMiniKQL::TType* outputType, ui64 maxStoredBytes,
+ bool collectProfileStats);
} // namespace NYql::NDq
diff --git a/ydb/library/yql/dq/runtime/dq_source.cpp b/ydb/library/yql/dq/runtime/dq_source.cpp
index 5cdf2451f7..b9e75a15e8 100644
--- a/ydb/library/yql/dq/runtime/dq_source.cpp
+++ b/ydb/library/yql/dq/runtime/dq_source.cpp
@@ -7,11 +7,11 @@ class TDqSource : public TDqInputImpl<TDqSource, IDqSource> {
using TBaseImpl = TDqInputImpl<TDqSource, IDqSource>;
friend TBaseImpl;
public:
- TDqSource(ui64 inputIndex, NKikimr::NMiniKQL::TType* inputType, ui64 maxBufferBytes, bool collectProfileStats)
+ TDqSource(ui64 inputIndex, NKikimr::NMiniKQL::TType* inputType, ui64 maxBufferBytes, bool collectProfileStats)
: TBaseImpl(inputType, maxBufferBytes)
, InputIndex(inputIndex)
- , BasicStats(InputIndex)
- , ProfileStats(collectProfileStats ? &BasicStats : nullptr) {}
+ , BasicStats(InputIndex)
+ , ProfileStats(collectProfileStats ? &BasicStats : nullptr) {}
ui64 GetInputIndex() const override {
return InputIndex;
@@ -25,13 +25,13 @@ public:
}
const TDqSourceStats* GetStats() const override {
- return &BasicStats;
+ return &BasicStats;
}
private:
const ui64 InputIndex;
- TDqSourceStats BasicStats;
- TDqSourceStats* ProfileStats = nullptr;
+ TDqSourceStats BasicStats;
+ TDqSourceStats* ProfileStats = nullptr;
};
IDqSource::TPtr CreateDqSource(
diff --git a/ydb/library/yql/dq/runtime/dq_source.h b/ydb/library/yql/dq/runtime/dq_source.h
index 5afbcd7635..01509873f5 100644
--- a/ydb/library/yql/dq/runtime/dq_source.h
+++ b/ydb/library/yql/dq/runtime/dq_source.h
@@ -23,7 +23,7 @@ public:
virtual const TDqSourceStats* GetStats() const = 0;
};
-IDqSource::TPtr CreateDqSource(ui64 inputIndex, NKikimr::NMiniKQL::TType* inputType, ui64 maxBufferBytes,
- bool collectProfileStats);
+IDqSource::TPtr CreateDqSource(ui64 inputIndex, NKikimr::NMiniKQL::TType* inputType, ui64 maxBufferBytes,
+ bool collectProfileStats);
} // namespace NYql::NDq
diff --git a/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp b/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp
index 42fae0f879..10900352c3 100644
--- a/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp
+++ b/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp
@@ -14,8 +14,8 @@
#include <ydb/library/yql/minikql/mkql_node_serialization.h>
#include <ydb/library/yql/minikql/mkql_node_visitor.h>
-#include <util/generic/scope.h>
-
+#include <util/generic/scope.h>
+
using namespace NKikimr;
using namespace NKikimr::NMiniKQL;
using namespace NYql::NDqProto;
@@ -118,16 +118,16 @@ void ValidateParamValue(std::string_view paramName, const TType* type, const NUd
} // namespace
-#define LOG(...) do { if (Y_UNLIKELY(LogFunc)) { LogFunc(__VA_ARGS__); } } while (0)
-
+#define LOG(...) do { if (Y_UNLIKELY(LogFunc)) { LogFunc(__VA_ARGS__); } } while (0)
+
NUdf::TUnboxedValue DqBuildInputValue(const NDqProto::TTaskInput& inputDesc, const NKikimr::NMiniKQL::TType* type,
TVector<IDqInput::TPtr>&& inputs, const THolderFactory& holderFactory)
-{
- switch (inputDesc.GetTypeCase()) {
+{
+ switch (inputDesc.GetTypeCase()) {
case NYql::NDqProto::TTaskInput::kSource:
Y_VERIFY(inputs.size() == 1);
[[fallthrough]];
- case NYql::NDqProto::TTaskInput::kUnionAll:
+ case NYql::NDqProto::TTaskInput::kUnionAll:
return CreateInputUnionValue(std::move(inputs), holderFactory);
case NYql::NDqProto::TTaskInput::kMerge: {
const auto& protoSortCols = inputDesc.GetMerge().GetSortColumns();
@@ -137,279 +137,279 @@ NUdf::TUnboxedValue DqBuildInputValue(const NDqProto::TTaskInput& inputDesc, con
return CreateInputMergeValue(std::move(inputs), std::move(sortColsInfo), holderFactory);
}
- default:
- YQL_ENSURE(false, "Unknown input type: " << (ui32) inputDesc.GetTypeCase());
- }
-}
-
-IDqOutputConsumer::TPtr DqBuildOutputConsumer(const NDqProto::TTaskOutput& outputDesc, const NMiniKQL::TType* type,
+ default:
+ YQL_ENSURE(false, "Unknown input type: " << (ui32) inputDesc.GetTypeCase());
+ }
+}
+
+IDqOutputConsumer::TPtr DqBuildOutputConsumer(const NDqProto::TTaskOutput& outputDesc, const NMiniKQL::TType* type,
const NMiniKQL::TTypeEnvironment& typeEnv, TVector<IDqOutput::TPtr>&& outputs)
-{
- switch (outputDesc.GetTypeCase()) {
+{
+ switch (outputDesc.GetTypeCase()) {
case NDqProto::TTaskOutput::kSink:
Y_VERIFY(outputDesc.ChannelsSize() == 0);
[[fallthrough]];
- case NDqProto::TTaskOutput::kMap: {
+ case NDqProto::TTaskOutput::kMap: {
YQL_ENSURE(outputs.size() == 1);
return CreateOutputMapConsumer(outputs[0]);
- }
-
- case NDqProto::TTaskOutput::kHashPartition: {
- TVector<NUdf::TDataTypeId> keyColumnTypes;
- TVector<ui32> keyColumnIndices;
- GetColumnsInfo(type, outputDesc.GetHashPartition().GetKeyColumns(), keyColumnTypes, keyColumnIndices);
- YQL_ENSURE(!keyColumnTypes.empty());
-
- YQL_ENSURE(outputDesc.GetHashPartition().GetPartitionsCount() == outputDesc.ChannelsSize());
- TVector<ui64> channelIds(outputDesc.GetHashPartition().GetPartitionsCount());
- for (ui32 i = 0; i < outputDesc.ChannelsSize(); ++i) {
- channelIds[i] = outputDesc.GetChannels(i).GetId();
- }
-
+ }
+
+ case NDqProto::TTaskOutput::kHashPartition: {
+ TVector<NUdf::TDataTypeId> keyColumnTypes;
+ TVector<ui32> keyColumnIndices;
+ GetColumnsInfo(type, outputDesc.GetHashPartition().GetKeyColumns(), keyColumnTypes, keyColumnIndices);
+ YQL_ENSURE(!keyColumnTypes.empty());
+
+ YQL_ENSURE(outputDesc.GetHashPartition().GetPartitionsCount() == outputDesc.ChannelsSize());
+ TVector<ui64> channelIds(outputDesc.GetHashPartition().GetPartitionsCount());
+ for (ui32 i = 0; i < outputDesc.ChannelsSize(); ++i) {
+ channelIds[i] = outputDesc.GetChannels(i).GetId();
+ }
+
return CreateOutputHashPartitionConsumer(std::move(outputs), std::move(keyColumnTypes),
- std::move(keyColumnIndices), typeEnv);
- }
-
- case NDqProto::TTaskOutput::kBroadcast: {
+ std::move(keyColumnIndices), typeEnv);
+ }
+
+ case NDqProto::TTaskOutput::kBroadcast: {
return CreateOutputBroadcastConsumer(std::move(outputs));
- }
-
- case NDqProto::TTaskOutput::kRangePartition: {
- YQL_ENSURE(false, "Unsupported partition type: `NYql::NDqProto::TTaskOutput::kRangePartition`");
- }
-
- case NDqProto::TTaskOutput::kEffects: {
- YQL_ENSURE(false, "Unsupported partition type: `NYql::NDqProto::TTaskOutput::kEffect`");
- }
-
- case NDqProto::TTaskOutput::TYPE_NOT_SET: {
- YQL_ENSURE(false, "Unexpected output type: `NYql::NDqProto::TDqTaskOutput::TYPE_NOT_SET`");
- }
- }
-}
-
-IDqOutputConsumer::TPtr TDqTaskRunnerExecutionContext::CreateOutputConsumer(const TTaskOutput& outputDesc,
- const NKikimr::NMiniKQL::TType* type, NUdf::IApplyContext*, const TTypeEnvironment& typeEnv,
+ }
+
+ case NDqProto::TTaskOutput::kRangePartition: {
+ YQL_ENSURE(false, "Unsupported partition type: `NYql::NDqProto::TTaskOutput::kRangePartition`");
+ }
+
+ case NDqProto::TTaskOutput::kEffects: {
+ YQL_ENSURE(false, "Unsupported partition type: `NYql::NDqProto::TTaskOutput::kEffect`");
+ }
+
+ case NDqProto::TTaskOutput::TYPE_NOT_SET: {
+ YQL_ENSURE(false, "Unexpected output type: `NYql::NDqProto::TDqTaskOutput::TYPE_NOT_SET`");
+ }
+ }
+}
+
+IDqOutputConsumer::TPtr TDqTaskRunnerExecutionContext::CreateOutputConsumer(const TTaskOutput& outputDesc,
+ const NKikimr::NMiniKQL::TType* type, NUdf::IApplyContext*, const TTypeEnvironment& typeEnv,
TVector<IDqOutput::TPtr>&& outputs) const
-{
+{
return DqBuildOutputConsumer(outputDesc, type, typeEnv, std::move(outputs));
-}
-
-IDqChannelStorage::TPtr TDqTaskRunnerExecutionContext::CreateChannelStorage(ui64 /* channelId */) const {
- return {};
-}
-
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-/// TDqTaskRunner
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+}
+
+IDqChannelStorage::TPtr TDqTaskRunnerExecutionContext::CreateChannelStorage(ui64 /* channelId */) const {
+ return {};
+}
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+/// TDqTaskRunner
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class TDqTaskRunner : public IDqTaskRunner {
public:
- TDqTaskRunner(const TDqTaskRunnerContext& context, const TDqTaskRunnerSettings& settings, const TLogFunc& logFunc)
- : Context(context)
- , Settings(settings)
- , LogFunc(logFunc)
- , CollectBasicStats(Settings.CollectBasicStats)
- , CollectProfileStats(Settings.CollectProfileStats)
- {
+ TDqTaskRunner(const TDqTaskRunnerContext& context, const TDqTaskRunnerSettings& settings, const TLogFunc& logFunc)
+ : Context(context)
+ , Settings(settings)
+ , LogFunc(logFunc)
+ , CollectBasicStats(Settings.CollectBasicStats)
+ , CollectProfileStats(Settings.CollectProfileStats)
+ {
if (CollectBasicStats) {
- Stats = std::make_unique<TDqTaskRunnerStats>();
+ Stats = std::make_unique<TDqTaskRunnerStats>();
if (Y_UNLIKELY(CollectProfileStats)) {
- Stats->ComputeCpuTimeByRun = NMonitoring::ExponentialHistogram(6, 10, 10);
+ Stats->ComputeCpuTimeByRun = NMonitoring::ExponentialHistogram(6, 10, 10);
}
} else {
YQL_ENSURE(!CollectProfileStats, "CollectProfileStats requires CollectBasicStats to be set as well");
- }
-
- if (!Context.Alloc) {
- SelfAlloc = std::make_unique<TScopedAlloc>(TAlignedPagePoolCounters(), Context.FuncRegistry->SupportsSizedAllocators());
- }
-
- if (!Context.TypeEnv) {
- SelfTypeEnv = std::make_unique<TTypeEnvironment>(Context.Alloc ? *Context.Alloc : *SelfAlloc);
- }
-
- if (SelfAlloc) {
- SelfAlloc->Release();
- }
- }
-
- ~TDqTaskRunner() {
- if (SelfAlloc) {
- SelfAlloc->Acquire();
- }
- }
-
- ui64 GetTaskId() const override {
- Y_VERIFY(TaskId, "Not prepared yet");
- return TaskId;
- }
-
- void BuildTask(const NDqProto::TDqTask& task, const TDqTaskRunnerParameterProvider& parameterProvider) {
- LOG(TStringBuilder() << "Build task: " << TaskId);
- auto startTime = TInstant::Now();
-
- auto& typeEnv = TypeEnv();
-
- auto& program = task.GetProgram();
- YQL_ENSURE(program.GetRuntimeVersion());
- YQL_ENSURE(program.GetRuntimeVersion() <= NYql::NDqProto::ERuntimeVersion::RUNTIME_VERSION_YQL_1_0);
-
- ProgramParsed.ProgramNode = DeserializeRuntimeNode(program.GetRaw(), typeEnv);
- YQL_ENSURE(ProgramParsed.ProgramNode.IsImmediate() && ProgramParsed.ProgramNode.GetNode()->GetType()->IsStruct());
- auto& programStruct = static_cast<TStructLiteral&>(*ProgramParsed.ProgramNode.GetNode());
- auto programType = programStruct.GetType();
- YQL_ENSURE(programType);
-
- auto programRootIdx = programType->FindMemberIndex("Program");
- YQL_ENSURE(programRootIdx);
- TRuntimeNode programRoot = programStruct.GetValue(*programRootIdx);
+ }
+
+ if (!Context.Alloc) {
+ SelfAlloc = std::make_unique<TScopedAlloc>(TAlignedPagePoolCounters(), Context.FuncRegistry->SupportsSizedAllocators());
+ }
+
+ if (!Context.TypeEnv) {
+ SelfTypeEnv = std::make_unique<TTypeEnvironment>(Context.Alloc ? *Context.Alloc : *SelfAlloc);
+ }
+
+ if (SelfAlloc) {
+ SelfAlloc->Release();
+ }
+ }
+
+ ~TDqTaskRunner() {
+ if (SelfAlloc) {
+ SelfAlloc->Acquire();
+ }
+ }
+
+ ui64 GetTaskId() const override {
+ Y_VERIFY(TaskId, "Not prepared yet");
+ return TaskId;
+ }
+
+ void BuildTask(const NDqProto::TDqTask& task, const TDqTaskRunnerParameterProvider& parameterProvider) {
+ LOG(TStringBuilder() << "Build task: " << TaskId);
+ auto startTime = TInstant::Now();
+
+ auto& typeEnv = TypeEnv();
+
+ auto& program = task.GetProgram();
+ YQL_ENSURE(program.GetRuntimeVersion());
+ YQL_ENSURE(program.GetRuntimeVersion() <= NYql::NDqProto::ERuntimeVersion::RUNTIME_VERSION_YQL_1_0);
+
+ ProgramParsed.ProgramNode = DeserializeRuntimeNode(program.GetRaw(), typeEnv);
+ YQL_ENSURE(ProgramParsed.ProgramNode.IsImmediate() && ProgramParsed.ProgramNode.GetNode()->GetType()->IsStruct());
+ auto& programStruct = static_cast<TStructLiteral&>(*ProgramParsed.ProgramNode.GetNode());
+ auto programType = programStruct.GetType();
+ YQL_ENSURE(programType);
+
+ auto programRootIdx = programType->FindMemberIndex("Program");
+ YQL_ENSURE(programRootIdx);
+ TRuntimeNode programRoot = programStruct.GetValue(*programRootIdx);
if (Context.FuncProvider) {
TExploringNodeVisitor explorer;
- explorer.Walk(programRoot.GetNode(), typeEnv);
+ explorer.Walk(programRoot.GetNode(), typeEnv);
bool wereChanges = false;
- programRoot = SinglePassVisitCallables(programRoot, explorer, Context.FuncProvider, typeEnv, true, wereChanges);
- }
-
- ProgramParsed.OutputItemTypes.resize(task.OutputsSize());
-
- if (programRoot.GetNode()->GetType()->IsCallable()) {
- auto programResultType = static_cast<const TCallableType*>(programRoot.GetNode()->GetType());
- YQL_ENSURE(programResultType->GetReturnType()->IsStream());
- auto programResultItemType = static_cast<const TStreamType*>(programResultType->GetReturnType())->GetItemType();
-
- if (programResultItemType->IsVariant()) {
- auto variantType = static_cast<const TVariantType*>(programResultItemType);
- YQL_ENSURE(variantType->GetUnderlyingType()->IsTuple());
- auto variantTupleType = static_cast<const TTupleType*>(variantType->GetUnderlyingType());
- YQL_ENSURE(task.OutputsSize() == variantTupleType->GetElementsCount(),
- "" << task.OutputsSize() << " != " << variantTupleType->GetElementsCount());
- for (ui32 i = 0; i < variantTupleType->GetElementsCount(); ++i) {
- ProgramParsed.OutputItemTypes[i] = variantTupleType->GetElementType(i);
- }
- }
- else {
- YQL_ENSURE(task.OutputsSize() == 1);
- ProgramParsed.OutputItemTypes[0] = programResultItemType;
- }
- }
- else {
- YQL_ENSURE(programRoot.GetNode()->GetType()->IsVoid());
- YQL_ENSURE(task.OutputsSize() == 0);
- }
-
- auto programInputsIdx = programType->FindMemberIndex("Inputs");
- YQL_ENSURE(programInputsIdx);
- TRuntimeNode programInputs = programStruct.GetValue(*programInputsIdx);
- YQL_ENSURE(programInputs.IsImmediate() && programInputs.GetNode()->GetType()->IsTuple());
- auto& programInputsTuple = static_cast<TTupleLiteral&>(*programInputs.GetNode());
- auto programInputsCount = programInputsTuple.GetValuesCount();
- YQL_ENSURE(task.InputsSize() == programInputsCount);
-
- ProgramParsed.InputItemTypes.resize(programInputsCount);
- ProgramParsed.EntryPoints.resize(programInputsCount + 1 /* parameters */);
- for (ui32 i = 0; i < programInputsCount; ++i) {
- auto input = programInputsTuple.GetValue(i);
- TType* type = input.GetStaticType();
- YQL_ENSURE(type->GetKind() == TType::EKind::Stream);
- ProgramParsed.InputItemTypes[i] = static_cast<TStreamType&>(*type).GetItemType();
- ProgramParsed.EntryPoints[i] = input.GetNode();
- }
-
- auto programParamsIdx = programType->FindMemberIndex("Parameters");
- YQL_ENSURE(programParamsIdx);
- TRuntimeNode programParams = programStruct.GetValue(*programParamsIdx);
- YQL_ENSURE(programParams.GetNode()->GetType()->IsCallable());
- auto paramsType = static_cast<TCallableType*>(programParams.GetNode()->GetType())->GetReturnType();
- YQL_ENSURE(paramsType->IsStruct());
- ProgramParsed.EntryPoints[programInputsCount] = programParams.GetNode();
- auto paramsStruct = static_cast<TStructType*>(paramsType);
- auto paramsCount = paramsStruct->GetMembersCount();
-
- TExploringNodeVisitor programExplorer;
- programExplorer.Walk(programRoot.GetNode(), typeEnv);
- auto programSize = programExplorer.GetNodes().size();
-
- LOG(TStringBuilder() << "task: " << TaskId << ", program size: " << programSize
- << ", llvm: `" << Settings.OptLLVM << "`.");
-
+ programRoot = SinglePassVisitCallables(programRoot, explorer, Context.FuncProvider, typeEnv, true, wereChanges);
+ }
+
+ ProgramParsed.OutputItemTypes.resize(task.OutputsSize());
+
+ if (programRoot.GetNode()->GetType()->IsCallable()) {
+ auto programResultType = static_cast<const TCallableType*>(programRoot.GetNode()->GetType());
+ YQL_ENSURE(programResultType->GetReturnType()->IsStream());
+ auto programResultItemType = static_cast<const TStreamType*>(programResultType->GetReturnType())->GetItemType();
+
+ if (programResultItemType->IsVariant()) {
+ auto variantType = static_cast<const TVariantType*>(programResultItemType);
+ YQL_ENSURE(variantType->GetUnderlyingType()->IsTuple());
+ auto variantTupleType = static_cast<const TTupleType*>(variantType->GetUnderlyingType());
+ YQL_ENSURE(task.OutputsSize() == variantTupleType->GetElementsCount(),
+ "" << task.OutputsSize() << " != " << variantTupleType->GetElementsCount());
+ for (ui32 i = 0; i < variantTupleType->GetElementsCount(); ++i) {
+ ProgramParsed.OutputItemTypes[i] = variantTupleType->GetElementType(i);
+ }
+ }
+ else {
+ YQL_ENSURE(task.OutputsSize() == 1);
+ ProgramParsed.OutputItemTypes[0] = programResultItemType;
+ }
+ }
+ else {
+ YQL_ENSURE(programRoot.GetNode()->GetType()->IsVoid());
+ YQL_ENSURE(task.OutputsSize() == 0);
+ }
+
+ auto programInputsIdx = programType->FindMemberIndex("Inputs");
+ YQL_ENSURE(programInputsIdx);
+ TRuntimeNode programInputs = programStruct.GetValue(*programInputsIdx);
+ YQL_ENSURE(programInputs.IsImmediate() && programInputs.GetNode()->GetType()->IsTuple());
+ auto& programInputsTuple = static_cast<TTupleLiteral&>(*programInputs.GetNode());
+ auto programInputsCount = programInputsTuple.GetValuesCount();
+ YQL_ENSURE(task.InputsSize() == programInputsCount);
+
+ ProgramParsed.InputItemTypes.resize(programInputsCount);
+ ProgramParsed.EntryPoints.resize(programInputsCount + 1 /* parameters */);
+ for (ui32 i = 0; i < programInputsCount; ++i) {
+ auto input = programInputsTuple.GetValue(i);
+ TType* type = input.GetStaticType();
+ YQL_ENSURE(type->GetKind() == TType::EKind::Stream);
+ ProgramParsed.InputItemTypes[i] = static_cast<TStreamType&>(*type).GetItemType();
+ ProgramParsed.EntryPoints[i] = input.GetNode();
+ }
+
+ auto programParamsIdx = programType->FindMemberIndex("Parameters");
+ YQL_ENSURE(programParamsIdx);
+ TRuntimeNode programParams = programStruct.GetValue(*programParamsIdx);
+ YQL_ENSURE(programParams.GetNode()->GetType()->IsCallable());
+ auto paramsType = static_cast<TCallableType*>(programParams.GetNode()->GetType())->GetReturnType();
+ YQL_ENSURE(paramsType->IsStruct());
+ ProgramParsed.EntryPoints[programInputsCount] = programParams.GetNode();
+ auto paramsStruct = static_cast<TStructType*>(paramsType);
+ auto paramsCount = paramsStruct->GetMembersCount();
+
+ TExploringNodeVisitor programExplorer;
+ programExplorer.Walk(programRoot.GetNode(), typeEnv);
+ auto programSize = programExplorer.GetNodes().size();
+
+ LOG(TStringBuilder() << "task: " << TaskId << ", program size: " << programSize
+ << ", llvm: `" << Settings.OptLLVM << "`.");
+
if (Y_UNLIKELY(CollectProfileStats)) {
ProgramParsed.StatsRegistry = NMiniKQL::CreateDefaultStatsRegistry();
}
auto validatePolicy = Settings.TerminateOnError ? NUdf::EValidatePolicy::Fail : NUdf::EValidatePolicy::Exception;
- TComputationPatternOpts opts(Alloc().Ref(), typeEnv, Context.ComputationFactory, Context.FuncRegistry,
+ TComputationPatternOpts opts(Alloc().Ref(), typeEnv, Context.ComputationFactory, Context.FuncRegistry,
NUdf::EValidateMode::None, validatePolicy, Settings.OptLLVM,
- EGraphPerProcess::Multi, ProgramParsed.StatsRegistry.Get());
+ EGraphPerProcess::Multi, ProgramParsed.StatsRegistry.Get());
SecureParamsProvider = MakeSimpleSecureParamsProvider(Settings.SecureParams);
opts.SecureParamsProvider = SecureParamsProvider.get();
- ProgramParsed.CompPattern = MakeComputationPattern(programExplorer, programRoot, ProgramParsed.EntryPoints, opts);
-
- ProgramParsed.CompGraph = ProgramParsed.CompPattern->Clone(
- opts.ToComputationOptions(*Context.RandomProvider, *Context.TimeProvider));
-
- TBindTerminator term(ProgramParsed.CompGraph->GetTerminator());
-
- auto paramNode = ProgramParsed.CompGraph->GetEntryPoint(programInputsCount, /* require */ false);
- if (paramNode) {
+ ProgramParsed.CompPattern = MakeComputationPattern(programExplorer, programRoot, ProgramParsed.EntryPoints, opts);
+
+ ProgramParsed.CompGraph = ProgramParsed.CompPattern->Clone(
+ opts.ToComputationOptions(*Context.RandomProvider, *Context.TimeProvider));
+
+ TBindTerminator term(ProgramParsed.CompGraph->GetTerminator());
+
+ auto paramNode = ProgramParsed.CompGraph->GetEntryPoint(programInputsCount, /* require */ false);
+ if (paramNode) {
// TODO: Remove serialized parameters that are used in OLAP program and not used in current program
- const auto& graphHolderFactory = ProgramParsed.CompGraph->GetHolderFactory();
- NUdf::TUnboxedValue* structMembers;
- auto paramsStructValue = graphHolderFactory.CreateDirectArrayHolder(paramsCount, structMembers);
-
- for (ui32 i = 0; i < paramsStruct->GetMembersCount(); ++i) {
- std::string_view name = paramsStruct->GetMemberName(i);
- TType* type = paramsStruct->GetMemberType(i);
-
- if (parameterProvider && parameterProvider(name, type, typeEnv, graphHolderFactory, structMembers[i])) {
-#ifndef NDEBUG
- YQL_ENSURE(!task.GetParameters().contains(name), "param: " << name);
-#endif
- } else {
- auto it = task.GetParameters().find(name);
- YQL_ENSURE(it != task.GetParameters().end());
-
- TDqDataSerializer::DeserializeParam(it->second, type, graphHolderFactory, structMembers[i]);
- }
+ const auto& graphHolderFactory = ProgramParsed.CompGraph->GetHolderFactory();
+ NUdf::TUnboxedValue* structMembers;
+ auto paramsStructValue = graphHolderFactory.CreateDirectArrayHolder(paramsCount, structMembers);
+
+ for (ui32 i = 0; i < paramsStruct->GetMembersCount(); ++i) {
+ std::string_view name = paramsStruct->GetMemberName(i);
+ TType* type = paramsStruct->GetMemberType(i);
+
+ if (parameterProvider && parameterProvider(name, type, typeEnv, graphHolderFactory, structMembers[i])) {
+#ifndef NDEBUG
+ YQL_ENSURE(!task.GetParameters().contains(name), "param: " << name);
+#endif
+ } else {
+ auto it = task.GetParameters().find(name);
+ YQL_ENSURE(it != task.GetParameters().end());
+
+ TDqDataSerializer::DeserializeParam(it->second, type, graphHolderFactory, structMembers[i]);
+ }
ValidateParamValue(name, type, structMembers[i]);
- }
-
- paramNode->SetValue(ProgramParsed.CompGraph->GetContext(), std::move(paramsStructValue));
- } else {
+ }
+
+ paramNode->SetValue(ProgramParsed.CompGraph->GetContext(), std::move(paramsStructValue));
+ } else {
/*
* This situation is ok, when there are OLAP parameters only. There is no parameter node
* because there is no parameters in program. But there are parameters in paramsStruct, they are
* serialized somewhere before in executor.
*/
- }
-
- auto buildTime = TInstant::Now() - startTime;
+ }
+
+ auto buildTime = TInstant::Now() - startTime;
if (Stats) {
Stats->BuildCpuTime = buildTime;
}
- LOG(TStringBuilder() << "Build task: " << TaskId << " takes " << buildTime.MicroSeconds() << " us");
- }
-
- void Prepare(const NDqProto::TDqTask& task, const TDqTaskRunnerMemoryLimits& memoryLimits,
- const IDqTaskRunnerExecutionContext& execCtx, const TDqTaskRunnerParameterProvider& parameterProvider) override
- {
- TaskId = task.GetId();
- BuildTask(task, parameterProvider);
-
- LOG(TStringBuilder() << "Prepare task: " << TaskId);
- auto startTime = TInstant::Now();
-
- auto& holderFactory = ProgramParsed.CompGraph->GetHolderFactory();
- TBindTerminator term(ProgramParsed.CompGraph->GetTerminator());
-
- auto& typeEnv = TypeEnv();
-
- for (ui32 i = 0; i < task.InputsSize(); ++i) {
- auto& inputDesc = task.GetInputs(i);
-
- TVector<IDqInput::TPtr> inputs{Reserve(std::max<ui64>(inputDesc.ChannelsSize(), 1))}; // 1 is for "source" type of input.
-
+ LOG(TStringBuilder() << "Build task: " << TaskId << " takes " << buildTime.MicroSeconds() << " us");
+ }
+
+ void Prepare(const NDqProto::TDqTask& task, const TDqTaskRunnerMemoryLimits& memoryLimits,
+ const IDqTaskRunnerExecutionContext& execCtx, const TDqTaskRunnerParameterProvider& parameterProvider) override
+ {
+ TaskId = task.GetId();
+ BuildTask(task, parameterProvider);
+
+ LOG(TStringBuilder() << "Prepare task: " << TaskId);
+ auto startTime = TInstant::Now();
+
+ auto& holderFactory = ProgramParsed.CompGraph->GetHolderFactory();
+ TBindTerminator term(ProgramParsed.CompGraph->GetTerminator());
+
+ auto& typeEnv = TypeEnv();
+
+ for (ui32 i = 0; i < task.InputsSize(); ++i) {
+ auto& inputDesc = task.GetInputs(i);
+
+ TVector<IDqInput::TPtr> inputs{Reserve(std::max<ui64>(inputDesc.ChannelsSize(), 1))}; // 1 is for "source" type of input.
+
if (inputDesc.HasSource()) {
auto source = CreateDqSource(i, ProgramParsed.InputItemTypes[i],
memoryLimits.ChannelBufferSize, Settings.CollectProfileStats);
@@ -423,27 +423,27 @@ public:
memoryLimits.ChannelBufferSize, Settings.CollectProfileStats, typeEnv, holderFactory,
inputChannelDesc.GetTransportVersion());
auto ret = InputChannels.emplace(channelId, inputChannel);
- YQL_ENSURE(ret.second, "task: " << TaskId << ", duplicated input channelId: " << channelId);
+ YQL_ENSURE(ret.second, "task: " << TaskId << ", duplicated input channelId: " << channelId);
inputs.emplace_back(inputChannel);
}
- }
-
- auto entryNode = ProgramParsed.CompGraph->GetEntryPoint(i, true);
- entryNode->SetValue(ProgramParsed.CompGraph->GetContext(),
+ }
+
+ auto entryNode = ProgramParsed.CompGraph->GetEntryPoint(i, true);
+ entryNode->SetValue(ProgramParsed.CompGraph->GetContext(),
DqBuildInputValue(inputDesc, ProgramParsed.InputItemTypes[i], std::move(inputs), holderFactory));
- }
-
- ResultStream = ProgramParsed.CompGraph->GetValue();
-
- TVector<IDqOutputConsumer::TPtr> outputConsumers(task.OutputsSize());
- for (ui32 i = 0; i < task.OutputsSize(); ++i) {
- auto& outputDesc = task.GetOutputs(i);
-
- if (outputDesc.GetTypeCase() == NDqProto::TTaskOutput::kEffects) {
- TaskHasEffects = true;
- }
-
- TVector<IDqOutput::TPtr> outputs{Reserve(std::max<ui64>(outputDesc.ChannelsSize(), 1))};
+ }
+
+ ResultStream = ProgramParsed.CompGraph->GetValue();
+
+ TVector<IDqOutputConsumer::TPtr> outputConsumers(task.OutputsSize());
+ for (ui32 i = 0; i < task.OutputsSize(); ++i) {
+ auto& outputDesc = task.GetOutputs(i);
+
+ if (outputDesc.GetTypeCase() == NDqProto::TTaskOutput::kEffects) {
+ TaskHasEffects = true;
+ }
+
+ TVector<IDqOutput::TPtr> outputs{Reserve(std::max<ui64>(outputDesc.ChannelsSize(), 1))};
if (outputDesc.HasSink()) {
auto sink = CreateDqSink(i, ProgramParsed.OutputItemTypes[i], memoryLimits.ChannelBufferSize,
Settings.CollectProfileStats);
@@ -453,52 +453,52 @@ public:
} else {
for (auto& outputChannelDesc : outputDesc.GetChannels()) {
ui64 channelId = outputChannelDesc.GetId();
-
- TDqOutputChannelSettings settings;
- settings.MaxStoredBytes = memoryLimits.ChannelBufferSize;
- settings.MaxChunkBytes = memoryLimits.OutputChunkMaxSize;
- settings.TransportVersion = outputChannelDesc.GetTransportVersion();
- settings.CollectProfileStats = Settings.CollectProfileStats;
- settings.AllowGeneratorsInUnboxedValues = Settings.AllowGeneratorsInUnboxedValues;
-
+
+ TDqOutputChannelSettings settings;
+ settings.MaxStoredBytes = memoryLimits.ChannelBufferSize;
+ settings.MaxChunkBytes = memoryLimits.OutputChunkMaxSize;
+ settings.TransportVersion = outputChannelDesc.GetTransportVersion();
+ settings.CollectProfileStats = Settings.CollectProfileStats;
+ settings.AllowGeneratorsInUnboxedValues = Settings.AllowGeneratorsInUnboxedValues;
+
if (!outputChannelDesc.GetInMemory()) {
- settings.ChannelStorage = execCtx.CreateChannelStorage(channelId);
+ settings.ChannelStorage = execCtx.CreateChannelStorage(channelId);
}
-
- auto outputChannel = CreateDqOutputChannel(channelId, ProgramParsed.OutputItemTypes[i], typeEnv,
- holderFactory, settings, LogFunc);
+
+ auto outputChannel = CreateDqOutputChannel(channelId, ProgramParsed.OutputItemTypes[i], typeEnv,
+ holderFactory, settings, LogFunc);
auto ret = OutputChannels.emplace(channelId, outputChannel);
- YQL_ENSURE(ret.second, "task: " << TaskId << ", duplicated output channelId: " << channelId);
+ YQL_ENSURE(ret.second, "task: " << TaskId << ", duplicated output channelId: " << channelId);
outputs.emplace_back(outputChannel);
- }
- }
-
- outputConsumers[i] = execCtx.CreateOutputConsumer(outputDesc, ProgramParsed.OutputItemTypes[i],
+ }
+ }
+
+ outputConsumers[i] = execCtx.CreateOutputConsumer(outputDesc, ProgramParsed.OutputItemTypes[i],
Context.ApplyCtx, typeEnv, std::move(outputs));
- }
-
- if (outputConsumers.empty()) {
- Output = nullptr;
- } else if (outputConsumers.size() == 1) {
- Output = std::move(outputConsumers[0]);
- } else {
- Output = CreateOutputMultiConsumer(std::move(outputConsumers));
- }
-
- auto prepareTime = TInstant::Now() - startTime;
- if (LogFunc) {
- TLogFunc logger = [taskId = TaskId, log = LogFunc](const TString& message) {
- log(TStringBuilder() << "Run task: " << taskId << ", " << message);
- };
- LogFunc = logger;
-
- }
-
- LOG(TStringBuilder() << "Prepare task: " << TaskId << ", takes " << prepareTime.MicroSeconds() << " us");
+ }
+
+ if (outputConsumers.empty()) {
+ Output = nullptr;
+ } else if (outputConsumers.size() == 1) {
+ Output = std::move(outputConsumers[0]);
+ } else {
+ Output = CreateOutputMultiConsumer(std::move(outputConsumers));
+ }
+
+ auto prepareTime = TInstant::Now() - startTime;
+ if (LogFunc) {
+ TLogFunc logger = [taskId = TaskId, log = LogFunc](const TString& message) {
+ log(TStringBuilder() << "Run task: " << taskId << ", " << message);
+ };
+ LogFunc = logger;
+
+ }
+
+ LOG(TStringBuilder() << "Prepare task: " << TaskId << ", takes " << prepareTime.MicroSeconds() << " us");
if (Stats) {
Stats->BuildCpuTime += prepareTime;
-
+
for (auto&[channelId, inputChannel] : InputChannels) {
Stats->InputChannels.emplace(channelId, inputChannel->GetStats());
}
@@ -508,21 +508,21 @@ public:
for (auto&[channelId, outputChannel] : OutputChannels) {
Stats->OutputChannels.emplace(channelId, outputChannel->GetStats());
}
- }
+ }
}
ERunStatus Run() final {
- LOG(TStringBuilder() << "Run task: " << TaskId);
-
- RunComputeTime = TDuration::Zero();
-
- auto runStatus = FetchAndDispatch();
+ LOG(TStringBuilder() << "Run task: " << TaskId);
+
+ RunComputeTime = TDuration::Zero();
+
+ auto runStatus = FetchAndDispatch();
if (Stats) {
Stats->RunStatusTimeMetrics.SetCurrentStatus(runStatus, RunComputeTime);
}
-
+
if (Y_UNLIKELY(CollectProfileStats)) {
- Stats->ComputeCpuTimeByRun->Collect(RunComputeTime.MilliSeconds());
+ Stats->ComputeCpuTimeByRun->Collect(RunComputeTime.MilliSeconds());
if (ProgramParsed.StatsRegistry) {
Stats->MkqlStats.clear();
@@ -536,22 +536,22 @@ public:
if (Stats) {
Stats->FinishTs = TInstant::Now();
}
- if (Y_UNLIKELY(CollectProfileStats)) {
- StopWaiting(Stats->FinishTs);
+ if (Y_UNLIKELY(CollectProfileStats)) {
+ StopWaiting(Stats->FinishTs);
}
return ERunStatus::Finished;
- }
-
- if (Y_UNLIKELY(CollectProfileStats)) {
- auto now = TInstant::Now();
- StartWaiting(now);
- if (runStatus == ERunStatus::PendingOutput) {
- StartWaitingOutput(now);
- }
- }
-
- return runStatus; // PendingInput or PendingOutput
+ }
+
+ if (Y_UNLIKELY(CollectProfileStats)) {
+ auto now = TInstant::Now();
+ StartWaiting(now);
+ if (runStatus == ERunStatus::PendingOutput) {
+ StartWaitingOutput(now);
+ }
+ }
+
+ return runStatus; // PendingInput or PendingOutput
}
bool HasEffects() const final {
@@ -559,43 +559,43 @@ public:
}
IDqInputChannel::TPtr GetInputChannel(ui64 channelId) override {
- auto ptr = InputChannels.FindPtr(channelId);
- YQL_ENSURE(ptr, "task: " << TaskId << " does not have input channelId: " << channelId);
- return *ptr;
+ auto ptr = InputChannels.FindPtr(channelId);
+ YQL_ENSURE(ptr, "task: " << TaskId << " does not have input channelId: " << channelId);
+ return *ptr;
}
IDqSource::TPtr GetSource(ui64 inputIndex) override {
auto ptr = Sources.FindPtr(inputIndex);
- YQL_ENSURE(ptr, "task: " << TaskId << " does not have input index: " << inputIndex);
+ YQL_ENSURE(ptr, "task: " << TaskId << " does not have input index: " << inputIndex);
return *ptr;
}
- IDqOutputChannel::TPtr GetOutputChannel(ui64 channelId) override {
- auto ptr = OutputChannels.FindPtr(channelId);
- YQL_ENSURE(ptr, "task: " << TaskId << " does not have output channelId: " << channelId);
- return *ptr;
+ IDqOutputChannel::TPtr GetOutputChannel(ui64 channelId) override {
+ auto ptr = OutputChannels.FindPtr(channelId);
+ YQL_ENSURE(ptr, "task: " << TaskId << " does not have output channelId: " << channelId);
+ return *ptr;
}
IDqSink::TPtr GetSink(ui64 outputIndex) override {
auto ptr = Sinks.FindPtr(outputIndex);
- YQL_ENSURE(ptr, "task: " << TaskId << " does not have output index: " << outputIndex);
+ YQL_ENSURE(ptr, "task: " << TaskId << " does not have output index: " << outputIndex);
return *ptr;
}
- TGuard<NKikimr::NMiniKQL::TScopedAlloc> BindAllocator(TMaybe<ui64> memoryLimit) override {
- auto guard = Context.TypeEnv ? Context.TypeEnv->BindAllocator() : SelfTypeEnv->BindAllocator();
- if (memoryLimit) {
- guard.GetMutex()->SetLimit(*memoryLimit);
- }
- return guard;
+ TGuard<NKikimr::NMiniKQL::TScopedAlloc> BindAllocator(TMaybe<ui64> memoryLimit) override {
+ auto guard = Context.TypeEnv ? Context.TypeEnv->BindAllocator() : SelfTypeEnv->BindAllocator();
+ if (memoryLimit) {
+ guard.GetMutex()->SetLimit(*memoryLimit);
+ }
+ return guard;
+ }
+
+ bool IsAllocatorAttached() override {
+ return Context.TypeEnv ? Context.TypeEnv->GetAllocator().IsAttached() : SelfTypeEnv->GetAllocator().IsAttached();
}
- bool IsAllocatorAttached() override {
- return Context.TypeEnv ? Context.TypeEnv->GetAllocator().IsAttached() : SelfTypeEnv->GetAllocator().IsAttached();
- }
-
- const NKikimr::NMiniKQL::TTypeEnvironment& GetTypeEnv() const override {
- return Context.TypeEnv ? *Context.TypeEnv : *SelfTypeEnv;
+ const NKikimr::NMiniKQL::TTypeEnvironment& GetTypeEnv() const override {
+ return Context.TypeEnv ? *Context.TypeEnv : *SelfTypeEnv;
}
const NKikimr::NMiniKQL::THolderFactory& GetHolderFactory() const override {
@@ -616,8 +616,8 @@ public:
}
}
- const TDqTaskRunnerStats* GetStats() const override {
- return Stats.get();
+ const TDqTaskRunnerStats* GetStats() const override {
+ return Stats.get();
}
TString Save() const override {
@@ -629,134 +629,134 @@ public:
}
private:
- NKikimr::NMiniKQL::TTypeEnvironment& TypeEnv() {
- return Context.TypeEnv ? *Context.TypeEnv : *SelfTypeEnv;
- }
-
- NKikimr::NMiniKQL::TScopedAlloc& Alloc() {
- return Context.Alloc ? *Context.Alloc : *SelfAlloc;
- }
-
- ERunStatus FetchAndDispatch() {
- if (!Output) {
- LOG("no consumers, Finish execution");
- return ERunStatus::Finished;
- }
-
- TBindTerminator term(ProgramParsed.CompGraph->GetTerminator());
-
- auto startComputeTime = TInstant::Now();
- Y_DEFER {
- if (Stats) {
- auto duration = TInstant::Now() - startComputeTime;
- Stats->ComputeCpuTime += duration;
- if (Y_UNLIKELY(CollectProfileStats)) {
- RunComputeTime = duration;
- }
- }
- };
-
- while (!Output->IsFull()) {
- if (Y_UNLIKELY(CollectProfileStats)) {
- auto now = TInstant::Now();
- StopWaitingOutput(now);
- StopWaiting(now);
- }
-
- NUdf::TUnboxedValue value;
- auto fetchStatus = ResultStream.Fetch(value);
-
- switch (fetchStatus) {
- case NUdf::EFetchStatus::Ok: {
- Output->Consume(std::move(value));
- break;
- }
- case NUdf::EFetchStatus::Finish: {
- LOG(TStringBuilder() << "task" << TaskId << ", execution finished, finish consumers");
- Output->Finish();
- return ERunStatus::Finished;
- }
- case NUdf::EFetchStatus::Yield: {
- return ERunStatus::PendingInput;
- }
- }
- }
-
- return ERunStatus::PendingOutput;
- }
-
-private:
- ui64 TaskId = 0;
- TDqTaskRunnerContext Context;
- TDqTaskRunnerSettings Settings;
- TLogFunc LogFunc;
+ NKikimr::NMiniKQL::TTypeEnvironment& TypeEnv() {
+ return Context.TypeEnv ? *Context.TypeEnv : *SelfTypeEnv;
+ }
+
+ NKikimr::NMiniKQL::TScopedAlloc& Alloc() {
+ return Context.Alloc ? *Context.Alloc : *SelfAlloc;
+ }
+
+ ERunStatus FetchAndDispatch() {
+ if (!Output) {
+ LOG("no consumers, Finish execution");
+ return ERunStatus::Finished;
+ }
+
+ TBindTerminator term(ProgramParsed.CompGraph->GetTerminator());
+
+ auto startComputeTime = TInstant::Now();
+ Y_DEFER {
+ if (Stats) {
+ auto duration = TInstant::Now() - startComputeTime;
+ Stats->ComputeCpuTime += duration;
+ if (Y_UNLIKELY(CollectProfileStats)) {
+ RunComputeTime = duration;
+ }
+ }
+ };
+
+ while (!Output->IsFull()) {
+ if (Y_UNLIKELY(CollectProfileStats)) {
+ auto now = TInstant::Now();
+ StopWaitingOutput(now);
+ StopWaiting(now);
+ }
+
+ NUdf::TUnboxedValue value;
+ auto fetchStatus = ResultStream.Fetch(value);
+
+ switch (fetchStatus) {
+ case NUdf::EFetchStatus::Ok: {
+ Output->Consume(std::move(value));
+ break;
+ }
+ case NUdf::EFetchStatus::Finish: {
+ LOG(TStringBuilder() << "task" << TaskId << ", execution finished, finish consumers");
+ Output->Finish();
+ return ERunStatus::Finished;
+ }
+ case NUdf::EFetchStatus::Yield: {
+ return ERunStatus::PendingInput;
+ }
+ }
+ }
+
+ return ERunStatus::PendingOutput;
+ }
+
+private:
+ ui64 TaskId = 0;
+ TDqTaskRunnerContext Context;
+ TDqTaskRunnerSettings Settings;
+ TLogFunc LogFunc;
std::unique_ptr<NUdf::ISecureParamsProvider> SecureParamsProvider;
-
- std::unique_ptr<NKikimr::NMiniKQL::TScopedAlloc> SelfAlloc; // if not set -> use Context.Alloc
- std::unique_ptr<NKikimr::NMiniKQL::TTypeEnvironment> SelfTypeEnv; // if not set -> use Context.TypeEnv
-
- struct TProgramParsed {
- TRuntimeNode ProgramNode;
- TVector<TType*> InputItemTypes;
- TVector<TType*> OutputItemTypes;
- TVector<TNode*> EntryPoints; // last entry node stands for parameters
-
- IStatsRegistryPtr StatsRegistry;
- IComputationPattern::TPtr CompPattern;
- THolder<IComputationGraph> CompGraph;
- };
- TProgramParsed ProgramParsed;
-
+
+ std::unique_ptr<NKikimr::NMiniKQL::TScopedAlloc> SelfAlloc; // if not set -> use Context.Alloc
+ std::unique_ptr<NKikimr::NMiniKQL::TTypeEnvironment> SelfTypeEnv; // if not set -> use Context.TypeEnv
+
+ struct TProgramParsed {
+ TRuntimeNode ProgramNode;
+ TVector<TType*> InputItemTypes;
+ TVector<TType*> OutputItemTypes;
+ TVector<TNode*> EntryPoints; // last entry node stands for parameters
+
+ IStatsRegistryPtr StatsRegistry;
+ IComputationPattern::TPtr CompPattern;
+ THolder<IComputationGraph> CompGraph;
+ };
+ TProgramParsed ProgramParsed;
+
THashMap<ui64, IDqInputChannel::TPtr> InputChannels; // Channel id -> Channel
THashMap<ui64, IDqSource::TPtr> Sources; // Input index -> Source
THashMap<ui64, IDqOutputChannel::TPtr> OutputChannels; // Channel id -> Channel
THashMap<ui64, IDqSink::TPtr> Sinks; // Output index -> Sink
- IDqOutputConsumer::TPtr Output;
- NUdf::TUnboxedValue ResultStream;
-
- bool TaskHasEffects = false;
-
- bool CollectBasicStats = false;
- bool CollectProfileStats = false;
- std::unique_ptr<TDqTaskRunnerStats> Stats;
+ IDqOutputConsumer::TPtr Output;
+ NUdf::TUnboxedValue ResultStream;
+
+ bool TaskHasEffects = false;
+
+ bool CollectBasicStats = false;
+ bool CollectProfileStats = false;
+ std::unique_ptr<TDqTaskRunnerStats> Stats;
TDuration RunComputeTime;
-
-private:
- // statistics support
- std::optional<TInstant> StartWaitOutputTime;
- std::optional<TInstant> StartWaitTime;
-
- void StartWaitingOutput(TInstant now) {
- if (Y_UNLIKELY(CollectProfileStats) && !StartWaitOutputTime) {
- StartWaitOutputTime = now;
- }
- }
-
- void StopWaitingOutput(TInstant now) {
- if (Y_UNLIKELY(CollectProfileStats) && StartWaitOutputTime) {
- Stats->WaitOutputTime += (now - *StartWaitOutputTime);
- StartWaitOutputTime.reset();
- }
- }
-
- void StartWaiting(TInstant now) {
- if (Y_UNLIKELY(CollectProfileStats) && !StartWaitTime) {
- StartWaitTime = now;
- }
- }
-
- void StopWaiting(TInstant now) {
- if (Y_UNLIKELY(CollectProfileStats) && StartWaitTime) {
- Stats->WaitTime += (now - *StartWaitTime);
- StartWaitTime.reset();
- }
- }
+
+private:
+ // statistics support
+ std::optional<TInstant> StartWaitOutputTime;
+ std::optional<TInstant> StartWaitTime;
+
+ void StartWaitingOutput(TInstant now) {
+ if (Y_UNLIKELY(CollectProfileStats) && !StartWaitOutputTime) {
+ StartWaitOutputTime = now;
+ }
+ }
+
+ void StopWaitingOutput(TInstant now) {
+ if (Y_UNLIKELY(CollectProfileStats) && StartWaitOutputTime) {
+ Stats->WaitOutputTime += (now - *StartWaitOutputTime);
+ StartWaitOutputTime.reset();
+ }
+ }
+
+ void StartWaiting(TInstant now) {
+ if (Y_UNLIKELY(CollectProfileStats) && !StartWaitTime) {
+ StartWaitTime = now;
+ }
+ }
+
+ void StopWaiting(TInstant now) {
+ if (Y_UNLIKELY(CollectProfileStats) && StartWaitTime) {
+ Stats->WaitTime += (now - *StartWaitTime);
+ StartWaitTime.reset();
+ }
+ }
};
-TIntrusivePtr<IDqTaskRunner> MakeDqTaskRunner(const TDqTaskRunnerContext& ctx, const TDqTaskRunnerSettings& settings,
- const TLogFunc& logFunc)
-{
- return new TDqTaskRunner(ctx, settings, logFunc);
+TIntrusivePtr<IDqTaskRunner> MakeDqTaskRunner(const TDqTaskRunnerContext& ctx, const TDqTaskRunnerSettings& settings,
+ const TLogFunc& logFunc)
+{
+ return new TDqTaskRunner(ctx, settings, logFunc);
}
} // namespace NYql::NDq
diff --git a/ydb/library/yql/dq/runtime/dq_tasks_runner.h b/ydb/library/yql/dq/runtime/dq_tasks_runner.h
index 4fbe6944ca..a0cc816e91 100644
--- a/ydb/library/yql/dq/runtime/dq_tasks_runner.h
+++ b/ydb/library/yql/dq/runtime/dq_tasks_runner.h
@@ -16,15 +16,15 @@
#include <library/cpp/monlib/metrics/histogram_collector.h>
-#include <util/generic/size_literals.h>
+#include <util/generic/size_literals.h>
#include <util/system/types.h>
namespace NYql::NDq {
enum class ERunStatus : ui32 {
Finished,
- PendingInput,
- PendingOutput
+ PendingInput,
+ PendingOutput
};
class TRunStatusTimeMetrics {
@@ -66,123 +66,123 @@ struct TMkqlStat {
i64 Value = 0;
};
-struct TDqTaskRunnerStats {
- // basic stats
- TDuration BuildCpuTime;
- TInstant FinishTs;
-
+struct TDqTaskRunnerStats {
+ // basic stats
+ TDuration BuildCpuTime;
+ TInstant FinishTs;
+
TDuration ComputeCpuTime;
TRunStatusTimeMetrics RunStatusTimeMetrics; // ComputeCpuTime + RunStatusTimeMetrics == 100% time
- // profile stats
- TDuration WaitTime; // wall time of waiting for input, scans & output
- TDuration WaitOutputTime;
+ // profile stats
+ TDuration WaitTime; // wall time of waiting for input, scans & output
+ TDuration WaitOutputTime;
- NMonitoring::IHistogramCollectorPtr ComputeCpuTimeByRun; // in millis
+ NMonitoring::IHistogramCollectorPtr ComputeCpuTimeByRun; // in millis
THashMap<ui64, const TDqInputChannelStats*> InputChannels; // Channel id -> Channel stats
THashMap<ui64, const TDqSourceStats*> Sources; // Input index -> Source stats
THashMap<ui64, const TDqOutputChannelStats*> OutputChannels; // Channel id -> Channel stats
TVector<TMkqlStat> MkqlStats;
-};
-
-struct TDqTaskRunnerContext {
- const NKikimr::NMiniKQL::IFunctionRegistry* FuncRegistry = nullptr;
- IRandomProvider* RandomProvider = nullptr;
- ITimeProvider* TimeProvider = nullptr;
- TDqComputeContextBase* ComputeCtx = nullptr;
- NKikimr::NMiniKQL::TComputationNodeFactory ComputationFactory;
- NUdf::IApplyContext* ApplyCtx = nullptr;
+};
+
+struct TDqTaskRunnerContext {
+ const NKikimr::NMiniKQL::IFunctionRegistry* FuncRegistry = nullptr;
+ IRandomProvider* RandomProvider = nullptr;
+ ITimeProvider* TimeProvider = nullptr;
+ TDqComputeContextBase* ComputeCtx = nullptr;
+ NKikimr::NMiniKQL::TComputationNodeFactory ComputationFactory;
+ NUdf::IApplyContext* ApplyCtx = nullptr;
NKikimr::NMiniKQL::TCallableVisitFuncProvider FuncProvider;
- NKikimr::NMiniKQL::TScopedAlloc* Alloc = nullptr;
- NKikimr::NMiniKQL::TTypeEnvironment* TypeEnv = nullptr;
-};
-
-class IDqTaskRunnerExecutionContext {
-public:
- virtual ~IDqTaskRunnerExecutionContext() = default;
-
- virtual IDqOutputConsumer::TPtr CreateOutputConsumer(const NDqProto::TTaskOutput& outputDesc,
- const NKikimr::NMiniKQL::TType* type, NUdf::IApplyContext* applyCtx,
+ NKikimr::NMiniKQL::TScopedAlloc* Alloc = nullptr;
+ NKikimr::NMiniKQL::TTypeEnvironment* TypeEnv = nullptr;
+};
+
+class IDqTaskRunnerExecutionContext {
+public:
+ virtual ~IDqTaskRunnerExecutionContext() = default;
+
+ virtual IDqOutputConsumer::TPtr CreateOutputConsumer(const NDqProto::TTaskOutput& outputDesc,
+ const NKikimr::NMiniKQL::TType* type, NUdf::IApplyContext* applyCtx,
const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv, TVector<IDqOutput::TPtr>&& outputs) const = 0;
-
- virtual IDqChannelStorage::TPtr CreateChannelStorage(ui64 channelId) const = 0;
-};
-
-class TDqTaskRunnerExecutionContext : public IDqTaskRunnerExecutionContext {
-public:
- IDqOutputConsumer::TPtr CreateOutputConsumer(const NDqProto::TTaskOutput& outputDesc,
- const NKikimr::NMiniKQL::TType* type, NUdf::IApplyContext* applyCtx,
+
+ virtual IDqChannelStorage::TPtr CreateChannelStorage(ui64 channelId) const = 0;
+};
+
+class TDqTaskRunnerExecutionContext : public IDqTaskRunnerExecutionContext {
+public:
+ IDqOutputConsumer::TPtr CreateOutputConsumer(const NDqProto::TTaskOutput& outputDesc,
+ const NKikimr::NMiniKQL::TType* type, NUdf::IApplyContext* applyCtx,
const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv, TVector<IDqOutput::TPtr>&& outputs) const override;
-
- IDqChannelStorage::TPtr CreateChannelStorage(ui64 channelId) const override;
-};
-
-struct TDqTaskRunnerSettings {
- bool CollectBasicStats = false;
- bool CollectProfileStats = false;
- bool TerminateOnError = false;
- bool AllowGeneratorsInUnboxedValues = true;
+
+ IDqChannelStorage::TPtr CreateChannelStorage(ui64 channelId) const override;
+};
+
+struct TDqTaskRunnerSettings {
+ bool CollectBasicStats = false;
+ bool CollectProfileStats = false;
+ bool TerminateOnError = false;
+ bool AllowGeneratorsInUnboxedValues = true;
TString OptLLVM = "";
THashMap<TString, TString> SecureParams;
THashMap<TString, TString> TaskParams;
-};
-
-struct TDqTaskRunnerMemoryLimits {
- ui32 ChannelBufferSize = 0;
- ui32 OutputChunkMaxSize = 0;
-};
-
+};
+
+struct TDqTaskRunnerMemoryLimits {
+ ui32 ChannelBufferSize = 0;
+ ui32 OutputChunkMaxSize = 0;
+};
+
NUdf::TUnboxedValue DqBuildInputValue(const NDqProto::TTaskInput& inputDesc, const NKikimr::NMiniKQL::TType* type,
TVector<IDqInputChannel::TPtr>&& channels, const NKikimr::NMiniKQL::THolderFactory& holderFactory);
-
-IDqOutputConsumer::TPtr DqBuildOutputConsumer(const NDqProto::TTaskOutput& outputDesc, const NKikimr::NMiniKQL::TType* type,
+
+IDqOutputConsumer::TPtr DqBuildOutputConsumer(const NDqProto::TTaskOutput& outputDesc, const NKikimr::NMiniKQL::TType* type,
const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv, TVector<IDqOutput::TPtr>&& channels);
-
-using TDqTaskRunnerParameterProvider = std::function<
- bool(std::string_view name, NKikimr::NMiniKQL::TType* type, const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv,
- const NKikimr::NMiniKQL::THolderFactory& holderFactory, NUdf::TUnboxedValue& value)
->;
-
-class IDqTaskRunner : public TSimpleRefCount<IDqTaskRunner>, private TNonCopyable {
-public:
- virtual ~IDqTaskRunner() = default;
-
- virtual ui64 GetTaskId() const = 0;
-
- virtual void Prepare(const NDqProto::TDqTask& task, const TDqTaskRunnerMemoryLimits& memoryLimits,
- const IDqTaskRunnerExecutionContext& execCtx = TDqTaskRunnerExecutionContext(),
- const TDqTaskRunnerParameterProvider& parameterProvider = {}) = 0;
+
+using TDqTaskRunnerParameterProvider = std::function<
+ bool(std::string_view name, NKikimr::NMiniKQL::TType* type, const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv,
+ const NKikimr::NMiniKQL::THolderFactory& holderFactory, NUdf::TUnboxedValue& value)
+>;
+
+class IDqTaskRunner : public TSimpleRefCount<IDqTaskRunner>, private TNonCopyable {
+public:
+ virtual ~IDqTaskRunner() = default;
+
+ virtual ui64 GetTaskId() const = 0;
+
+ virtual void Prepare(const NDqProto::TDqTask& task, const TDqTaskRunnerMemoryLimits& memoryLimits,
+ const IDqTaskRunnerExecutionContext& execCtx = TDqTaskRunnerExecutionContext(),
+ const TDqTaskRunnerParameterProvider& parameterProvider = {}) = 0;
virtual ERunStatus Run() = 0;
virtual bool HasEffects() const = 0;
virtual IDqInputChannel::TPtr GetInputChannel(ui64 channelId) = 0;
virtual IDqSource::TPtr GetSource(ui64 inputIndex) = 0;
- virtual IDqOutputChannel::TPtr GetOutputChannel(ui64 channelId) = 0;
+ virtual IDqOutputChannel::TPtr GetOutputChannel(ui64 channelId) = 0;
virtual IDqSink::TPtr GetSink(ui64 outputIndex) = 0;
- // if memoryLimit = Nothing() then don't set memory limit, use existing one (if any)
- // if memoryLimit = 0 then set unlimited
- // otherwise use particular memory limit
- virtual TGuard<NKikimr::NMiniKQL::TScopedAlloc> BindAllocator(TMaybe<ui64> memoryLimit = Nothing()) = 0;
- virtual bool IsAllocatorAttached() = 0;
- virtual const NKikimr::NMiniKQL::TTypeEnvironment& GetTypeEnv() const = 0;
+ // if memoryLimit = Nothing() then don't set memory limit, use existing one (if any)
+ // if memoryLimit = 0 then set unlimited
+ // otherwise use particular memory limit
+ virtual TGuard<NKikimr::NMiniKQL::TScopedAlloc> BindAllocator(TMaybe<ui64> memoryLimit = Nothing()) = 0;
+ virtual bool IsAllocatorAttached() = 0;
+ virtual const NKikimr::NMiniKQL::TTypeEnvironment& GetTypeEnv() const = 0;
virtual const NKikimr::NMiniKQL::THolderFactory& GetHolderFactory() const = 0;
virtual const THashMap<TString, TString>& GetSecureParams() const = 0;
virtual const THashMap<TString, TString>& GetTaskParams() const = 0;
virtual void UpdateStats() = 0;
- virtual const TDqTaskRunnerStats* GetStats() const = 0;
+ virtual const TDqTaskRunnerStats* GetStats() const = 0;
[[nodiscard]]
virtual TString Save() const = 0;
virtual void Load(TStringBuf in) = 0;
};
-TIntrusivePtr<IDqTaskRunner> MakeDqTaskRunner(const TDqTaskRunnerContext& ctx, const TDqTaskRunnerSettings& settings,
- const TLogFunc& logFunc);
-
+TIntrusivePtr<IDqTaskRunner> MakeDqTaskRunner(const TDqTaskRunnerContext& ctx, const TDqTaskRunnerSettings& settings,
+ const TLogFunc& logFunc);
+
} // namespace NYql::NDq
diff --git a/ydb/library/yql/dq/runtime/dq_transport.cpp b/ydb/library/yql/dq/runtime/dq_transport.cpp
index ba9f843c1e..93b85b0c6f 100644
--- a/ydb/library/yql/dq/runtime/dq_transport.cpp
+++ b/ydb/library/yql/dq/runtime/dq_transport.cpp
@@ -17,57 +17,57 @@ using namespace NYql;
namespace {
-NDqProto::TData SerializeBufferArrowV1(TUnboxedValueVector& buffer, const TType* itemType);
+NDqProto::TData SerializeBufferArrowV1(TUnboxedValueVector& buffer, const TType* itemType);
-void DeserializeBufferArrowV1(const NDqProto::TData& data, const TType* itemType,
+void DeserializeBufferArrowV1(const NDqProto::TData& data, const TType* itemType,
const THolderFactory& holderFactory, TUnboxedValueVector& buffer);
-NDqProto::TData SerializeValuePickleV1(const TType* type, const NUdf::TUnboxedValuePod& value) {
- TValuePacker packer(/* stable */ false, type);
+NDqProto::TData SerializeValuePickleV1(const TType* type, const NUdf::TUnboxedValuePod& value) {
+ TValuePacker packer(/* stable */ false, type);
TStringBuf packResult = packer.Pack(value);
NDqProto::TData data;
data.SetTransportVersion(NDqProto::DATA_TRANSPORT_UV_PICKLE_1_0);
data.SetRaw(packResult.data(), packResult.size());
- data.SetRows(1);
+ data.SetRows(1);
return data;
}
-NDqProto::TData SerializeValueArrowV1(const TType* type, const NUdf::TUnboxedValuePod& value) {
+NDqProto::TData SerializeValueArrowV1(const TType* type, const NUdf::TUnboxedValuePod& value) {
TUnboxedValueVector buffer;
buffer.push_back(value);
return SerializeBufferArrowV1(buffer, type);
}
-void DeserializeValuePickleV1(const TType* type, const NDqProto::TData& data, NUdf::TUnboxedValue& value,
- const THolderFactory& holderFactory)
-{
+void DeserializeValuePickleV1(const TType* type, const NDqProto::TData& data, NUdf::TUnboxedValue& value,
+ const THolderFactory& holderFactory)
+{
YQL_ENSURE(data.GetTransportVersion() == (ui32) NDqProto::DATA_TRANSPORT_UV_PICKLE_1_0);
- TValuePacker packer(/* stable */ false, type);
+ TValuePacker packer(/* stable */ false, type);
value = packer.Unpack(data.GetRaw(), holderFactory);
}
-void DeserializeValueArrowV1(const TType* type, const NDqProto::TData& data, NUdf::TUnboxedValue& value,
- const THolderFactory& holderFactory)
-{
+void DeserializeValueArrowV1(const TType* type, const NDqProto::TData& data, NUdf::TUnboxedValue& value,
+ const THolderFactory& holderFactory)
+{
TUnboxedValueVector buffer;
DeserializeBufferArrowV1(data, type, holderFactory, buffer);
value = buffer[0];
}
-NDqProto::TData SerializeBufferPickleV1(TUnboxedValueVector& buffer, const TType* itemType,
- const TTypeEnvironment& typeEnv, const THolderFactory& holderFactory)
-{
- const auto listType = TListType::Create(const_cast<TType*>(itemType), typeEnv);
+NDqProto::TData SerializeBufferPickleV1(TUnboxedValueVector& buffer, const TType* itemType,
+ const TTypeEnvironment& typeEnv, const THolderFactory& holderFactory)
+{
+ const auto listType = TListType::Create(const_cast<TType*>(itemType), typeEnv);
const NUdf::TUnboxedValue listValue = holderFactory.VectorAsArray(buffer);
auto data = SerializeValuePickleV1(listType, listValue);
- data.SetRows(buffer.size());
- return data;
+ data.SetRows(buffer.size());
+ return data;
}
-NDqProto::TData SerializeBufferArrowV1(TUnboxedValueVector& buffer, const TType* itemType) {
+NDqProto::TData SerializeBufferArrowV1(TUnboxedValueVector& buffer, const TType* itemType) {
auto array = NArrow::MakeArray(buffer, itemType);
auto serialized = NArrow::SerializeArray(array);
@@ -79,10 +79,10 @@ NDqProto::TData SerializeBufferArrowV1(TUnboxedValueVector& buffer, const TType*
return data;
}
-void DeserializeBufferPickleV1(const NDqProto::TData& data, const TType* itemType, const TTypeEnvironment& typeEnv,
- const THolderFactory& holderFactory, TUnboxedValueVector& buffer)
-{
- auto listType = TListType::Create(const_cast<TType*>(itemType), typeEnv);
+void DeserializeBufferPickleV1(const NDqProto::TData& data, const TType* itemType, const TTypeEnvironment& typeEnv,
+ const THolderFactory& holderFactory, TUnboxedValueVector& buffer)
+{
+ auto listType = TListType::Create(const_cast<TType*>(itemType), typeEnv);
NUdf::TUnboxedValue value;
DeserializeValuePickleV1(listType, data, value, holderFactory);
@@ -93,9 +93,9 @@ void DeserializeBufferPickleV1(const NDqProto::TData& data, const TType* itemTyp
}
}
-void DeserializeBufferArrowV1(const NDqProto::TData& data, const TType* itemType, const THolderFactory& holderFactory,
- TUnboxedValueVector& buffer)
-{
+void DeserializeBufferArrowV1(const NDqProto::TData& data, const TType* itemType, const THolderFactory& holderFactory,
+ TUnboxedValueVector& buffer)
+{
YQL_ENSURE(data.GetTransportVersion() == (ui32) NDqProto::DATA_TRANSPORT_ARROW_1_0);
auto array = NArrow::DeserializeArray(data.GetRaw(), NArrow::GetArrowType(itemType));
@@ -107,26 +107,26 @@ void DeserializeBufferArrowV1(const NDqProto::TData& data, const TType* itemType
}
NDqProto::TData SerializeParamV1(const TMkqlValueRef& param, const TTypeEnvironment& typeEnv,
- const THolderFactory& holderFactory)
-{
+ const THolderFactory& holderFactory)
+{
auto [type, value] = ImportValueFromProto(param.GetType(), param.GetValue(), typeEnv, holderFactory);
return SerializeValuePickleV1(type, value);
}
-void DeserializeParamV1(const NDqProto::TData& data, const TType* type, const THolderFactory& holderFactory,
- NUdf::TUnboxedValue& value)
-{
+void DeserializeParamV1(const NDqProto::TData& data, const TType* type, const THolderFactory& holderFactory,
+ NUdf::TUnboxedValue& value)
+{
DeserializeValuePickleV1(type, data, value, holderFactory);
}
} // namespace
-NDqProto::EDataTransportVersion TDqDataSerializer::GetTransportVersion() const {
+NDqProto::EDataTransportVersion TDqDataSerializer::GetTransportVersion() const {
return TransportVersion;
}
-NDqProto::TData TDqDataSerializer::Serialize(const NUdf::TUnboxedValue& value, const TType* itemType) const {
+NDqProto::TData TDqDataSerializer::Serialize(const NUdf::TUnboxedValue& value, const TType* itemType) const {
switch (TransportVersion) {
case NDqProto::DATA_TRANSPORT_VERSION_UNSPECIFIED:
case NDqProto::DATA_TRANSPORT_UV_PICKLE_1_0:
@@ -136,9 +136,9 @@ NDqProto::TData TDqDataSerializer::Serialize(const NUdf::TUnboxedValue& value, c
default:
YQL_ENSURE(false, "Unsupported TransportVersion");
}
-}
+}
-NDqProto::TData TDqDataSerializer::Serialize(TUnboxedValueVector& buffer, const TType* itemType) const {
+NDqProto::TData TDqDataSerializer::Serialize(TUnboxedValueVector& buffer, const TType* itemType) const {
switch (TransportVersion) {
case NDqProto::DATA_TRANSPORT_VERSION_UNSPECIFIED:
case NDqProto::DATA_TRANSPORT_UV_PICKLE_1_0:
@@ -148,11 +148,11 @@ NDqProto::TData TDqDataSerializer::Serialize(TUnboxedValueVector& buffer, const
default:
YQL_ENSURE(false, "Unsupported TransportVersion");
}
-}
-
-void TDqDataSerializer::Deserialize(const NDqProto::TData& data, const TType* itemType,
- TUnboxedValueVector& buffer) const
-{
+}
+
+void TDqDataSerializer::Deserialize(const NDqProto::TData& data, const TType* itemType,
+ TUnboxedValueVector& buffer) const
+{
switch (TransportVersion) {
case NDqProto::DATA_TRANSPORT_VERSION_UNSPECIFIED:
case NDqProto::DATA_TRANSPORT_UV_PICKLE_1_0: {
@@ -168,9 +168,9 @@ void TDqDataSerializer::Deserialize(const NDqProto::TData& data, const TType* it
}
}
-void TDqDataSerializer::Deserialize(const NDqProto::TData& data, const TType* itemType,
- NUdf::TUnboxedValue& value) const
-{
+void TDqDataSerializer::Deserialize(const NDqProto::TData& data, const TType* itemType,
+ NUdf::TUnboxedValue& value) const
+{
switch (TransportVersion) {
case NDqProto::DATA_TRANSPORT_VERSION_UNSPECIFIED:
case NDqProto::DATA_TRANSPORT_UV_PICKLE_1_0: {
@@ -187,220 +187,220 @@ void TDqDataSerializer::Deserialize(const NDqProto::TData& data, const TType* it
}
-NDqProto::TData TDqDataSerializer::SerializeParam(const TMkqlValueRef& param, const TTypeEnvironment& typeEnv,
- const NKikimr::NMiniKQL::THolderFactory& holderFactory)
-{
- return SerializeParamV1(param, typeEnv, holderFactory);
+NDqProto::TData TDqDataSerializer::SerializeParam(const TMkqlValueRef& param, const TTypeEnvironment& typeEnv,
+ const NKikimr::NMiniKQL::THolderFactory& holderFactory)
+{
+ return SerializeParamV1(param, typeEnv, holderFactory);
}
-void TDqDataSerializer::DeserializeParam(const NDqProto::TData& data, const TType* type,
- const NKikimr::NMiniKQL::THolderFactory& holderFactory, NUdf::TUnboxedValue& value)
-{
- YQL_ENSURE(data.GetTransportVersion() == (ui32) NDqProto::DATA_TRANSPORT_UV_PICKLE_1_0);
+void TDqDataSerializer::DeserializeParam(const NDqProto::TData& data, const TType* type,
+ const NKikimr::NMiniKQL::THolderFactory& holderFactory, NUdf::TUnboxedValue& value)
+{
+ YQL_ENSURE(data.GetTransportVersion() == (ui32) NDqProto::DATA_TRANSPORT_UV_PICKLE_1_0);
- return DeserializeParamV1(data, type, holderFactory, value);
+ return DeserializeParamV1(data, type, holderFactory, value);
}
-NDqProto::TData TDqDataSerializer::SerializeParamValue(const TType* type, const NUdf::TUnboxedValuePod& value) {
- return SerializeValuePickleV1(type, value);
-}
-
-ui64 TDqDataSerializer::CalcSerializedSize(NUdf::TUnboxedValue& value, const NKikimr::NMiniKQL::TType* itemType) {
+NDqProto::TData TDqDataSerializer::SerializeParamValue(const TType* type, const NUdf::TUnboxedValuePod& value) {
+ return SerializeValuePickleV1(type, value);
+}
+
+ui64 TDqDataSerializer::CalcSerializedSize(NUdf::TUnboxedValue& value, const NKikimr::NMiniKQL::TType* itemType) {
auto data = SerializeValuePickleV1(itemType, value);
// YQL-9648
DeserializeValuePickleV1(itemType, data, value, HolderFactory);
return data.GetRaw().size();
}
-namespace {
-
-std::optional<ui64> EstimateIntegralDataSize(const TDataType* dataType) {
- switch (*dataType->GetDataSlot()) {
- case NUdf::EDataSlot::Bool:
- case NUdf::EDataSlot::Int8:
- case NUdf::EDataSlot::Uint8:
- return 1;
- case NUdf::EDataSlot::Int16:
- case NUdf::EDataSlot::Uint16:
- return 2;
- case NUdf::EDataSlot::Int32:
- case NUdf::EDataSlot::Uint32:
- case NUdf::EDataSlot::Float:
- case NUdf::EDataSlot::Date:
- case NUdf::EDataSlot::TzDate:
- case NUdf::EDataSlot::Datetime:
- case NUdf::EDataSlot::TzDatetime:
- return 4;
- case NUdf::EDataSlot::Int64:
- case NUdf::EDataSlot::Uint64:
- case NUdf::EDataSlot::Double:
- case NUdf::EDataSlot::Timestamp:
- case NUdf::EDataSlot::TzTimestamp:
- case NUdf::EDataSlot::Interval:
- return 8;
- case NUdf::EDataSlot::Uuid:
- case NUdf::EDataSlot::Decimal:
- return 16;
- case NUdf::EDataSlot::String:
- case NUdf::EDataSlot::Utf8:
- case NUdf::EDataSlot::DyNumber:
- case NUdf::EDataSlot::Json:
- case NUdf::EDataSlot::JsonDocument:
- case NUdf::EDataSlot::Yson:
- return std::nullopt;
- }
-}
-
-ui64 EstimateSizeImpl(const NUdf::TUnboxedValuePod& value, const NKikimr::NMiniKQL::TType* type, bool* fixed) {
- switch (type->GetKind()) {
- case TType::EKind::Void:
- case TType::EKind::Null:
- case TType::EKind::EmptyList:
- case TType::EKind::EmptyDict:
- return 0;
-
- case TType::EKind::Data: {
- auto dataType = static_cast<const TDataType*>(type);
- if (auto size = EstimateIntegralDataSize(dataType); size.has_value()) {
- return *size;
- }
- if (fixed) {
- *fixed = false;
- }
- switch (*dataType->GetDataSlot()) {
- case NUdf::EDataSlot::String:
- case NUdf::EDataSlot::Utf8:
- case NUdf::EDataSlot::DyNumber:
- case NUdf::EDataSlot::Json:
- case NUdf::EDataSlot::JsonDocument:
- case NUdf::EDataSlot::Yson:
- return 2 + value.AsStringRef().Size();
- default:
- YQL_ENSURE(false, "" << dataType->GetKindAsStr());
- }
- }
-
- case TType::EKind::Optional: {
- auto optionalType = static_cast<const TOptionalType*>(type);
- if (value) {
- if (optionalType->GetItemType()->GetKind() == TType::EKind::Data) {
- auto dataType = static_cast<const TDataType*>(optionalType->GetItemType());
- if (auto size = EstimateIntegralDataSize(dataType); size.has_value()) {
- return *size;
- }
- }
- return EstimateSizeImpl(value.GetOptionalValue(), optionalType->GetItemType(), fixed);
- }
- return 0;
- }
-
- case TType::EKind::List: {
- auto listType = static_cast<const TListType*>(type);
- auto itemType = listType->GetItemType();
- ui64 size = 2;
- if (value.HasFastListLength() && value.GetListLength() > 0 && value.GetElements()) {
- auto len = value.GetListLength();
- auto p = value.GetElements();
- do {
- size += EstimateSizeImpl(*p++, itemType, fixed);
- }
- while (--len);
- } else {
- const auto iter = value.GetListIterator();
- for (NUdf::TUnboxedValue item; iter.Next(item);) {
- size += EstimateSizeImpl(item, itemType, fixed);
- }
- }
- return size;
- }
-
- case TType::EKind::Struct: {
- auto structType = static_cast<const TStructType*>(type);
- ui64 size = 2;
- for (ui32 index = 0; index < structType->GetMembersCount(); ++index) {
- auto memberType = structType->GetMemberType(index);
-
- if (memberType->GetKind() == TType::EKind::Data) {
- auto dataType = static_cast<const TDataType*>(memberType);
- if (auto s = EstimateIntegralDataSize(dataType); s.has_value()) {
- size += *s;
- continue;
- }
- }
-
- size += EstimateSizeImpl(value.GetElement(index), memberType, fixed);
- }
-
- return size;
- }
-
- case TType::EKind::Tuple: {
- auto tupleType = static_cast<const TTupleType*>(type);
- ui64 size = 2;
- for (ui32 index = 0; index < tupleType->GetElementsCount(); ++index) {
- auto elementType = tupleType->GetElementType(index);
-
- if (elementType->GetKind() == TType::EKind::Data) {
- auto dataType = static_cast<const TDataType*>(elementType);
- if (auto s = EstimateIntegralDataSize(dataType); s.has_value()) {
- size += *s;
- continue;
- }
- }
-
- size += EstimateSizeImpl(value.GetElement(index), elementType, fixed);
- }
- return size;
- }
-
- case TType::EKind::Dict: {
- auto dictType = static_cast<const TDictType*>(type);
- auto keyType = dictType->GetKeyType();
- auto payloadType = dictType->GetPayloadType();
-
- ui64 size = 2;
- const auto iter = value.GetDictIterator();
- for (NUdf::TUnboxedValue key, payload; iter.NextPair(key, payload);) {
- size += EstimateSizeImpl(key, keyType, fixed);
- size += EstimateSizeImpl(payload, payloadType, fixed);
- }
- return size;
- }
-
- case TType::EKind::Variant: {
- auto variantType = static_cast<const TVariantType*>(type);
- ui32 variantIndex = value.GetVariantIndex();
- TType* innerType = variantType->GetUnderlyingType();
- if (innerType->IsStruct()) {
- innerType = static_cast<TStructType*>(innerType)->GetMemberType(variantIndex);
- } else {
- MKQL_ENSURE(innerType->IsTuple(), "Unexpected underlying variant type: " << innerType->GetKindAsStr());
- innerType = static_cast<TTupleType*>(innerType)->GetElementType(variantIndex);
- }
- return 2 + EstimateSizeImpl(value.GetVariantItem(), innerType, fixed);
- }
-
- case TType::EKind::Type:
- case TType::EKind::Stream:
- case TType::EKind::Callable:
- case TType::EKind::Any:
- case TType::EKind::Resource:
- case TType::EKind::Flow:
- case TType::EKind::ReservedKind:
- case TType::EKind::Tagged:
- case TType::EKind::Block:
- THROW yexception() << "Unsupported type: " << type->GetKindAsStr();
- }
-}
-
-} // namespace
-
-ui64 TDqDataSerializer::EstimateSize(const NUdf::TUnboxedValue& value, const NKikimr::NMiniKQL::TType* type, bool* fixed)
-{
- if (fixed) {
- *fixed = true;
- }
- return EstimateSizeImpl(value, type, fixed);
-}
-
+namespace {
+
+std::optional<ui64> EstimateIntegralDataSize(const TDataType* dataType) {
+ switch (*dataType->GetDataSlot()) {
+ case NUdf::EDataSlot::Bool:
+ case NUdf::EDataSlot::Int8:
+ case NUdf::EDataSlot::Uint8:
+ return 1;
+ case NUdf::EDataSlot::Int16:
+ case NUdf::EDataSlot::Uint16:
+ return 2;
+ case NUdf::EDataSlot::Int32:
+ case NUdf::EDataSlot::Uint32:
+ case NUdf::EDataSlot::Float:
+ case NUdf::EDataSlot::Date:
+ case NUdf::EDataSlot::TzDate:
+ case NUdf::EDataSlot::Datetime:
+ case NUdf::EDataSlot::TzDatetime:
+ return 4;
+ case NUdf::EDataSlot::Int64:
+ case NUdf::EDataSlot::Uint64:
+ case NUdf::EDataSlot::Double:
+ case NUdf::EDataSlot::Timestamp:
+ case NUdf::EDataSlot::TzTimestamp:
+ case NUdf::EDataSlot::Interval:
+ return 8;
+ case NUdf::EDataSlot::Uuid:
+ case NUdf::EDataSlot::Decimal:
+ return 16;
+ case NUdf::EDataSlot::String:
+ case NUdf::EDataSlot::Utf8:
+ case NUdf::EDataSlot::DyNumber:
+ case NUdf::EDataSlot::Json:
+ case NUdf::EDataSlot::JsonDocument:
+ case NUdf::EDataSlot::Yson:
+ return std::nullopt;
+ }
+}
+
+ui64 EstimateSizeImpl(const NUdf::TUnboxedValuePod& value, const NKikimr::NMiniKQL::TType* type, bool* fixed) {
+ switch (type->GetKind()) {
+ case TType::EKind::Void:
+ case TType::EKind::Null:
+ case TType::EKind::EmptyList:
+ case TType::EKind::EmptyDict:
+ return 0;
+
+ case TType::EKind::Data: {
+ auto dataType = static_cast<const TDataType*>(type);
+ if (auto size = EstimateIntegralDataSize(dataType); size.has_value()) {
+ return *size;
+ }
+ if (fixed) {
+ *fixed = false;
+ }
+ switch (*dataType->GetDataSlot()) {
+ case NUdf::EDataSlot::String:
+ case NUdf::EDataSlot::Utf8:
+ case NUdf::EDataSlot::DyNumber:
+ case NUdf::EDataSlot::Json:
+ case NUdf::EDataSlot::JsonDocument:
+ case NUdf::EDataSlot::Yson:
+ return 2 + value.AsStringRef().Size();
+ default:
+ YQL_ENSURE(false, "" << dataType->GetKindAsStr());
+ }
+ }
+
+ case TType::EKind::Optional: {
+ auto optionalType = static_cast<const TOptionalType*>(type);
+ if (value) {
+ if (optionalType->GetItemType()->GetKind() == TType::EKind::Data) {
+ auto dataType = static_cast<const TDataType*>(optionalType->GetItemType());
+ if (auto size = EstimateIntegralDataSize(dataType); size.has_value()) {
+ return *size;
+ }
+ }
+ return EstimateSizeImpl(value.GetOptionalValue(), optionalType->GetItemType(), fixed);
+ }
+ return 0;
+ }
+
+ case TType::EKind::List: {
+ auto listType = static_cast<const TListType*>(type);
+ auto itemType = listType->GetItemType();
+ ui64 size = 2;
+ if (value.HasFastListLength() && value.GetListLength() > 0 && value.GetElements()) {
+ auto len = value.GetListLength();
+ auto p = value.GetElements();
+ do {
+ size += EstimateSizeImpl(*p++, itemType, fixed);
+ }
+ while (--len);
+ } else {
+ const auto iter = value.GetListIterator();
+ for (NUdf::TUnboxedValue item; iter.Next(item);) {
+ size += EstimateSizeImpl(item, itemType, fixed);
+ }
+ }
+ return size;
+ }
+
+ case TType::EKind::Struct: {
+ auto structType = static_cast<const TStructType*>(type);
+ ui64 size = 2;
+ for (ui32 index = 0; index < structType->GetMembersCount(); ++index) {
+ auto memberType = structType->GetMemberType(index);
+
+ if (memberType->GetKind() == TType::EKind::Data) {
+ auto dataType = static_cast<const TDataType*>(memberType);
+ if (auto s = EstimateIntegralDataSize(dataType); s.has_value()) {
+ size += *s;
+ continue;
+ }
+ }
+
+ size += EstimateSizeImpl(value.GetElement(index), memberType, fixed);
+ }
+
+ return size;
+ }
+
+ case TType::EKind::Tuple: {
+ auto tupleType = static_cast<const TTupleType*>(type);
+ ui64 size = 2;
+ for (ui32 index = 0; index < tupleType->GetElementsCount(); ++index) {
+ auto elementType = tupleType->GetElementType(index);
+
+ if (elementType->GetKind() == TType::EKind::Data) {
+ auto dataType = static_cast<const TDataType*>(elementType);
+ if (auto s = EstimateIntegralDataSize(dataType); s.has_value()) {
+ size += *s;
+ continue;
+ }
+ }
+
+ size += EstimateSizeImpl(value.GetElement(index), elementType, fixed);
+ }
+ return size;
+ }
+
+ case TType::EKind::Dict: {
+ auto dictType = static_cast<const TDictType*>(type);
+ auto keyType = dictType->GetKeyType();
+ auto payloadType = dictType->GetPayloadType();
+
+ ui64 size = 2;
+ const auto iter = value.GetDictIterator();
+ for (NUdf::TUnboxedValue key, payload; iter.NextPair(key, payload);) {
+ size += EstimateSizeImpl(key, keyType, fixed);
+ size += EstimateSizeImpl(payload, payloadType, fixed);
+ }
+ return size;
+ }
+
+ case TType::EKind::Variant: {
+ auto variantType = static_cast<const TVariantType*>(type);
+ ui32 variantIndex = value.GetVariantIndex();
+ TType* innerType = variantType->GetUnderlyingType();
+ if (innerType->IsStruct()) {
+ innerType = static_cast<TStructType*>(innerType)->GetMemberType(variantIndex);
+ } else {
+ MKQL_ENSURE(innerType->IsTuple(), "Unexpected underlying variant type: " << innerType->GetKindAsStr());
+ innerType = static_cast<TTupleType*>(innerType)->GetElementType(variantIndex);
+ }
+ return 2 + EstimateSizeImpl(value.GetVariantItem(), innerType, fixed);
+ }
+
+ case TType::EKind::Type:
+ case TType::EKind::Stream:
+ case TType::EKind::Callable:
+ case TType::EKind::Any:
+ case TType::EKind::Resource:
+ case TType::EKind::Flow:
+ case TType::EKind::ReservedKind:
+ case TType::EKind::Tagged:
+ case TType::EKind::Block:
+ THROW yexception() << "Unsupported type: " << type->GetKindAsStr();
+ }
+}
+
+} // namespace
+
+ui64 TDqDataSerializer::EstimateSize(const NUdf::TUnboxedValue& value, const NKikimr::NMiniKQL::TType* type, bool* fixed)
+{
+ if (fixed) {
+ *fixed = true;
+ }
+ return EstimateSizeImpl(value, type, fixed);
+}
+
} // namespace NYql::NDq
diff --git a/ydb/library/yql/dq/runtime/dq_transport.h b/ydb/library/yql/dq/runtime/dq_transport.h
index 89049fc3a7..8f6b6f947f 100644
--- a/ydb/library/yql/dq/runtime/dq_transport.h
+++ b/ydb/library/yql/dq/runtime/dq_transport.h
@@ -14,27 +14,27 @@ namespace NYql::NDq {
class TDqDataSerializer : private TNonCopyable {
public:
- TDqDataSerializer(const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv,
- const NKikimr::NMiniKQL::THolderFactory& holderFactory, NDqProto::EDataTransportVersion transportVersion)
- : TypeEnv(typeEnv)
- , HolderFactory(holderFactory)
- , TransportVersion(transportVersion) {}
+ TDqDataSerializer(const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv,
+ const NKikimr::NMiniKQL::THolderFactory& holderFactory, NDqProto::EDataTransportVersion transportVersion)
+ : TypeEnv(typeEnv)
+ , HolderFactory(holderFactory)
+ , TransportVersion(transportVersion) {}
- NDqProto::EDataTransportVersion GetTransportVersion() const;
+ NDqProto::EDataTransportVersion GetTransportVersion() const;
- NDqProto::TData Serialize(const NUdf::TUnboxedValue& value, const NKikimr::NMiniKQL::TType* itemType) const;
- NDqProto::TData Serialize(NKikimr::NMiniKQL::TUnboxedValueVector& buffer, const NKikimr::NMiniKQL::TType* itemType) const;
+ NDqProto::TData Serialize(const NUdf::TUnboxedValue& value, const NKikimr::NMiniKQL::TType* itemType) const;
+ NDqProto::TData Serialize(NKikimr::NMiniKQL::TUnboxedValueVector& buffer, const NKikimr::NMiniKQL::TType* itemType) const;
- template <class TForwardIterator>
- NDqProto::TData Serialize(TForwardIterator first, TForwardIterator last, const NKikimr::NMiniKQL::TType* itemType) const {
+ template <class TForwardIterator>
+ NDqProto::TData Serialize(TForwardIterator first, TForwardIterator last, const NKikimr::NMiniKQL::TType* itemType) const {
switch (TransportVersion) {
case NDqProto::DATA_TRANSPORT_VERSION_UNSPECIFIED:
case NDqProto::DATA_TRANSPORT_UV_PICKLE_1_0: {
auto count = std::distance(first, last);
- const auto listType = NKikimr::NMiniKQL::TListType::Create(
- const_cast<NKikimr::NMiniKQL::TType*>(itemType), TypeEnv);
+ const auto listType = NKikimr::NMiniKQL::TListType::Create(
+ const_cast<NKikimr::NMiniKQL::TType*>(itemType), TypeEnv);
const NUdf::TUnboxedValue listValue = HolderFactory.RangeAsArray(first, last);
-
+
auto data = Serialize(listValue, listType);
data.SetRows(count);
return data;
@@ -46,23 +46,23 @@ public:
default:
YQL_ENSURE(false, "Unsupported TransportVersion");
}
- }
-
- void Deserialize(const NDqProto::TData& data, const NKikimr::NMiniKQL::TType* itemType,
- NKikimr::NMiniKQL::TUnboxedValueVector& buffer) const;
- void Deserialize(const NDqProto::TData& data, const NKikimr::NMiniKQL::TType* itemType, NUdf::TUnboxedValue& value) const;
-
- ui64 CalcSerializedSize(NUdf::TUnboxedValue& value, const NKikimr::NMiniKQL::TType* type);
- static ui64 EstimateSize(const NUdf::TUnboxedValue& value, const NKikimr::NMiniKQL::TType* type, bool* fixed = nullptr);
+ }
+
+ void Deserialize(const NDqProto::TData& data, const NKikimr::NMiniKQL::TType* itemType,
+ NKikimr::NMiniKQL::TUnboxedValueVector& buffer) const;
+ void Deserialize(const NDqProto::TData& data, const NKikimr::NMiniKQL::TType* itemType, NUdf::TUnboxedValue& value) const;
+
+ ui64 CalcSerializedSize(NUdf::TUnboxedValue& value, const NKikimr::NMiniKQL::TType* type);
+ static ui64 EstimateSize(const NUdf::TUnboxedValue& value, const NKikimr::NMiniKQL::TType* type, bool* fixed = nullptr);
- static NDqProto::TData SerializeParam(const TMkqlValueRef& param, const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv,
- const NKikimr::NMiniKQL::THolderFactory& holderFactory);
- static void DeserializeParam(const NDqProto::TData& data, const NKikimr::NMiniKQL::TType* type,
- const NKikimr::NMiniKQL::THolderFactory& holderFactory, NKikimr::NUdf::TUnboxedValue& value);
-
- static NDqProto::TData SerializeParamValue(const NKikimr::NMiniKQL::TType* type, const NUdf::TUnboxedValuePod& value);
-
-public:
+ static NDqProto::TData SerializeParam(const TMkqlValueRef& param, const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv,
+ const NKikimr::NMiniKQL::THolderFactory& holderFactory);
+ static void DeserializeParam(const NDqProto::TData& data, const NKikimr::NMiniKQL::TType* type,
+ const NKikimr::NMiniKQL::THolderFactory& holderFactory, NKikimr::NUdf::TUnboxedValue& value);
+
+ static NDqProto::TData SerializeParamValue(const NKikimr::NMiniKQL::TType* type, const NUdf::TUnboxedValuePod& value);
+
+public:
const NKikimr::NMiniKQL::TTypeEnvironment& TypeEnv;
const NKikimr::NMiniKQL::THolderFactory& HolderFactory;
const NDqProto::EDataTransportVersion TransportVersion;
diff --git a/ydb/library/yql/dq/runtime/ut/ut_helper.cpp b/ydb/library/yql/dq/runtime/ut/ut_helper.cpp
index f91e54775f..ae2f2eaa02 100644
--- a/ydb/library/yql/dq/runtime/ut/ut_helper.cpp
+++ b/ydb/library/yql/dq/runtime/ut/ut_helper.cpp
@@ -1 +1 @@
-#include "ut_helper.h"
+#include "ut_helper.h"
diff --git a/ydb/library/yql/dq/runtime/ut/ut_helper.h b/ydb/library/yql/dq/runtime/ut/ut_helper.h
index f6fc8f2e6d..003c419e7f 100644
--- a/ydb/library/yql/dq/runtime/ut/ut_helper.h
+++ b/ydb/library/yql/dq/runtime/ut/ut_helper.h
@@ -1,62 +1,62 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/dq/runtime/dq_channel_storage.h>
-
-#include <util/generic/hash.h>
-
-namespace NYql::NDq {
-
-
-class TMockChannelStorage : public IDqChannelStorage {
-public:
- TMockChannelStorage(ui64 capacity)
- : Capacity(capacity) {}
-
- bool IsEmpty() const override {
- return Blobs.empty();
- }
-
- bool IsFull() const override {
- return Capacity <= UsedSpace;
- }
-
- void Put(ui64 blobId, TBuffer&& blob) override {
- if (UsedSpace + blob.size() > Capacity) {
- ythrow yexception() << "Space limit exceeded";
- }
-
- auto result = Blobs.emplace(blobId, std::move(blob));
- Y_VERIFY(result.second);
- UsedSpace += result.first->second.size();
- }
-
- bool Get(ui64 blobId, TBuffer& data) override {
- if (!Blobs.contains(blobId)) {
- ythrow yexception() << "Not found";
- }
-
- if (GetBlankRequests) {
- --GetBlankRequests;
- return false;
- }
-
- data = std::move(Blobs[blobId]);
- Blobs.erase(blobId);
- UsedSpace -= data.size();
-
- return true;
- }
-
-public:
- void SetBlankGetRequests(ui32 count) {
- GetBlankRequests = count;
- }
-
-private:
- const ui64 Capacity;
- THashMap<ui64, TBuffer> Blobs;
- ui64 UsedSpace = 0;
- ui32 GetBlankRequests = 0;
-};
-
-} // namespace NYql::NDq
+
+#include <util/generic/hash.h>
+
+namespace NYql::NDq {
+
+
+class TMockChannelStorage : public IDqChannelStorage {
+public:
+ TMockChannelStorage(ui64 capacity)
+ : Capacity(capacity) {}
+
+ bool IsEmpty() const override {
+ return Blobs.empty();
+ }
+
+ bool IsFull() const override {
+ return Capacity <= UsedSpace;
+ }
+
+ void Put(ui64 blobId, TBuffer&& blob) override {
+ if (UsedSpace + blob.size() > Capacity) {
+ ythrow yexception() << "Space limit exceeded";
+ }
+
+ auto result = Blobs.emplace(blobId, std::move(blob));
+ Y_VERIFY(result.second);
+ UsedSpace += result.first->second.size();
+ }
+
+ bool Get(ui64 blobId, TBuffer& data) override {
+ if (!Blobs.contains(blobId)) {
+ ythrow yexception() << "Not found";
+ }
+
+ if (GetBlankRequests) {
+ --GetBlankRequests;
+ return false;
+ }
+
+ data = std::move(Blobs[blobId]);
+ Blobs.erase(blobId);
+ UsedSpace -= data.size();
+
+ return true;
+ }
+
+public:
+ void SetBlankGetRequests(ui32 count) {
+ GetBlankRequests = count;
+ }
+
+private:
+ const ui64 Capacity;
+ THashMap<ui64, TBuffer> Blobs;
+ ui64 UsedSpace = 0;
+ ui32 GetBlankRequests = 0;
+};
+
+} // namespace NYql::NDq
diff --git a/ydb/library/yql/dq/runtime/ut/ya.make b/ydb/library/yql/dq/runtime/ut/ya.make
index 22a050fc31..d0b21e0c24 100644
--- a/ydb/library/yql/dq/runtime/ut/ya.make
+++ b/ydb/library/yql/dq/runtime/ut/ya.make
@@ -1,24 +1,24 @@
UNITTEST_FOR(ydb/library/yql/dq/runtime)
-
-OWNER(g:yql_ydb_core)
-
-FORK_SUBTESTS()
-
-IF (SANITIZER_TYPE OR WITH_VALGRIND)
- SIZE(MEDIUM)
-ENDIF()
-
-SRCS(
+
+OWNER(g:yql_ydb_core)
+
+FORK_SUBTESTS()
+
+IF (SANITIZER_TYPE OR WITH_VALGRIND)
+ SIZE(MEDIUM)
+ENDIF()
+
+SRCS(
dq_arrow_helpers_ut.cpp
- dq_output_channel_ut.cpp
- ut_helper.cpp
-)
-
-PEERDIR(
+ dq_output_channel_ut.cpp
+ ut_helper.cpp
+)
+
+PEERDIR(
library/cpp/testing/unittest
ydb/library/yql/public/udf/service/exception_policy
-)
-
+)
+
YQL_LAST_ABI_VERSION()
-END()
+END()
diff --git a/ydb/library/yql/dq/runtime/ya.make b/ydb/library/yql/dq/runtime/ya.make
index 79bce89cda..4cc8585431 100644
--- a/ydb/library/yql/dq/runtime/ya.make
+++ b/ydb/library/yql/dq/runtime/ya.make
@@ -22,10 +22,10 @@ SRCS(
dq_arrow_helpers.cpp
dq_columns_resolve.cpp
dq_compute.cpp
- dq_input_channel.cpp
- dq_input_producer.cpp
- dq_output_channel.cpp
- dq_output_consumer.cpp
+ dq_input_channel.cpp
+ dq_input_producer.cpp
+ dq_output_channel.cpp
+ dq_output_consumer.cpp
dq_source.cpp
dq_sink.cpp
dq_tasks_runner.cpp
@@ -37,7 +37,7 @@ GENERATE_ENUM_SERIALIZATION(dq_tasks_runner.h)
YQL_LAST_ABI_VERSION()
END()
-
-RECURSE_FOR_TESTS(
- ut
-)
+
+RECURSE_FOR_TESTS(
+ ut
+)
diff --git a/ydb/library/yql/dq/tasks/dq_connection_builder.h b/ydb/library/yql/dq/tasks/dq_connection_builder.h
index 7aced63ab3..2c46061f60 100644
--- a/ydb/library/yql/dq/tasks/dq_connection_builder.h
+++ b/ydb/library/yql/dq/tasks/dq_connection_builder.h
@@ -5,10 +5,10 @@
namespace NYql::NDq {
-using TChannelLogFunc = std::function<void(ui64 channel, ui64 from, ui64 to, TStringBuf type, bool enableSpilling)>;
+using TChannelLogFunc = std::function<void(ui64 channel, ui64 from, ui64 to, TStringBuf type, bool enableSpilling)>;
template <class TStageInfoMeta, class TTaskMeta, class TInputMeta, class TOutputMeta>
-void CommonBuildTasks(TDqTasksGraph<TStageInfoMeta, TTaskMeta, TInputMeta, TOutputMeta>& graph, const NNodes::TDqPhyStage& stage) {
+void CommonBuildTasks(TDqTasksGraph<TStageInfoMeta, TTaskMeta, TInputMeta, TOutputMeta>& graph, const NNodes::TDqPhyStage& stage) {
ui32 partitionsCount = 1;
const auto stageSettings = NDq::TDqStageSettings::Parse(stage);
@@ -18,10 +18,10 @@ void CommonBuildTasks(TDqTasksGraph<TStageInfoMeta, TTaskMeta, TInputMeta, TOutp
YQL_ENSURE(input.Maybe<NNodes::TCallable>());
// Current assumptions:
- // 1. `Broadcast` can not be the 1st stage input unless it's a single input
+ // 1. `Broadcast` can not be the 1st stage input unless it's a single input
// 2. All stage's inputs, except 1st one, must be a `Broadcast` or `UnionAll` !!! or `HashShuffle` !!!
if (inputIndex == 0) {
- YQL_ENSURE(stage.Inputs().Size() == 1 || !input.Maybe<NNodes::TDqCnBroadcast>());
+ YQL_ENSURE(stage.Inputs().Size() == 1 || !input.Maybe<NNodes::TDqCnBroadcast>());
} else {
YQL_ENSURE(
input.Maybe<NNodes::TDqCnBroadcast>() ||
@@ -29,7 +29,7 @@ void CommonBuildTasks(TDqTasksGraph<TStageInfoMeta, TTaskMeta, TInputMeta, TOutp
input.Maybe<NNodes::TDqCnHashShuffle>(), "" << input.Ref().Content());
}
- if (auto maybeCnShuffle = input.Maybe<NNodes::TDqCnHashShuffle>()) {
+ if (auto maybeCnShuffle = input.Maybe<NNodes::TDqCnHashShuffle>()) {
auto shuffle = maybeCnShuffle.Cast();
auto& originStageInfo = graph.GetStageInfo(shuffle.Output().Stage());
if (stageSettings.IsExternalFunction) {
@@ -55,8 +55,8 @@ void CommonBuildTasks(TDqTasksGraph<TStageInfoMeta, TTaskMeta, TInputMeta, TOutp
template <typename TGraph>
void BuildUnionAllChannels(TGraph& graph, const typename TGraph::TStageInfoType& stageInfo, ui32 inputIndex,
- const typename TGraph::TStageInfoType& inputStageInfo, ui32 outputIndex, bool enableSpilling,
- const TChannelLogFunc& logFunc)
+ const typename TGraph::TStageInfoType& inputStageInfo, ui32 outputIndex, bool enableSpilling,
+ const TChannelLogFunc& logFunc)
{
YQL_ENSURE(stageInfo.Tasks.size() == 1, "Multiple tasks on union all input. StageId: " << stageInfo.Id);
auto& targetTask = graph.GetTask(stageInfo.Tasks[0]);
@@ -69,7 +69,7 @@ void BuildUnionAllChannels(TGraph& graph, const typename TGraph::TStageInfoType&
channel.SrcOutputIndex = outputIndex;
channel.DstTask = targetTask.Id;
channel.DstInputIndex = inputIndex;
- channel.InMemory = !enableSpilling || inputStageInfo.OutputsCount == 1;
+ channel.InMemory = !enableSpilling || inputStageInfo.OutputsCount == 1;
auto& taskInput = targetTask.Inputs[inputIndex];
taskInput.Channels.push_back(channel.Id);
@@ -78,7 +78,7 @@ void BuildUnionAllChannels(TGraph& graph, const typename TGraph::TStageInfoType&
taskOutput.Type = TTaskOutputType::Map;
taskOutput.Channels.push_back(channel.Id);
- logFunc(channel.Id, originTaskId, targetTask.Id, "UnionAll/Map", !channel.InMemory);
+ logFunc(channel.Id, originTaskId, targetTask.Id, "UnionAll/Map", !channel.InMemory);
}
}
@@ -92,13 +92,13 @@ void BuildUnionAllChannels(TGraph& graph, const NNodes::TDqPhyStage& stage, ui32
auto& inputStageInfo = graph.GetStageInfo(dqUnion.Output().Stage());
auto outputIndex = FromString<ui32>(dqUnion.Output().Index().Value());
- BuildUnionAllChannels(graph, stageInfo, inputIndex, inputStageInfo, outputIndex, false, logFunc);
+ BuildUnionAllChannels(graph, stageInfo, inputIndex, inputStageInfo, outputIndex, false, logFunc);
}
template <typename TGraph, typename TKeyColumns>
-void BuildHashShuffleChannels(TGraph& graph, const typename TGraph::TStageInfoType& stageInfo, ui32 inputIndex,
+void BuildHashShuffleChannels(TGraph& graph, const typename TGraph::TStageInfoType& stageInfo, ui32 inputIndex,
const typename TGraph::TStageInfoType& inputStageInfo, ui32 outputIndex, const TKeyColumns& keyColumns,
- bool enableSpilling, const TChannelLogFunc& logFunc)
+ bool enableSpilling, const TChannelLogFunc& logFunc)
{
for (auto& originTaskId : inputStageInfo.Tasks) {
auto& originTask = graph.GetTask(originTaskId);
@@ -118,23 +118,23 @@ void BuildHashShuffleChannels(TGraph& graph, const typename TGraph::TStageInfoTy
channel.SrcOutputIndex = outputIndex;
channel.DstTask = targetTask.Id;
channel.DstInputIndex = inputIndex;
- channel.InMemory = !enableSpilling || inputStageInfo.OutputsCount == 1;
+ channel.InMemory = !enableSpilling || inputStageInfo.OutputsCount == 1;
taskOutput.Channels.push_back(channel.Id);
auto& taskInput = targetTask.Inputs[inputIndex];
taskInput.Channels.push_back(channel.Id);
- logFunc(channel.Id, originTask.Id, targetTask.Id, "Shuffle/HashPartition", !channel.InMemory);
+ logFunc(channel.Id, originTask.Id, targetTask.Id, "Shuffle/HashPartition", !channel.InMemory);
}
}
}
template <typename TGraph>
-void BuildHashShuffleChannels(TGraph& graph, const NNodes::TDqPhyStage& stage, ui32 inputIndex,
+void BuildHashShuffleChannels(TGraph& graph, const NNodes::TDqPhyStage& stage, ui32 inputIndex,
const TChannelLogFunc& logFunc)
{
auto& stageInfo = graph.GetStageInfo(stage);
- auto shuffle = stage.Inputs().Item(inputIndex).Cast<NNodes::TDqCnHashShuffle>();
+ auto shuffle = stage.Inputs().Item(inputIndex).Cast<NNodes::TDqCnHashShuffle>();
auto& originStageInfo = graph.GetStageInfo(shuffle.Output().Stage());
auto outputIndex = FromString<ui32>(shuffle.Output().Index().Value());
@@ -143,13 +143,13 @@ void BuildHashShuffleChannels(TGraph& graph, const NNodes::TDqPhyStage& stage, u
keyColumns.push_back(TString(keyColumn));
}
- BuildHashShuffleChannels(graph, stageInfo, inputIndex, originStageInfo, outputIndex, keyColumns, false, logFunc);
+ BuildHashShuffleChannels(graph, stageInfo, inputIndex, originStageInfo, outputIndex, keyColumns, false, logFunc);
}
template <typename TGraph>
void BuildMapChannels(TGraph& graph, const typename TGraph::TStageInfoType& stageInfo, ui32 inputIndex,
- const typename TGraph::TStageInfoType& inputStageInfo, ui32 outputIndex, bool enableSpilling,
- const TChannelLogFunc& logFunc)
+ const typename TGraph::TStageInfoType& inputStageInfo, ui32 outputIndex, bool enableSpilling,
+ const TChannelLogFunc& logFunc)
{
auto& originTasks = inputStageInfo.Tasks;
auto& targetTasks = stageInfo.Tasks;
@@ -166,7 +166,7 @@ void BuildMapChannels(TGraph& graph, const typename TGraph::TStageInfoType& stag
channel.SrcOutputIndex = outputIndex;
channel.DstTask = targetTaskId;
channel.DstInputIndex = inputIndex;
- channel.InMemory = !enableSpilling || inputStageInfo.OutputsCount == 1;
+ channel.InMemory = !enableSpilling || inputStageInfo.OutputsCount == 1;
auto& originTask = graph.GetTask(originTaskId);
auto& targetTask = graph.GetTask(targetTaskId);
@@ -178,7 +178,7 @@ void BuildMapChannels(TGraph& graph, const typename TGraph::TStageInfoType& stag
taskOutput.Type = TTaskOutputType::Map;
taskOutput.Channels.push_back(channel.Id);
- logFunc(channel.Id, originTaskId, targetTaskId, "Map/Map", !channel.InMemory);
+ logFunc(channel.Id, originTaskId, targetTaskId, "Map/Map", !channel.InMemory);
}
}
@@ -197,8 +197,8 @@ void BuildMapChannels(TGraph& graph, const NNodes::TDqPhyStage& stage, ui32 inpu
template <typename TGraph>
void BuildBroadcastChannels(TGraph& graph, const typename TGraph::TStageInfoType& stageInfo, ui32 inputIndex,
- const typename TGraph::TStageInfoType& inputStageInfo, ui32 outputIndex, bool enableSpilling,
- const TChannelLogFunc& logFunc)
+ const typename TGraph::TStageInfoType& inputStageInfo, ui32 outputIndex, bool enableSpilling,
+ const TChannelLogFunc& logFunc)
{
YQL_ENSURE(inputStageInfo.Tasks.size() == 1);
@@ -213,7 +213,7 @@ void BuildBroadcastChannels(TGraph& graph, const typename TGraph::TStageInfoType
channel.SrcOutputIndex = outputIndex;
channel.DstTask = targetTaskId;
channel.DstInputIndex = inputIndex;
- channel.InMemory = !enableSpilling || inputStageInfo.OutputsCount == 1;
+ channel.InMemory = !enableSpilling || inputStageInfo.OutputsCount == 1;
auto& originTask = graph.GetTask(originTaskId);
auto& targetTask = graph.GetTask(targetTaskId);
@@ -225,7 +225,7 @@ void BuildBroadcastChannels(TGraph& graph, const typename TGraph::TStageInfoType
taskOutput.Type = TTaskOutputType::Broadcast;
taskOutput.Channels.push_back(channel.Id);
- logFunc(channel.Id, originTaskId, targetTaskId, "Broadcast/Broadcast", !channel.InMemory);
+ logFunc(channel.Id, originTaskId, targetTaskId, "Broadcast/Broadcast", !channel.InMemory);
}
}
@@ -239,7 +239,7 @@ void BuildBroadcastChannels(TGraph& graph, const NNodes::TDqPhyStage& stage, ui3
auto& originStageInfo = graph.GetStageInfo(cnBroadcast.Output().Stage());
auto outputIndex = FromString<ui32>(cnBroadcast.Output().Index().Value());
- BuildBroadcastChannels(graph, stageInfo, inputIndex, originStageInfo, outputIndex, false, logFunc);
+ BuildBroadcastChannels(graph, stageInfo, inputIndex, originStageInfo, outputIndex, false, logFunc);
}
template <typename TGraph>
diff --git a/ydb/library/yql/dq/tasks/dq_task_program.cpp b/ydb/library/yql/dq/tasks/dq_task_program.cpp
index d22d361406..d06a6023b3 100644
--- a/ydb/library/yql/dq/tasks/dq_task_program.cpp
+++ b/ydb/library/yql/dq/tasks/dq_task_program.cpp
@@ -27,7 +27,7 @@ const TStructExprType* CollectParameters(NNodes::TCoLambda program, TExprContext
TString BuildProgram(NNodes::TCoLambda program, const TStructExprType& paramsType,
const NCommon::IMkqlCallableCompiler& compiler, const TTypeEnvironment& typeEnv,
- const IFunctionRegistry& funcRegistry, TExprContext& exprCtx, const TVector<TExprBase>& reads)
+ const IFunctionRegistry& funcRegistry, TExprContext& exprCtx, const TVector<TExprBase>& reads)
{
TProgramBuilder pgmBuilder(typeEnv, funcRegistry);
@@ -37,7 +37,7 @@ TString BuildProgram(NNodes::TCoLambda program, const TStructExprType& paramsTyp
auto paramsNode = pgmBuilder.Arg(NCommon::BuildType(program.Ref(), paramsType, pgmBuilder));
for (const auto& arg : program.Args()) {
- YQL_ENSURE(arg.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Stream, "program: " << program.Ref().Dump());
+ YQL_ENSURE(arg.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Stream, "program: " << program.Ref().Dump());
auto itemType = NCommon::BuildType(arg.Ref(), *arg.Ref().GetTypeAnn(), pgmBuilder);
TRuntimeNode inputNode = pgmBuilder.Arg(itemType);
arguments[arg.Raw()] = inputNode;
@@ -53,15 +53,15 @@ TString BuildProgram(NNodes::TCoLambda program, const TStructExprType& paramsTyp
structBuilder.Add("Program", rootNode);
structBuilder.Add("Inputs", pgmBuilder.NewTuple(inputNodes));
structBuilder.Add("Parameters", paramsNode);
-
- if (!reads.empty()) {
- auto readsExpr = Build<TExprList>(exprCtx, program.Pos())
- .Add(reads)
- .Done();
- TRuntimeNode readsNode = MkqlBuildExpr(readsExpr.Ref(), ctx);
- structBuilder.Add("Reads", readsNode);
- }
-
+
+ if (!reads.empty()) {
+ auto readsExpr = Build<TExprList>(exprCtx, program.Pos())
+ .Add(reads)
+ .Done();
+ TRuntimeNode readsNode = MkqlBuildExpr(readsExpr.Ref(), ctx);
+ structBuilder.Add("Reads", readsNode);
+ }
+
auto programNode = structBuilder.Build();
TExploringNodeVisitor explorer;
diff --git a/ydb/library/yql/dq/tasks/dq_task_program.h b/ydb/library/yql/dq/tasks/dq_task_program.h
index 835985c26c..9bdf0de319 100644
--- a/ydb/library/yql/dq/tasks/dq_task_program.h
+++ b/ydb/library/yql/dq/tasks/dq_task_program.h
@@ -13,7 +13,7 @@ const TStructExprType* CollectParameters(NNodes::TCoLambda program, TExprContext
TString BuildProgram(NNodes::TCoLambda program, const TStructExprType& paramsType,
const NCommon::IMkqlCallableCompiler& compiler, const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv,
- const NKikimr::NMiniKQL::IFunctionRegistry& funcRegistry, TExprContext& exprCtx,
- const TVector<NNodes::TExprBase>& reads);
+ const NKikimr::NMiniKQL::IFunctionRegistry& funcRegistry, TExprContext& exprCtx,
+ const TVector<NNodes::TExprBase>& reads);
} // namespace NYql::NDq
diff --git a/ydb/library/yql/dq/tasks/dq_tasks_graph.h b/ydb/library/yql/dq/tasks/dq_tasks_graph.h
index 0f1bd83b7f..e5c772310b 100644
--- a/ydb/library/yql/dq/tasks/dq_tasks_graph.h
+++ b/ydb/library/yql/dq/tasks/dq_tasks_graph.h
@@ -57,7 +57,7 @@ struct TStageInfo : private TMoveOnly {
OutputsCount = 1;
}
} else {
- YQL_ENSURE(resultType->GetKind() == ETypeAnnotationKind::Void, "got " << *resultType);
+ YQL_ENSURE(resultType->GetKind() == ETypeAnnotationKind::Void, "got " << *resultType);
OutputsCount = 0;
}
}
@@ -86,7 +86,7 @@ struct TChannel {
ui32 SrcOutputIndex = 0;
ui64 DstTask = 0;
ui32 DstInputIndex = 0;
- bool InMemory = true;
+ bool InMemory = true;
NDqProto::ECheckpointingMode CheckpointingMode = NDqProto::CHECKPOINTING_MODE_DEFAULT;
};
@@ -226,11 +226,11 @@ public:
return GetStageInfo(TStageId(txId, stage.Ref().UniqueId()));
}
- const TStageInfoType& GetStageInfo(const NNodes::TDqStageBase& stage) const {
+ const TStageInfoType& GetStageInfo(const NNodes::TDqStageBase& stage) const {
return GetStageInfo(0, stage);
}
- TStageInfoType& GetStageInfo(const NNodes::TDqStageBase& stage) {
+ TStageInfoType& GetStageInfo(const NNodes::TDqStageBase& stage) {
return GetStageInfo(0, stage);
}
diff --git a/ydb/library/yql/dq/type_ann/dq_type_ann.cpp b/ydb/library/yql/dq/type_ann/dq_type_ann.cpp
index 9d9c5995c9..fcbd3bb9ec 100644
--- a/ydb/library/yql/dq/type_ann/dq_type_ann.cpp
+++ b/ydb/library/yql/dq/type_ann/dq_type_ann.cpp
@@ -7,7 +7,7 @@
#include <ydb/library/yql/utils/log/log.h>
#include <ydb/library/yql/providers/common/provider/yql_provider.h>
-
+
namespace NYql::NDq {
using namespace NYql::NNodes;
@@ -20,8 +20,8 @@ const TTypeAnnotationNode* GetDqOutputType(const TDqOutput& output, TExprContext
ui32 resultIndex;
if (!TryFromString(output.Index().Value(), resultIndex)) {
- ctx.AddError(TIssue(ctx.GetPosition(output.Pos()),
- TStringBuilder() << "Failed to convert to integer: " << output.Index().Value()));
+ ctx.AddError(TIssue(ctx.GetPosition(output.Pos()),
+ TStringBuilder() << "Failed to convert to integer: " << output.Index().Value()));
return nullptr;
}
@@ -31,8 +31,8 @@ const TTypeAnnotationNode* GetDqOutputType(const TDqOutput& output, TExprContext
}
if (resultIndex >= stageResultTuple->GetSize()) {
- ctx.AddError(TIssue(ctx.GetPosition(output.Pos()),
- TStringBuilder() << "Stage result index out of bounds: " << resultIndex));
+ ctx.AddError(TIssue(ctx.GetPosition(output.Pos()),
+ TStringBuilder() << "Stage result index out of bounds: " << resultIndex));
return nullptr;
}
@@ -49,74 +49,74 @@ const TTypeAnnotationNode* GetDqConnectionType(const TDqConnection& node, TExprC
}
template <typename TStage>
-TStatus AnnotateStage(const TExprNode::TPtr& stage, TExprContext& ctx) {
- if (!EnsureMinMaxArgsCount(*stage, 3, 4, ctx)) {
+TStatus AnnotateStage(const TExprNode::TPtr& stage, TExprContext& ctx) {
+ if (!EnsureMinMaxArgsCount(*stage, 3, 4, ctx)) {
return TStatus::Error;
}
- auto* inputsTuple = stage->Child(TDqStageBase::idx_Inputs);
- auto& programLambda = stage->ChildRef(TDqStageBase::idx_Program);
- auto* settingsTuple = stage->Child(TDqPhyStage::idx_Settings);
-
- if (!EnsureTuple(*inputsTuple, ctx)) {
+ auto* inputsTuple = stage->Child(TDqStageBase::idx_Inputs);
+ auto& programLambda = stage->ChildRef(TDqStageBase::idx_Program);
+ auto* settingsTuple = stage->Child(TDqPhyStage::idx_Settings);
+
+ if (!EnsureTuple(*inputsTuple, ctx)) {
return TStatus::Error;
}
if constexpr (std::is_same_v<TStage, TDqPhyStage>) {
- if (!EnsureTuple(*settingsTuple, ctx)) {
+ if (!EnsureTuple(*settingsTuple, ctx)) {
return TStatus::Error;
}
- for (auto& setting: settingsTuple->Children()) {
- if (!EnsureTupleMinSize(*setting, 1, ctx)) {
+ for (auto& setting: settingsTuple->Children()) {
+ if (!EnsureTupleMinSize(*setting, 1, ctx)) {
return TStatus::Error;
}
- if (!EnsureAtom(*setting->Child(0), ctx)) {
+ if (!EnsureAtom(*setting->Child(0), ctx)) {
return TStatus::Error;
}
}
}
- if (!EnsureLambda(*programLambda, ctx)) {
+ if (!EnsureLambda(*programLambda, ctx)) {
return TStatus::Error;
}
- if (!EnsureArgsCount(programLambda->Head(), inputsTuple->ChildrenSize(), ctx)) {
+ if (!EnsureArgsCount(programLambda->Head(), inputsTuple->ChildrenSize(), ctx)) {
return TStatus::Error;
}
TVector<const TTypeAnnotationNode*> argTypes;
- argTypes.reserve(inputsTuple->ChildrenSize());
-
- for (const auto& input: inputsTuple->Children()) {
- if (!TDqPhyPrecompute::Match(input.Get()) &&
- !(TDqConnection::Match(input.Get()) && !TDqCnValue::Match(input.Get())) &&
- !TDqSource::Match(input.Get()) &&
- !(input->Content() == "KqpTxResultBinding"sv))
- {
- ctx.AddError(TIssue(TStringBuilder() << "Unexpected stage input " << input->Content()));
- return TStatus::Error;
- }
-
- auto* argType = input->GetTypeAnn();
- if (!TDqPhyPrecompute::Match(input.Get()) && input->Content() != "KqpTxResultBinding") {
- if (argType->GetKind() == ETypeAnnotationKind::List) {
- auto* listItemType = argType->Cast<TListExprType>()->GetItemType();
- if constexpr (std::is_same_v<TStage, TDqPhyStage>) {
- argType = ctx.MakeType<TStreamExprType>(listItemType);
- } else {
- argType = ctx.MakeType<TFlowExprType>(listItemType);
- }
+ argTypes.reserve(inputsTuple->ChildrenSize());
+
+ for (const auto& input: inputsTuple->Children()) {
+ if (!TDqPhyPrecompute::Match(input.Get()) &&
+ !(TDqConnection::Match(input.Get()) && !TDqCnValue::Match(input.Get())) &&
+ !TDqSource::Match(input.Get()) &&
+ !(input->Content() == "KqpTxResultBinding"sv))
+ {
+ ctx.AddError(TIssue(TStringBuilder() << "Unexpected stage input " << input->Content()));
+ return TStatus::Error;
+ }
+
+ auto* argType = input->GetTypeAnn();
+ if (!TDqPhyPrecompute::Match(input.Get()) && input->Content() != "KqpTxResultBinding") {
+ if (argType->GetKind() == ETypeAnnotationKind::List) {
+ auto* listItemType = argType->Cast<TListExprType>()->GetItemType();
+ if constexpr (std::is_same_v<TStage, TDqPhyStage>) {
+ argType = ctx.MakeType<TStreamExprType>(listItemType);
+ } else {
+ argType = ctx.MakeType<TFlowExprType>(listItemType);
+ }
}
- }
+ }
argTypes.emplace_back(argType);
}
- if (!UpdateLambdaAllArgumentsTypes(programLambda, argTypes, ctx)) {
+ if (!UpdateLambdaAllArgumentsTypes(programLambda, argTypes, ctx)) {
return TStatus::Error;
}
-
- auto* resultType = programLambda->GetTypeAnn();
- if (!resultType) {
+
+ auto* resultType = programLambda->GetTypeAnn();
+ if (!resultType) {
return TStatus::Repeat;
}
@@ -130,7 +130,7 @@ TStatus AnnotateStage(const TExprNode::TPtr& stage, TExprContext& ctx) {
itemType = resultType->template Cast<TFlowExprType>()->GetItemType();
} else if (resultType->GetKind() == ETypeAnnotationKind::Stream) {
itemType = resultType->template Cast<TStreamExprType>()->GetItemType();
- }
+ }
if (itemType) {
if (itemType->GetKind() == ETypeAnnotationKind::Variant) {
auto variantType = itemType->Cast<TVariantExprType>()->GetUnderlyingType();
@@ -144,21 +144,21 @@ TStatus AnnotateStage(const TExprNode::TPtr& stage, TExprContext& ctx) {
resultTypesTuple.emplace_back(ctx.MakeType<TListExprType>(itemType));
}
} else {
- YQL_ENSURE(resultType->GetKind() != ETypeAnnotationKind::List, "stage: " << stage->Dump());
+ YQL_ENSURE(resultType->GetKind() != ETypeAnnotationKind::List, "stage: " << stage->Dump());
resultTypesTuple.emplace_back(resultType);
- }
- }
+ }
+ }
- if (TDqStageBase::idx_Sinks < stage->ChildrenSize()) {
- for (const auto& sink: stage->Child(TDqStageBase::idx_Sinks)->Children()) {
+ if (TDqStageBase::idx_Sinks < stage->ChildrenSize()) {
+ for (const auto& sink: stage->Child(TDqStageBase::idx_Sinks)->Children()) {
sink->SetTypeAnn(resultType);
}
}
- stage->SetTypeAnn(ctx.MakeType<TTupleExprType>(resultTypesTuple));
+ stage->SetTypeAnn(ctx.MakeType<TTupleExprType>(resultTypesTuple));
return TStatus::Ok;
-}
-
+}
+
THashMap<TStringBuf, THashMap<TStringBuf, const TTypeAnnotationNode*>>
ParseJoinInputType(const TStructExprType& rowType, TStringBuf tableLabel, TExprContext& ctx, bool optional) {
THashMap<TStringBuf, THashMap<TStringBuf, const TTypeAnnotationNode*>> result;
@@ -187,7 +187,7 @@ ParseJoinInputType(const TStructExprType& rowType, TStringBuf tableLabel, TExprC
return result;
}
-template <bool IsMapJoin>
+template <bool IsMapJoin>
const TStructExprType* GetDqJoinResultType(TPositionHandle pos, const TStructExprType& leftRowType,
const TStringBuf& leftLabel, const TStructExprType& rightRowType, const TStringBuf& rightLabel,
const TStringBuf& joinType, const TDqJoinKeyTupleList& joinKeys, TExprContext& ctx)
@@ -212,12 +212,12 @@ const TStructExprType* GetDqJoinResultType(TPositionHandle pos, const TStructExp
return nullptr;
}
- if constexpr (IsMapJoin) {
- if (joinType.StartsWith("Right") || joinType == "Cross") {
- ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder()
- << "Unsupported map join type: " << joinType));
- return nullptr;
- }
+ if constexpr (IsMapJoin) {
+ if (joinType.StartsWith("Right") || joinType == "Cross") {
+ ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder()
+ << "Unsupported map join type: " << joinType));
+ return nullptr;
+ }
}
// check join keys
@@ -281,10 +281,10 @@ const TStructExprType* GetDqJoinResultType(TPositionHandle pos, const TStructExp
};
TVector<const TItemExprType*> resultStructItems;
- if (joinType != "RightOnly" && joinType != "RightSemi") {
- addAllMembersFrom(leftType, &resultStructItems, joinType == "Right");
- }
- if (joinType != "LeftOnly" && joinType != "LeftSemi") {
+ if (joinType != "RightOnly" && joinType != "RightSemi") {
+ addAllMembersFrom(leftType, &resultStructItems, joinType == "Right");
+ }
+ if (joinType != "LeftOnly" && joinType != "LeftSemi") {
addAllMembersFrom(rightType, &resultStructItems, joinType == "Left");
}
@@ -292,7 +292,7 @@ const TStructExprType* GetDqJoinResultType(TPositionHandle pos, const TStructExp
return rowType;
}
-template <bool IsMapJoin>
+template <bool IsMapJoin>
const TStructExprType* GetDqJoinResultType(const TExprNode::TPtr& input, bool stream, TExprContext& ctx) {
if (!EnsureArgsCount(*input, 6, ctx)) {
return nullptr;
@@ -368,8 +368,8 @@ const TStructExprType* GetDqJoinResultType(const TExprNode::TPtr& input, bool st
? join.RightLabel().Cast<TCoAtom>().Value()
: TStringBuf("");
- return GetDqJoinResultType<IsMapJoin>(join.Pos(), *leftStructType, leftTableLabel, *rightStructType,
- rightTableLabel, join.JoinType(), join.JoinKeys(), ctx);
+ return GetDqJoinResultType<IsMapJoin>(join.Pos(), *leftStructType, leftTableLabel, *rightStructType,
+ rightTableLabel, join.JoinType(), join.JoinKeys(), ctx);
}
TStatus AnnotateDqPrecompute(const TExprNode::TPtr& node, TExprContext& ctx) {
@@ -386,13 +386,13 @@ TStatus AnnotateDqPhyPrecompute(const TExprNode::TPtr& node, TExprContext& ctx)
return TStatus::Error;
}
- auto* cn = node->Child(TDqPhyPrecompute::idx_Connection);
- if (!TDqConnection::Match(cn)) {
- ctx.AddError(TIssue(ctx.GetPosition(cn->Pos()), TStringBuilder() << "Expected DqConnection, got " << cn->Content()));
+ auto* cn = node->Child(TDqPhyPrecompute::idx_Connection);
+ if (!TDqConnection::Match(cn)) {
+ ctx.AddError(TIssue(ctx.GetPosition(cn->Pos()), TStringBuilder() << "Expected DqConnection, got " << cn->Content()));
return TStatus::Error;
}
- node->SetTypeAnn(cn->GetTypeAnn());
+ node->SetTypeAnn(cn->GetTypeAnn());
return TStatus::Ok;
}
@@ -522,21 +522,21 @@ TStatus AnnotateDqCnMerge(const TExprNode::TPtr& node, TExprContext& ctx) {
return TStatus::Ok;
}
-TStatus AnnotateDqCnHashShuffle(const TExprNode::TPtr& input, TExprContext& ctx) {
+TStatus AnnotateDqCnHashShuffle(const TExprNode::TPtr& input, TExprContext& ctx) {
if (!EnsureArgsCount(*input, 2, ctx)) {
return TStatus::Error;
}
- if (!EnsureCallable(*input->Child(TDqCnHashShuffle::idx_Output), ctx)) {
+ if (!EnsureCallable(*input->Child(TDqCnHashShuffle::idx_Output), ctx)) {
return TStatus::Error;
}
- if (!TDqOutput::Match(input->Child(TDqCnHashShuffle::idx_Output))) {
- ctx.AddError(TIssue(ctx.GetPosition(input->Child(TDqCnHashShuffle::idx_Output)->Pos()), TStringBuilder() << "Expected " << TDqOutput::CallableName()));
+ if (!TDqOutput::Match(input->Child(TDqCnHashShuffle::idx_Output))) {
+ ctx.AddError(TIssue(ctx.GetPosition(input->Child(TDqCnHashShuffle::idx_Output)->Pos()), TStringBuilder() << "Expected " << TDqOutput::CallableName()));
return TStatus::Error;
}
- if (!EnsureTupleMinSize(*input->Child(TDqCnHashShuffle::idx_KeyColumns), 1, ctx)) {
+ if (!EnsureTupleMinSize(*input->Child(TDqCnHashShuffle::idx_KeyColumns), 1, ctx)) {
return TStatus::Error;
}
@@ -551,7 +551,7 @@ TStatus AnnotateDqCnHashShuffle(const TExprNode::TPtr& input, TExprContext& ctx)
}
auto structType = itemType->Cast<TStructExprType>();
- for (const auto& column: input->Child(TDqCnHashShuffle::idx_KeyColumns)->Children()) {
+ for (const auto& column: input->Child(TDqCnHashShuffle::idx_KeyColumns)->Children()) {
if (!EnsureAtom(*column, ctx)) {
return TStatus::Error;
}
@@ -566,19 +566,19 @@ TStatus AnnotateDqCnHashShuffle(const TExprNode::TPtr& input, TExprContext& ctx)
return TStatus::Ok;
}
-TStatus AnnotateDqCnValue(const TExprNode::TPtr& cnValue, TExprContext& ctx) {
- if (!EnsureArgsCount(*cnValue, 1, ctx)) {
+TStatus AnnotateDqCnValue(const TExprNode::TPtr& cnValue, TExprContext& ctx) {
+ if (!EnsureArgsCount(*cnValue, 1, ctx)) {
return TStatus::Error;
}
- auto& output = cnValue->ChildRef(TDqCnValue::idx_Output);
- if (!TDqOutput::Match(output.Get())) {
- ctx.AddError(TIssue(ctx.GetPosition(output->Pos()), TStringBuilder() << "Expected " << TDqOutput::CallableName()
- << ", got " << output->Content()));
+ auto& output = cnValue->ChildRef(TDqCnValue::idx_Output);
+ if (!TDqOutput::Match(output.Get())) {
+ ctx.AddError(TIssue(ctx.GetPosition(output->Pos()), TStringBuilder() << "Expected " << TDqOutput::CallableName()
+ << ", got " << output->Content()));
return TStatus::Error;
}
- auto* resultType = GetDqOutputType(TDqOutput(output), ctx);
+ auto* resultType = GetDqOutputType(TDqOutput(output), ctx);
if (!resultType) {
return TStatus::Error;
}
@@ -588,7 +588,7 @@ TStatus AnnotateDqCnValue(const TExprNode::TPtr& cnValue, TExprContext& ctx) {
outputType = resultType->Cast<TListExprType>()->GetItemType();
}
- cnValue->SetTypeAnn(outputType);
+ cnValue->SetTypeAnn(outputType);
return TStatus::Ok;
}
@@ -674,7 +674,7 @@ TStatus AnnotateDqReplicate(const TExprNode::TPtr& input, TExprContext& ctx) {
}
TStatus AnnotateDqJoin(const TExprNode::TPtr& input, TExprContext& ctx) {
- auto resultRowType = GetDqJoinResultType<false>(input, false, ctx);
+ auto resultRowType = GetDqJoinResultType<false>(input, false, ctx);
if (!resultRowType) {
return TStatus::Error;
}
@@ -684,7 +684,7 @@ TStatus AnnotateDqJoin(const TExprNode::TPtr& input, TExprContext& ctx) {
}
TStatus AnnotateDqMapOrDictJoin(const TExprNode::TPtr& input, TExprContext& ctx) {
- auto resultRowType = GetDqJoinResultType<true>(input, true, ctx);
+ auto resultRowType = GetDqJoinResultType<true>(input, true, ctx);
if (!resultRowType) {
return TStatus::Error;
}
@@ -694,7 +694,7 @@ TStatus AnnotateDqMapOrDictJoin(const TExprNode::TPtr& input, TExprContext& ctx)
}
TStatus AnnotateDqCrossJoin(const TExprNode::TPtr& input, TExprContext& ctx) {
- auto resultRowType = GetDqJoinResultType<false>(input, true, ctx);
+ auto resultRowType = GetDqJoinResultType<false>(input, true, ctx);
if (!resultRowType) {
return TStatus::Error;
}
@@ -773,11 +773,11 @@ THolder<IGraphTransformer> CreateDqTypeAnnotationTransformer(TTypeAnnotationCont
auto coreTransformer = CreateExtCallableTypeAnnotationTransformer(typesCtx);
return CreateFunctorTransformer(
- [coreTransformer](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
+ [coreTransformer](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
output = input;
TIssueScopeGuard issueScope(ctx.IssueManager, [&input, &ctx] {
return MakeIntrusive<TIssue>(ctx.GetPosition(input->Pos()),
- TStringBuilder() << "At function: " << input->Content());
+ TStringBuilder() << "At function: " << input->Content());
});
if (TDqStage::Match(input.Get())) {
@@ -786,8 +786,8 @@ THolder<IGraphTransformer> CreateDqTypeAnnotationTransformer(TTypeAnnotationCont
if (TDqPhyStage::Match(input.Get())) {
return AnnotateDqPhyStage(input, ctx);
- }
-
+ }
+
if (TDqOutput::Match(input.Get())) {
return AnnotateDqOutput(input, ctx);
}
@@ -796,8 +796,8 @@ THolder<IGraphTransformer> CreateDqTypeAnnotationTransformer(TTypeAnnotationCont
return AnnotateDqConnection(input, ctx);
}
- if (TDqCnHashShuffle::Match(input.Get())) {
- return AnnotateDqCnHashShuffle(input, ctx);
+ if (TDqCnHashShuffle::Match(input.Get())) {
+ return AnnotateDqCnHashShuffle(input, ctx);
}
if (TDqCnMap::Match(input.Get())) {
@@ -822,8 +822,8 @@ THolder<IGraphTransformer> CreateDqTypeAnnotationTransformer(TTypeAnnotationCont
if (TDqReplicate::Match(input.Get())) {
return AnnotateDqReplicate(input, ctx);
- }
-
+ }
+
if (TDqJoin::Match(input.Get())) {
return AnnotateDqJoin(input, ctx);
}
@@ -916,24 +916,24 @@ bool IsMergeConnectionApplicable(const TVector<const TTypeAnnotationNode*>& sort
return true;
}
-TString PrintDqStageOnly(const TDqStageBase& stage, TExprContext& ctx) {
- if (stage.Inputs().Empty()) {
- return NCommon::ExprToPrettyString(ctx, stage.Ref());
- }
-
- TNodeOnNodeOwnedMap replaces;
- for (ui64 i = 0; i < stage.Inputs().Size(); ++i) {
- auto input = stage.Inputs().Item(i);
- auto param = Build<NNodes::TCoParameter>(ctx, input.Pos())
- .Name().Build(TStringBuilder() << "stage_input_" << i)
- .Type(ExpandType(input.Pos(), *input.Ref().GetTypeAnn(), ctx))
- .Done();
-
- replaces[input.Raw()] = param.Ptr();
- }
-
- auto newStage = ctx.ReplaceNodes(stage.Ptr(), replaces);
- return NCommon::ExprToPrettyString(ctx, *newStage);
-}
-
+TString PrintDqStageOnly(const TDqStageBase& stage, TExprContext& ctx) {
+ if (stage.Inputs().Empty()) {
+ return NCommon::ExprToPrettyString(ctx, stage.Ref());
+ }
+
+ TNodeOnNodeOwnedMap replaces;
+ for (ui64 i = 0; i < stage.Inputs().Size(); ++i) {
+ auto input = stage.Inputs().Item(i);
+ auto param = Build<NNodes::TCoParameter>(ctx, input.Pos())
+ .Name().Build(TStringBuilder() << "stage_input_" << i)
+ .Type(ExpandType(input.Pos(), *input.Ref().GetTypeAnn(), ctx))
+ .Done();
+
+ replaces[input.Raw()] = param.Ptr();
+ }
+
+ auto newStage = ctx.ReplaceNodes(stage.Ptr(), replaces);
+ return NCommon::ExprToPrettyString(ctx, *newStage);
+}
+
} // namespace NYql::NDq
diff --git a/ydb/library/yql/dq/type_ann/dq_type_ann.h b/ydb/library/yql/dq/type_ann/dq_type_ann.h
index b74cf86527..03aefe701f 100644
--- a/ydb/library/yql/dq/type_ann/dq_type_ann.h
+++ b/ydb/library/yql/dq/type_ann/dq_type_ann.h
@@ -4,13 +4,13 @@
#include <ydb/library/yql/core/yql_type_annotation.h>
#include <ydb/library/yql/dq/expr_nodes/dq_expr_nodes.h>
-
+
namespace NYql::NDq {
IGraphTransformer::TStatus AnnotateDqStage(const TExprNode::TPtr& input, TExprContext& ctx);
IGraphTransformer::TStatus AnnotateDqPhyStage(const TExprNode::TPtr& input, TExprContext& ctx);
IGraphTransformer::TStatus AnnotateDqOutput(const TExprNode::TPtr& input, TExprContext& ctx);
-IGraphTransformer::TStatus AnnotateDqCnHashShuffle(const TExprNode::TPtr& input, TExprContext& ctx);
+IGraphTransformer::TStatus AnnotateDqCnHashShuffle(const TExprNode::TPtr& input, TExprContext& ctx);
IGraphTransformer::TStatus AnnotateDqCnValue(const TExprNode::TPtr& input, TExprContext& ctx);
IGraphTransformer::TStatus AnnotateDqCnResult(const TExprNode::TPtr& input, TExprContext& ctx);
IGraphTransformer::TStatus AnnotateDqReplicate(const TExprNode::TPtr& input, TExprContext& ctx);
@@ -29,6 +29,6 @@ bool IsTypeSupportedInMergeCn(EDataSlot type);
bool IsTypeSupportedInMergeCn(const TDataExprType* dataType);
bool IsMergeConnectionApplicable(const TVector<const TTypeAnnotationNode*>& sortKeyTypes);
-TString PrintDqStageOnly(const NNodes::TDqStageBase& stage, TExprContext& ctx);
-
+TString PrintDqStageOnly(const NNodes::TDqStageBase& stage, TExprContext& ctx);
+
} // namespace NYql::NDq
diff --git a/ydb/library/yql/dq/ya.make b/ydb/library/yql/dq/ya.make
index 27ed0fb5db..c206ab90ef 100644
--- a/ydb/library/yql/dq/ya.make
+++ b/ydb/library/yql/dq/ya.make
@@ -1,6 +1,6 @@
RECURSE(
actors
- common
+ common
comp_nodes
expr_nodes
opt
diff --git a/ydb/library/yql/minikql/aligned_page_pool.cpp b/ydb/library/yql/minikql/aligned_page_pool.cpp
index 1be3dba680..e015c255c8 100644
--- a/ydb/library/yql/minikql/aligned_page_pool.cpp
+++ b/ydb/library/yql/minikql/aligned_page_pool.cpp
@@ -160,7 +160,7 @@ void TAlignedPagePool::ReleaseFreePages() {
}
void TAlignedPagePool::OffloadAlloc(ui64 size) {
- if (Limit && TotalAllocated + size > Limit && !TryIncreaseLimit(TotalAllocated + size)) {
+ if (Limit && TotalAllocated + size > Limit && !TryIncreaseLimit(TotalAllocated + size)) {
throw TMemoryLimitExceededException();
}
@@ -206,7 +206,7 @@ void* TAlignedPagePool::GetPage() {
return res;
}
- if (Limit && TotalAllocated + POOL_PAGE_SIZE > Limit && !TryIncreaseLimit(TotalAllocated + POOL_PAGE_SIZE)) {
+ if (Limit && TotalAllocated + POOL_PAGE_SIZE > Limit && !TryIncreaseLimit(TotalAllocated + POOL_PAGE_SIZE)) {
throw TMemoryLimitExceededException();
}
@@ -220,8 +220,8 @@ void* TAlignedPagePool::GetPage() {
}
++PageGlobalHitCount;
AllPages.emplace(ptr);
-
- UpdatePeaks();
+
+ UpdatePeaks();
return ptr;
}
@@ -261,7 +261,7 @@ void* TAlignedPagePool::Alloc(size_t size) {
void* res = nullptr;
size = AlignUp(size, SYS_PAGE_SIZE);
- if (Limit && TotalAllocated + size > Limit && !TryIncreaseLimit(TotalAllocated + size)) {
+ if (Limit && TotalAllocated + size > Limit && !TryIncreaseLimit(TotalAllocated + size)) {
throw TMemoryLimitExceededException();
}
@@ -393,14 +393,14 @@ void TAlignedPagePool::Free(void* ptr, size_t size) noexcept {
}
}
-bool TAlignedPagePool::TryIncreaseLimit(ui64 required) {
+bool TAlignedPagePool::TryIncreaseLimit(ui64 required) {
if (!IncreaseMemoryLimitCallback) {
- return false;
- }
+ return false;
+ }
IncreaseMemoryLimitCallback(Limit, required);
- return Limit >= required;
-}
-
+ return Limit >= required;
+}
+
ui64 TAlignedPagePool::GetGlobalPagePoolSize() {
ui64 size = 0;
for (size_t level = 0; level <= MidLevels; ++level) {
diff --git a/ydb/library/yql/minikql/aligned_page_pool.h b/ydb/library/yql/minikql/aligned_page_pool.h
index 195ca2b729..2ef2878655 100644
--- a/ydb/library/yql/minikql/aligned_page_pool.h
+++ b/ydb/library/yql/minikql/aligned_page_pool.h
@@ -170,12 +170,12 @@ public:
AllocNotifyCurrentBytes = 0;
}
- using TIncreaseMemoryLimitCallback = std::function<void(ui64 currentLimit, ui64 required)>;
-
- void SetIncreaseMemoryLimitCallback(TIncreaseMemoryLimitCallback&& callback) {
+ using TIncreaseMemoryLimitCallback = std::function<void(ui64 currentLimit, ui64 required)>;
+
+ void SetIncreaseMemoryLimitCallback(TIncreaseMemoryLimitCallback&& callback) {
IncreaseMemoryLimitCallback = std::move(callback);
- }
-
+ }
+
protected:
void* Alloc(size_t size);
void Free(void* ptr, size_t size) noexcept;
@@ -185,8 +185,8 @@ protected:
PeakUsed = Max(PeakUsed, GetUsed());
}
- bool TryIncreaseLimit(ui64 required);
-
+ bool TryIncreaseLimit(ui64 required);
+
protected:
std::stack<void*, std::vector<void*>> FreePages;
std::unordered_set<void*> AllPages;
@@ -208,7 +208,7 @@ protected:
TAlignedPagePoolCounters Counters;
bool CheckLostMem = true;
-
+
TAllocNotifyCallback AllocNotifyCallback;
ui64 AllocNotifyBytes = 0;
ui64 AllocNotifyCurrentBytes = 0;
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp
index 1bab036adb..57599acb64 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp
@@ -164,8 +164,8 @@ struct TCallableComputationNodeBuilderFuncMapFiller {
{"ToSortedDict", &WrapToSortedDict},
{"ToHashedDict", &WrapToHashedDict},
{"SqueezeToList", &WrapSqueezeToList},
- {"SqueezeToSortedDict", &WrapSqueezeToSortedDict},
- {"SqueezeToHashedDict", &WrapSqueezeToHashedDict},
+ {"SqueezeToSortedDict", &WrapSqueezeToSortedDict},
+ {"SqueezeToHashedDict", &WrapSqueezeToHashedDict},
{"NarrowSqueezeToSortedDict", &WrapSqueezeToSortedDict},
{"NarrowSqueezeToHashedDict", &WrapSqueezeToHashedDict},
{"Coalesce", &WrapCoalesce},
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_skip.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_skip.cpp
index 91a16e993d..62a7f10f24 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_skip.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_skip.cpp
@@ -87,8 +87,8 @@ public:
block = good;
- ValueCleanup(Flow->GetRepresentation(), item, ctx, block);
-
+ ValueCleanup(Flow->GetRepresentation(), item, ctx, block);
+
const auto decr = BinaryOperator::CreateSub(count, ConstantInt::get(count->getType(), 1ULL), "decr", block);
count->addIncoming(decr, block);
const auto next = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_UGT, decr, ConstantInt::get(decr->getType(), 0ULL), "next", block);
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_todict.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_todict.cpp
index 1aa326e493..2e299e487d 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_todict.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_todict.cpp
@@ -630,55 +630,55 @@ public:
}
};
-template <typename TSetAccumulator, bool IsStream>
-class TSetWrapper : public TMutableComputationNode<TSetWrapper<TSetAccumulator, IsStream>> {
- typedef TMutableComputationNode<TSetWrapper<TSetAccumulator, IsStream>> TBaseComputation;
+template <typename TSetAccumulator, bool IsStream>
+class TSetWrapper : public TMutableComputationNode<TSetWrapper<TSetAccumulator, IsStream>> {
+ typedef TMutableComputationNode<TSetWrapper<TSetAccumulator, IsStream>> TBaseComputation;
public:
- class TStreamValue : public TComputationValue<TStreamValue> {
- public:
- TStreamValue(TMemoryUsageInfo* memInfo, NUdf::TUnboxedValue&& input, IComputationExternalNode* const item,
- IComputationNode* const key, TSetAccumulator&& setAccum, TComputationContext& ctx)
- : TComputationValue<TStreamValue>(memInfo)
- , Input(std::move(input))
- , Item(item)
- , Key(key)
- , SetAccum(std::move(setAccum))
- , Ctx(ctx) {}
-
- private:
- NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) override {
- if (Finished) {
- return NUdf::EFetchStatus::Finish;
- }
-
- for (;;) {
- NUdf::TUnboxedValue item;
- switch (auto status = Input.Fetch(item)) {
- case NUdf::EFetchStatus::Ok: {
- Item->SetValue(Ctx, std::move(item));
- SetAccum.Add(Key->GetValue(Ctx));
- break; // and continue
- }
- case NUdf::EFetchStatus::Finish: {
- result = SetAccum.Build();
- Finished = true;
- return NUdf::EFetchStatus::Ok;
- }
- case NUdf::EFetchStatus::Yield: {
- return NUdf::EFetchStatus::Yield;
- }
- }
- }
- }
-
- NUdf::TUnboxedValue Input;
- IComputationExternalNode* const Item;
- IComputationNode* const Key;
- TSetAccumulator SetAccum;
- TComputationContext& Ctx;
- bool Finished = false;
- };
-
+ class TStreamValue : public TComputationValue<TStreamValue> {
+ public:
+ TStreamValue(TMemoryUsageInfo* memInfo, NUdf::TUnboxedValue&& input, IComputationExternalNode* const item,
+ IComputationNode* const key, TSetAccumulator&& setAccum, TComputationContext& ctx)
+ : TComputationValue<TStreamValue>(memInfo)
+ , Input(std::move(input))
+ , Item(item)
+ , Key(key)
+ , SetAccum(std::move(setAccum))
+ , Ctx(ctx) {}
+
+ private:
+ NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) override {
+ if (Finished) {
+ return NUdf::EFetchStatus::Finish;
+ }
+
+ for (;;) {
+ NUdf::TUnboxedValue item;
+ switch (auto status = Input.Fetch(item)) {
+ case NUdf::EFetchStatus::Ok: {
+ Item->SetValue(Ctx, std::move(item));
+ SetAccum.Add(Key->GetValue(Ctx));
+ break; // and continue
+ }
+ case NUdf::EFetchStatus::Finish: {
+ result = SetAccum.Build();
+ Finished = true;
+ return NUdf::EFetchStatus::Ok;
+ }
+ case NUdf::EFetchStatus::Yield: {
+ return NUdf::EFetchStatus::Yield;
+ }
+ }
+ }
+ }
+
+ NUdf::TUnboxedValue Input;
+ IComputationExternalNode* const Item;
+ IComputationNode* const Key;
+ TSetAccumulator SetAccum;
+ TComputationContext& Ctx;
+ bool Finished = false;
+ };
+
TSetWrapper(TComputationMutables& mutables, TType* keyType, IComputationNode* list, IComputationExternalNode* item,
IComputationNode* key, ui64 itemsCountHint)
: TBaseComputation(mutables, EValueRepresentation::Boxed)
@@ -692,11 +692,11 @@ public:
}
NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const {
- if constexpr (IsStream) {
- return ctx.HolderFactory.Create<TStreamValue>(List->GetValue(ctx), Item, Key,
- TSetAccumulator(KeyType, KeyTypes, IsTuple, Encoded, ctx, ItemsCountHint), ctx);
- }
-
+ if constexpr (IsStream) {
+ return ctx.HolderFactory.Create<TStreamValue>(List->GetValue(ctx), Item, Key,
+ TSetAccumulator(KeyType, KeyTypes, IsTuple, Encoded, ctx, ItemsCountHint), ctx);
+ }
+
const auto& list = List->GetValue(ctx);
auto itemsCountHint = ItemsCountHint;
if (list.HasFastListLength()) {
@@ -735,47 +735,47 @@ private:
bool Encoded;
};
-template <typename TSetAccumulator>
+template <typename TSetAccumulator>
class TSqueezeSetFlowWrapper : public TStatefulFlowCodegeneratorNode<TSqueezeSetFlowWrapper<TSetAccumulator>> {
using TBase = TStatefulFlowCodegeneratorNode<TSqueezeSetFlowWrapper<TSetAccumulator>>;
public:
- class TState : public TComputationValue<TState> {
- using TBase = TComputationValue<TState>;
- public:
- TState(TMemoryUsageInfo* memInfo, TSetAccumulator&& setAccum)
- : TBase(memInfo), SetAccum(std::move(setAccum)) {}
-
+ class TState : public TComputationValue<TState> {
+ using TBase = TComputationValue<TState>;
+ public:
+ TState(TMemoryUsageInfo* memInfo, TSetAccumulator&& setAccum)
+ : TBase(memInfo), SetAccum(std::move(setAccum)) {}
+
NUdf::TUnboxedValuePod Build() {
return SetAccum.Build().Release();
- }
-
+ }
+
void Insert(NUdf::TUnboxedValuePod value) {
SetAccum.Add(value);
- }
-
- private:
- TSetAccumulator SetAccum;
- };
-
+ }
+
+ private:
+ TSetAccumulator SetAccum;
+ };
+
TSqueezeSetFlowWrapper(TComputationMutables& mutables, TType* keyType,
- IComputationNode* flow, IComputationExternalNode* item, IComputationNode* key, ui64 itemsCountHint)
+ IComputationNode* flow, IComputationExternalNode* item, IComputationNode* key, ui64 itemsCountHint)
: TBase(mutables, flow, EValueRepresentation::Boxed, EValueRepresentation::Any)
- , KeyType(keyType)
- , Flow(flow)
- , Item(item)
- , Key(key)
- , ItemsCountHint(itemsCountHint)
- {
- GetDictionaryKeyTypes(KeyType, KeyTypes, IsTuple, Encoded);
- }
-
+ , KeyType(keyType)
+ , Flow(flow)
+ , Item(item)
+ , Key(key)
+ , ItemsCountHint(itemsCountHint)
+ {
+ GetDictionaryKeyTypes(KeyType, KeyTypes, IsTuple, Encoded);
+ }
+
NUdf::TUnboxedValuePod DoCalculate(NUdf::TUnboxedValue& state, TComputationContext& ctx) const {
if (state.IsFinish()) {
return state.Release();
} else if (!state.HasValue()) {
MakeState(ctx, state);
- }
-
+ }
+
while (const auto statePtr = static_cast<TState*>(state.AsBoxed().Get())) {
if (auto item = Flow->GetValue(ctx); item.IsYield()) {
return item.Release();
@@ -787,19 +787,19 @@ public:
Item->SetValue(ctx, std::move(item));
statePtr->Insert(Key->GetValue(ctx).Release());
}
- }
+ }
Y_UNREACHABLE();
}
#ifndef MKQL_DISABLE_CODEGEN
Value* DoGenerateGetValue(const TCodegenContext& ctx, Value* statePtr, BasicBlock*& block) const {
auto& context = ctx.Codegen->GetContext();
-
+
const auto codegenItemArg = dynamic_cast<ICodegeneratorExternalNode*>(Item);
MKQL_ENSURE(codegenItemArg, "Item must be codegenerator node.");
-
+
const auto valueType = Type::getInt128Ty(context);
const auto structPtrType = PointerType::getUnqual(StructType::get(context));
-
+
const auto stateType = StructType::get(context, {
structPtrType, // vtbl
Type::getInt32Ty(context), // ref
@@ -884,37 +884,37 @@ public:
const auto dict = new LoadInst(ptr, "dict", block);
UnRefBoxed(state, ctx, block);
result->addIncoming(dict, block);
- }
-
+ }
+
new StoreInst(item, statePtr, block);
BranchInst::Create(over, block);
block = over;
return result;
- }
-#endif
+ }
+#endif
private:
void MakeState(TComputationContext& ctx, NUdf::TUnboxedValue& state) const {
state = ctx.HolderFactory.Create<TState>(TSetAccumulator(KeyType, KeyTypes, IsTuple, Encoded, ctx, ItemsCountHint));
}
-
- void RegisterDependencies() const final {
+
+ void RegisterDependencies() const final {
if (const auto flow = this->FlowDependsOn(Flow)) {
this->Own(flow, Item);
this->DependsOn(flow, Key);
}
- }
-
- TType* const KeyType;
- IComputationNode* const Flow;
- IComputationExternalNode* const Item;
- IComputationNode* const Key;
- const ui64 ItemsCountHint;
- TKeyTypes KeyTypes;
- bool IsTuple;
- bool Encoded;
-};
-
+ }
+
+ TType* const KeyType;
+ IComputationNode* const Flow;
+ IComputationExternalNode* const Item;
+ IComputationNode* const Key;
+ const ui64 ItemsCountHint;
+ TKeyTypes KeyTypes;
+ bool IsTuple;
+ bool Encoded;
+};
+
template <typename TSetAccumulator>
class TSqueezeSetWideWrapper : public TStatefulFlowCodegeneratorNode<TSqueezeSetWideWrapper<TSetAccumulator>> {
using TBase = TStatefulFlowCodegeneratorNode<TSqueezeSetWideWrapper<TSetAccumulator>>;
@@ -1110,57 +1110,57 @@ private:
mutable std::vector<NUdf::TUnboxedValue*> Fields;
};
-template <typename TMapAccumulator, bool IsStream>
-class TMapWrapper : public TMutableComputationNode<TMapWrapper<TMapAccumulator, IsStream>> {
- typedef TMutableComputationNode<TMapWrapper<TMapAccumulator, IsStream>> TBaseComputation;
-public:
- class TStreamValue : public TComputationValue<TStreamValue> {
- public:
- TStreamValue(TMemoryUsageInfo* memInfo, NUdf::TUnboxedValue&& input, IComputationExternalNode* const item,
- IComputationNode* const key, IComputationNode* const payload, TMapAccumulator&& mapAccum, TComputationContext& ctx)
- : TComputationValue<TStreamValue>(memInfo)
- , Input(std::move(input))
- , Item(item)
- , Key(key)
- , Payload(payload)
- , MapAccum(std::move(mapAccum))
- , Ctx(ctx) {}
-
- private:
- NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) override {
- if (Finished) {
- return NUdf::EFetchStatus::Finish;
- }
-
- for (;;) {
- NUdf::TUnboxedValue item;
- switch (auto status = Input.Fetch(item)) {
- case NUdf::EFetchStatus::Ok: {
- Item->SetValue(Ctx, std::move(item));
- MapAccum.Add(Key->GetValue(Ctx), Payload->GetValue(Ctx));
- break; // and continue
- }
- case NUdf::EFetchStatus::Finish: {
- result = MapAccum.Build();
- Finished = true;
- return NUdf::EFetchStatus::Ok;
- }
- case NUdf::EFetchStatus::Yield: {
- return NUdf::EFetchStatus::Yield;
- }
- }
- }
- }
-
- NUdf::TUnboxedValue Input;
- IComputationExternalNode* const Item;
- IComputationNode* const Key;
- IComputationNode* const Payload;
- TMapAccumulator MapAccum;
- TComputationContext& Ctx;
- bool Finished = false;
- };
-
+template <typename TMapAccumulator, bool IsStream>
+class TMapWrapper : public TMutableComputationNode<TMapWrapper<TMapAccumulator, IsStream>> {
+ typedef TMutableComputationNode<TMapWrapper<TMapAccumulator, IsStream>> TBaseComputation;
+public:
+ class TStreamValue : public TComputationValue<TStreamValue> {
+ public:
+ TStreamValue(TMemoryUsageInfo* memInfo, NUdf::TUnboxedValue&& input, IComputationExternalNode* const item,
+ IComputationNode* const key, IComputationNode* const payload, TMapAccumulator&& mapAccum, TComputationContext& ctx)
+ : TComputationValue<TStreamValue>(memInfo)
+ , Input(std::move(input))
+ , Item(item)
+ , Key(key)
+ , Payload(payload)
+ , MapAccum(std::move(mapAccum))
+ , Ctx(ctx) {}
+
+ private:
+ NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) override {
+ if (Finished) {
+ return NUdf::EFetchStatus::Finish;
+ }
+
+ for (;;) {
+ NUdf::TUnboxedValue item;
+ switch (auto status = Input.Fetch(item)) {
+ case NUdf::EFetchStatus::Ok: {
+ Item->SetValue(Ctx, std::move(item));
+ MapAccum.Add(Key->GetValue(Ctx), Payload->GetValue(Ctx));
+ break; // and continue
+ }
+ case NUdf::EFetchStatus::Finish: {
+ result = MapAccum.Build();
+ Finished = true;
+ return NUdf::EFetchStatus::Ok;
+ }
+ case NUdf::EFetchStatus::Yield: {
+ return NUdf::EFetchStatus::Yield;
+ }
+ }
+ }
+ }
+
+ NUdf::TUnboxedValue Input;
+ IComputationExternalNode* const Item;
+ IComputationNode* const Key;
+ IComputationNode* const Payload;
+ TMapAccumulator MapAccum;
+ TComputationContext& Ctx;
+ bool Finished = false;
+ };
+
TMapWrapper(TComputationMutables& mutables, TType* keyType, TType* payloadType, IComputationNode* list, IComputationExternalNode* item,
IComputationNode* key, IComputationNode* payload, ui64 itemsCountHint)
: TBaseComputation(mutables, EValueRepresentation::Boxed)
@@ -1176,11 +1176,11 @@ public:
}
NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const {
- if constexpr (IsStream) {
- return ctx.HolderFactory.Create<TStreamValue>(List->GetValue(ctx), Item, Key, Payload,
- TMapAccumulator(KeyType, PayloadType, KeyTypes, IsTuple, Encoded, ctx, ItemsCountHint), ctx);
- }
-
+ if constexpr (IsStream) {
+ return ctx.HolderFactory.Create<TStreamValue>(List->GetValue(ctx), Item, Key, Payload,
+ TMapAccumulator(KeyType, PayloadType, KeyTypes, IsTuple, Encoded, ctx, ItemsCountHint), ctx);
+ }
+
const auto& list = List->GetValue(ctx);
auto itemsCountHint = ItemsCountHint;
@@ -1223,50 +1223,50 @@ private:
bool Encoded;
};
-template <typename TMapAccumulator>
+template <typename TMapAccumulator>
class TSqueezeMapFlowWrapper : public TStatefulFlowCodegeneratorNode<TSqueezeMapFlowWrapper<TMapAccumulator>> {
using TBase = TStatefulFlowCodegeneratorNode<TSqueezeMapFlowWrapper<TMapAccumulator>>;
-public:
- class TState : public TComputationValue<TState> {
- using TBase = TComputationValue<TState>;
- public:
- TState(TMemoryUsageInfo* memInfo, TMapAccumulator&& mapAccum)
- : TBase(memInfo), MapAccum(std::move(mapAccum)) {}
-
+public:
+ class TState : public TComputationValue<TState> {
+ using TBase = TComputationValue<TState>;
+ public:
+ TState(TMemoryUsageInfo* memInfo, TMapAccumulator&& mapAccum)
+ : TBase(memInfo), MapAccum(std::move(mapAccum)) {}
+
NUdf::TUnboxedValuePod Build() {
return MapAccum.Build().Release();
- }
-
+ }
+
void Insert(NUdf::TUnboxedValuePod key, NUdf::TUnboxedValuePod value) {
MapAccum.Add(key, value);
- }
-
- private:
- TMapAccumulator MapAccum;
- };
-
+ }
+
+ private:
+ TMapAccumulator MapAccum;
+ };
+
TSqueezeMapFlowWrapper(TComputationMutables& mutables, TType* keyType, TType* payloadType,
- IComputationNode* flow, IComputationExternalNode* item, IComputationNode* key, IComputationNode* payload,
- ui64 itemsCountHint)
+ IComputationNode* flow, IComputationExternalNode* item, IComputationNode* key, IComputationNode* payload,
+ ui64 itemsCountHint)
: TBase(mutables, flow, EValueRepresentation::Boxed, EValueRepresentation::Any)
- , KeyType(keyType)
- , PayloadType(payloadType)
- , Flow(flow)
- , Item(item)
- , Key(key)
- , Payload(payload)
- , ItemsCountHint(itemsCountHint)
- {
- GetDictionaryKeyTypes(KeyType, KeyTypes, IsTuple, Encoded);
- }
-
+ , KeyType(keyType)
+ , PayloadType(payloadType)
+ , Flow(flow)
+ , Item(item)
+ , Key(key)
+ , Payload(payload)
+ , ItemsCountHint(itemsCountHint)
+ {
+ GetDictionaryKeyTypes(KeyType, KeyTypes, IsTuple, Encoded);
+ }
+
NUdf::TUnboxedValuePod DoCalculate(NUdf::TUnboxedValue& state, TComputationContext& ctx) const {
if (state.IsFinish()) {
return state;
} else if (!state.HasValue()) {
MakeState(ctx, state);
- }
-
+ }
+
while (const auto statePtr = static_cast<TState*>(state.AsBoxed().Get())) {
if (auto item = Flow->GetValue(ctx); item.IsYield()) {
return item.Release();
@@ -1278,19 +1278,19 @@ public:
Item->SetValue(ctx, std::move(item));
statePtr->Insert(Key->GetValue(ctx).Release(), Payload->GetValue(ctx).Release());
}
- }
+ }
Y_UNREACHABLE();
}
#ifndef MKQL_DISABLE_CODEGEN
Value* DoGenerateGetValue(const TCodegenContext& ctx, Value* statePtr, BasicBlock*& block) const {
auto& context = ctx.Codegen->GetContext();
-
+
const auto codegenItemArg = dynamic_cast<ICodegeneratorExternalNode*>(Item);
MKQL_ENSURE(codegenItemArg, "Item must be codegenerator node.");
-
+
const auto valueType = Type::getInt128Ty(context);
const auto structPtrType = PointerType::getUnqual(StructType::get(context));
-
+
const auto stateType = StructType::get(context, {
structPtrType, // vtbl
Type::getInt32Ty(context), // ref
@@ -1377,40 +1377,40 @@ public:
const auto dict = new LoadInst(ptr, "dict", block);
UnRefBoxed(state, ctx, block);
result->addIncoming(dict, block);
- }
-
+ }
+
new StoreInst(item, statePtr, block);
BranchInst::Create(over, block);
block = over;
return result;
- }
-#endif
+ }
+#endif
private:
void MakeState(TComputationContext& ctx, NUdf::TUnboxedValue& state) const {
state = ctx.HolderFactory.Create<TState>(TMapAccumulator(KeyType, PayloadType, KeyTypes, IsTuple, Encoded, ctx, ItemsCountHint));
}
-
- void RegisterDependencies() const final {
+
+ void RegisterDependencies() const final {
if (const auto flow = this->FlowDependsOn(Flow)) {
this->Own(flow, Item);
this->DependsOn(flow, Key);
this->DependsOn(flow, Payload);
}
- }
-
- TType* const KeyType;
- TType* PayloadType;
- IComputationNode* const Flow;
- IComputationExternalNode* const Item;
- IComputationNode* const Key;
- IComputationNode* const Payload;
- const ui64 ItemsCountHint;
- TKeyTypes KeyTypes;
- bool IsTuple;
- bool Encoded;
-};
-
+ }
+
+ TType* const KeyType;
+ TType* PayloadType;
+ IComputationNode* const Flow;
+ IComputationExternalNode* const Item;
+ IComputationNode* const Key;
+ IComputationNode* const Payload;
+ const ui64 ItemsCountHint;
+ TKeyTypes KeyTypes;
+ bool IsTuple;
+ bool Encoded;
+};
+
template <typename TMapAccumulator>
class TSqueezeMapWideWrapper : public TStatefulFlowCodegeneratorNode<TSqueezeMapWideWrapper<TMapAccumulator>> {
using TBase = TStatefulFlowCodegeneratorNode<TSqueezeMapWideWrapper<TMapAccumulator>>;
@@ -1617,7 +1617,7 @@ private:
};
template <typename TAccumulator>
-IComputationNode* WrapToSet(TCallable& callable, const TNodeLocator& nodeLocator, TComputationMutables& mutables) {
+IComputationNode* WrapToSet(TCallable& callable, const TNodeLocator& nodeLocator, TComputationMutables& mutables) {
const auto keyType = callable.GetInput(callable.GetInputsCount() - 5U).GetStaticType();
const auto itemsCountHint = AS_VALUE(TDataLiteral, callable.GetInput(callable.GetInputsCount() - 1U))->AsValue().Get<ui64>();
@@ -1635,22 +1635,22 @@ IComputationNode* WrapToSet(TCallable& callable, const TNodeLocator& nodeLocator
const auto itemArg = LocateExternalNode(nodeLocator, callable, 1U);
const auto type = callable.GetInput(0U).GetStaticType();
-
- if (type->IsList()) {
+
+ if (type->IsList()) {
return new TSetWrapper<TAccumulator, false>(mutables, keyType, flow, itemArg, keySelector, itemsCountHint);
- }
- if (type->IsFlow()) {
+ }
+ if (type->IsFlow()) {
return new TSqueezeSetFlowWrapper<TAccumulator>(mutables, keyType, flow, itemArg, keySelector, itemsCountHint);
- }
- if (type->IsStream()) {
+ }
+ if (type->IsStream()) {
return new TSetWrapper<TAccumulator, true>(mutables, keyType, flow, itemArg, keySelector, itemsCountHint);
- }
-
- THROW yexception() << "Expected list, flow or stream.";
+ }
+
+ THROW yexception() << "Expected list, flow or stream.";
}
template <typename TAccumulator>
-IComputationNode* WrapToMap(TCallable& callable, const TNodeLocator& nodeLocator, TComputationMutables& mutables) {
+IComputationNode* WrapToMap(TCallable& callable, const TNodeLocator& nodeLocator, TComputationMutables& mutables) {
const auto keyType = callable.GetInput(callable.GetInputsCount() - 5U).GetStaticType();
const auto payloadType = callable.GetInput(callable.GetInputsCount() - 4U).GetStaticType();
@@ -1672,30 +1672,30 @@ IComputationNode* WrapToMap(TCallable& callable, const TNodeLocator& nodeLocator
const auto itemArg = LocateExternalNode(nodeLocator, callable, 1U);
const auto type = callable.GetInput(0U).GetStaticType();
- if (type->IsList()) {
+ if (type->IsList()) {
return new TMapWrapper<TAccumulator, false>(mutables, keyType, payloadType, flow, itemArg, keySelector, payloadSelector, itemsCountHint);
- }
- if (type->IsFlow()) {
+ }
+ if (type->IsFlow()) {
return new TSqueezeMapFlowWrapper<TAccumulator>(mutables, keyType, payloadType, flow, itemArg, keySelector, payloadSelector, itemsCountHint);
- }
- if (type->IsStream()) {
+ }
+ if (type->IsStream()) {
return new TMapWrapper<TAccumulator, true>(mutables, keyType, payloadType, flow, itemArg, keySelector, payloadSelector, itemsCountHint);
- }
-
- THROW yexception() << "Expected list, flow or stream.";
+ }
+
+ THROW yexception() << "Expected list, flow or stream.";
}
-template <bool IsList>
-IComputationNode* WrapToSortedDictInternal(TCallable& callable, const TComputationNodeFactoryContext& ctx) {
+template <bool IsList>
+IComputationNode* WrapToSortedDictInternal(TCallable& callable, const TComputationNodeFactoryContext& ctx) {
MKQL_ENSURE(callable.GetInputsCount() >= 6U, "Expected six or more args.");
const auto type = callable.GetInput(0U).GetStaticType();
- if constexpr (IsList) {
- MKQL_ENSURE(type->IsList(), "Expected list.");
- } else {
- MKQL_ENSURE(type->IsFlow() || type->IsStream(), "Expected flow or stream.");
- }
-
+ if constexpr (IsList) {
+ MKQL_ENSURE(type->IsList(), "Expected list.");
+ } else {
+ MKQL_ENSURE(type->IsFlow() || type->IsStream(), "Expected flow or stream.");
+ }
+
const auto keyType = callable.GetInput(callable.GetInputsCount() - 5U).GetStaticType();
const auto payloadType = callable.GetInput(callable.GetInputsCount() - 4U).GetStaticType();
@@ -1706,7 +1706,7 @@ IComputationNode* WrapToSortedDictInternal(TCallable& callable, const TComputati
const auto flow = LocateNode(ctx.NodeLocator, callable, 0U);
const auto keySelector = LocateNode(ctx.NodeLocator, callable, callable.GetInputsCount() -5U);
const auto payloadSelector = LocateNode(ctx.NodeLocator, callable, callable.GetInputsCount() -4U);
-
+
if (const auto wide = dynamic_cast<IComputationWideFlowNode*>(flow)) {
const auto width = callable.GetInputsCount() - 6U;
TComputationExternalNodePtrVector args(width, nullptr);
@@ -1724,15 +1724,15 @@ IComputationNode* WrapToSortedDictInternal(TCallable& callable, const TComputati
const auto itemArg = LocateExternalNode(ctx.NodeLocator, callable, 1U);
if (!isMulti && payloadType->IsVoid()) {
- if (type->IsList()) {
+ if (type->IsList()) {
return new TSetWrapper<TSortedSetAccumulator, false>(ctx.Mutables, keyType, flow, itemArg, keySelector, itemsCountHint);
- }
- if (type->IsFlow()) {
+ }
+ if (type->IsFlow()) {
return new TSqueezeSetFlowWrapper<TSortedSetAccumulator>(ctx.Mutables, keyType, flow, itemArg, keySelector, itemsCountHint);
- }
- if (type->IsStream()) {
+ }
+ if (type->IsStream()) {
return new TSetWrapper<TSortedSetAccumulator, true>(ctx.Mutables, keyType, flow, itemArg, keySelector, itemsCountHint);
- }
+ }
} else if (isMulti) {
if (type->IsList()) {
return new TMapWrapper<TSortedMapAccumulator<true>, false>(ctx.Mutables, keyType, payloadType, flow, itemArg, keySelector, payloadSelector, itemsCountHint);
@@ -1744,37 +1744,37 @@ IComputationNode* WrapToSortedDictInternal(TCallable& callable, const TComputati
return new TMapWrapper<TSortedMapAccumulator<true>, true>(ctx.Mutables, keyType, payloadType, flow, itemArg, keySelector, payloadSelector, itemsCountHint);
}
} else {
- if (type->IsList()) {
+ if (type->IsList()) {
return new TMapWrapper<TSortedMapAccumulator<false>, false>(ctx.Mutables, keyType, payloadType, flow, itemArg, keySelector, payloadSelector, itemsCountHint);
- }
- if (type->IsFlow()) {
+ }
+ if (type->IsFlow()) {
return new TSqueezeMapFlowWrapper<TSortedMapAccumulator<false>>(ctx.Mutables, keyType, payloadType, flow, itemArg, keySelector, payloadSelector, itemsCountHint);
- }
- if (type->IsStream()) {
+ }
+ if (type->IsStream()) {
return new TMapWrapper<TSortedMapAccumulator<false>, true>(ctx.Mutables, keyType, payloadType, flow, itemArg, keySelector, payloadSelector, itemsCountHint);
- }
+ }
}
-
- THROW yexception() << "Expected list, flow or stream.";
+
+ THROW yexception() << "Expected list, flow or stream.";
}
-template <bool IsList>
-IComputationNode* WrapToHashedDictInternal(TCallable& callable, const TComputationNodeFactoryContext& ctx) {
+template <bool IsList>
+IComputationNode* WrapToHashedDictInternal(TCallable& callable, const TComputationNodeFactoryContext& ctx) {
MKQL_ENSURE(callable.GetInputsCount() >= 6U, "Expected six or more args.");
-
+
const auto type = callable.GetInput(0U).GetStaticType();
- if constexpr (IsList) {
- MKQL_ENSURE(type->IsList(), "Expected list.");
- } else {
- MKQL_ENSURE(type->IsFlow() || type->IsStream(), "Expected flow or stream.");
- }
-
+ if constexpr (IsList) {
+ MKQL_ENSURE(type->IsList(), "Expected list.");
+ } else {
+ MKQL_ENSURE(type->IsFlow() || type->IsStream(), "Expected flow or stream.");
+ }
+
const auto keyType = callable.GetInput(callable.GetInputsCount() - 5U).GetStaticType();
const auto payloadType = callable.GetInput(callable.GetInputsCount() - 4U).GetStaticType();
const bool multi = AS_VALUE(TDataLiteral, callable.GetInput(callable.GetInputsCount() - 3U))->AsValue().Get<bool>();
const bool isCompact = AS_VALUE(TDataLiteral, callable.GetInput(callable.GetInputsCount() - 2U))->AsValue().Get<bool>();
const auto payloadSelectorNode = callable.GetInput(callable.GetInputsCount() - 4U);
-
+
if (!multi && payloadType->IsVoid()) {
if (isCompact) {
if (keyType->IsData()) {
@@ -1856,22 +1856,22 @@ IComputationNode* WrapToHashedDictInternal(TCallable& callable, const TComputati
}
}
-
-IComputationNode* WrapToSortedDict(TCallable& callable, const TComputationNodeFactoryContext& ctx) {
- return WrapToSortedDictInternal<true>(callable, ctx);
+
+IComputationNode* WrapToSortedDict(TCallable& callable, const TComputationNodeFactoryContext& ctx) {
+ return WrapToSortedDictInternal<true>(callable, ctx);
+}
+
+IComputationNode* WrapToHashedDict(TCallable& callable, const TComputationNodeFactoryContext& ctx) {
+ return WrapToHashedDictInternal<true>(callable, ctx);
+}
+
+IComputationNode* WrapSqueezeToSortedDict(TCallable& callable, const TComputationNodeFactoryContext& ctx) {
+ return WrapToSortedDictInternal<false>(callable, ctx);
+}
+
+IComputationNode* WrapSqueezeToHashedDict(TCallable& callable, const TComputationNodeFactoryContext& ctx) {
+ return WrapToHashedDictInternal<false>(callable, ctx);
+}
+
+}
}
-
-IComputationNode* WrapToHashedDict(TCallable& callable, const TComputationNodeFactoryContext& ctx) {
- return WrapToHashedDictInternal<true>(callable, ctx);
-}
-
-IComputationNode* WrapSqueezeToSortedDict(TCallable& callable, const TComputationNodeFactoryContext& ctx) {
- return WrapToSortedDictInternal<false>(callable, ctx);
-}
-
-IComputationNode* WrapSqueezeToHashedDict(TCallable& callable, const TComputationNodeFactoryContext& ctx) {
- return WrapToHashedDictInternal<false>(callable, ctx);
-}
-
-}
-}
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_todict.h b/ydb/library/yql/minikql/comp_nodes/mkql_todict.h
index 815baf6527..6930c5a41c 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_todict.h
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_todict.h
@@ -7,8 +7,8 @@ namespace NMiniKQL {
IComputationNode* WrapToSortedDict(TCallable& callable, const TComputationNodeFactoryContext& ctx);
IComputationNode* WrapToHashedDict(TCallable& callable, const TComputationNodeFactoryContext& ctx);
-IComputationNode* WrapSqueezeToSortedDict(TCallable& callable, const TComputationNodeFactoryContext& ctx);
-IComputationNode* WrapSqueezeToHashedDict(TCallable& callable, const TComputationNodeFactoryContext& ctx);
-
+IComputationNode* WrapSqueezeToSortedDict(TCallable& callable, const TComputationNodeFactoryContext& ctx);
+IComputationNode* WrapSqueezeToHashedDict(TCallable& callable, const TComputationNodeFactoryContext& ctx);
+
}
}
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_todict_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_todict_ut.cpp
index 287b370279..5a5cd7e2de 100644
--- a/ydb/library/yql/minikql/comp_nodes/ut/mkql_todict_ut.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_todict_ut.cpp
@@ -104,63 +104,63 @@ Y_UNIT_TEST_SUITE(TMiniKQLToDictTest) {
auto res = graph->GetValue().template Get<bool>();
UNIT_ASSERT_VALUES_EQUAL(res, true);
}
-
- Y_UNIT_TEST_LLVM(TestSqueezeToDict) {
- auto test = [](bool stream, bool hashed, bool multi, bool compact, bool withPayload) {
- Cerr << "TestSqueezeToDict [on: " << (stream ? "stream" : "flow")
- << "type: " << (hashed ? "hashed" : "sorted") << ", multi: " << multi
- << ", compact: " << compact << ", payload: " << withPayload << "]" << Endl;
-
- TSetup<LLVM> setup;
- TProgramBuilder& pb = *setup.PgmBuilder;
-
- TVector<TRuntimeNode> items;
- for (auto s : data) {
- items.push_back(pb.NewDataLiteral<NUdf::EDataSlot::Utf8>(s));
- }
- Shuffle(items.begin(), items.end());
-
- auto dataType = pb.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id);
- auto list = pb.NewList(dataType, items);
- auto input = stream ? pb.Iterator(list, items) : pb.ToFlow(list);
- auto pgmReturn = hashed
- ? pb.SqueezeToHashedDict(input, multi, [](TRuntimeNode n) { return n; },
- [&pb, withPayload](TRuntimeNode n) { return withPayload ? n : pb.NewVoid(); }, compact)
- : pb.SqueezeToSortedDict(input, multi, [](TRuntimeNode n) { return n; },
- [&pb, withPayload](TRuntimeNode n) { return withPayload ? n : pb.NewVoid(); }, compact);
- if (!stream) {
- pgmReturn = pb.FromFlow(pgmReturn);
- }
-
- auto graph = setup.BuildGraph(pgmReturn);
- NUdf::TUnboxedValue res = graph->GetValue();
- UNIT_ASSERT(!res.IsSpecial());
-
- NUdf::TUnboxedValue v;
- auto status = res.Fetch(v);
- UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, status);
-
- for (auto s : data) {
- UNIT_ASSERT_C(v.Contains(NUdf::TUnboxedValue(MakeString(s))), s);
- }
- UNIT_ASSERT(!v.Contains(NUdf::TUnboxedValue(MakeString("green cucumber"))));
-
- status = res.Fetch(v);
- UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, status);
- };
-
- for (auto stream : {true, false}) {
- for (auto hashed : {true, false}) {
- for (auto multi : {true, false}) {
- for (auto compact : {true, false}) {
- for (auto withPayload : {true, false}) {
- test(stream, hashed, multi, compact, withPayload);
- }
- }
- }
- }
- }
- }
+
+ Y_UNIT_TEST_LLVM(TestSqueezeToDict) {
+ auto test = [](bool stream, bool hashed, bool multi, bool compact, bool withPayload) {
+ Cerr << "TestSqueezeToDict [on: " << (stream ? "stream" : "flow")
+ << "type: " << (hashed ? "hashed" : "sorted") << ", multi: " << multi
+ << ", compact: " << compact << ", payload: " << withPayload << "]" << Endl;
+
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ TVector<TRuntimeNode> items;
+ for (auto s : data) {
+ items.push_back(pb.NewDataLiteral<NUdf::EDataSlot::Utf8>(s));
+ }
+ Shuffle(items.begin(), items.end());
+
+ auto dataType = pb.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id);
+ auto list = pb.NewList(dataType, items);
+ auto input = stream ? pb.Iterator(list, items) : pb.ToFlow(list);
+ auto pgmReturn = hashed
+ ? pb.SqueezeToHashedDict(input, multi, [](TRuntimeNode n) { return n; },
+ [&pb, withPayload](TRuntimeNode n) { return withPayload ? n : pb.NewVoid(); }, compact)
+ : pb.SqueezeToSortedDict(input, multi, [](TRuntimeNode n) { return n; },
+ [&pb, withPayload](TRuntimeNode n) { return withPayload ? n : pb.NewVoid(); }, compact);
+ if (!stream) {
+ pgmReturn = pb.FromFlow(pgmReturn);
+ }
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ NUdf::TUnboxedValue res = graph->GetValue();
+ UNIT_ASSERT(!res.IsSpecial());
+
+ NUdf::TUnboxedValue v;
+ auto status = res.Fetch(v);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, status);
+
+ for (auto s : data) {
+ UNIT_ASSERT_C(v.Contains(NUdf::TUnboxedValue(MakeString(s))), s);
+ }
+ UNIT_ASSERT(!v.Contains(NUdf::TUnboxedValue(MakeString("green cucumber"))));
+
+ status = res.Fetch(v);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, status);
+ };
+
+ for (auto stream : {true, false}) {
+ for (auto hashed : {true, false}) {
+ for (auto multi : {true, false}) {
+ for (auto compact : {true, false}) {
+ for (auto withPayload : {true, false}) {
+ test(stream, hashed, multi, compact, withPayload);
+ }
+ }
+ }
+ }
+ }
+ }
#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 23u
Y_UNIT_TEST_LLVM(TestNarrowSqueezeToDict) {
auto test = [](bool hashed, bool multi, bool compact, bool withPayload) {
diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_holders.h b/ydb/library/yql/minikql/computation/mkql_computation_node_holders.h
index 50b81a5053..4486d6f43a 100644
--- a/ydb/library/yql/minikql/computation/mkql_computation_node_holders.h
+++ b/ydb/library/yql/minikql/computation/mkql_computation_node_holders.h
@@ -379,22 +379,22 @@ public:
NUdf::TUnboxedValuePod VectorAsArray(TUnboxedValueVector& values) const;
- template <class TForwardIterator>
- NUdf::TUnboxedValuePod RangeAsArray(TForwardIterator first, TForwardIterator last) const {
- auto count = std::distance(first, last);
- if (count == 0)
- return GetEmptyContainer();
-
- NUdf::TUnboxedValue* itemsPtr = nullptr;
- auto tuple = CreateDirectArrayHolder(count, itemsPtr);
- while (first != last) {
- *itemsPtr++ = std::move(*first);
- ++first;
- }
-
- return tuple;
- }
-
+ template <class TForwardIterator>
+ NUdf::TUnboxedValuePod RangeAsArray(TForwardIterator first, TForwardIterator last) const {
+ auto count = std::distance(first, last);
+ if (count == 0)
+ return GetEmptyContainer();
+
+ NUdf::TUnboxedValue* itemsPtr = nullptr;
+ auto tuple = CreateDirectArrayHolder(count, itemsPtr);
+ while (first != last) {
+ *itemsPtr++ = std::move(*first);
+ ++first;
+ }
+
+ return tuple;
+ }
+
NUdf::TUnboxedValuePod CreateDirectSortedSetHolder(
TSortedSetFiller filler,
const TKeyTypes& types,
diff --git a/ydb/library/yql/minikql/mkql_alloc.h b/ydb/library/yql/minikql/mkql_alloc.h
index 5fa22218f8..5e864cdc4c 100644
--- a/ydb/library/yql/minikql/mkql_alloc.h
+++ b/ydb/library/yql/minikql/mkql_alloc.h
@@ -108,9 +108,9 @@ public:
}
size_t GetUsed() const { return MyState_.GetUsed(); }
- size_t GetPeakUsed() const { return MyState_.GetPeakUsed(); }
+ size_t GetPeakUsed() const { return MyState_.GetPeakUsed(); }
size_t GetAllocated() const { return MyState_.GetAllocated(); }
- size_t GetPeakAllocated() const { return MyState_.GetPeakAllocated(); }
+ size_t GetPeakAllocated() const { return MyState_.GetPeakAllocated(); }
size_t GetLimit() const { return MyState_.GetLimit(); }
void SetLimit(size_t limit) { MyState_.SetLimit(limit); }
diff --git a/ydb/library/yql/minikql/mkql_program_builder.cpp b/ydb/library/yql/minikql/mkql_program_builder.cpp
index 84adca1925..47ae6e4c8f 100644
--- a/ydb/library/yql/minikql/mkql_program_builder.cpp
+++ b/ydb/library/yql/minikql/mkql_program_builder.cpp
@@ -1754,16 +1754,16 @@ TRuntimeNode TProgramBuilder::ToHashedDict(TRuntimeNode list, bool all, const TU
return ToDict(list, all, keySelector, payloadSelector, __func__, isCompact, itemsCountHint);
}
-TRuntimeNode TProgramBuilder::SqueezeToSortedDict(TRuntimeNode stream, bool all, const TUnaryLambda& keySelector,
- const TUnaryLambda& payloadSelector, bool isCompact, ui64 itemsCountHint) {
- return SqueezeToDict(stream, all, keySelector, payloadSelector, __func__, isCompact, itemsCountHint);
-}
-
-TRuntimeNode TProgramBuilder::SqueezeToHashedDict(TRuntimeNode stream, bool all, const TUnaryLambda& keySelector,
- const TUnaryLambda& payloadSelector, bool isCompact, ui64 itemsCountHint) {
- return SqueezeToDict(stream, all, keySelector, payloadSelector, __func__, isCompact, itemsCountHint);
-}
-
+TRuntimeNode TProgramBuilder::SqueezeToSortedDict(TRuntimeNode stream, bool all, const TUnaryLambda& keySelector,
+ const TUnaryLambda& payloadSelector, bool isCompact, ui64 itemsCountHint) {
+ return SqueezeToDict(stream, all, keySelector, payloadSelector, __func__, isCompact, itemsCountHint);
+}
+
+TRuntimeNode TProgramBuilder::SqueezeToHashedDict(TRuntimeNode stream, bool all, const TUnaryLambda& keySelector,
+ const TUnaryLambda& payloadSelector, bool isCompact, ui64 itemsCountHint) {
+ return SqueezeToDict(stream, all, keySelector, payloadSelector, __func__, isCompact, itemsCountHint);
+}
+
TRuntimeNode TProgramBuilder::NarrowSqueezeToSortedDict(TRuntimeNode stream, bool all, const TNarrowLambda& keySelector,
const TNarrowLambda& payloadSelector, bool isCompact, ui64 itemsCountHint) {
return NarrowSqueezeToDict(stream, all, keySelector, payloadSelector, __func__, isCompact, itemsCountHint);
@@ -3073,7 +3073,7 @@ TRuntimeNode TProgramBuilder::UnaryDataFunction(TRuntimeNode data, const std::st
return TRuntimeNode(callableBuilder.Build(), false);
}
-TRuntimeNode TProgramBuilder::ToDict(TRuntimeNode list, bool multi, const TUnaryLambda& keySelector,
+TRuntimeNode TProgramBuilder::ToDict(TRuntimeNode list, bool multi, const TUnaryLambda& keySelector,
const TUnaryLambda& payloadSelector, std::string_view callableName, bool isCompact, ui64 itemsCountHint)
{
bool isOptional;
@@ -3109,44 +3109,44 @@ TRuntimeNode TProgramBuilder::ToDict(TRuntimeNode list, bool multi, const TUnary
return TRuntimeNode(callableBuilder.Build(), false);
}
-TRuntimeNode TProgramBuilder::SqueezeToDict(TRuntimeNode stream, bool multi, const TUnaryLambda& keySelector,
+TRuntimeNode TProgramBuilder::SqueezeToDict(TRuntimeNode stream, bool multi, const TUnaryLambda& keySelector,
const TUnaryLambda& payloadSelector, std::string_view callableName, bool isCompact, ui64 itemsCountHint)
-{
- if constexpr (RuntimeVersion < 21U) {
- THROW yexception() << "Runtime version (" << RuntimeVersion << ") too old for " << __func__;
- }
-
- const auto type = stream.GetStaticType();
- MKQL_ENSURE(type->IsStream() || type->IsFlow(), "Expected stream or flow.");
-
- const auto itemType = type->IsFlow() ? AS_TYPE(TFlowType, type)->GetItemType() : AS_TYPE(TStreamType, type)->GetItemType();
- ThrowIfListOfVoid(itemType);
- const auto itemArg = Arg(itemType);
-
- const auto key = keySelector(itemArg);
- const auto keyType = key.GetStaticType();
-
- auto payload = payloadSelector(itemArg);
- auto payloadType = payload.GetStaticType();
- if (multi) {
- payloadType = TListType::Create(payloadType, Env);
- }
-
- auto dictType = TDictType::Create(keyType, payloadType, Env);
- auto returnType = type->IsFlow()
- ? (TType*) TFlowType::Create(dictType, Env)
- : (TType*) TStreamType::Create(dictType, Env);
- TCallableBuilder callableBuilder(Env, callableName, returnType);
- callableBuilder.Add(stream);
- callableBuilder.Add(itemArg);
- callableBuilder.Add(key);
- callableBuilder.Add(payload);
- callableBuilder.Add(NewDataLiteral(multi));
- callableBuilder.Add(NewDataLiteral(isCompact));
- callableBuilder.Add(NewDataLiteral(itemsCountHint));
- return TRuntimeNode(callableBuilder.Build(), false);
-}
-
+{
+ if constexpr (RuntimeVersion < 21U) {
+ THROW yexception() << "Runtime version (" << RuntimeVersion << ") too old for " << __func__;
+ }
+
+ const auto type = stream.GetStaticType();
+ MKQL_ENSURE(type->IsStream() || type->IsFlow(), "Expected stream or flow.");
+
+ const auto itemType = type->IsFlow() ? AS_TYPE(TFlowType, type)->GetItemType() : AS_TYPE(TStreamType, type)->GetItemType();
+ ThrowIfListOfVoid(itemType);
+ const auto itemArg = Arg(itemType);
+
+ const auto key = keySelector(itemArg);
+ const auto keyType = key.GetStaticType();
+
+ auto payload = payloadSelector(itemArg);
+ auto payloadType = payload.GetStaticType();
+ if (multi) {
+ payloadType = TListType::Create(payloadType, Env);
+ }
+
+ auto dictType = TDictType::Create(keyType, payloadType, Env);
+ auto returnType = type->IsFlow()
+ ? (TType*) TFlowType::Create(dictType, Env)
+ : (TType*) TStreamType::Create(dictType, Env);
+ TCallableBuilder callableBuilder(Env, callableName, returnType);
+ callableBuilder.Add(stream);
+ callableBuilder.Add(itemArg);
+ callableBuilder.Add(key);
+ callableBuilder.Add(payload);
+ callableBuilder.Add(NewDataLiteral(multi));
+ callableBuilder.Add(NewDataLiteral(isCompact));
+ callableBuilder.Add(NewDataLiteral(itemsCountHint));
+ return TRuntimeNode(callableBuilder.Build(), false);
+}
+
TRuntimeNode TProgramBuilder::NarrowSqueezeToDict(TRuntimeNode flow, bool multi, const TNarrowLambda& keySelector,
const TNarrowLambda& payloadSelector, std::string_view callableName, bool isCompact, ui64 itemsCountHint)
{
diff --git a/ydb/library/yql/minikql/mkql_program_builder.h b/ydb/library/yql/minikql/mkql_program_builder.h
index 2363eecf45..10f1ad7ccf 100644
--- a/ydb/library/yql/minikql/mkql_program_builder.h
+++ b/ydb/library/yql/minikql/mkql_program_builder.h
@@ -450,10 +450,10 @@ public:
const TUnaryLambda& payloadSelector, bool isCompact = false, ui64 itemsCountHint = 0);
TRuntimeNode ToHashedDict(TRuntimeNode list, bool all, const TUnaryLambda& keySelector,
const TUnaryLambda& payloadSelector, bool isCompact = false, ui64 itemsCountHint = 0);
- TRuntimeNode SqueezeToSortedDict(TRuntimeNode stream, bool all, const TUnaryLambda& keySelector,
- const TUnaryLambda& payloadSelector, bool isCompact = false, ui64 itemsCountHint = 0);
- TRuntimeNode SqueezeToHashedDict(TRuntimeNode stream, bool all, const TUnaryLambda& keySelector,
- const TUnaryLambda& payloadSelector, bool isCompact = false, ui64 itemsCountHint = 0);
+ TRuntimeNode SqueezeToSortedDict(TRuntimeNode stream, bool all, const TUnaryLambda& keySelector,
+ const TUnaryLambda& payloadSelector, bool isCompact = false, ui64 itemsCountHint = 0);
+ TRuntimeNode SqueezeToHashedDict(TRuntimeNode stream, bool all, const TUnaryLambda& keySelector,
+ const TUnaryLambda& payloadSelector, bool isCompact = false, ui64 itemsCountHint = 0);
TRuntimeNode NarrowSqueezeToSortedDict(TRuntimeNode stream, bool all, const TNarrowLambda& keySelector,
const TNarrowLambda& payloadSelector, bool isCompact = false, ui64 itemsCountHint = 0);
TRuntimeNode NarrowSqueezeToHashedDict(TRuntimeNode stream, bool all, const TNarrowLambda& keySelector,
@@ -656,7 +656,7 @@ private:
TRuntimeNode ToDict(TRuntimeNode list, bool multi, const TUnaryLambda& keySelector,
const TUnaryLambda& payloadSelector, std::string_view callableName, bool isCompact, ui64 itemsCountHint);
- TRuntimeNode SqueezeToDict(TRuntimeNode stream, bool multi, const TUnaryLambda& keySelector,
+ TRuntimeNode SqueezeToDict(TRuntimeNode stream, bool multi, const TUnaryLambda& keySelector,
const TUnaryLambda& payloadSelector, std::string_view callableName, bool isCompact, ui64 itemsCountHint);
TRuntimeNode NarrowSqueezeToDict(TRuntimeNode stream, bool multi, const TNarrowLambda& keySelector,
const TNarrowLambda& payloadSelector, std::string_view callableName, bool isCompact, ui64 itemsCountHint);
diff --git a/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp b/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp
index 0bd61649b9..dd1f162d78 100644
--- a/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp
+++ b/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp
@@ -26,37 +26,37 @@ namespace NCommon {
TRuntimeNode CombineByKeyImpl(const TExprNode& node, TMkqlBuildContext& ctx) {
NNodes::TCoCombineByKey combine(&node);
- const bool isStreamOrFlow = combine.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Stream ||
- combine.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Flow;
-
- YQL_ENSURE(!isStreamOrFlow);
-
+ const bool isStreamOrFlow = combine.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Stream ||
+ combine.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Flow;
+
+ YQL_ENSURE(!isStreamOrFlow);
+
const auto input = MkqlBuildExpr(combine.Input().Ref(), ctx);
TRuntimeNode preMapList = ctx.ProgramBuilder.FlatMap(input, [&](TRuntimeNode item) {
return MkqlBuildLambda(combine.PreMapLambda().Ref(), ctx, {item});
});
- const auto dict = ctx.ProgramBuilder.ToHashedDict(preMapList, true, [&](TRuntimeNode item) {
- return MkqlBuildLambda(combine.KeySelectorLambda().Ref(), ctx, {item});
- }, [&](TRuntimeNode item) {
- return item;
- });
-
- const auto values = ctx.ProgramBuilder.DictItems(dict);
- return ctx.ProgramBuilder.FlatMap(values, [&](TRuntimeNode item) {
- auto key = ctx.ProgramBuilder.Nth(item, 0);
- auto payloadList = ctx.ProgramBuilder.Nth(item, 1);
- auto fold1 = ctx.ProgramBuilder.Fold1(payloadList, [&](TRuntimeNode item2) {
- return MkqlBuildLambda(combine.InitHandlerLambda().Ref(), ctx, {key, item2});
- }, [&](TRuntimeNode item2, TRuntimeNode state) {
- return MkqlBuildLambda(combine.UpdateHandlerLambda().Ref(), ctx, {key, item2, state});
+ const auto dict = ctx.ProgramBuilder.ToHashedDict(preMapList, true, [&](TRuntimeNode item) {
+ return MkqlBuildLambda(combine.KeySelectorLambda().Ref(), ctx, {item});
+ }, [&](TRuntimeNode item) {
+ return item;
+ });
+
+ const auto values = ctx.ProgramBuilder.DictItems(dict);
+ return ctx.ProgramBuilder.FlatMap(values, [&](TRuntimeNode item) {
+ auto key = ctx.ProgramBuilder.Nth(item, 0);
+ auto payloadList = ctx.ProgramBuilder.Nth(item, 1);
+ auto fold1 = ctx.ProgramBuilder.Fold1(payloadList, [&](TRuntimeNode item2) {
+ return MkqlBuildLambda(combine.InitHandlerLambda().Ref(), ctx, {key, item2});
+ }, [&](TRuntimeNode item2, TRuntimeNode state) {
+ return MkqlBuildLambda(combine.UpdateHandlerLambda().Ref(), ctx, {key, item2, state});
});
- auto res = ctx.ProgramBuilder.FlatMap(fold1, [&](TRuntimeNode state) {
- return MkqlBuildLambda(combine.FinishHandlerLambda().Ref(), ctx, {key, state});
+ auto res = ctx.ProgramBuilder.FlatMap(fold1, [&](TRuntimeNode state) {
+ return MkqlBuildLambda(combine.FinishHandlerLambda().Ref(), ctx, {key, state});
});
- return res;
- });
+ return res;
+ });
}
namespace {
@@ -1656,24 +1656,24 @@ TMkqlCommonCallableCompiler::TShared::TShared() {
}, isCompact, itemsCount.GetOrElse(0));
});
- AddCallable("SqueezeToDict", [](const TExprNode& node, TMkqlBuildContext& ctx) {
- const auto stream = MkqlBuildExpr(node.Head(), ctx);
- TMaybe<bool> isMany;
- TMaybe<bool> isHashed;
- TMaybe<ui64> itemsCount;
- bool isCompact;
- if (const auto error = ParseToDictSettings(node, ctx.ExprCtx, isMany, isHashed, itemsCount, isCompact)) {
- ythrow TNodeException(node) << error->Message;
- }
-
- const auto factory = *isHashed ? &TProgramBuilder::SqueezeToHashedDict : &TProgramBuilder::SqueezeToSortedDict;
- return (ctx.ProgramBuilder.*factory)(stream, *isMany, [&](TRuntimeNode item) {
- return MkqlBuildLambda(*node.Child(1), ctx, {item});
- }, [&](TRuntimeNode item) {
- return MkqlBuildLambda(*node.Child(2), ctx, {item});
- }, isCompact, itemsCount.GetOrElse(0));
- });
-
+ AddCallable("SqueezeToDict", [](const TExprNode& node, TMkqlBuildContext& ctx) {
+ const auto stream = MkqlBuildExpr(node.Head(), ctx);
+ TMaybe<bool> isMany;
+ TMaybe<bool> isHashed;
+ TMaybe<ui64> itemsCount;
+ bool isCompact;
+ if (const auto error = ParseToDictSettings(node, ctx.ExprCtx, isMany, isHashed, itemsCount, isCompact)) {
+ ythrow TNodeException(node) << error->Message;
+ }
+
+ const auto factory = *isHashed ? &TProgramBuilder::SqueezeToHashedDict : &TProgramBuilder::SqueezeToSortedDict;
+ return (ctx.ProgramBuilder.*factory)(stream, *isMany, [&](TRuntimeNode item) {
+ return MkqlBuildLambda(*node.Child(1), ctx, {item});
+ }, [&](TRuntimeNode item) {
+ return MkqlBuildLambda(*node.Child(2), ctx, {item});
+ }, isCompact, itemsCount.GetOrElse(0));
+ });
+
AddCallable("NarrowSqueezeToDict", [](const TExprNode& node, TMkqlBuildContext& ctx) {
const auto stream = MkqlBuildExpr(node.Head(), ctx);
TMaybe<bool> isMany;
diff --git a/ydb/library/yql/providers/common/mkql/yql_provider_mkql.h b/ydb/library/yql/providers/common/mkql/yql_provider_mkql.h
index 95b8a5867a..a25c03b488 100644
--- a/ydb/library/yql/providers/common/mkql/yql_provider_mkql.h
+++ b/ydb/library/yql/providers/common/mkql/yql_provider_mkql.h
@@ -38,7 +38,7 @@ struct TMkqlBuildContext {
, ParentCtx(&parent)
, Level(parent.Level + 1U)
, LambdaId(lambdaId)
- , Parameters(parent.Parameters)
+ , Parameters(parent.Parameters)
{}
};
diff --git a/ydb/library/yql/providers/dq/actors/compute_actor.cpp b/ydb/library/yql/providers/dq/actors/compute_actor.cpp
index 834cdcda14..a2ae923d16 100644
--- a/ydb/library/yql/providers/dq/actors/compute_actor.cpp
+++ b/ydb/library/yql/providers/dq/actors/compute_actor.cpp
@@ -24,7 +24,7 @@ IActor* CreateComputeActor(
const TString& computeActorType,
const NDq::NTaskRunnerActor::ITaskRunnerActorFactory::TPtr& taskRunnerActorFactory)
{
- auto memoryLimits = NDq::TComputeMemoryLimits();
+ auto memoryLimits = NDq::TComputeMemoryLimits();
memoryLimits.ChannelBufferSize = 1000000;
memoryLimits.ScanBufferSize = 16_MB;
// light == heavy since we allow extra allocation
@@ -36,10 +36,10 @@ IActor* CreateComputeActor(
memoryLimits.MinMemAllocSize = options.MkqlMinAllocSize;
memoryLimits.MinMemFreeSize = options.MkqlMinAllocSize;
- auto computeRuntimeSettings = NDq::TComputeRuntimeSettings();
+ auto computeRuntimeSettings = NDq::TComputeRuntimeSettings();
computeRuntimeSettings.ExtraMemoryAllocationPool = 3;
- computeRuntimeSettings.FailOnUndelivery = false;
- computeRuntimeSettings.StatsMode = NDqProto::DQ_STATS_MODE_PROFILE;
+ computeRuntimeSettings.FailOnUndelivery = false;
+ computeRuntimeSettings.StatsMode = NDqProto::DQ_STATS_MODE_PROFILE;
// clear fake actorids
for (auto& input : *task.MutableInputs()) {
@@ -55,7 +55,7 @@ IActor* CreateComputeActor(
}
}
- auto taskRunnerFactory = [=](const NDqProto::TDqTask& task, const NDq::TLogFunc& logger) {
+ auto taskRunnerFactory = [=](const NDqProto::TDqTask& task, const NDq::TLogFunc& logger) {
Y_UNUSED(logger);
return options.Factory->Get(task, {});
};
diff --git a/ydb/library/yql/providers/dq/actors/result_receiver.cpp b/ydb/library/yql/providers/dq/actors/result_receiver.cpp
index 33a014f7ab..7fdb73f257 100644
--- a/ydb/library/yql/providers/dq/actors/result_receiver.cpp
+++ b/ydb/library/yql/providers/dq/actors/result_receiver.cpp
@@ -66,16 +66,16 @@ public:
private:
STRICT_STFUNC(Handler, {
- HFunc(NDq::TEvDqCompute::TEvChannelData, OnChannelData)
+ HFunc(NDq::TEvDqCompute::TEvChannelData, OnChannelData)
HFunc(TEvReadyState, OnReadyState);
HFunc(TEvQueryResponse, OnQueryResult);
cFunc(TEvents::TEvPoison::EventType, PassAway)
})
- void OnChannelData(NDq::TEvDqCompute::TEvChannelData::TPtr& ev, const TActorContext&) {
+ void OnChannelData(NDq::TEvDqCompute::TEvChannelData::TPtr& ev, const TActorContext&) {
YQL_LOG_CTX_SCOPE(TraceId);
- auto res = MakeHolder<NDq::TEvDqCompute::TEvChannelDataAck>();
+ auto res = MakeHolder<NDq::TEvDqCompute::TEvChannelDataAck>();
if (!Discard) {
if (!Finished && ev->Get()->Record.GetChannelData().GetData().GetRaw().size() > 0) {
diff --git a/ydb/library/yql/providers/dq/actors/task_controller.cpp b/ydb/library/yql/providers/dq/actors/task_controller.cpp
index daf55c90f9..238828e453 100644
--- a/ydb/library/yql/providers/dq/actors/task_controller.cpp
+++ b/ydb/library/yql/providers/dq/actors/task_controller.cpp
@@ -123,26 +123,26 @@ private:
if (state.HasStats() && state.GetStats().GetTasks().size()) {
YQL_LOG(DEBUG) << " " << SelfId() << " AddStats " << taskId;
- AddStats(state.GetStats());
+ AddStats(state.GetStats());
if (ServiceCounters.Counters && !AggrPeriod) {
ExportStats(TaskStat, taskId);
}
}
switch (ev->Get()->Record.GetState()) {
- case NDqProto::COMPUTE_STATE_UNKNOWN: {
+ case NDqProto::COMPUTE_STATE_UNKNOWN: {
// TODO: use issues
TString message = "unexpected state from " + ToString(computeActor) + ", task: " + ToString(taskId);
OnError(message, false, false);
break;
}
- case NDqProto::COMPUTE_STATE_FAILURE: {
+ case NDqProto::COMPUTE_STATE_FAILURE: {
// TODO: don't convert issues to string
NYql::IssuesFromMessage(state.GetIssues(), Issues);
OnError(Issues.ToString(), false, false);
break;
}
- case NDqProto::COMPUTE_STATE_EXECUTING: {
+ case NDqProto::COMPUTE_STATE_EXECUTING: {
YQL_LOG(DEBUG) << " " << SelfId() << " Executing TaskId: " << taskId;
if (!FinishedTasks.contains(taskId)) {
// may get late/reordered? message
@@ -150,7 +150,7 @@ private:
}
break;
}
- case NDqProto::COMPUTE_STATE_FINISHED: {
+ case NDqProto::COMPUTE_STATE_FINISHED: {
YQL_LOG(DEBUG) << " " << SelfId() << " Finish TaskId: " << taskId;
Executing.erase(taskId);
FinishedTasks.insert(taskId);
@@ -243,11 +243,11 @@ private:
}
}
- void AddStats(const NDqProto::TDqComputeActorStats& x) {
- YQL_ENSURE(x.GetTasks().size() == 1);
- auto& s = x.GetTasks(0);
- ui64 taskId = s.GetTaskId();
-
+ void AddStats(const NDqProto::TDqComputeActorStats& x) {
+ YQL_ENSURE(x.GetTasks().size() == 1);
+ auto& s = x.GetTasks(0);
+ ui64 taskId = s.GetTaskId();
+
#define ADD_COUNTER(name) \
if (stats.Get ## name()) { \
TaskStat.SetCounter(TaskStat.GetCounterName("TaskRunner", labels, #name), stats.Get ## name ()); \
@@ -259,44 +259,44 @@ private:
auto& stats = s;
// basic stats
- ADD_COUNTER(ComputeCpuTimeUs)
+ ADD_COUNTER(ComputeCpuTimeUs)
ADD_COUNTER(PendingInputTimeUs)
ADD_COUNTER(PendingOutputTimeUs)
ADD_COUNTER(FinishTimeUs)
// profile stats
- ADD_COUNTER(BuildCpuTimeUs)
- ADD_COUNTER(WaitTimeUs)
- ADD_COUNTER(WaitOutputTimeUs)
-
- if (auto v = x.GetMkqlMaxMemoryUsage()) {
- TaskStat.SetCounter(TaskStat.GetCounterName("TaskRunner", labels, "MkqlMaxMemoryUsage"), v);
- }
-
+ ADD_COUNTER(BuildCpuTimeUs)
+ ADD_COUNTER(WaitTimeUs)
+ ADD_COUNTER(WaitOutputTimeUs)
+
+ if (auto v = x.GetMkqlMaxMemoryUsage()) {
+ TaskStat.SetCounter(TaskStat.GetCounterName("TaskRunner", labels, "MkqlMaxMemoryUsage"), v);
+ }
+
for (const auto& stat : s.GetMkqlStats()) {
TaskStat.SetCounter(TaskStat.GetCounterName("TaskRunner", labels, stat.GetName()), stat.GetValue());
}
- if (stats.ComputeCpuTimeByRunSize()) {
+ if (stats.ComputeCpuTimeByRunSize()) {
auto& hist = TaskStat.GetHistogram(TaskStat.GetCounterName("TaskRunner", labels, "ComputeTimeByRunMs"));
- for (const auto& bucket : s.GetComputeCpuTimeByRun()) {
+ for (const auto& bucket : s.GetComputeCpuTimeByRun()) {
hist[bucket.GetBound()] = bucket.GetValue();
}
}
// compilation stats
-// ADD_COUNTER(MkqlTotalNodes)
-// ADD_COUNTER(MkqlCodegenFunctions)
-// ADD_COUNTER(CodeGenTotalInstructions)
-// ADD_COUNTER(CodeGenTotalFunctions)
-//
-// ADD_COUNTER(CodeGenFullTime)
-// ADD_COUNTER(CodeGenFinalizeTime)
-// ADD_COUNTER(CodeGenModulePassTime)
-
-// if (stats.GetFinishTs() >= stats.GetStartTs()) {
-// TaskStat.SetCounter(TaskStat.GetCounterName("TaskRunner", labels, "Total"), stats.GetFinishTs() - stats.GetStartTs());
-// }
+// ADD_COUNTER(MkqlTotalNodes)
+// ADD_COUNTER(MkqlCodegenFunctions)
+// ADD_COUNTER(CodeGenTotalInstructions)
+// ADD_COUNTER(CodeGenTotalFunctions)
+//
+// ADD_COUNTER(CodeGenFullTime)
+// ADD_COUNTER(CodeGenFinalizeTime)
+// ADD_COUNTER(CodeGenModulePassTime)
+
+// if (stats.GetFinishTs() >= stats.GetStartTs()) {
+// TaskStat.SetCounter(TaskStat.GetCounterName("TaskRunner", labels, "Total"), stats.GetFinishTs() - stats.GetStartTs());
+// }
for (const auto& stats : s.GetInputChannels()) {
std::map<TString, TString> labels = {
@@ -309,11 +309,11 @@ private:
ADD_COUNTER(RowsIn);
ADD_COUNTER(RowsOut);
ADD_COUNTER(MaxMemoryUsage);
- ADD_COUNTER(DeserializationTimeUs);
+ ADD_COUNTER(DeserializationTimeUs);
-// if (stats.GetFinishTs() >= stats.GetStartTs()) {
-// TaskStat.SetCounter(TaskStat.GetCounterName("TaskRunner", labels, "Total"), stats.GetFinishTs() - stats.GetStartTs());
-// }
+// if (stats.GetFinishTs() >= stats.GetStartTs()) {
+// TaskStat.SetCounter(TaskStat.GetCounterName("TaskRunner", labels, "Total"), stats.GetFinishTs() - stats.GetStartTs());
+// }
}
for (const auto& stats : s.GetOutputChannels()) {
@@ -328,22 +328,22 @@ private:
ADD_COUNTER(RowsOut);
ADD_COUNTER(MaxMemoryUsage);
- ADD_COUNTER(SerializationTimeUs);
+ ADD_COUNTER(SerializationTimeUs);
ADD_COUNTER(BlockedByCapacity);
ADD_COUNTER(SpilledBytes);
ADD_COUNTER(SpilledRows);
ADD_COUNTER(SpilledBlobs);
-// if (stats.GetFinishTs() >= stats.GetStartTs()) {
-// TaskStat.SetCounter(TaskStat.GetCounterName("TaskRunner", labels, "Total"), stats.GetFinishTs() - stats.GetStartTs());
-// }
+// if (stats.GetFinishTs() >= stats.GetStartTs()) {
+// TaskStat.SetCounter(TaskStat.GetCounterName("TaskRunner", labels, "Total"), stats.GetFinishTs() - stats.GetStartTs());
+// }
}
for (const auto& stats : s.GetSources()) {
std::map<TString, TString> labels = {
{"Task", ToString(taskId)},
- {"Source", ToString(stats.GetInputIndex())}
+ {"Source", ToString(stats.GetInputIndex())}
};
ADD_COUNTER(Chunks);
@@ -354,15 +354,15 @@ private:
ADD_COUNTER(ErrorsCount);
-// if (stats.GetFinishTs() >= stats.GetStartTs()) {
-// TaskStat.SetCounter(TaskStat.GetCounterName("TaskRunner", labels, "Total"), stats.GetFinishTs() - stats.GetStartTs());
-// }
+// if (stats.GetFinishTs() >= stats.GetStartTs()) {
+// TaskStat.SetCounter(TaskStat.GetCounterName("TaskRunner", labels, "Total"), stats.GetFinishTs() - stats.GetStartTs());
+// }
}
for (const auto& stats : s.GetSinks()) {
std::map<TString, TString> labels = {
{"Task", ToString(taskId)},
- {"Sink", ToString(stats.GetOutputIndex())}
+ {"Sink", ToString(stats.GetOutputIndex())}
};
ADD_COUNTER(Chunks)
@@ -373,9 +373,9 @@ private:
ADD_COUNTER(ErrorsCount);
-// if (stats.GetFinishTs() >= stats.GetStartTs()) {
-// TaskStat.SetCounter(TaskStat.GetCounterName("TaskRunner", labels, "Total"), stats.GetFinishTs() - stats.GetStartTs());
-// }
+// if (stats.GetFinishTs() >= stats.GetStartTs()) {
+// TaskStat.SetCounter(TaskStat.GetCounterName("TaskRunner", labels, "Total"), stats.GetFinishTs() - stats.GetStartTs());
+// }
}
#undef ADD_COUNTER
@@ -440,7 +440,7 @@ private:
YQL_LOG(DEBUG) << "Update channels";
for (const auto& [task, actorId] : Tasks) {
- auto ev = MakeHolder<NDq::TEvDqCompute::TEvChannelsInfo>();
+ auto ev = MakeHolder<NDq::TEvDqCompute::TEvChannelsInfo>();
for (const auto& input : task.GetInputs()) {
for (const auto& channel : input.GetChannels()) {
diff --git a/ydb/library/yql/providers/dq/api/protos/task_command_executor.proto b/ydb/library/yql/providers/dq/api/protos/task_command_executor.proto
index 989444e921..6e3d08e595 100644
--- a/ydb/library/yql/providers/dq/api/protos/task_command_executor.proto
+++ b/ydb/library/yql/providers/dq/api/protos/task_command_executor.proto
@@ -104,23 +104,23 @@ message TGetStoredBytesResponse {
}
message TGetStatsResponse {
- reserved 1; // NKqpProto.TKqpStatsTask Stats = 1;
- NDqProto.TDqTaskStats Stats = 2;
+ reserved 1; // NKqpProto.TKqpStatsTask Stats = 1;
+ NDqProto.TDqTaskStats Stats = 2;
}
message TGetStatsInputResponse {
- reserved 1; // NKqpProto.TKqpStatsTask.TInputChannelStats Stats = 1;
- NDqProto.TDqInputChannelStats Stats = 2;
+ reserved 1; // NKqpProto.TKqpStatsTask.TInputChannelStats Stats = 1;
+ NDqProto.TDqInputChannelStats Stats = 2;
}
message TGetStatsOutputResponse {
- reserved 1; // NKqpProto.TKqpStatsTask.TOutputChannelStats Stats = 1;
- NDqProto.TDqOutputChannelStats Stats = 2;
+ reserved 1; // NKqpProto.TKqpStatsTask.TOutputChannelStats Stats = 1;
+ NDqProto.TDqOutputChannelStats Stats = 2;
}
message TGetStatsSourceResponse {
- reserved 1; // NKqpProto.TKqpStatsTask.TSourceStats Stats = 1;
- NDqProto.TDqSourceStats Stats = 2;
+ reserved 1; // NKqpProto.TKqpStatsTask.TSourceStats Stats = 1;
+ NDqProto.TDqSourceStats Stats = 2;
}
message TSinkStatsResponse {
diff --git a/ydb/library/yql/providers/dq/counters/counters.h b/ydb/library/yql/providers/dq/counters/counters.h
index 5cf5643842..4a457d24fc 100644
--- a/ydb/library/yql/providers/dq/counters/counters.h
+++ b/ydb/library/yql/providers/dq/counters/counters.h
@@ -272,8 +272,8 @@ struct TCounters {
};
// basic stats
- ADD_COUNTER(ComputeCpuTime)
- ADD_COUNTER(BuildCpuTime)
+ ADD_COUNTER(ComputeCpuTime)
+ ADD_COUNTER(BuildCpuTime)
// profile stats
ADD_COUNTER(WaitTime)
diff --git a/ydb/library/yql/providers/dq/opt/dqs_opt.cpp b/ydb/library/yql/providers/dq/opt/dqs_opt.cpp
index b3481a7718..b4bc07fbef 100644
--- a/ydb/library/yql/providers/dq/opt/dqs_opt.cpp
+++ b/ydb/library/yql/providers/dq/opt/dqs_opt.cpp
@@ -44,7 +44,7 @@ namespace NYql::NDqs {
output = Build<TDqCnUnionAll>(ctx, input->Pos()) // clang-format off
.Output()
- .Stage<TDqStage>()
+ .Stage<TDqStage>()
.Inputs()
.Build()
.Program()
@@ -79,7 +79,7 @@ namespace NYql::NDqs {
output = Build<TDqCnResult>(ctx, input->Pos()) // clang-format off
.Output()
- .Stage<TDqStage>()
+ .Stage<TDqStage>()
.Inputs()
.Add(node.Cast<TDqCnUnionAll>())
.Build()
@@ -92,8 +92,8 @@ namespace NYql::NDqs {
.Index()
.Build("0")
.Build()
- .ColumnHints() // TODO: set column hints
- .Build()
+ .ColumnHints() // TODO: set column hints
+ .Build()
.Done().Ptr(); // clang-format on
return TStatus(IGraphTransformer::TStatus::Repeat, true);
});
@@ -118,10 +118,10 @@ namespace NYql::NDqs {
namespace NPeephole {
- class TDqsPeepholeTransformer: public TSyncTransformerBase {
+ class TDqsPeepholeTransformer: public TSyncTransformerBase {
public:
- TDqsPeepholeTransformer(THolder<IGraphTransformer>&& typeAnnTransformer,
- TTypeAnnotationContext& typesCtx)
+ TDqsPeepholeTransformer(THolder<IGraphTransformer>&& typeAnnTransformer,
+ TTypeAnnotationContext& typesCtx)
: TypeAnnTransformer(std::move(typeAnnTransformer))
, TypesCtx(typesCtx)
{
@@ -164,17 +164,17 @@ namespace NYql::NDqs {
}
THolder<IGraphTransformer> CreateDqsPeepholeTransformer(THolder<IGraphTransformer>&& typeAnnTransformer, TTypeAnnotationContext& typesCtx) {
- return MakeHolder<NPeephole::TDqsPeepholeTransformer>(std::move(typeAnnTransformer), typesCtx);
+ return MakeHolder<NPeephole::TDqsPeepholeTransformer>(std::move(typeAnnTransformer), typesCtx);
}
THolder<IGraphTransformer> CreateDqsFinalizingOptTransformer() {
return CreateFunctorTransformer(
[](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
- auto status = NDq::DqReplicateStageMultiOutput(input, output, ctx);
+ auto status = NDq::DqReplicateStageMultiOutput(input, output, ctx);
if (status.Level != TStatus::Error && input != output) {
YQL_CVLOG(NLog::ELevel::INFO, NLog::EComponent::ProviderDq) << "DqReplicateStageMultiOutput";
}
return status;
- });
+ });
}
-}
+}
diff --git a/ydb/library/yql/providers/dq/opt/logical_optimize.cpp b/ydb/library/yql/providers/dq/opt/logical_optimize.cpp
index e49cc9fbe0..9e87f81e7c 100644
--- a/ydb/library/yql/providers/dq/opt/logical_optimize.cpp
+++ b/ydb/library/yql/providers/dq/opt/logical_optimize.cpp
@@ -429,7 +429,7 @@ private:
.Done();
}
- return Build<TDqCnHashShuffle>(ctx, pos)
+ return Build<TDqCnHashShuffle>(ctx, pos)
.Output()
.Stage(mappedInput)
.Index().Value("0").Build()
diff --git a/ydb/library/yql/providers/dq/opt/physical_optimize.cpp b/ydb/library/yql/providers/dq/opt/physical_optimize.cpp
index 60afafdbe3..858e3da45a 100644
--- a/ydb/library/yql/providers/dq/opt/physical_optimize.cpp
+++ b/ydb/library/yql/providers/dq/opt/physical_optimize.cpp
@@ -189,27 +189,27 @@ protected:
template <bool IsGlobal>
TMaybeNode<TExprBase> PushSkipNullMembersToStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) {
- return DqPushSkipNullMembersToStage(node, ctx, optCtx, *getParents(), IsGlobal);
+ return DqPushSkipNullMembersToStage(node, ctx, optCtx, *getParents(), IsGlobal);
}
template <bool IsGlobal>
TMaybeNode<TExprBase> PushExtractMembersToStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) {
- return DqPushExtractMembersToStage(node, ctx, optCtx, *getParents(), IsGlobal);
+ return DqPushExtractMembersToStage(node, ctx, optCtx, *getParents(), IsGlobal);
}
template <bool IsGlobal>
TMaybeNode<TExprBase> BuildFlatmapStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) {
- return DqBuildFlatmapStage(node, ctx, optCtx, *getParents(), IsGlobal);
+ return DqBuildFlatmapStage(node, ctx, optCtx, *getParents(), IsGlobal);
}
template <bool IsGlobal>
TMaybeNode<TExprBase> PushOrderedLMapToStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) {
- return DqPushOrderedLMapToStage(node, ctx, optCtx, *getParents(), IsGlobal);
+ return DqPushOrderedLMapToStage(node, ctx, optCtx, *getParents(), IsGlobal);
}
template <bool IsGlobal>
TMaybeNode<TExprBase> PushLMapToStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) {
- return DqPushLMapToStage(node, ctx, optCtx, *getParents(), IsGlobal);
+ return DqPushLMapToStage(node, ctx, optCtx, *getParents(), IsGlobal);
}
template <bool IsGlobal>
@@ -219,7 +219,7 @@ protected:
template <bool IsGlobal>
TMaybeNode<TExprBase> PushCombineToStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) {
- return DqPushCombineToStage(node, ctx, optCtx, *getParents(), IsGlobal);
+ return DqPushCombineToStage(node, ctx, optCtx, *getParents(), IsGlobal);
}
TMaybeNode<TExprBase> BuildPartitionsStage(TExprBase node, TExprContext& ctx, const TGetParents& getParents) {
@@ -236,20 +236,20 @@ protected:
template <bool IsGlobal>
TMaybeNode<TExprBase> BuildTopSortStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) {
- return DqBuildTopSortStage(node, ctx, optCtx, *getParents(), IsGlobal);
+ return DqBuildTopSortStage(node, ctx, optCtx, *getParents(), IsGlobal);
}
template <bool IsGlobal>
TMaybeNode<TExprBase> BuildSortStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) {
- return DqBuildSortStage(node, ctx, optCtx, *getParents(), IsGlobal);
+ return DqBuildSortStage(node, ctx, optCtx, *getParents(), IsGlobal);
}
template <bool IsGlobal>
TMaybeNode<TExprBase> BuildTakeOrTakeSkipStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) {
if (node.Maybe<TCoTake>().Input().Maybe<TCoSkip>()) {
- return DqBuildTakeSkipStage(node, ctx, optCtx, *getParents(), IsGlobal);
+ return DqBuildTakeSkipStage(node, ctx, optCtx, *getParents(), IsGlobal);
} else {
- return DqBuildTakeStage(node, ctx, optCtx, *getParents(), IsGlobal);
+ return DqBuildTakeStage(node, ctx, optCtx, *getParents(), IsGlobal);
}
}
@@ -267,7 +267,7 @@ protected:
template <bool IsGlobal>
TMaybeNode<TExprBase> PushJoinToStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) {
- return DqPushJoinToStage(node, ctx, optCtx, *getParents(), IsGlobal);
+ return DqPushJoinToStage(node, ctx, optCtx, *getParents(), IsGlobal);
}
template <bool IsGlobal>
@@ -278,7 +278,7 @@ protected:
return node;
}
- return DqBuildPhyJoin(join, false /* TODO */, ctx, optCtx);
+ return DqBuildPhyJoin(join, false /* TODO */, ctx, optCtx);
}
template <bool IsGlobal>
@@ -291,15 +291,15 @@ protected:
return node;
}
- return DqBuildJoinDict(join, ctx); // , optCtx);
+ return DqBuildJoinDict(join, ctx); // , optCtx);
}
- TMaybeNode<TExprBase> BuildHasItems(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx) {
- return DqBuildHasItems(node, ctx, optCtx);
+ TMaybeNode<TExprBase> BuildHasItems(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx) {
+ return DqBuildHasItems(node, ctx, optCtx);
}
- TMaybeNode<TExprBase> BuildScalarPrecompute(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx) {
- return DqBuildScalarPrecompute(node, ctx, optCtx);
+ TMaybeNode<TExprBase> BuildScalarPrecompute(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx) {
+ return DqBuildScalarPrecompute(node, ctx, optCtx);
}
private:
diff --git a/ydb/library/yql/providers/dq/planner/dqs_task_graph.h b/ydb/library/yql/providers/dq/planner/dqs_task_graph.h
index 0c021ec53c..0b62dfce4a 100644
--- a/ydb/library/yql/providers/dq/planner/dqs_task_graph.h
+++ b/ydb/library/yql/providers/dq/planner/dqs_task_graph.h
@@ -7,7 +7,7 @@
namespace NYql::NDqs {
struct TStageInfoMeta {
- NNodes::TDqPhyStage Stage;
+ NNodes::TDqPhyStage Stage;
};
struct TTaskInputMeta {
diff --git a/ydb/library/yql/providers/dq/planner/execution_planner.cpp b/ydb/library/yql/providers/dq/planner/execution_planner.cpp
index 1352fbba71..ec404c0a6d 100644
--- a/ydb/library/yql/providers/dq/planner/execution_planner.cpp
+++ b/ydb/library/yql/providers/dq/planner/execution_planner.cpp
@@ -41,7 +41,7 @@ using namespace Yql::DqsProto;
namespace NYql::NDqs {
namespace {
TVector<TDqPhyStage> GetStages(const TExprNode::TPtr& exprRoot) {
- TVector<TDqPhyStage> stages;
+ TVector<TDqPhyStage> stages;
VisitExpr(
exprRoot,
[](const TExprNode::TPtr& exprNode) {
@@ -50,7 +50,7 @@ namespace NYql::NDqs {
},
[&stages](const TExprNode::TPtr& exprNode) {
const auto& node = TExprBase(exprNode);
- if (auto maybeStage = node.Maybe<TDqPhyStage>()) {
+ if (auto maybeStage = node.Maybe<TDqPhyStage>()) {
stages.push_back(maybeStage.Cast());
}
@@ -371,7 +371,7 @@ namespace NYql::NDqs {
program.SetRaw(programStr);
taskMeta.SetStageId(stageId);
taskDesc.MutableMeta()->PackFrom(taskMeta);
- taskDesc.SetStageId(stageId);
+ taskDesc.SetStageId(stageId);
plan.emplace_back(std::move(taskDesc));
}
@@ -493,14 +493,14 @@ namespace NYql::NDqs {
continue; \
}
- void TDqsExecutionPlanner::BuildConnections(const NNodes::TDqPhyStage& stage) {
- NDq::TChannelLogFunc logFunc = [](ui64, ui64, ui64, TStringBuf, bool) {};
+ void TDqsExecutionPlanner::BuildConnections(const NNodes::TDqPhyStage& stage) {
+ NDq::TChannelLogFunc logFunc = [](ui64, ui64, ui64, TStringBuf, bool) {};
for (ui32 inputIndex = 0; inputIndex < stage.Inputs().Size(); ++inputIndex) {
const auto& input = stage.Inputs().Item(inputIndex);
if (input.Maybe<TDqConnection>()) {
BUILD_CONNECTION(TDqCnUnionAll, BuildUnionAllChannels);
- BUILD_CONNECTION(TDqCnHashShuffle, BuildHashShuffleChannels);
+ BUILD_CONNECTION(TDqCnHashShuffle, BuildHashShuffleChannels);
BUILD_CONNECTION(TDqCnBroadcast, BuildBroadcastChannels);
BUILD_CONNECTION(TDqCnMap, BuildMapChannels);
BUILD_CONNECTION(TDqCnMerge, BuildMergeChannels);
@@ -613,7 +613,7 @@ namespace NYql::NDqs {
switch (output.Type) {
case TTaskOutputType::Map:
YQL_ENSURE(output.Channels.size() == 1);
- outputDesc.MutableMap(); //->SetChannelId(output.Channels[0]);
+ outputDesc.MutableMap(); //->SetChannelId(output.Channels[0]);
break;
case TTaskOutputType::HashPartition: {
@@ -627,9 +627,9 @@ namespace NYql::NDqs {
}
case TTaskOutputType::Broadcast: {
- //for (const auto channel : output.Channels) {
- outputDesc.MutableBroadcast(); //->AddChannelIds(channel);
- //}
+ //for (const auto channel : output.Channels) {
+ outputDesc.MutableBroadcast(); //->AddChannelIds(channel);
+ //}
break;
}
diff --git a/ydb/library/yql/providers/dq/planner/execution_planner.h b/ydb/library/yql/providers/dq/planner/execution_planner.h
index fdf104ee5c..280600c8f7 100644
--- a/ydb/library/yql/providers/dq/planner/execution_planner.h
+++ b/ydb/library/yql/providers/dq/planner/execution_planner.h
@@ -51,7 +51,7 @@ namespace NYql::NDqs {
private:
bool BuildReadStage(const TDqSettings::TPtr& settings, const NNodes::TDqPhyStage& stage, bool dqSource, bool canFallback);
- void BuildConnections(const NNodes::TDqPhyStage& stage);
+ void BuildConnections(const NNodes::TDqPhyStage& stage);
THashMap<NDq::TStageId, std::tuple<TString,ui64>> BuildAllPrograms();
void FillChannelDesc(NDqProto::TChannel& channelDesc, const NDq::TChannel& channel);
void FillInputDesc(NDqProto::TTaskInput& inputDesc, const TTaskInput& input);
diff --git a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp
index 17ecc7a057..8759f44221 100644
--- a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp
+++ b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp
@@ -95,7 +95,7 @@ public:
IDqGateway::TResult result;
NDqProto::TDqTask task;
task.SetId(0);
- task.SetStageId(0);
+ task.SetStageId(0);
auto& program = *task.MutableProgram();
program.SetRuntimeVersion(NYql::NDqProto::ERuntimeVersion::RUNTIME_VERSION_YQL_1_0);
@@ -134,17 +134,17 @@ public:
settings.SecureParams = secureParams;
settings.CollectBasicStats = true;
settings.CollectProfileStats = true;
- settings.AllowGeneratorsInUnboxedValues = true;
- auto runner = NDq::MakeDqTaskRunner(executionContext, settings, {});
-
- {
- auto guard = runner->BindAllocator(State->Settings->MemoryLimit.Get().GetOrElse(0));
- runner->Prepare(task, limits);
- }
-
+ settings.AllowGeneratorsInUnboxedValues = true;
+ auto runner = NDq::MakeDqTaskRunner(executionContext, settings, {});
+
+ {
+ auto guard = runner->BindAllocator(State->Settings->MemoryLimit.Get().GetOrElse(0));
+ runner->Prepare(task, limits);
+ }
+
TVector<NDqProto::TData> rows;
{
- auto guard = runner->BindAllocator(State->Settings->MemoryLimit.Get().GetOrElse(0));
+ auto guard = runner->BindAllocator(State->Settings->MemoryLimit.Get().GetOrElse(0));
YQL_LOG(DEBUG) << " NDq::ERunStatus " << runner->Run();
NDq::ERunStatus status;
@@ -211,7 +211,7 @@ private:
void AfterCreate(TTransformationPipeline*) const final {}
void AfterTypeAnnotation(TTransformationPipeline* pipeline) const final {
- pipeline->Add(NDq::CreateDqBuildPhyStagesTransformer(false), "Build-Phy");
+ pipeline->Add(NDq::CreateDqBuildPhyStagesTransformer(false), "Build-Phy");
pipeline->Add(NDqs::CreateDqsRewritePhyCallablesTransformer(), "Rewrite-Phy-Callables");
}
diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_datasink_type_ann.cpp b/ydb/library/yql/providers/dq/provider/yql_dq_datasink_type_ann.cpp
index 13b12b89d2..c8d9786a7a 100644
--- a/ydb/library/yql/providers/dq/provider/yql_dq_datasink_type_ann.cpp
+++ b/ydb/library/yql/providers/dq/provider/yql_dq_datasink_type_ann.cpp
@@ -20,7 +20,7 @@ public:
AddHandler({TDqPhyStage::CallableName()}, Hndl(&NDq::AnnotateDqPhyStage));
AddHandler({TDqOutput::CallableName()}, Hndl(&NDq::AnnotateDqOutput));
AddHandler({TDqCnUnionAll::CallableName()}, Hndl(&NDq::AnnotateDqConnection));
- AddHandler({TDqCnHashShuffle::CallableName()}, Hndl(&NDq::AnnotateDqCnHashShuffle));
+ AddHandler({TDqCnHashShuffle::CallableName()}, Hndl(&NDq::AnnotateDqCnHashShuffle));
AddHandler({TDqCnResult::CallableName()}, Hndl(&NDq::AnnotateDqCnResult));
AddHandler({TDqCnMap::CallableName()}, Hndl(&NDq::AnnotateDqConnection));
AddHandler({TDqCnBroadcast::CallableName()}, Hndl(&NDq::AnnotateDqConnection));
diff --git a/ydb/library/yql/providers/dq/runtime/task_command_executor.cpp b/ydb/library/yql/providers/dq/runtime/task_command_executor.cpp
index 1109497004..9b41920b17 100644
--- a/ydb/library/yql/providers/dq/runtime/task_command_executor.cpp
+++ b/ydb/library/yql/providers/dq/runtime/task_command_executor.cpp
@@ -51,7 +51,7 @@ void ToProto(T* s1, const NDq::TDqInputChannelStats* ss)
s1->SetRowsIn(ss->RowsIn);
s1->SetRowsOut(ss->RowsOut);
s1->SetMaxMemoryUsage(ss->MaxMemoryUsage);
- s1->SetDeserializationTimeUs(ss->DeserializationTime.MicroSeconds());
+ s1->SetDeserializationTimeUs(ss->DeserializationTime.MicroSeconds());
}
template<typename T>
@@ -102,7 +102,7 @@ public:
}
if (JobStats) {
- auto taskId = Runner->GetTaskId();
+ auto taskId = Runner->GetTaskId();
std::map<TString, TString> labels = {
{"Task", ToString(taskId)}
};
@@ -288,7 +288,7 @@ public:
}
case NDqProto::TCommandHeader::PUSH: {
Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion);
- Y_ENSURE(taskId == Runner->GetTaskId());
+ Y_ENSURE(taskId == Runner->GetTaskId());
auto channel = Runner->GetInputChannel(channelId);
NDqProto::TData data;
@@ -301,7 +301,7 @@ public:
}
case NDqProto::TCommandHeader::PUSH_SOURCE: {
Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion);
- Y_ENSURE(taskId == Runner->GetTaskId());
+ Y_ENSURE(taskId == Runner->GetTaskId());
auto source = Runner->GetSource(channelId);
NDqProto::TSourcePushRequest request;
@@ -363,28 +363,28 @@ public:
}
case NDqProto::TCommandHeader::FINISH: {
Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion);
- Y_ENSURE(taskId == Runner->GetTaskId());
+ Y_ENSURE(taskId == Runner->GetTaskId());
Runner->GetInputChannel(channelId)->Finish();
UpdateInputChannelStats(channelId);
break;
}
case NDqProto::TCommandHeader::FINISH_OUTPUT: {
Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion);
- Y_ENSURE(taskId == Runner->GetTaskId());
+ Y_ENSURE(taskId == Runner->GetTaskId());
Runner->GetOutputChannel(channelId)->Finish();
UpdateOutputChannelStats(channelId);
break;
}
case NDqProto::TCommandHeader::FINISH_SOURCE: {
Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion);
- Y_ENSURE(taskId == Runner->GetTaskId());
+ Y_ENSURE(taskId == Runner->GetTaskId());
Runner->GetSource(channelId)->Finish();
UpdateSourceStats(channelId);
break;
}
case NDqProto::TCommandHeader::DROP_OUTPUT: {
Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion);
- Y_ENSURE(taskId == Runner->GetTaskId());
+ Y_ENSURE(taskId == Runner->GetTaskId());
auto res = Runner->GetOutputChannel(channelId)->Drop();
NDqProto::TDropOutputResponse response;
@@ -394,14 +394,14 @@ public:
}
case NDqProto::TCommandHeader::TERMINATE_OUTPUT: {
Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion);
- Y_ENSURE(taskId == Runner->GetTaskId());
+ Y_ENSURE(taskId == Runner->GetTaskId());
Runner->GetOutputChannel(channelId)->Terminate();
UpdateOutputChannelStats(channelId);
break;
}
case NDqProto::TCommandHeader::GET_STORED_BYTES: {
Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion);
- Y_ENSURE(taskId == Runner->GetTaskId());
+ Y_ENSURE(taskId == Runner->GetTaskId());
auto res = Runner->GetInputChannel(channelId)->GetStoredBytes();
NDqProto::TGetStoredBytesResponse response;
@@ -411,7 +411,7 @@ public:
}
case NDqProto::TCommandHeader::GET_STORED_BYTES_SOURCE: {
Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion);
- Y_ENSURE(taskId == Runner->GetTaskId());
+ Y_ENSURE(taskId == Runner->GetTaskId());
auto res = Runner->GetSource(channelId)->GetStoredBytes();
NDqProto::TGetStoredBytesResponse response;
@@ -423,13 +423,13 @@ public:
Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion);
auto guard = Runner->BindAllocator(0); // Explicitly reset memory limit
- Y_ENSURE(taskId == Runner->GetTaskId());
+ Y_ENSURE(taskId == Runner->GetTaskId());
auto channel = Runner->GetOutputChannel(channelId);
NDqProto::TPopResponse response;
- response.SetResult(channel->Pop(*response.MutableData(), 5 << 20));
+ response.SetResult(channel->Pop(*response.MutableData(), 5 << 20));
UpdateOutputChannelStats(channelId);
QueryStat.FlushCounters(response);
response.Save(&output);
@@ -438,7 +438,7 @@ public:
}
case NDqProto::TCommandHeader::IS_FINISHED: {
Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion);
- Y_ENSURE(taskId == Runner->GetTaskId());
+ Y_ENSURE(taskId == Runner->GetTaskId());
auto channel = Runner->GetOutputChannel(channelId);
NDqProto::TIsFinishedResponse response;
response.SetResult(channel->IsFinished());
@@ -447,9 +447,9 @@ public:
}
case NDqProto::TCommandHeader::RUN: {
Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion);
- auto guard = Runner->BindAllocator(DqConfiguration->MemoryLimit.Get().GetOrElse(0));
+ auto guard = Runner->BindAllocator(DqConfiguration->MemoryLimit.Get().GetOrElse(0));
- Y_ENSURE(taskId == Runner->GetTaskId());
+ Y_ENSURE(taskId == Runner->GetTaskId());
try {
NDqProto::TRunResponse response;
auto status = Runner->Run();
@@ -484,7 +484,7 @@ public:
Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion);
auto guard = Runner->BindAllocator(0); // Explicitly reset memory limit
- Y_ENSURE(taskId == Runner->GetTaskId());
+ Y_ENSURE(taskId == Runner->GetTaskId());
auto channel = Runner->GetInputChannel(channelId);
auto inputType = channel->GetInputType();
@@ -498,7 +498,7 @@ public:
Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion);
auto guard = Runner->BindAllocator(0); // Explicitly reset memory limit
- Y_ENSURE(taskId == Runner->GetTaskId());
+ Y_ENSURE(taskId == Runner->GetTaskId());
auto source = Runner->GetSource(channelId);
auto inputType = source->GetInputType();
@@ -511,7 +511,7 @@ public:
case NDqProto::TCommandHeader::GET_FREE_SPACE: {
Y_ENSURE(header.GetVersion() >= 2);
- Y_ENSURE(taskId == Runner->GetTaskId());
+ Y_ENSURE(taskId == Runner->GetTaskId());
auto channel = Runner->GetInputChannel(channelId);
NDqProto::TGetFreeSpaceResponse response;
response.SetFreeSpace(channel->GetFreeSpace());
@@ -522,7 +522,7 @@ public:
case NDqProto::TCommandHeader::GET_FREE_SPACE_SOURCE: {
Y_ENSURE(header.GetVersion() >= 4);
- Y_ENSURE(taskId == Runner->GetTaskId());
+ Y_ENSURE(taskId == Runner->GetTaskId());
auto source = Runner->GetSource(channelId);
NDqProto::TGetFreeSpaceResponse response;
response.SetFreeSpace(source->GetFreeSpace());
@@ -532,16 +532,16 @@ public:
}
case NDqProto::TCommandHeader::GET_STATS: {
Y_ENSURE(header.GetVersion() >= 3);
- Y_ENSURE(taskId == Runner->GetTaskId());
+ Y_ENSURE(taskId == Runner->GetTaskId());
auto stats = Runner->GetStats();
NDqProto::TGetStatsResponse response;
auto* s = response.MutableStats();
s->SetTaskId(taskId);
- //s->SetStartTs(stats->StartTs.MilliSeconds());
- //s->SetFinishTs(stats->FinishTs.MilliSeconds());
- s->SetBuildCpuTimeUs(stats->BuildCpuTime.MicroSeconds());
- s->SetComputeCpuTimeUs(stats->ComputeCpuTime.MicroSeconds());
+ //s->SetStartTs(stats->StartTs.MilliSeconds());
+ //s->SetFinishTs(stats->FinishTs.MilliSeconds());
+ s->SetBuildCpuTimeUs(stats->BuildCpuTime.MicroSeconds());
+ s->SetComputeCpuTimeUs(stats->ComputeCpuTime.MicroSeconds());
// All run statuses metrics
s->SetPendingInputTimeUs(stats->RunStatusTimeMetrics[NDq::ERunStatus::PendingInput].MicroSeconds());
@@ -549,17 +549,17 @@ public:
s->SetFinishTimeUs(stats->RunStatusTimeMetrics[NDq::ERunStatus::Finished].MicroSeconds());
static_assert(NDq::TRunStatusTimeMetrics::StatusesCount == 3); // Add all statuses here
- //s->SetTotalTime(stats->TotalTime.MilliSeconds());
- s->SetWaitTimeUs(stats->WaitTime.MicroSeconds());
- s->SetWaitOutputTimeUs(stats->WaitOutputTime.MicroSeconds());
+ //s->SetTotalTime(stats->TotalTime.MilliSeconds());
+ s->SetWaitTimeUs(stats->WaitTime.MicroSeconds());
+ s->SetWaitOutputTimeUs(stats->WaitOutputTime.MicroSeconds());
- //s->SetMkqlTotalNodes(stats->MkqlTotalNodes);
- //s->SetMkqlCodegenFunctions(stats->MkqlCodegenFunctions);
- //s->SetCodeGenTotalInstructions(stats->CodeGenTotalInstructions);
- //s->SetCodeGenTotalFunctions(stats->CodeGenTotalFunctions);
- //s->SetCodeGenFullTime(stats->CodeGenFullTime);
- //s->SetCodeGenFinalizeTime(stats->CodeGenFinalizeTime);
- //s->SetCodeGenModulePassTime(stats->CodeGenModulePassTime);
+ //s->SetMkqlTotalNodes(stats->MkqlTotalNodes);
+ //s->SetMkqlCodegenFunctions(stats->MkqlCodegenFunctions);
+ //s->SetCodeGenTotalInstructions(stats->CodeGenTotalInstructions);
+ //s->SetCodeGenTotalFunctions(stats->CodeGenTotalFunctions);
+ //s->SetCodeGenFullTime(stats->CodeGenFullTime);
+ //s->SetCodeGenFinalizeTime(stats->CodeGenFinalizeTime);
+ //s->SetCodeGenModulePassTime(stats->CodeGenModulePassTime);
for (const auto& [id, ss] : stats->OutputChannels) {
ToProto(s->AddOutputChannels(), ss);
@@ -579,7 +579,7 @@ public:
}
case NDqProto::TCommandHeader::GET_STATS_INPUT: {
Y_ENSURE(header.GetVersion() >= 3);
- Y_ENSURE(taskId == Runner->GetTaskId());
+ Y_ENSURE(taskId == Runner->GetTaskId());
auto ss = Runner->GetInputChannel(channelId)->GetStats();
NDqProto::TGetStatsInputResponse response;
ToProto(response.MutableStats(), ss);
@@ -588,7 +588,7 @@ public:
}
case NDqProto::TCommandHeader::GET_STATS_SOURCE: {
Y_ENSURE(header.GetVersion() >= 4);
- Y_ENSURE(taskId == Runner->GetTaskId());
+ Y_ENSURE(taskId == Runner->GetTaskId());
auto ss = Runner->GetSource(channelId)->GetStats();
NDqProto::TGetStatsSourceResponse response;
ToProto(response.MutableStats(), ss);
@@ -597,7 +597,7 @@ public:
}
case NDqProto::TCommandHeader::GET_STATS_OUTPUT: {
Y_ENSURE(header.GetVersion() >= 3);
- Y_ENSURE(taskId == Runner->GetTaskId());
+ Y_ENSURE(taskId == Runner->GetTaskId());
auto ss = Runner->GetOutputChannel(channelId)->GetStats();
NDqProto::TGetStatsOutputResponse response;
ToProto(response.MutableStats(), ss);
@@ -740,7 +740,7 @@ public:
settings.CollectBasicStats = true;
settings.CollectProfileStats = true;
settings.TerminateOnError = TerminateOnError;
- settings.AllowGeneratorsInUnboxedValues = true;
+ settings.AllowGeneratorsInUnboxedValues = true;
for (const auto& x: taskMeta.GetSecureParams()) {
settings.SecureParams[x.first] = x.second;
YQL_LOG(DEBUG) << "SecureParam " << x.first << ":XXX";
@@ -749,16 +749,16 @@ public:
Ctx.FuncProvider = TaskTransformFactory(taskParams, Ctx.FuncRegistry);
- Runner = MakeDqTaskRunner(Ctx, settings, nullptr);
+ Runner = MakeDqTaskRunner(Ctx, settings, nullptr);
});
auto guard = Runner->BindAllocator(DqConfiguration->MemoryLimit.Get().GetOrElse(0));
-
+
QueryStat.Measure<void>("Prepare", [&]() {
NDq::TDqTaskRunnerMemoryLimits limits;
limits.ChannelBufferSize = DqConfiguration->ChannelBufferSize.Get().GetOrElse(2000_MB);
limits.OutputChunkMaxSize = DqConfiguration->OutputChunkMaxSize.Get().GetOrElse(4_MB);
- Runner->Prepare(task, limits);
+ Runner->Prepare(task, limits);
});
result.Save(&output);
diff --git a/ydb/library/yql/providers/dq/task_runner/tasks_runner_local.cpp b/ydb/library/yql/providers/dq/task_runner/tasks_runner_local.cpp
index 71063d1a66..ab8769f44e 100644
--- a/ydb/library/yql/providers/dq/task_runner/tasks_runner_local.cpp
+++ b/ydb/library/yql/providers/dq/task_runner/tasks_runner_local.cpp
@@ -106,16 +106,16 @@ private:
ui64 TaskId;
ui64 ChannelId;
- IDqOutputChannel::TPtr Channel;
+ IDqOutputChannel::TPtr Channel;
TCounters& QueryStat;
TDqOutputChannelStats Stats;
};
-class TLocalTaskRunner: public ITaskRunner {
+class TLocalTaskRunner: public ITaskRunner {
public:
TLocalTaskRunner(const NDqProto::TDqTask& task, TIntrusivePtr<IDqTaskRunner> runner)
- : Task(task)
- , Runner(runner)
+ : Task(task)
+ , Runner(runner)
{ }
~TLocalTaskRunner()
@@ -125,13 +125,13 @@ public:
return 1;
}
- ui64 GetTaskId() const override {
- return Task.GetId();
+ ui64 GetTaskId() const override {
+ return Task.GetId();
}
NYql::NDqProto::TPrepareResponse Prepare() override {
NYql::NDqProto::TPrepareResponse ret;
- Runner->Prepare(Task, DefaultMemoryLimits());
+ Runner->Prepare(Task, DefaultMemoryLimits());
return ret;
}
@@ -145,16 +145,16 @@ public:
QueryStat.FlushCounters(response);
}
return response;
- }
-
- IInputChannel::TPtr GetInputChannel(ui64 channelId) override {
- return new TLocalInputChannel(Runner->GetInputChannel(channelId), Task.GetId(), channelId, &QueryStat);
- }
-
- IOutputChannel::TPtr GetOutputChannel(ui64 channelId) override {
- return new TLocalOutputChannel(Runner->GetOutputChannel(channelId), Task.GetId(), channelId, &QueryStat);
- }
-
+ }
+
+ IInputChannel::TPtr GetInputChannel(ui64 channelId) override {
+ return new TLocalInputChannel(Runner->GetInputChannel(channelId), Task.GetId(), channelId, &QueryStat);
+ }
+
+ IOutputChannel::TPtr GetOutputChannel(ui64 channelId) override {
+ return new TLocalOutputChannel(Runner->GetOutputChannel(channelId), Task.GetId(), channelId, &QueryStat);
+ }
+
IDqSource::TPtr GetSource(ui64 index) override {
return Runner->GetSource(index);
}
@@ -171,7 +171,7 @@ public:
return Runner->GetSecureParams();
}
- const NMiniKQL::TTypeEnvironment& GetTypeEnv() const override {
+ const NMiniKQL::TTypeEnvironment& GetTypeEnv() const override {
return Runner->GetTypeEnv();
}
@@ -179,14 +179,14 @@ public:
return Runner->GetHolderFactory();
}
- TGuard<NKikimr::NMiniKQL::TScopedAlloc> BindAllocator(TMaybe<ui64> memoryLimit) override {
- return Runner->BindAllocator(memoryLimit);
- }
-
- bool IsAllocatorAttached() override {
- return Runner->IsAllocatorAttached();
- }
-
+ TGuard<NKikimr::NMiniKQL::TScopedAlloc> BindAllocator(TMaybe<ui64> memoryLimit) override {
+ return Runner->BindAllocator(memoryLimit);
+ }
+
+ bool IsAllocatorAttached() override {
+ return Runner->IsAllocatorAttached();
+ }
+
TStatus GetStatus() override {
return {0, ""};
}
@@ -196,8 +196,8 @@ private:
QueryStat.AddTaskRunnerStats(*Runner->GetStats(), Stats, Task.GetId());
}
- NDqProto::TDqTask Task;
- TIntrusivePtr<IDqTaskRunner> Runner;
+ NDqProto::TDqTask Task;
+ TIntrusivePtr<IDqTaskRunner> Runner;
TCounters QueryStat;
TDqTaskRunnerStats Stats;
};
@@ -232,7 +232,7 @@ protected:
const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry;
TTaskTransformFactory TaskTransformFactory;
- NDq::TDqTaskRunnerContext ExecutionContext;
+ NDq::TDqTaskRunnerContext ExecutionContext;
};
/*______________________________________________________________________________________________*/
@@ -257,7 +257,7 @@ public:
settings.TerminateOnError = TerminateOnError;
settings.CollectBasicStats = true;
settings.CollectProfileStats = true;
- settings.AllowGeneratorsInUnboxedValues = true;
+ settings.AllowGeneratorsInUnboxedValues = true;
Yql::DqsProto::TTaskMeta taskMeta;
task.GetMeta().UnpackTo(&taskMeta);
@@ -275,7 +275,7 @@ public:
}
auto ctx = ExecutionContext;
ctx.FuncProvider = TaskTransformFactory(settings.TaskParams, ctx.FuncRegistry);
- return MakeDqTaskRunner(ctx, settings, { });
+ return MakeDqTaskRunner(ctx, settings, { });
}
private:
diff --git a/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.cpp b/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.cpp
index db90740fe7..cdcbd32ef5 100644
--- a/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.cpp
+++ b/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.cpp
@@ -54,8 +54,8 @@ void FromProto(TDqOutputChannelStats* s, const T& f)
s->RowsIn = f.GetRowsIn();
s->RowsOut = f.GetRowsOut();
s->MaxMemoryUsage = f.GetMaxMemoryUsage();
- //s->StartTs = TInstant::MilliSeconds(f.GetStartTs());
- //s->FinishTs = TInstant::MilliSeconds(f.GetFinishTs());
+ //s->StartTs = TInstant::MilliSeconds(f.GetStartTs());
+ //s->FinishTs = TInstant::MilliSeconds(f.GetFinishTs());
}
template<typename T>
@@ -67,9 +67,9 @@ void FromProto(TDqInputChannelStats* s, const T& f)
s->RowsIn = f.GetRowsIn();
s->RowsOut = f.GetRowsOut();
s->MaxMemoryUsage = f.GetMaxMemoryUsage();
- //s->StartTs = TInstant::MilliSeconds(f.GetStartTs());
- //s->FinishTs = TInstant::MilliSeconds(f.GetFinishTs());
- s->DeserializationTime = TDuration::MicroSeconds(f.GetDeserializationTimeUs());
+ //s->StartTs = TInstant::MilliSeconds(f.GetStartTs());
+ //s->FinishTs = TInstant::MilliSeconds(f.GetFinishTs());
+ s->DeserializationTime = TDuration::MicroSeconds(f.GetDeserializationTimeUs());
}
template<typename T>
@@ -81,8 +81,8 @@ void FromProto(TDqSourceStats* s, const T& f)
s->RowsIn = f.GetRowsIn();
s->RowsOut = f.GetRowsOut();
s->MaxMemoryUsage = f.GetMaxMemoryUsage();
- //s->StartTs = TInstant::MilliSeconds(f.GetStartTs());
- //s->FinishTs = TInstant::MilliSeconds(f.GetFinishTs());
+ //s->StartTs = TInstant::MilliSeconds(f.GetStartTs());
+ //s->FinishTs = TInstant::MilliSeconds(f.GetFinishTs());
}
template<typename T>
@@ -98,24 +98,24 @@ void FromProto(TDqSinkStats* s, const T& f)
template<typename T>
void FromProto(TDqTaskRunnerStats* s, const T& f)
{
- //s->StartTs = TInstant::MilliSeconds(f.GetStartTs());
- //s->FinishTs = TInstant::MilliSeconds(f.GetFinishTs());
- s->BuildCpuTime = TDuration::MicroSeconds(f.GetBuildCpuTimeUs());
- s->ComputeCpuTime = TDuration::MicroSeconds(f.GetComputeCpuTimeUs());
+ //s->StartTs = TInstant::MilliSeconds(f.GetStartTs());
+ //s->FinishTs = TInstant::MilliSeconds(f.GetFinishTs());
+ s->BuildCpuTime = TDuration::MicroSeconds(f.GetBuildCpuTimeUs());
+ s->ComputeCpuTime = TDuration::MicroSeconds(f.GetComputeCpuTimeUs());
s->RunStatusTimeMetrics.Load(ERunStatus::PendingInput, TDuration::MicroSeconds(f.GetPendingInputTimeUs()));
s->RunStatusTimeMetrics.Load(ERunStatus::PendingOutput, TDuration::MicroSeconds(f.GetPendingOutputTimeUs()));
s->RunStatusTimeMetrics.Load(ERunStatus::Finished, TDuration::MicroSeconds(f.GetFinishTimeUs()));
- //s->TotalTime = TDuration::MilliSeconds(f.GetTotalTime());
- s->WaitTime = TDuration::MicroSeconds(f.GetWaitTimeUs());
- s->WaitOutputTime = TDuration::MicroSeconds(f.GetWaitOutputTimeUs());
-
- //s->MkqlTotalNodes = f.GetMkqlTotalNodes();
- //s->MkqlCodegenFunctions = f.GetMkqlCodegenFunctions();
- //s->CodeGenTotalInstructions = f.GetCodeGenTotalInstructions();
- //s->CodeGenTotalFunctions = f.GetCodeGenTotalFunctions();
- //s->CodeGenFullTime = f.GetCodeGenFullTime();
- //s->CodeGenFinalizeTime = f.GetCodeGenFinalizeTime();
- //s->CodeGenModulePassTime = f.GetCodeGenModulePassTime();
+ //s->TotalTime = TDuration::MilliSeconds(f.GetTotalTime());
+ s->WaitTime = TDuration::MicroSeconds(f.GetWaitTimeUs());
+ s->WaitOutputTime = TDuration::MicroSeconds(f.GetWaitOutputTimeUs());
+
+ //s->MkqlTotalNodes = f.GetMkqlTotalNodes();
+ //s->MkqlCodegenFunctions = f.GetMkqlCodegenFunctions();
+ //s->CodeGenTotalInstructions = f.GetCodeGenTotalInstructions();
+ //s->CodeGenTotalFunctions = f.GetCodeGenTotalFunctions();
+ //s->CodeGenFullTime = f.GetCodeGenFullTime();
+ //s->CodeGenFinalizeTime = f.GetCodeGenFinalizeTime();
+ //s->CodeGenModulePassTime = f.GetCodeGenModulePassTime();
for (const auto& input : f.GetInputChannels()) {
FromProto(const_cast<TDqInputChannelStats*>(s->InputChannels[input.GetChannelId()]), input);
@@ -956,7 +956,7 @@ public:
ythrow yexception() << "unimplemented";
}
- void Push(NDqProto::TCheckpoint&& checkpoint) override {
+ void Push(NDqProto::TCheckpoint&& checkpoint) override {
Y_UNUSED(checkpoint);
ythrow yexception() << "unimplemented";
}
@@ -999,7 +999,7 @@ public:
}
}
- bool Pop(NDqProto::TCheckpoint&) override {
+ bool Pop(NDqProto::TCheckpoint&) override {
return false;
}
@@ -1013,11 +1013,11 @@ public:
Y_UNUSED(data);
ythrow yexception() << "unimplemented";
}
-
- bool PopAll(NKikimr::NMiniKQL::TUnboxedValueVector& rows) override {
- Y_UNUSED(rows);
- ythrow yexception() << "unimplemented";
- }
+
+ bool PopAll(NKikimr::NMiniKQL::TUnboxedValueVector& rows) override {
+ Y_UNUSED(rows);
+ ythrow yexception() << "unimplemented";
+ }
// |>
ui64 Drop() override {
@@ -1285,13 +1285,13 @@ public:
InitTaskMeta();
}
- ~TTaskRunner() {
+ ~TTaskRunner() {
Alloc.Acquire();
Command->Kill();
Command->Wait(TDuration::Seconds(0));
}
- void ReadStderr() {
+ void ReadStderr() {
auto& input = Command->GetStderr();
char buf[1024];
size_t size;
@@ -1303,10 +1303,10 @@ public:
YQL_LOG(DEBUG) << "stderr (" << StageId << " " << TraceId << " ) finished";
}
- ui64 GetTaskId() const override {
+ ui64 GetTaskId() const override {
return TaskId;
- }
-
+ }
+
i32 GetProtocolVersion() override {
if (ProtocolVersion < 0) {
NDqProto::TCommandHeader header;
@@ -1344,20 +1344,20 @@ public:
header.SetCommand(NDqProto::TCommandHeader::RUN);
header.SetTaskId(Task.GetId());
header.Save(&Output);
-
+
NDqProto::TRunResponse response;
response.Load(&Input);
return response;
}
- IInputChannel::TPtr GetInputChannel(ui64 channelId) override {
+ IInputChannel::TPtr GetInputChannel(ui64 channelId) override {
return new TInputChannel(this, Task.GetId(), channelId, Input, Output);
- }
-
- IOutputChannel::TPtr GetOutputChannel(ui64 channelId) override {
+ }
+
+ IOutputChannel::TPtr GetOutputChannel(ui64 channelId) override {
return new TOutputChannel(Task.GetId(), channelId, Input, Output);
- }
-
+ }
+
IDqSource::TPtr GetSource(ui64 index) override {
return new TDqSource(Task.GetId(), index, this);
}
@@ -1366,7 +1366,7 @@ public:
return new TDqSink(Task.GetId(), index, this);
}
- const NMiniKQL::TTypeEnvironment& GetTypeEnv() const override {
+ const NMiniKQL::TTypeEnvironment& GetTypeEnv() const override {
return TypeEnv;
}
@@ -1382,18 +1382,18 @@ public:
return TaskParams;
}
- TGuard<NKikimr::NMiniKQL::TScopedAlloc> BindAllocator(TMaybe<ui64> memoryLimit) override {
+ TGuard<NKikimr::NMiniKQL::TScopedAlloc> BindAllocator(TMaybe<ui64> memoryLimit) override {
auto guard = TypeEnv.BindAllocator();
- if (memoryLimit) {
- guard.GetMutex()->SetLimit(*memoryLimit);
- }
- return guard;
- }
-
- bool IsAllocatorAttached() override {
+ if (memoryLimit) {
+ guard.GetMutex()->SetLimit(*memoryLimit);
+ }
+ return guard;
+ }
+
+ bool IsAllocatorAttached() override {
return TypeEnv.GetAllocator().IsAttached();
- }
-
+ }
+
void Kill() override {
bool expected = true;
if (!Running.compare_exchange_strong(expected, false)) {
@@ -1488,16 +1488,16 @@ public:
, Task(task)
{ }
- ui64 GetTaskId() const override {
- return Task.GetId();
+ ui64 GetTaskId() const override {
+ return Task.GetId();
}
- void Prepare(const NDqProto::TDqTask& task, const TDqTaskRunnerMemoryLimits& memoryLimits,
- const IDqTaskRunnerExecutionContext& execCtx, const TDqTaskRunnerParameterProvider&) override
- {
- Y_UNUSED(memoryLimits);
- Y_UNUSED(execCtx);
- Y_VERIFY(Task.GetId() == task.GetId());
+ void Prepare(const NDqProto::TDqTask& task, const TDqTaskRunnerMemoryLimits& memoryLimits,
+ const IDqTaskRunnerExecutionContext& execCtx, const TDqTaskRunnerParameterProvider&) override
+ {
+ Y_UNUSED(memoryLimits);
+ Y_UNUSED(execCtx);
+ Y_VERIFY(Task.GetId() == task.GetId());
try {
auto result = Delegate->Prepare();
Y_UNUSED(result);
@@ -1573,8 +1573,8 @@ public:
return sink;
}
- const NKikimr::NMiniKQL::TTypeEnvironment& GetTypeEnv() const override {
- return Delegate->GetTypeEnv();
+ const NKikimr::NMiniKQL::TTypeEnvironment& GetTypeEnv() const override {
+ return Delegate->GetTypeEnv();
}
const NKikimr::NMiniKQL::THolderFactory& GetHolderFactory() const override {
@@ -1589,14 +1589,14 @@ public:
return Delegate->GetTaskParams();
}
- TGuard<NKikimr::NMiniKQL::TScopedAlloc> BindAllocator(TMaybe<ui64> memoryLimit) override {
- return Delegate->BindAllocator(memoryLimit);
+ TGuard<NKikimr::NMiniKQL::TScopedAlloc> BindAllocator(TMaybe<ui64> memoryLimit) override {
+ return Delegate->BindAllocator(memoryLimit);
+ }
+
+ bool IsAllocatorAttached() override {
+ return Delegate->IsAllocatorAttached();
}
- bool IsAllocatorAttached() override {
- return Delegate->IsAllocatorAttached();
- }
-
void UpdateStats() override {
}
@@ -1877,7 +1877,7 @@ private:
YQL_LOG(DEBUG) << "Arg: " << arg;
}
command->Run();
- return command;
+ return command;
}
std::atomic<i64> ContainerId = 1;
diff --git a/ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.h b/ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.h
index ea51a66b2e..31a6e82784 100644
--- a/ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.h
+++ b/ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.h
@@ -54,8 +54,8 @@ public:
virtual ~ITaskRunner() = default;
- virtual ui64 GetTaskId() const = 0;
-
+ virtual ui64 GetTaskId() const = 0;
+
virtual NYql::NDqProto::TPrepareResponse Prepare() = 0;
virtual NYql::NDqProto::TRunResponse Run() = 0;
@@ -66,16 +66,16 @@ public:
virtual const THashMap<TString,TString>& GetTaskParams() const = 0;
virtual const THashMap<TString,TString>& GetSecureParams() const = 0;
- virtual const NKikimr::NMiniKQL::TTypeEnvironment& GetTypeEnv() const = 0;
+ virtual const NKikimr::NMiniKQL::TTypeEnvironment& GetTypeEnv() const = 0;
virtual const NKikimr::NMiniKQL::THolderFactory& GetHolderFactory() const = 0;
- // if memoryLimit = Nothing() then don't set memory limit, use existing one (if any)
- // if memoryLimit = 0 then set unlimited
- // otherwise use particular memory limit
- virtual TGuard<NKikimr::NMiniKQL::TScopedAlloc> BindAllocator(TMaybe<ui64> memoryLimit = Nothing()) = 0;
- virtual bool IsAllocatorAttached() = 0;
-
- struct TStatus {
+ // if memoryLimit = Nothing() then don't set memory limit, use existing one (if any)
+ // if memoryLimit = 0 then set unlimited
+ // otherwise use particular memory limit
+ virtual TGuard<NKikimr::NMiniKQL::TScopedAlloc> BindAllocator(TMaybe<ui64> memoryLimit = Nothing()) = 0;
+ virtual bool IsAllocatorAttached() = 0;
+
+ struct TStatus {
int ExitCode;
TString Stderr;
};
diff --git a/ydb/library/yql/utils/log/log_component.h b/ydb/library/yql/utils/log/log_component.h
index ef5cfb093e..d7a1e10ca1 100644
--- a/ydb/library/yql/utils/log/log_component.h
+++ b/ydb/library/yql/utils/log/log_component.h
@@ -31,7 +31,7 @@ enum class EComponent {
ProviderYdb,
ProviderPq,
ProviderS3,
- CoreDq,
+ CoreDq,
// <--- put other log components here
MaxValue
};
@@ -72,7 +72,7 @@ struct EComponentHelpers {
case EComponent::ProviderYdb: return TStringBuf("YDB");
case EComponent::ProviderPq: return TStringBuf("PQ");
case EComponent::ProviderS3: return TStringBuf("S3");
- case EComponent::CoreDq: return TStringBuf("core dq");
+ case EComponent::CoreDq: return TStringBuf("core dq");
default:
ythrow yexception() << "invalid log component value: "
<< ToInt(component);
@@ -102,7 +102,7 @@ struct EComponentHelpers {
if (str == TStringBuf("YDB")) return EComponent::ProviderYdb;
if (str == TStringBuf("PQ")) return EComponent::ProviderPq;
if (str == TStringBuf("S3")) return EComponent::ProviderS3;
- if (str == TStringBuf("core dq")) return EComponent::CoreDq;
+ if (str == TStringBuf("core dq")) return EComponent::CoreDq;
ythrow yexception() << "unknown log component: '" << str << '\'';
}
@@ -117,5 +117,5 @@ struct EComponentHelpers {
}
};
-} // namespace NLog
-} // namespace NYql
+} // namespace NLog
+} // namespace NYql
diff --git a/ydb/public/api/protos/ydb_experimental.proto b/ydb/public/api/protos/ydb_experimental.proto
index c2990986ed..94eed9671b 100644
--- a/ydb/public/api/protos/ydb_experimental.proto
+++ b/ydb/public/api/protos/ydb_experimental.proto
@@ -13,15 +13,15 @@ import "ydb/public/api/protos/ydb_value.proto";
message ExecuteStreamQueryRequest {
string yql_text = 1;
map<string, TypedValue> parameters = 2;
-
- enum ProfileMode {
- PROFILE_MODE_UNSPECIFIED = 0;
- NONE = 1;
+
+ enum ProfileMode {
+ PROFILE_MODE_UNSPECIFIED = 0;
+ NONE = 1;
BASIC = 2;
FULL = 3;
- }
-
- ProfileMode profile_mode = 3;
+ }
+
+ ProfileMode profile_mode = 3;
bool explain = 4;
}
@@ -31,14 +31,14 @@ message ExecuteStreamQueryResponse {
ExecuteStreamQueryResult result = 3;
}
-message StreamQueryProgress {
-}
-
+message StreamQueryProgress {
+}
+
message ExecuteStreamQueryResult {
- oneof result {
- Ydb.ResultSet result_set = 1;
- string profile = 2;
- StreamQueryProgress progress = 3;
+ oneof result {
+ Ydb.ResultSet result_set = 1;
+ string profile = 2;
+ StreamQueryProgress progress = 3;
string query_plan = 4;
- }
+ }
}
diff --git a/ydb/public/api/protos/ydb_query_stats.proto b/ydb/public/api/protos/ydb_query_stats.proto
index ad6e5ecdde..300d5d9837 100644
--- a/ydb/public/api/protos/ydb_query_stats.proto
+++ b/ydb/public/api/protos/ydb_query_stats.proto
@@ -17,15 +17,15 @@ message TableAccessStats {
OperationStats reads = 3;
OperationStats updates = 4;
OperationStats deletes = 5;
- uint64 partitions_count = 6;
+ uint64 partitions_count = 6;
}
message QueryPhaseStats {
uint64 duration_us = 1;
repeated TableAccessStats table_access = 2;
uint64 cpu_time_us = 3;
- uint64 affected_shards = 4;
- bool literal_phase = 5;
+ uint64 affected_shards = 4;
+ bool literal_phase = 5;
}
message CompilationStats {
diff --git a/ydb/public/lib/experimental/ydb_experimental.cpp b/ydb/public/lib/experimental/ydb_experimental.cpp
index 56aef28f70..8765c0c750 100644
--- a/ydb/public/lib/experimental/ydb_experimental.cpp
+++ b/ydb/public/lib/experimental/ydb_experimental.cpp
@@ -46,8 +46,8 @@ public:
return Finished_;
}
- TAsyncStreamPart ReadNext(std::shared_ptr<TSelf> self) {
- auto promise = NThreading::NewPromise<TStreamPart>();
+ TAsyncStreamPart ReadNext(std::shared_ptr<TSelf> self) {
+ auto promise = NThreading::NewPromise<TStreamPart>();
// Capture self - guarantee no dtor call during the read
auto readCb = [self, promise](TGRpcStatus&& grpcStatus) mutable {
if (!grpcStatus.Ok()) {
@@ -60,20 +60,20 @@ public:
TPlainStatus plainStatus{clientStatus, std::move(issues), self->Endpoint_, {}};
TStatus status{std::move(plainStatus)};
- if (self->Response_.result().has_result_set()) {
- promise.SetValue({TResultSet(std::move(*self->Response_.mutable_result()->mutable_result_set())),
- std::move(status)});
- } else if (!self->Response_.result().profile().empty()) {
- promise.SetValue({std::move(*self->Response_.mutable_result()->mutable_profile()),
- std::move(status)});
+ if (self->Response_.result().has_result_set()) {
+ promise.SetValue({TResultSet(std::move(*self->Response_.mutable_result()->mutable_result_set())),
+ std::move(status)});
+ } else if (!self->Response_.result().profile().empty()) {
+ promise.SetValue({std::move(*self->Response_.mutable_result()->mutable_profile()),
+ std::move(status)});
} else if (!self->Response_.result().query_plan().Empty()) {
TMaybe<TString> queryPlan = self->Response_.result().query_plan();
promise.SetValue({queryPlan, std::move(status)});
- } else if (self->Response_.result().has_progress()) {
- // skip, not supported yet
- } else {
- promise.SetValue(std::move(status));
- }
+ } else if (self->Response_.result().has_progress()) {
+ // skip, not supported yet
+ } else {
+ promise.SetValue(std::move(status));
+ }
}
};
StreamProcessor_->Read(&Response_, readCb);
@@ -93,7 +93,7 @@ TStreamPartIterator::TStreamPartIterator(
, ReaderImpl_(impl)
{}
-TAsyncStreamPart TStreamPartIterator::ReadNext() {
+TAsyncStreamPart TStreamPartIterator::ReadNext() {
if (ReaderImpl_->IsFinished())
RaiseError("Attempt to perform read on invalid or finished stream");
return ReaderImpl_->ReadNext(ReaderImpl_);
@@ -115,18 +115,18 @@ public:
*request.mutable_parameters() = params->GetProtoMap();
}
- switch (settings.ProfileMode_) {
- case EStreamQueryProfileMode::None:
- request.set_profile_mode(Ydb::Experimental::ExecuteStreamQueryRequest_ProfileMode_NONE);
- break;
- case EStreamQueryProfileMode::Basic:
- request.set_profile_mode(Ydb::Experimental::ExecuteStreamQueryRequest_ProfileMode_BASIC);
- break;
+ switch (settings.ProfileMode_) {
+ case EStreamQueryProfileMode::None:
+ request.set_profile_mode(Ydb::Experimental::ExecuteStreamQueryRequest_ProfileMode_NONE);
+ break;
+ case EStreamQueryProfileMode::Basic:
+ request.set_profile_mode(Ydb::Experimental::ExecuteStreamQueryRequest_ProfileMode_BASIC);
+ break;
case EStreamQueryProfileMode::Full:
request.set_profile_mode(Ydb::Experimental::ExecuteStreamQueryRequest_ProfileMode_FULL);
break;
- }
-
+ }
+
request.set_explain(settings.Explain_);
auto promise = NewPromise<std::pair<TPlainStatus, TStreamProcessorPtr>>();
@@ -158,12 +158,12 @@ public:
{
Y_ASSERT(future.HasValue());
auto pair = future.ExtractValue();
- promise.SetValue(TStreamPartIterator(
- pair.second
+ promise.SetValue(TStreamPartIterator(
+ pair.second
? std::make_shared<TStreamPartIterator::TReaderImpl>(pair.second, pair.first.Endpoint)
- : nullptr,
+ : nullptr,
std::move(pair.first))
- );
+ );
};
ExecuteStreamQueryInternal(query, params, settings).Subscribe(iteratorCallback);
diff --git a/ydb/public/lib/experimental/ydb_experimental.h b/ydb/public/lib/experimental/ydb_experimental.h
index 22403cdbd0..3cdd8f6dc8 100644
--- a/ydb/public/lib/experimental/ydb_experimental.h
+++ b/ydb/public/lib/experimental/ydb_experimental.h
@@ -19,49 +19,49 @@ public:
class TStreamPart : public TStreamPartStatus {
public:
- bool HasResultSet() const { return ResultSet_.Defined(); }
- const TResultSet& GetResultSet() const { return *ResultSet_; }
- TResultSet ExtractResultSet() { return std::move(*ResultSet_); }
+ bool HasResultSet() const { return ResultSet_.Defined(); }
+ const TResultSet& GetResultSet() const { return *ResultSet_; }
+ TResultSet ExtractResultSet() { return std::move(*ResultSet_); }
- bool HasProfile() const { return Profile_.Defined(); }
- const TString& GetProfile() const { return *Profile_; }
- TString ExtractProfile() { return std::move(*Profile_); }
+ bool HasProfile() const { return Profile_.Defined(); }
+ const TString& GetProfile() const { return *Profile_; }
+ TString ExtractProfile() { return std::move(*Profile_); }
bool HasPlan() const { return Plan_.Defined(); }
const TString& GetPlan() const { return *Plan_; }
TString ExtractPlan() { return std::move(*Plan_); }
- TStreamPart(TStatus&& status)
+ TStreamPart(TStatus&& status)
: TStreamPartStatus(std::move(status))
{}
- TStreamPart(TResultSet&& resultSet, TStatus&& status)
- : TStreamPartStatus(std::move(status))
- , ResultSet_(std::move(resultSet))
- {}
-
- TStreamPart(TString&& profile, TStatus&& status)
- : TStreamPartStatus(std::move(status))
- , Profile_(std::move(profile))
- {}
-
+ TStreamPart(TResultSet&& resultSet, TStatus&& status)
+ : TStreamPartStatus(std::move(status))
+ , ResultSet_(std::move(resultSet))
+ {}
+
+ TStreamPart(TString&& profile, TStatus&& status)
+ : TStreamPartStatus(std::move(status))
+ , Profile_(std::move(profile))
+ {}
+
TStreamPart(const TMaybe<TString>& plan, TStatus&& status)
: TStreamPartStatus(std::move(status))
, Plan_(plan)
{}
private:
- TMaybe<TResultSet> ResultSet_;
- TMaybe<TString> Profile_;
+ TMaybe<TResultSet> ResultSet_;
+ TMaybe<TString> Profile_;
TMaybe<TString> Plan_;
};
-using TAsyncStreamPart = NThreading::TFuture<TStreamPart>;
+using TAsyncStreamPart = NThreading::TFuture<TStreamPart>;
class TStreamPartIterator : public TStatus {
friend class TStreamQueryClient;
public:
- TAsyncStreamPart ReadNext();
+ TAsyncStreamPart ReadNext();
class TReaderImpl;
private:
TStreamPartIterator(
@@ -71,16 +71,16 @@ private:
std::shared_ptr<TReaderImpl> ReaderImpl_;
};
-enum class EStreamQueryProfileMode {
- None,
+enum class EStreamQueryProfileMode {
+ None,
Basic,
Full
-};
-
+};
+
struct TExecuteStreamQuerySettings : public TRequestSettings<TExecuteStreamQuerySettings> {
using TSelf = TExecuteStreamQuerySettings;
-
- FLUENT_SETTING_DEFAULT(EStreamQueryProfileMode, ProfileMode, EStreamQueryProfileMode::None);
+
+ FLUENT_SETTING_DEFAULT(EStreamQueryProfileMode, ProfileMode, EStreamQueryProfileMode::None);
FLUENT_SETTING_DEFAULT(bool, Explain, false);
};
diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp
index 1835f9e0d4..2f94b2fc03 100644
--- a/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp
+++ b/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp
@@ -519,7 +519,7 @@ void TCommandExecuteQuery::PrintScanQueryResponse(NTable::TScanQueryPartIterator
if (streamPart.HasResultSet()) {
printer.Print(streamPart.GetResultSet());
}
-
+
if (streamPart.HasQueryStats()) {
const auto& queryStats = streamPart.GetQueryStats();
statsStr << Endl << queryStats.ToString(false) << Endl;
@@ -529,7 +529,7 @@ void TCommandExecuteQuery::PrintScanQueryResponse(NTable::TScanQueryPartIterator
statsStr << "Full statistics:" << Endl << *plan << Endl;
}
}
- }
+ }
} // TResultSetPrinter destructor should be called before printing stats
if (statsStr.Size()) {
diff --git a/ydb/public/lib/yson_value/ydb_yson_value.cpp b/ydb/public/lib/yson_value/ydb_yson_value.cpp
index ce87776bac..c855acf0a7 100644
--- a/ydb/public/lib/yson_value/ydb_yson_value.cpp
+++ b/ydb/public/lib/yson_value/ydb_yson_value.cpp
@@ -94,10 +94,10 @@ static void FormatValueYsonInternal(TValueParser& parser, NYson::TYsonWriter& wr
PrimitiveValueToYson(parser.GetPrimitiveType(), parser, writer);
break;
- case TTypeParser::ETypeKind::Decimal:
- writer.OnStringScalar(parser.GetDecimal().ToString());
- break;
-
+ case TTypeParser::ETypeKind::Decimal:
+ writer.OnStringScalar(parser.GetDecimal().ToString());
+ break;
+
case TTypeParser::ETypeKind::Optional:
parser.OpenOptional();
if (parser.IsNull()) {
@@ -175,10 +175,10 @@ static void FormatValueYsonInternal(TValueParser& parser, NYson::TYsonWriter& wr
writer.OnStringScalar("Void");
break;
- case TTypeParser::ETypeKind::Null:
- writer.OnEntity();
- break;
-
+ case TTypeParser::ETypeKind::Null:
+ writer.OnEntity();
+ break;
+
default:
ThrowFatalError(TStringBuilder() << "Unsupported type kind: " << parser.GetKind());
}
diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.cpp b/ydb/public/sdk/cpp/client/ydb_table/table.cpp
index 8c73cc9f5c..610fec3884 100644
--- a/ydb/public/sdk/cpp/client/ydb_table/table.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_table/table.cpp
@@ -3581,14 +3581,14 @@ static void ConvertCreateTableSettingsToProto(const TCreateTableSettings& settin
if (policy.UniformPartitions_) {
proto->mutable_partitioning_policy()->set_uniform_partitions(policy.UniformPartitions_.GetRef());
}
- if (policy.ExplicitPartitions_) {
- auto* borders = proto->mutable_partitioning_policy()->mutable_explicit_partitions();
- for (const auto& splitPoint : policy.ExplicitPartitions_->SplitPoints_) {
- auto* border = borders->Addsplit_points();
- border->mutable_type()->CopyFrom(TProtoAccessor::GetProto(splitPoint.GetType()));
- border->mutable_value()->CopyFrom(TProtoAccessor::GetProto(splitPoint));
- }
- }
+ if (policy.ExplicitPartitions_) {
+ auto* borders = proto->mutable_partitioning_policy()->mutable_explicit_partitions();
+ for (const auto& splitPoint : policy.ExplicitPartitions_->SplitPoints_) {
+ auto* border = borders->Addsplit_points();
+ border->mutable_type()->CopyFrom(TProtoAccessor::GetProto(splitPoint.GetType()));
+ border->mutable_value()->CopyFrom(TProtoAccessor::GetProto(splitPoint));
+ }
+ }
}
if (settings.StoragePolicy_) {
const auto& policy = settings.StoragePolicy_.GetRef();
diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.h b/ydb/public/sdk/cpp/client/ydb_table/table.h
index 08bbd1e9f5..42d21d2fec 100644
--- a/ydb/public/sdk/cpp/client/ydb_table/table.h
+++ b/ydb/public/sdk/cpp/client/ydb_table/table.h
@@ -1132,12 +1132,12 @@ struct TStoragePolicy {
FLUENT_SETTING_VECTOR(TColumnFamilyPolicy, ColumnFamilies);
};
-struct TExplicitPartitions {
- using TSelf = TExplicitPartitions;
-
- FLUENT_SETTING_VECTOR(TValue, SplitPoints);
-};
-
+struct TExplicitPartitions {
+ using TSelf = TExplicitPartitions;
+
+ FLUENT_SETTING_VECTOR(TValue, SplitPoints);
+};
+
struct TPartitioningPolicy {
using TSelf = TPartitioningPolicy;
@@ -1146,8 +1146,8 @@ struct TPartitioningPolicy {
FLUENT_SETTING_OPTIONAL(EAutoPartitioningPolicy, AutoPartitioning);
FLUENT_SETTING_OPTIONAL(ui64, UniformPartitions);
-
- FLUENT_SETTING_OPTIONAL(TExplicitPartitions, ExplicitPartitions);
+
+ FLUENT_SETTING_OPTIONAL(TExplicitPartitions, ExplicitPartitions);
};
struct TReplicationPolicy {
diff --git a/ydb/services/ydb/ydb_table_ut.cpp b/ydb/services/ydb/ydb_table_ut.cpp
index c2d5d071b0..4861c06b92 100644
--- a/ydb/services/ydb/ydb_table_ut.cpp
+++ b/ydb/services/ydb/ydb_table_ut.cpp
@@ -3893,38 +3893,38 @@ R"___(<main>: Error: Transaction not found: , code: 2015
UNIT_ASSERT_VALUES_EQUAL(res.GetTableDescription().GetPartitionsCount(), 2);
UNIT_ASSERT_VALUES_EQUAL(res.GetTableDescription().GetPartitionStats().size(), 2);
}
- }
-
- Y_UNIT_TEST(TestExplicitPartitioning) {
- TKikimrWithGrpcAndRootSchema server;
-
- auto connection = TDriver(
- TDriverConfig()
- .SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort()));
-
- TTableClient client(connection);
-
- auto sessionResult = client.CreateSession().ExtractValueSync();
- UNIT_ASSERT_C(sessionResult.IsSuccess(), sessionResult.GetIssues().ToString());
- auto session = sessionResult.GetSession();
-
- {
- auto tableBuilder = client.GetTableBuilder()
+ }
+
+ Y_UNIT_TEST(TestExplicitPartitioning) {
+ TKikimrWithGrpcAndRootSchema server;
+
+ auto connection = TDriver(
+ TDriverConfig()
+ .SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort()));
+
+ TTableClient client(connection);
+
+ auto sessionResult = client.CreateSession().ExtractValueSync();
+ UNIT_ASSERT_C(sessionResult.IsSuccess(), sessionResult.GetIssues().ToString());
+ auto session = sessionResult.GetSession();
+
+ {
+ auto tableBuilder = client.GetTableBuilder()
.AddNullableColumn("Value", EPrimitiveType::Utf8)
.AddNullableColumn("SubKey", EPrimitiveType::Utf8)
- .AddNullableColumn("Key", EPrimitiveType::Uint32)
- .SetPrimaryKeyColumn("Key");
-
- TExplicitPartitions partitions;
- partitions.AppendSplitPoints(TValueBuilder().BeginTuple().AddElement().OptionalUint32(10).EndTuple().Build());
- partitions.AppendSplitPoints(TValueBuilder().BeginTuple().AddElement().OptionalUint32(20).EndTuple().Build());
-
- auto tableSettings = TCreateTableSettings().PartitioningPolicy(
+ .AddNullableColumn("Key", EPrimitiveType::Uint32)
+ .SetPrimaryKeyColumn("Key");
+
+ TExplicitPartitions partitions;
+ partitions.AppendSplitPoints(TValueBuilder().BeginTuple().AddElement().OptionalUint32(10).EndTuple().Build());
+ partitions.AppendSplitPoints(TValueBuilder().BeginTuple().AddElement().OptionalUint32(20).EndTuple().Build());
+
+ auto tableSettings = TCreateTableSettings().PartitioningPolicy(
TPartitioningPolicy().ExplicitPartitions(partitions).AutoPartitioning(
EAutoPartitioningPolicy::AutoSplitMerge));
-
- auto result = session.CreateTable("/Root/Foo", tableBuilder.Build(), tableSettings).ExtractValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ auto result = session.CreateTable("/Root/Foo", tableBuilder.Build(), tableSettings).ExtractValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
for(ui32 idx = 0; idx < 25; ++idx) {
// trying to wait for split merge to complete (if enabled...???)
@@ -3932,32 +3932,32 @@ R"___(<main>: Error: Transaction not found: , code: 2015
Sleep(TDuration::Seconds(1));
}
- }
-
- {
- auto result = session.ExecuteDataQuery(R"___(
- UPSERT INTO [Root/Foo] (Key, Value) VALUES
- (1, "one"),
- (2, "two"),
- (12, "twelve"),
- (15, "fifteen"),
- (17, "seventeen"),
- (100500, "too much")
- )___", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
-
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
- }
-
- {
- auto res = session.ExecuteDataQuery("select count(*) from [Root/Foo]", TTxControl::BeginTx().CommitTx())
- .GetValueSync();
- UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString());
- TResultSetParser parser{res.GetResultSet(0)};
- while (parser.TryNextRow()) {
- UNIT_ASSERT_EQUAL(6, TValueParser{parser.GetValue(0)}.GetUint64());
- }
- }
-
+ }
+
+ {
+ auto result = session.ExecuteDataQuery(R"___(
+ UPSERT INTO [Root/Foo] (Key, Value) VALUES
+ (1, "one"),
+ (2, "two"),
+ (12, "twelve"),
+ (15, "fifteen"),
+ (17, "seventeen"),
+ (100500, "too much")
+ )___", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ }
+
+ {
+ auto res = session.ExecuteDataQuery("select count(*) from [Root/Foo]", TTxControl::BeginTx().CommitTx())
+ .GetValueSync();
+ UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString());
+ TResultSetParser parser{res.GetResultSet(0)};
+ while (parser.TryNextRow()) {
+ UNIT_ASSERT_EQUAL(6, TValueParser{parser.GetValue(0)}.GetUint64());
+ }
+ }
+
EnsureTablePartitions(client, "/Root/Foo", 3);
}
diff --git a/ydb/services/ydb/ydb_ut.cpp b/ydb/services/ydb/ydb_ut.cpp
index 79ecbb5ce7..2d37063079 100644
--- a/ydb/services/ydb/ydb_ut.cpp
+++ b/ydb/services/ydb/ydb_ut.cpp
@@ -2169,10 +2169,10 @@ tx_meta {
{
const TString type =
"type_id: UUID\n";
- UNIT_ASSERT(google::protobuf::TextFormat::ParseFromString(type, parameter.mutable_type()));
- const TString value = R"(low_128: 1290426546294828066
- high_128: 13600338575655354541)";
- UNIT_ASSERT(google::protobuf::TextFormat::ParseFromString(value, parameter.mutable_value()));
+ UNIT_ASSERT(google::protobuf::TextFormat::ParseFromString(type, parameter.mutable_type()));
+ const TString value = R"(low_128: 1290426546294828066
+ high_128: 13600338575655354541)";
+ UNIT_ASSERT(google::protobuf::TextFormat::ParseFromString(value, parameter.mutable_value()));
}
auto& map = *request.mutable_parameters();
diff --git a/ydb/tests/functional/api/test_isolation.py b/ydb/tests/functional/api/test_isolation.py
index 0765145182..1d85e4ce3e 100644
--- a/ydb/tests/functional/api/test_isolation.py
+++ b/ydb/tests/functional/api/test_isolation.py
@@ -2,7 +2,7 @@
from hamcrest import assert_that, equal_to, raises, contains_string
from ydb.tests.library.harness.kikimr_cluster import kikimr_cluster_factory
-import pytest
+import pytest
import ydb
@@ -41,30 +41,30 @@ class TestTransactionIsolation(object):
'upsert into %s (id, value) values (1, 10), (2, 20); ' % table, commit_tx=True)
return table, session
- @staticmethod
- def _prepare_engine(new_engine):
- if new_engine:
- return 'pragma kikimr.UseNewEngine = "true";'
- else:
- return 'pragma kikimr.UseNewEngine = "false";'
-
- @pytest.mark.parametrize('new_engine', [True, False])
- def test_prevents_write_cycles_g0(self, new_engine):
+ @staticmethod
+ def _prepare_engine(new_engine):
+ if new_engine:
+ return 'pragma kikimr.UseNewEngine = "true";'
+ else:
+ return 'pragma kikimr.UseNewEngine = "false";'
+
+ @pytest.mark.parametrize('new_engine', [True, False])
+ def test_prevents_write_cycles_g0(self, new_engine):
"""Write Cycles (G0), locking updated rows"""
table_name, session = self._prepare("test_prevents_write_cycles_g0")
t1 = session.transaction()
t2 = session.transaction()
- prefix = self._prepare_engine(new_engine)
-
- t1.execute('{} upsert into {} (id, value) values (1, 11)'.format(prefix, table_name))
- t2.execute('{} select * from {} where id=1 or id=2;'.format(prefix, table_name))
- t2.execute('{} upsert into {} (id, value) values (1, 12)'.format(prefix, table_name))
- t1.execute('{} upsert into {} (id, value) values (2, 21)'.format(prefix, table_name))
+ prefix = self._prepare_engine(new_engine)
+
+ t1.execute('{} upsert into {} (id, value) values (1, 11)'.format(prefix, table_name))
+ t2.execute('{} select * from {} where id=1 or id=2;'.format(prefix, table_name))
+ t2.execute('{} upsert into {} (id, value) values (1, 12)'.format(prefix, table_name))
+ t1.execute('{} upsert into {} (id, value) values (2, 21)'.format(prefix, table_name))
t1.commit()
- result_rows = t1.execute("{} select id, value from {} order by id;".format(prefix, table_name), commit_tx=True)
+ result_rows = t1.execute("{} select id, value from {} order by id;".format(prefix, table_name), commit_tx=True)
assert_that(
result_rows[0].rows,
equal_to(
@@ -76,7 +76,7 @@ class TestTransactionIsolation(object):
)
def callee():
- t2.execute("{} upsert into {} (id, value) values (2, 22);".format(prefix, table_name))
+ t2.execute("{} upsert into {} (id, value) values (2, 22);".format(prefix, table_name))
t2.commit()
assert_that(
@@ -88,7 +88,7 @@ class TestTransactionIsolation(object):
)
for e_thread in (t1, ):
- result_rows = e_thread.execute("{} select id, value from {} order by id;".format(prefix, table_name))
+ result_rows = e_thread.execute("{} select id, value from {} order by id;".format(prefix, table_name))
assert_that(
result_rows[0].rows,
equal_to(
@@ -99,17 +99,17 @@ class TestTransactionIsolation(object):
)
)
- @pytest.mark.parametrize('new_engine', [True, False])
- def test_prevents_aborted_reads_g1a(self, new_engine):
+ @pytest.mark.parametrize('new_engine', [True, False])
+ def test_prevents_aborted_reads_g1a(self, new_engine):
table_name, session = self._prepare("test_prevents_aborted_reads_g1a")
t1 = session.transaction()
t2 = session.transaction()
- prefix = self._prepare_engine(new_engine)
-
- t1.execute('{} update {} set value = 101 where id = 1;'.format(prefix, table_name))
- result_rows = t2.execute('{} select id, value from {} order by id;'.format(prefix, table_name))
+ prefix = self._prepare_engine(new_engine)
+
+ t1.execute('{} update {} set value = 101 where id = 1;'.format(prefix, table_name))
+ result_rows = t2.execute('{} select id, value from {} order by id;'.format(prefix, table_name))
assert_that(
result_rows[0].rows,
equal_to(
@@ -122,7 +122,7 @@ class TestTransactionIsolation(object):
# abort; -- T1 ----> replaced to rollback
t1.rollback()
- result_rows = t2.execute('{} select id, value from {} order by id;'.format(prefix, table_name))
+ result_rows = t2.execute('{} select id, value from {} order by id;'.format(prefix, table_name))
assert_that(
result_rows[0].rows,
equal_to(
@@ -134,18 +134,18 @@ class TestTransactionIsolation(object):
)
t2.commit()
- @pytest.mark.parametrize('new_engine', [True, False])
- def test_prevents_intermediate_reads_g1b(self, new_engine):
+ @pytest.mark.parametrize('new_engine', [True, False])
+ def test_prevents_intermediate_reads_g1b(self, new_engine):
table_name, session = self._prepare("test_prevents_intermediate_reads_g1b")
t1 = session.transaction()
t2 = session.transaction()
- prefix = self._prepare_engine(new_engine)
-
- t1.execute('{} select * from {} where id=1'.format(prefix, table_name))
- t1.execute('{} upsert into {} (id, value) values (1, 101);'.format(prefix, table_name))
- result_rows = t2.execute('{} select id, value from {} order by id;'.format(prefix, table_name), commit_tx=True)
+ prefix = self._prepare_engine(new_engine)
+
+ t1.execute('{} select * from {} where id=1'.format(prefix, table_name))
+ t1.execute('{} upsert into {} (id, value) values (1, 101);'.format(prefix, table_name))
+ result_rows = t2.execute('{} select id, value from {} order by id;'.format(prefix, table_name), commit_tx=True)
assert_that(
result_rows[0].rows,
equal_to(
@@ -156,10 +156,10 @@ class TestTransactionIsolation(object):
)
)
- t1.execute('{} upsert into {} (id, value) values (1, 11);'.format(prefix, table_name))
+ t1.execute('{} upsert into {} (id, value) values (1, 11);'.format(prefix, table_name))
t1.commit()
- result_rows = t2.execute('{} select id, value from {} order by id;'.format(prefix, table_name), commit_tx=True)
+ result_rows = t2.execute('{} select id, value from {} order by id;'.format(prefix, table_name), commit_tx=True)
assert_that(
result_rows[0].rows,
equal_to(
@@ -171,24 +171,24 @@ class TestTransactionIsolation(object):
)
t2.commit()
- @pytest.mark.parametrize('new_engine', [True, False])
- def test_isolation_mailing_list_example(self, new_engine):
+ @pytest.mark.parametrize('new_engine', [True, False])
+ def test_isolation_mailing_list_example(self, new_engine):
table_name, session = self._prepare("test_isolation_mailing_list_example")
t1 = session.transaction()
t2 = session.transaction()
- prefix = self._prepare_engine(new_engine)
+ prefix = self._prepare_engine(new_engine)
- t1.execute(session.prepare('{} upsert into {} (id, value) values (1, 3);'.format(prefix, table_name)), commit_tx=True)
-
- t1.execute(session.prepare('{} select id, value FROM {} WHERE id = 1'.format(prefix, table_name)))
- t2.execute(session.prepare('{} select id, value FROM {} WHERE id = 1'.format(prefix, table_name)))
- t1.execute(session.prepare('{} upsert into {} (id, value) values (1, 4);'.format(prefix, table_name)))
+ t1.execute(session.prepare('{} upsert into {} (id, value) values (1, 3);'.format(prefix, table_name)), commit_tx=True)
+
+ t1.execute(session.prepare('{} select id, value FROM {} WHERE id = 1'.format(prefix, table_name)))
+ t2.execute(session.prepare('{} select id, value FROM {} WHERE id = 1'.format(prefix, table_name)))
+ t1.execute(session.prepare('{} upsert into {} (id, value) values (1, 4);'.format(prefix, table_name)))
t1.commit()
def callee():
- t2.execute(session.prepare('{} upsert into {} (id, value) values (1, 5);'.format(prefix, table_name)))
+ t2.execute(session.prepare('{} upsert into {} (id, value) values (1, 5);'.format(prefix, table_name)))
t2.commit()
assert_that(
@@ -200,7 +200,7 @@ class TestTransactionIsolation(object):
)
for thr in (t1, ):
- result_rows = thr.execute('{} select id, value from {} where id = 1;'.format(prefix, table_name), commit_tx=True)
+ result_rows = thr.execute('{} select id, value from {} where id = 1;'.format(prefix, table_name), commit_tx=True)
assert_that(
result_rows[0].rows[0].value,
equal_to(
@@ -208,23 +208,23 @@ class TestTransactionIsolation(object):
)
)
- @pytest.mark.parametrize('new_engine', [True, False])
- def test_prevents_observed_transaction_vanishes_otv(self, new_engine):
+ @pytest.mark.parametrize('new_engine', [True, False])
+ def test_prevents_observed_transaction_vanishes_otv(self, new_engine):
table_name, session = self._prepare("test_prevents_observed_transaction_vanishes_otv")
t1 = session.transaction()
t2 = session.transaction()
t3 = session.transaction()
- prefix = self._prepare_engine(new_engine)
-
- t1.execute('{} select * from {} where id=1 or id=2;'.format(prefix, table_name))
- t1.execute('{} upsert into {} (id, value) values (1, 11);'.format(prefix, table_name))
- t1.execute('{} upsert into {} (id, value) values (2, 19);'.format(prefix, table_name))
- t2.execute('{} select * from {} where id=1'.format(prefix, table_name))
- t2.execute('{} upsert into {} (id, value) values (1, 12);'.format(prefix, table_name))
+ prefix = self._prepare_engine(new_engine)
+
+ t1.execute('{} select * from {} where id=1 or id=2;'.format(prefix, table_name))
+ t1.execute('{} upsert into {} (id, value) values (1, 11);'.format(prefix, table_name))
+ t1.execute('{} upsert into {} (id, value) values (2, 19);'.format(prefix, table_name))
+ t2.execute('{} select * from {} where id=1'.format(prefix, table_name))
+ t2.execute('{} upsert into {} (id, value) values (1, 12);'.format(prefix, table_name))
t1.commit()
- result_rows = t3.execute('{} select value from {} where id = 1;'.format(prefix, table_name))
+ result_rows = t3.execute('{} select value from {} where id = 1;'.format(prefix, table_name))
assert_that(
result_rows[0].rows,
equal_to(
@@ -235,7 +235,7 @@ class TestTransactionIsolation(object):
)
def callee():
- t2.execute('{} upsert into {} (id, value) values (2, 18);'.format(prefix, table_name))
+ t2.execute('{} upsert into {} (id, value) values (2, 18);'.format(prefix, table_name))
t2.commit()
assert_that(
@@ -246,10 +246,10 @@ class TestTransactionIsolation(object):
)
)
- t3.execute('{} select value from {} where id = 2;'.format(prefix, table_name))
+ t3.execute('{} select value from {} where id = 2;'.format(prefix, table_name))
# commit; -- T2 we already got transaction locks invalidated error
- result_rows = t3.execute('{} select value from {} where id = 2;'.format(prefix, table_name))
+ result_rows = t3.execute('{} select value from {} where id = 2;'.format(prefix, table_name))
assert_that(
result_rows[0].rows,
equal_to(
@@ -258,7 +258,7 @@ class TestTransactionIsolation(object):
]
)
)
- result_rows = t3.execute('{} select value from {} where id = 1;'.format(prefix, table_name))
+ result_rows = t3.execute('{} select value from {} where id = 1;'.format(prefix, table_name))
assert_that(
result_rows[0].rows,
equal_to(
@@ -269,19 +269,19 @@ class TestTransactionIsolation(object):
)
t3.commit()
- @pytest.mark.parametrize('new_engine', [True, False])
- def test_does_not_prevent_predicate_many_preceders_pmp(self, new_engine):
+ @pytest.mark.parametrize('new_engine', [True, False])
+ def test_does_not_prevent_predicate_many_preceders_pmp(self, new_engine):
table_name, session = self._prepare("test_does_not_prevent_predicate_many_preceders_pmp")
t1 = session.transaction()
t2 = session.transaction()
- prefix = self._prepare_engine(new_engine)
-
- result_rows = t1.execute('{} select id from {} where value = 30;'.format(prefix, table_name))
+ prefix = self._prepare_engine(new_engine)
+
+ result_rows = t1.execute('{} select id from {} where value = 30;'.format(prefix, table_name))
assert_that(result_rows[0].rows, equal_to([]))
- t2.execute('{} upsert into {} (id, value) values(3, 30);'.format(prefix, table_name))
+ t2.execute('{} upsert into {} (id, value) values(3, 30);'.format(prefix, table_name))
t2.commit()
try:
@@ -292,31 +292,31 @@ class TestTransactionIsolation(object):
else:
assert_that(result_rows[0].rows, equal_to([]))
- @pytest.mark.parametrize('new_engine', [True, False])
- def test_lost_update_p4(self, new_engine):
+ @pytest.mark.parametrize('new_engine', [True, False])
+ def test_lost_update_p4(self, new_engine):
table_name, session = self._prepare("test_lost_update_p4")
t1 = session.transaction()
t2 = session.transaction()
- prefix = self._prepare_engine(new_engine)
-
- result_rows = t1.execute('{} select id from {} where id = 1;'.format(prefix, table_name))
+ prefix = self._prepare_engine(new_engine)
+
+ result_rows = t1.execute('{} select id from {} where id = 1;'.format(prefix, table_name))
assert_that(
result_rows[0].rows, equal_to(
[{'id': 1}]
)
)
- result_rows = t2.execute('{} select id from {} where id = 1;'.format(prefix, table_name))
+ result_rows = t2.execute('{} select id from {} where id = 1;'.format(prefix, table_name))
assert_that(
result_rows[0].rows, equal_to(
[{'id': 1}]
)
)
- t1.execute('{} update {} set value = 11 where id = 1;'.format(prefix, table_name))
- t2.execute('{} update {} set value = 11 where id = 1;'.format(prefix, table_name))
+ t1.execute('{} update {} set value = 11 where id = 1;'.format(prefix, table_name))
+ t2.execute('{} update {} set value = 11 where id = 1;'.format(prefix, table_name))
t1.commit()
@@ -331,16 +331,16 @@ class TestTransactionIsolation(object):
)
)
- @pytest.mark.parametrize('new_engine', [True, False])
- def test_lost_update_on_value_p4(self, new_engine):
+ @pytest.mark.parametrize('new_engine', [True, False])
+ def test_lost_update_on_value_p4(self, new_engine):
table_name, session = self._prepare("test_lost_update_on_value_p4")
t1 = session.transaction()
t2 = session.transaction()
- prefix = self._prepare_engine(new_engine)
-
- result_rows = t1.execute('{} select id, value from {} where id = 1;'.format(prefix, table_name))
+ prefix = self._prepare_engine(new_engine)
+
+ result_rows = t1.execute('{} select id, value from {} where id = 1;'.format(prefix, table_name))
assert_that(
result_rows[0].rows,
equal_to(
@@ -350,7 +350,7 @@ class TestTransactionIsolation(object):
)
)
- result_rows = t2.execute('{} select id, value from {} where id = 1;'.format(prefix, table_name))
+ result_rows = t2.execute('{} select id, value from {} where id = 1;'.format(prefix, table_name))
assert_that(
result_rows[0].rows,
equal_to(
@@ -360,11 +360,11 @@ class TestTransactionIsolation(object):
)
)
- t1.execute('{} update {} set value = 11 where id = 1;'.format(prefix, table_name))
+ t1.execute('{} update {} set value = 11 where id = 1;'.format(prefix, table_name))
t1.commit()
def callee():
- t2.execute('{} update {} set value = 12 where id = 1;'.format(prefix, table_name))
+ t2.execute('{} update {} set value = 12 where id = 1;'.format(prefix, table_name))
t2.commit()
assert_that(
@@ -375,16 +375,16 @@ class TestTransactionIsolation(object):
)
)
- @pytest.mark.parametrize('new_engine', [True, False])
- def test_lost_update_on_value_with_upsert_p4(self, new_engine):
+ @pytest.mark.parametrize('new_engine', [True, False])
+ def test_lost_update_on_value_with_upsert_p4(self, new_engine):
table_name, session = self._prepare("test_lost_update_on_value_with_upsert_p4")
t1 = session.transaction()
t2 = session.transaction()
- prefix = self._prepare_engine(new_engine)
-
- result_rows = t1.execute('{} select id, value from {} where id = 1;'.format(prefix, table_name))
+ prefix = self._prepare_engine(new_engine)
+
+ result_rows = t1.execute('{} select id, value from {} where id = 1;'.format(prefix, table_name))
assert_that(
result_rows[0].rows, equal_to(
[
@@ -393,7 +393,7 @@ class TestTransactionIsolation(object):
)
)
- result_rows = t2.execute('{} select id, value from {} where id = 1;'.format(prefix, table_name))
+ result_rows = t2.execute('{} select id, value from {} where id = 1;'.format(prefix, table_name))
assert_that(
result_rows[0].rows, equal_to(
[
@@ -402,9 +402,9 @@ class TestTransactionIsolation(object):
)
)
- t1.execute('{} upsert into {} (id, value) VALUES (1, 11);'.format(prefix, table_name))
+ t1.execute('{} upsert into {} (id, value) VALUES (1, 11);'.format(prefix, table_name))
t1.commit()
- t2.execute('{} upsert into {} (id, value) VALUES (1, 12);'.format(prefix, table_name))
+ t2.execute('{} upsert into {} (id, value) VALUES (1, 12);'.format(prefix, table_name))
def callee():
t2.commit()
@@ -417,16 +417,16 @@ class TestTransactionIsolation(object):
)
)
- @pytest.mark.parametrize('new_engine', [True, False])
- def test_read_skew_g_single(self, new_engine):
+ @pytest.mark.parametrize('new_engine', [True, False])
+ def test_read_skew_g_single(self, new_engine):
table_name, session = self._prepare("test_read_skew_g_single")
t1 = session.transaction()
t2 = session.transaction()
- prefix = self._prepare_engine(new_engine)
-
- result_rows = t1.execute('{} select value from {} where id = 1;'.format(prefix, table_name))
+ prefix = self._prepare_engine(new_engine)
+
+ result_rows = t1.execute('{} select value from {} where id = 1;'.format(prefix, table_name))
assert_that(
result_rows[0].rows, equal_to(
[
@@ -435,14 +435,14 @@ class TestTransactionIsolation(object):
)
)
- result_rows = t2.execute('{} select value from {} where id = 1;'.format(prefix, table_name))
+ result_rows = t2.execute('{} select value from {} where id = 1;'.format(prefix, table_name))
assert_that(
result_rows[0].rows, equal_to(
[{'value': 10}]
)
)
- result_rows = t2.execute('{} select value from {} where id = 2;'.format(prefix, table_name))
+ result_rows = t2.execute('{} select value from {} where id = 2;'.format(prefix, table_name))
assert_that(
result_rows[0].rows, equal_to(
[
@@ -451,8 +451,8 @@ class TestTransactionIsolation(object):
)
)
- t2.execute('{} upsert into {} (id, value) values (1, 12);'.format(prefix, table_name))
- t2.execute('{} upsert into {} (id, value) values (2, 18);'.format(prefix, table_name))
+ t2.execute('{} upsert into {} (id, value) values (1, 12);'.format(prefix, table_name))
+ t2.execute('{} upsert into {} (id, value) values (2, 18);'.format(prefix, table_name))
t2.commit()
try:
@@ -469,18 +469,18 @@ class TestTransactionIsolation(object):
)
)
- @pytest.mark.parametrize('new_engine', [True, False])
- def test_read_skew_g_single_predicate_deps(self, new_engine):
+ @pytest.mark.parametrize('new_engine', [True, False])
+ def test_read_skew_g_single_predicate_deps(self, new_engine):
table_name, session = self._prepare("test_read_skew_g_single_predicate_deps")
t1 = session.transaction()
t2 = session.transaction()
- prefix = self._prepare_engine(new_engine)
+ prefix = self._prepare_engine(new_engine)
+
+ t1.execute('{} select value from {} where value % 5 = 0;'.format(prefix, table_name))
+ t2.execute('{} update {} set value = 12 where value = 10;'.format(prefix, table_name))
- t1.execute('{} select value from {} where value % 5 = 0;'.format(prefix, table_name))
- t2.execute('{} update {} set value = 12 where value = 10;'.format(prefix, table_name))
-
t2.commit()
try:
@@ -491,16 +491,16 @@ class TestTransactionIsolation(object):
else:
assert_that(result_rows[0].rows, equal_to([]))
- @pytest.mark.parametrize('new_engine', [True, False])
- def test_read_skew_g_single_write_predicate(self, new_engine):
+ @pytest.mark.parametrize('new_engine', [True, False])
+ def test_read_skew_g_single_write_predicate(self, new_engine):
table_name, session = self._prepare("test_read_skew_g_single_write_predicate")
t1 = session.transaction()
t2 = session.transaction()
- prefix = self._prepare_engine(new_engine)
-
- result_rows = t1.execute('{} select value from {} where id = 1;'.format(prefix, table_name))
+ prefix = self._prepare_engine(new_engine)
+
+ result_rows = t1.execute('{} select value from {} where id = 1;'.format(prefix, table_name))
assert_that(
result_rows[0].rows, equal_to(
[
@@ -509,9 +509,9 @@ class TestTransactionIsolation(object):
)
)
- t2.execute('{} select * from {};'.format(prefix, table_name))
- t2.execute('{} upsert into {} (id, value) values (1, 12)'.format(prefix, table_name))
- t2.execute('{} upsert into {} (id, value) values (2, 18);'.format(prefix, table_name))
+ t2.execute('{} select * from {};'.format(prefix, table_name))
+ t2.execute('{} upsert into {} (id, value) values (1, 12)'.format(prefix, table_name))
+ t2.execute('{} upsert into {} (id, value) values (2, 18);'.format(prefix, table_name))
t2.commit()
@@ -526,19 +526,19 @@ class TestTransactionIsolation(object):
)
)
- @pytest.mark.parametrize('new_engine', [True, False])
- def test_write_skew_g2_item(self, new_engine):
+ @pytest.mark.parametrize('new_engine', [True, False])
+ def test_write_skew_g2_item(self, new_engine):
table_name, session = self._prepare("test_write_skew_g2_item")
t1 = session.transaction()
t2 = session.transaction()
- prefix = self._prepare_engine(new_engine)
-
- t1.execute('{} select value from {} where id in (1,2);'.format(prefix, table_name))
- t2.execute('{} select value from {} where id in (1,2);'.format(prefix, table_name))
- t1.execute('{} update {} set value = 11 where id = 1;'.format(prefix, table_name))
- t2.execute('{} update {} set value = 21 where id = 2;'.format(prefix, table_name))
+ prefix = self._prepare_engine(new_engine)
+
+ t1.execute('{} select value from {} where id in (1,2);'.format(prefix, table_name))
+ t2.execute('{} select value from {} where id in (1,2);'.format(prefix, table_name))
+ t1.execute('{} update {} set value = 11 where id = 1;'.format(prefix, table_name))
+ t2.execute('{} update {} set value = 21 where id = 2;'.format(prefix, table_name))
t1.commit()
def callee():
@@ -552,21 +552,21 @@ class TestTransactionIsolation(object):
)
)
- @pytest.mark.parametrize('new_engine', [True, False])
- def test_anti_dependency_cycles_g2(self, new_engine):
+ @pytest.mark.parametrize('new_engine', [True, False])
+ def test_anti_dependency_cycles_g2(self, new_engine):
table_name, session = self._prepare("test_anti_dependency_cycles_g2")
t1 = session.transaction()
t2 = session.transaction()
- prefix = self._prepare_engine(new_engine)
+ prefix = self._prepare_engine(new_engine)
- t1.execute('{} select value from {} where value % 3 = 0;'.format(prefix, table_name))
- t2.execute('{} select value from {} where value % 3 = 0;'.format(prefix, table_name))
+ t1.execute('{} select value from {} where value % 3 = 0;'.format(prefix, table_name))
+ t2.execute('{} select value from {} where value % 3 = 0;'.format(prefix, table_name))
+
+ t1.execute('{} upsert into {} (id, value) values(3, 30);'.format(prefix, table_name))
+ t2.execute('{} upsert into {} (id, value) values(4, 42);'.format(prefix, table_name))
- t1.execute('{} upsert into {} (id, value) values(3, 30);'.format(prefix, table_name))
- t2.execute('{} upsert into {} (id, value) values(4, 42);'.format(prefix, table_name))
-
t1.commit()
def callee():
@@ -580,17 +580,17 @@ class TestTransactionIsolation(object):
)
)
- @pytest.mark.parametrize('new_engine', [True, False])
- def test_anti_dependency_cycles_g2_two_edges(self, new_engine):
+ @pytest.mark.parametrize('new_engine', [True, False])
+ def test_anti_dependency_cycles_g2_two_edges(self, new_engine):
table_name, session = self._prepare("test_anti_dependency_cycles_g2_two_edges")
t1 = session.transaction()
t2 = session.transaction()
t3 = session.transaction()
- prefix = self._prepare_engine(new_engine)
-
- result_rows = t1.execute('{} select id, value from {};'.format(prefix, table_name))
+ prefix = self._prepare_engine(new_engine)
+
+ result_rows = t1.execute('{} select id, value from {};'.format(prefix, table_name))
assert_that(
result_rows[0].rows,
equal_to(
@@ -601,11 +601,11 @@ class TestTransactionIsolation(object):
)
)
- t2.execute('{} update {} set value = value + 5 where id = 2;'.format(prefix, table_name))
+ t2.execute('{} update {} set value = value + 5 where id = 2;'.format(prefix, table_name))
t2.commit()
- result_rows = t3.execute('{} select id, value from {};'.format(prefix, table_name))
+ result_rows = t3.execute('{} select id, value from {};'.format(prefix, table_name))
assert_that(
result_rows[0].rows,
equal_to(
@@ -619,7 +619,7 @@ class TestTransactionIsolation(object):
t3.commit()
def callee():
- t1.execute('{} update {} set value = 0 where id = 1;'.format(prefix, table_name))
+ t1.execute('{} update {} set value = 0 where id = 1;'.format(prefix, table_name))
assert_that(
callee,
diff --git a/ydb/tests/functional/canonical/sql/topsort/topsort.sql b/ydb/tests/functional/canonical/sql/topsort/topsort.sql
index 129eac471b..32d5594f22 100644
--- a/ydb/tests/functional/canonical/sql/topsort/topsort.sql
+++ b/ydb/tests/functional/canonical/sql/topsort/topsort.sql
@@ -1 +1 @@
-SELECT * FROM Input1 WHERE Name != "Name1" ORDER BY Amount DESC LIMIT 4;
+SELECT * FROM Input1 WHERE Name != "Name1" ORDER BY Amount DESC LIMIT 4;
diff --git a/ydb/tests/functional/canonical/sql/topsort/topsort_multi_range.sql b/ydb/tests/functional/canonical/sql/topsort/topsort_multi_range.sql
index c046cbc918..549c34d1b9 100644
--- a/ydb/tests/functional/canonical/sql/topsort/topsort_multi_range.sql
+++ b/ydb/tests/functional/canonical/sql/topsort/topsort_multi_range.sql
@@ -6,4 +6,4 @@ $fetch = (
);
SELECT * FROM $fetch
-ORDER BY Amount DESC, Group, Name LIMIT 5;
+ORDER BY Amount DESC, Group, Name LIMIT 5;
diff --git a/ydb/tests/functional/canonical/test_sql.py b/ydb/tests/functional/canonical/test_sql.py
index 290553ca93..af5cdc2b9c 100644
--- a/ydb/tests/functional/canonical/test_sql.py
+++ b/ydb/tests/functional/canonical/test_sql.py
@@ -128,8 +128,8 @@ class BaseCanonicalTest(object):
is_v1 = True
if use_new_engine:
- query_parts = ["PRAGMA Kikimr.UseNewEngine=\"True\";"] + query_parts
-
+ query_parts = ["PRAGMA Kikimr.UseNewEngine=\"True\";"] + query_parts
+
query_parts = ["PRAGMA TablePathPrefix=\"{}\";".format(cls.prefix)] + query_parts
if is_v1:
query_parts = ['--!syntax_v1'] + query_parts
@@ -521,7 +521,7 @@ class TestCanonicalFolder1(BaseCanonicalTest):
data_folder = 'ydb/tests/functional/canonical/sql'
uniq_idx = 'base'
- @pytest.mark.parametrize(['query_name', 'kind'],
+ @pytest.mark.parametrize(['query_name', 'kind'],
get_queries('ydb/tests/functional/canonical/sql'))
def test_case(self, query_name, kind):
- return self.run_test_case(query_name, kind)
+ return self.run_test_case(query_name, kind)
diff --git a/ydb/tests/functional/suite_tests/canondata/result.json b/ydb/tests/functional/suite_tests/canondata/result.json
index 22d1db72f8..ebfb6f8144 100644
--- a/ydb/tests/functional/suite_tests/canondata/result.json
+++ b/ydb/tests/functional/suite_tests/canondata/result.json
@@ -19071,4 +19071,4 @@
"uri": "file://test_stream_query.TestStreamQuery.test_sql_suite_results-window.test_/window.test.out"
}
}
-}
+}
diff --git a/ydb/tests/tools/ydb_serializable/__main__.py b/ydb/tests/tools/ydb_serializable/__main__.py
index 7b919bbf54..47dc68f64f 100644
--- a/ydb/tests/tools/ydb_serializable/__main__.py
+++ b/ydb/tests/tools/ydb_serializable/__main__.py
@@ -32,7 +32,7 @@ def main():
parser.add_argument('--rt-snapshot', dest='read_table_snapshot', action='store_true', default=None, help='Use server-side snapshots for read-table transactions')
parser.add_argument('--ignore-rt', dest='ignore_read_table', action='store_true', help='Ignore read-table results (e.g. for interference only, legacy option)')
parser.add_argument('--processes', type=int, default=1, help='Number of processes to fork into, default is 1')
- parser.add_argument('--new-engine', dest='force_new_engine', action='store_true', default=False, help='Force NewEngine')
+ parser.add_argument('--new-engine', dest='force_new_engine', action='store_true', default=False, help='Force NewEngine')
args = parser.parse_args()
logger = DummyLogger()
@@ -49,7 +49,7 @@ def main():
options.read_table_ranges = args.read_table_ranges
options.ignore_read_table = args.ignore_read_table
options.read_table_snapshot = args.read_table_snapshot
- options.force_new_engine = args.force_new_engine
+ options.force_new_engine = args.force_new_engine
def run_single():
def handler(signum, frame, sys=sys, logger=logger):
diff --git a/ydb/tests/tools/ydb_serializable/lib/__init__.py b/ydb/tests/tools/ydb_serializable/lib/__init__.py
index 716384fc46..0909a02d07 100644
--- a/ydb/tests/tools/ydb_serializable/lib/__init__.py
+++ b/ydb/tests/tools/ydb_serializable/lib/__init__.py
@@ -12,69 +12,69 @@ KEY_PREFIX_TYPE = ydb.TupleType().add_element(ydb.OptionalType(ydb.PrimitiveType
QUERY_POINT_READS = '''\
---!syntax_v1
-{FORCE_NEW_ENGINE}
+--!syntax_v1
+{FORCE_NEW_ENGINE}
+
+DECLARE $data AS List<Struct<
+ key: Uint64>>;
-DECLARE $data AS List<Struct<
- key: Uint64>>;
-
SELECT t.key AS key, t.value AS value
FROM AS_TABLE($data) AS d
-INNER JOIN `{TABLE}` AS t ON t.key = d.key;
+INNER JOIN `{TABLE}` AS t ON t.key = d.key;
'''
QUERY_POINT_WRITES = '''\
---!syntax_v1
-{FORCE_NEW_ENGINE}
-
-DECLARE $data AS List<Struct<
+--!syntax_v1
+{FORCE_NEW_ENGINE}
+
+DECLARE $data AS List<Struct<
key: Uint64,
- value: Uint64>>;
+ value: Uint64>>;
-UPSERT INTO `{TABLE}`
+UPSERT INTO `{TABLE}`
SELECT key, value
FROM AS_TABLE($data);
'''
QUERY_POINT_READS_WRITES = '''\
---!syntax_v1
-{FORCE_NEW_ENGINE}
-
-DECLARE $reads AS List<Struct<
- key: Uint64>>;
-DECLARE $writes AS List<Struct<
+--!syntax_v1
+{FORCE_NEW_ENGINE}
+
+DECLARE $reads AS List<Struct<
+ key: Uint64>>;
+DECLARE $writes AS List<Struct<
key: Uint64,
- value: Uint64>>;
+ value: Uint64>>;
SELECT t.key AS key, t.value AS value
FROM AS_TABLE($reads) AS r
-INNER JOIN `{TABLE}` AS t ON t.key = r.key;
+INNER JOIN `{TABLE}` AS t ON t.key = r.key;
-UPSERT INTO `{TABLE}`
+UPSERT INTO `{TABLE}`
SELECT key, value
FROM AS_TABLE($writes);
'''
QUERY_RANGE_READS = '''\
---!syntax_v1
-{FORCE_NEW_ENGINE}
-
+--!syntax_v1
+{FORCE_NEW_ENGINE}
+
DECLARE $minKey AS Uint64;
DECLARE $maxKey AS Uint64;
SELECT key, value
-FROM `{TABLE}`
+FROM `{TABLE}`
WHERE key >= $minKey AND key <= $maxKey;
'''
-def new_engine_pragma(force_new_engine):
- return 'PRAGMA Kikimr.UseNewEngine = "true";' if force_new_engine else ''
-
-
+def new_engine_pragma(force_new_engine):
+ return 'PRAGMA Kikimr.UseNewEngine = "true";' if force_new_engine else ''
+
+
def generate_random_name(cnt=20):
return ''.join(
random.choice('abcdefghijklmnopqrstuvwxyz')
@@ -246,7 +246,7 @@ class DatabaseCheckerOptions(object):
self.read_table_ranges = False
self.ignore_read_table = False
self.read_table_snapshot = None
- self.force_new_engine = False
+ self.force_new_engine = False
class DatabaseChecker(object):
@@ -312,8 +312,8 @@ class DatabaseChecker(object):
@gen.coroutine
def async_perform_point_reads(self, history, table, options, checker, deadline):
with (yield self.async_session()) as session:
- read_query = yield session.async_prepare(QUERY_POINT_READS.format(
- TABLE=table, FORCE_NEW_ENGINE=new_engine_pragma(options.force_new_engine)))
+ read_query = yield session.async_prepare(QUERY_POINT_READS.format(
+ TABLE=table, FORCE_NEW_ENGINE=new_engine_pragma(options.force_new_engine)))
while time.time() < deadline:
keys = checker.select_read_from_write_keys(cnt=random.randint(1, options.shards))
@@ -358,8 +358,8 @@ class DatabaseChecker(object):
@gen.coroutine
def async_perform_point_writes(self, history, table, options, checker, deadline):
with (yield self.async_session()) as session:
- write_query = yield session.async_prepare(QUERY_POINT_WRITES.format(
- TABLE=table, FORCE_NEW_ENGINE=new_engine_pragma(options.force_new_engine)))
+ write_query = yield session.async_prepare(QUERY_POINT_WRITES.format(
+ TABLE=table, FORCE_NEW_ENGINE=new_engine_pragma(options.force_new_engine)))
while time.time() < deadline:
keys = checker.select_write_keys(cnt=random.randint(1, options.shards))
@@ -402,8 +402,8 @@ class DatabaseChecker(object):
TABLE=table, FORCE_NEW_ENGINE=new_engine_pragma(options.force_new_engine)))
write_query = yield session.async_prepare(QUERY_POINT_WRITES.format(
TABLE=table, FORCE_NEW_ENGINE=new_engine_pragma(options.force_new_engine)))
- read_write_query = yield session.async_prepare(QUERY_POINT_READS_WRITES.format(
- TABLE=table, FORCE_NEW_ENGINE=new_engine_pragma(options.force_new_engine)))
+ read_write_query = yield session.async_prepare(QUERY_POINT_READS_WRITES.format(
+ TABLE=table, FORCE_NEW_ENGINE=new_engine_pragma(options.force_new_engine)))
while time.time() < deadline:
read_keys = checker.select_read_keys(cnt=random.randint(1, options.shards))
@@ -473,8 +473,8 @@ class DatabaseChecker(object):
@gen.coroutine
def async_perform_verifying_reads(self, history, table, options, checker, deadline, keysets):
with (yield self.async_session()) as session:
- read_query = yield session.async_prepare(QUERY_POINT_READS.format(
- TABLE=table, FORCE_NEW_ENGINE=new_engine_pragma(options.force_new_engine)))
+ read_query = yield session.async_prepare(QUERY_POINT_READS.format(
+ TABLE=table, FORCE_NEW_ENGINE=new_engine_pragma(options.force_new_engine)))
while time.time() < deadline:
if not keysets:
@@ -520,8 +520,8 @@ class DatabaseChecker(object):
@gen.coroutine
def async_perform_range_reads(self, history, table, options, checker, deadline):
with (yield self.async_session()) as session:
- range_query = yield session.async_prepare(QUERY_RANGE_READS.format(
- TABLE=table, FORCE_NEW_ENGINE=new_engine_pragma(options.force_new_engine)))
+ range_query = yield session.async_prepare(QUERY_RANGE_READS.format(
+ TABLE=table, FORCE_NEW_ENGINE=new_engine_pragma(options.force_new_engine)))
while time.time() < deadline:
min_key = random.randint(0, options.keys)