aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authormonster <monster@yandex-team.ru>2022-02-10 16:47:19 +0300
committerDaniil Cherednik <dcherednik@yandex-team.ru>2022-02-10 16:47:19 +0300
commitdd76ae1f6213d065375ab296699f764faafbe5bd (patch)
tree075a4553d0011d8f92752c0891794febb15912cf
parentf02a874a7290593efbe4b3aeae69a04b46c1cc86 (diff)
downloadydb-dd76ae1f6213d065375ab296699f764faafbe5bd.tar.gz
Restoring authorship annotation for <monster@yandex-team.ru>. Commit 1 of 2.
-rw-r--r--contrib/libs/tcmalloc/patches/userdata.patch440
-rw-r--r--contrib/libs/tcmalloc/tcmalloc/internal/logging.h4
-rw-r--r--contrib/libs/tcmalloc/tcmalloc/internal_malloc_extension.h12
-rw-r--r--contrib/libs/tcmalloc/tcmalloc/malloc_extension.cc30
-rw-r--r--contrib/libs/tcmalloc/tcmalloc/malloc_extension.h24
-rw-r--r--contrib/libs/tcmalloc/tcmalloc/peak_heap_tracker.cc8
-rw-r--r--contrib/libs/tcmalloc/tcmalloc/stack_trace_table.cc8
-rw-r--r--contrib/libs/tcmalloc/tcmalloc/static_vars.cc12
-rw-r--r--contrib/libs/tcmalloc/tcmalloc/static_vars.h62
-rw-r--r--contrib/libs/tcmalloc/tcmalloc/tcmalloc.cc18
-rw-r--r--library/cpp/actors/core/README.md198
-rw-r--r--library/cpp/actors/core/actorsystem.h4
-rw-r--r--library/cpp/actors/core/event.h4
-rw-r--r--library/cpp/actors/core/executor_thread.cpp16
-rw-r--r--library/cpp/actors/core/mailbox.h4
-rw-r--r--library/cpp/actors/core/memory_track.cpp76
-rw-r--r--library/cpp/actors/core/memory_track.h584
-rw-r--r--library/cpp/actors/core/memory_tracker.cpp206
-rw-r--r--library/cpp/actors/core/memory_tracker.h106
-rw-r--r--library/cpp/actors/core/memory_tracker_ut.cpp524
-rw-r--r--library/cpp/actors/core/mon_stats.h48
-rw-r--r--library/cpp/actors/core/process_stats.cpp12
-rw-r--r--library/cpp/actors/core/process_stats.h2
-rw-r--r--library/cpp/actors/core/scheduler_queue.h2
-rw-r--r--library/cpp/actors/core/ut/ya.make2
-rw-r--r--library/cpp/actors/core/ya.make8
-rw-r--r--library/cpp/actors/helpers/mon_histogram_helper.h38
-rw-r--r--library/cpp/actors/interconnect/interconnect_counters.cpp18
-rw-r--r--library/cpp/actors/interconnect/interconnect_tcp_input_session.cpp6
-rw-r--r--library/cpp/actors/prof/tag.cpp44
-rw-r--r--library/cpp/actors/prof/tag.h26
-rw-r--r--library/cpp/actors/prof/tcmalloc.cpp64
-rw-r--r--library/cpp/actors/prof/tcmalloc.h18
-rw-r--r--library/cpp/actors/prof/tcmalloc_null.cpp20
-rw-r--r--library/cpp/actors/prof/ya.make14
-rw-r--r--library/cpp/binsaver/bin_saver.cpp14
-rw-r--r--library/cpp/binsaver/bin_saver.h28
-rw-r--r--library/cpp/binsaver/buffered_io.h82
-rw-r--r--library/cpp/binsaver/class_factory.h24
-rw-r--r--library/cpp/binsaver/mem_io.h4
-rw-r--r--library/cpp/digest/sfh/sfh.h52
-rw-r--r--library/cpp/grpc/server/grpc_counters.h18
-rw-r--r--library/cpp/grpc/server/grpc_request.h8
-rw-r--r--library/cpp/grpc/server/grpc_request_base.h6
-rw-r--r--library/cpp/grpc/server/grpc_server.cpp2
-rw-r--r--library/cpp/monlib/dynamic_counters/contention_ut.cpp2
-rw-r--r--library/cpp/monlib/dynamic_counters/counters.cpp64
-rw-r--r--library/cpp/monlib/dynamic_counters/counters.h66
-rw-r--r--library/cpp/monlib/dynamic_counters/counters_ut.cpp58
-rw-r--r--library/cpp/monlib/dynamic_counters/encode.cpp16
-rw-r--r--library/cpp/monlib/dynamic_counters/golovan_page.cpp2
-rw-r--r--library/cpp/monlib/dynamic_counters/page.cpp8
-rw-r--r--library/cpp/monlib/dynamic_counters/ya.make2
-rw-r--r--library/cpp/monlib/service/pages/mon_page.h2
-rw-r--r--library/cpp/yson/consumer.cpp10
-rw-r--r--library/cpp/yson/consumer.h12
-rw-r--r--library/cpp/yson/detail.h262
-rw-r--r--library/cpp/yson/format.h22
-rw-r--r--library/cpp/yson/json/json_writer.cpp104
-rw-r--r--library/cpp/yson/json/json_writer.h44
-rw-r--r--library/cpp/yson/lexer.cpp32
-rw-r--r--library/cpp/yson/lexer.h26
-rw-r--r--library/cpp/yson/lexer_detail.h130
-rw-r--r--library/cpp/yson/node/node.cpp222
-rw-r--r--library/cpp/yson/node/node.h116
-rw-r--r--library/cpp/yson/node/node_builder.cpp174
-rw-r--r--library/cpp/yson/node/node_builder.h48
-rw-r--r--library/cpp/yson/node/node_io.cpp66
-rw-r--r--library/cpp/yson/node/node_io.h22
-rw-r--r--library/cpp/yson/node/node_visitor.cpp178
-rw-r--r--library/cpp/yson/node/node_visitor.h58
-rw-r--r--library/cpp/yson/parser.cpp70
-rw-r--r--library/cpp/yson/parser.h42
-rw-r--r--library/cpp/yson/parser_detail.h106
-rw-r--r--library/cpp/yson/public.h18
-rw-r--r--library/cpp/yson/token.cpp80
-rw-r--r--library/cpp/yson/token.h52
-rw-r--r--library/cpp/yson/tokenizer.cpp18
-rw-r--r--library/cpp/yson/tokenizer.h20
-rw-r--r--library/cpp/yson/varint.cpp28
-rw-r--r--library/cpp/yson/writer.cpp136
-rw-r--r--library/cpp/yson/writer.h50
-rw-r--r--library/cpp/yson/ya.make22
-rw-r--r--library/cpp/yson/zigzag.h20
-rw-r--r--util/generic/guid.cpp2
-rw-r--r--ydb/core/base/appdata.cpp2
-rw-r--r--ydb/core/base/appdata.h2
-rw-r--r--ydb/core/base/blobstorage.h4
-rw-r--r--ydb/core/base/counters.cpp4
-rw-r--r--ydb/core/base/events.h2
-rw-r--r--ydb/core/base/pool_stats_collector.cpp4
-rw-r--r--ydb/core/base/pool_stats_collector.h2
-rw-r--r--ydb/core/base/shared_data.cpp42
-rw-r--r--ydb/core/base/shared_data.h24
-rw-r--r--ydb/core/base/tablet_types.h2
-rw-r--r--ydb/core/blobstorage/dsproxy/dsproxy_nodemon.cpp6
-rw-r--r--ydb/core/blobstorage/incrhuge/incrhuge.h2
-rw-r--r--ydb/core/blobstorage/incrhuge/incrhuge_keeper_common.h2
-rw-r--r--ydb/core/blobstorage/pdisk/blobstorage_pdisk.h2
-rw-r--r--ydb/core/blobstorage/pdisk/blobstorage_pdisk_util_countedqueuemanyone.h2
-rw-r--r--ydb/core/blobstorage/testload/test_load_actor.cpp26
-rw-r--r--ydb/core/blobstorage/testload/test_load_actor.h10
-rw-r--r--ydb/core/blobstorage/testload/test_load_memory.cpp278
-rw-r--r--ydb/core/blobstorage/testload/ya.make2
-rw-r--r--ydb/core/blobstorage/vdisk/common/vdisk_histograms.cpp2
-rw-r--r--ydb/core/client/flat_ut.cpp6
-rw-r--r--ydb/core/client/server/msgbus_server_s3_listing.cpp2
-rw-r--r--ydb/core/cms/console/console__create_tenant.cpp6
-rw-r--r--ydb/core/cms/console/console__scheme.h2
-rw-r--r--ydb/core/cms/console/console_tenants_manager.cpp22
-rw-r--r--ydb/core/cms/console/console_tenants_manager.h2
-rw-r--r--ydb/core/control/immediate_control_board_impl.cpp98
-rw-r--r--ydb/core/control/immediate_control_board_impl.h6
-rw-r--r--ydb/core/driver_lib/cli_base/cli_cmds_db.cpp8
-rw-r--r--ydb/core/driver_lib/run/config.h2
-rw-r--r--ydb/core/driver_lib/run/kikimr_services_initializers.cpp64
-rw-r--r--ydb/core/driver_lib/run/kikimr_services_initializers.h20
-rw-r--r--ydb/core/driver_lib/run/run.cpp18
-rw-r--r--ydb/core/engine/minikql/flat_local_minikql_host.h2
-rw-r--r--ydb/core/engine/minikql/minikql_engine_host.cpp2
-rw-r--r--ydb/core/engine/mkql_engine_flat_ut.cpp2
-rw-r--r--ydb/core/grpc_services/base/base.h26
-rw-r--r--ydb/core/grpc_services/counters/counters.cpp1264
-rw-r--r--ydb/core/grpc_services/counters/counters.h62
-rw-r--r--ydb/core/grpc_services/counters/ya.make34
-rw-r--r--ydb/core/grpc_services/grpc_helper.cpp2
-rw-r--r--ydb/core/grpc_services/rpc_read_columns.cpp42
-rw-r--r--ydb/core/grpc_streaming/grpc_streaming.h20
-rw-r--r--ydb/core/kqp/common/kqp_tx_info.h64
-rw-r--r--ydb/core/kqp/counters/kqp_counters.cpp1544
-rw-r--r--ydb/core/kqp/counters/kqp_counters.h456
-rw-r--r--ydb/core/kqp/counters/kqp_db_counters.h232
-rw-r--r--ydb/core/kqp/counters/ya.make34
-rw-r--r--ydb/core/kqp/executer/kqp_scan_executer.cpp6
-rw-r--r--ydb/core/kqp/host/kqp_host.cpp12
-rw-r--r--ydb/core/kqp/host/kqp_runner.cpp16
-rw-r--r--ydb/core/kqp/kqp.h18
-rw-r--r--ydb/core/kqp/kqp_compile_actor.cpp26
-rw-r--r--ydb/core/kqp/kqp_compile_request.cpp16
-rw-r--r--ydb/core/kqp/kqp_compile_service.cpp46
-rw-r--r--ydb/core/kqp/kqp_ic_gateway.cpp8
-rw-r--r--ydb/core/kqp/kqp_impl.h14
-rw-r--r--ydb/core/kqp/kqp_worker_actor.cpp134
-rw-r--r--ydb/core/kqp/node/kqp_node.cpp2
-rw-r--r--ydb/core/kqp/prepare/kqp_prepare.h2
-rw-r--r--ydb/core/kqp/prepare/kqp_query_exec.cpp12
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_datasink.cpp78
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_expr_nodes.json36
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_gateway.h2
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_opt_build.cpp6
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_provider.cpp2
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_provider_impl.h2
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_type_ann.cpp24
-rw-r--r--ydb/core/kqp/proxy/kqp_proxy_service.cpp176
-rw-r--r--ydb/core/kqp/proxy/kqp_proxy_service.h24
-rw-r--r--ydb/core/kqp/rm/kqp_rm.cpp2
-rw-r--r--ydb/core/kqp/runtime/kqp_compute.h14
-rw-r--r--ydb/core/kqp/runtime/kqp_program_builder.cpp4
-rw-r--r--ydb/core/kqp/runtime/kqp_read_table.cpp110
-rw-r--r--ydb/core/kqp/runtime/kqp_read_table.h38
-rw-r--r--ydb/core/kqp/runtime/kqp_scan_data.cpp100
-rw-r--r--ydb/core/kqp/runtime/kqp_scan_data.h58
-rw-r--r--ydb/core/kqp/runtime/ya.make8
-rw-r--r--ydb/core/kqp/ut/common/kqp_ut_common.cpp6
-rw-r--r--ydb/core/kqp/ut/common/kqp_ut_common.h8
-rw-r--r--ydb/core/kqp/ut/kqp_acl_ut.cpp210
-rw-r--r--ydb/core/kqp/ut/kqp_scan_ut.cpp2
-rw-r--r--ydb/core/kqp/ut/kqp_sys_view_ut.cpp480
-rw-r--r--ydb/core/kqp/ut/ya.make4
-rw-r--r--ydb/core/mind/bscontroller/bsc.cpp6
-rw-r--r--ydb/core/mind/bscontroller/impl.h24
-rw-r--r--ydb/core/mind/bscontroller/sys_view.cpp354
-rw-r--r--ydb/core/mind/bscontroller/sys_view.h34
-rw-r--r--ydb/core/mind/bscontroller/ya.make4
-rw-r--r--ydb/core/mind/configured_tablet_bootstrapper.cpp6
-rw-r--r--ydb/core/mind/hive/hive_impl.cpp156
-rw-r--r--ydb/core/mind/hive/hive_impl.h4
-rw-r--r--ydb/core/mind/hive/hive_schema.h14
-rw-r--r--ydb/core/mind/hive/monitoring.cpp4
-rw-r--r--ydb/core/mind/hive/tx__delete_tablet_result.cpp4
-rw-r--r--ydb/core/mind/hive/tx__load_everything.cpp22
-rw-r--r--ydb/core/mind/labels_maintainer.cpp40
-rw-r--r--ydb/core/mind/local.cpp8
-rw-r--r--ydb/core/mind/tenant_ut_pool.cpp74
-rw-r--r--ydb/core/mon_alloc/monitor.cpp240
-rw-r--r--ydb/core/mon_alloc/monitor.h64
-rw-r--r--ydb/core/mon_alloc/profiler.cpp140
-rw-r--r--ydb/core/mon_alloc/profiler.h2
-rw-r--r--ydb/core/mon_alloc/stats.cpp116
-rw-r--r--ydb/core/mon_alloc/stats.h36
-rw-r--r--ydb/core/mon_alloc/tcmalloc.cpp1580
-rw-r--r--ydb/core/mon_alloc/tcmalloc.h40
-rw-r--r--ydb/core/mon_alloc/tcmalloc_null.cpp54
-rw-r--r--ydb/core/mon_alloc/ya.make6
-rw-r--r--ydb/core/protos/blobstorage.proto18
-rw-r--r--ydb/core/protos/config.proto6
-rw-r--r--ydb/core/protos/counters_schemeshard.proto2
-rw-r--r--ydb/core/protos/counters_sysview_processor.proto32
-rw-r--r--ydb/core/protos/flat_tx_scheme.proto4
-rw-r--r--ydb/core/protos/node_whiteboard.proto2
-rw-r--r--ydb/core/protos/query_stats.proto2
-rw-r--r--ydb/core/protos/services.proto12
-rw-r--r--ydb/core/protos/subdomains.proto4
-rw-r--r--ydb/core/protos/sys_view.proto972
-rw-r--r--ydb/core/protos/tablet.proto2
-rw-r--r--ydb/core/protos/tablet_database.proto2
-rw-r--r--ydb/core/protos/tx_datashard.proto6
-rw-r--r--ydb/core/protos/ya.make4
-rw-r--r--ydb/core/scheme/scheme_tabledefs.h34
-rw-r--r--ydb/core/sys_view/common/common.h64
-rw-r--r--ydb/core/sys_view/common/db_counters.h156
-rw-r--r--ydb/core/sys_view/common/events.h666
-rw-r--r--ydb/core/sys_view/common/keys.h94
-rw-r--r--ydb/core/sys_view/common/path.h20
-rw-r--r--ydb/core/sys_view/common/scan_actor_base_impl.h634
-rw-r--r--ydb/core/sys_view/common/schema.cpp390
-rw-r--r--ydb/core/sys_view/common/schema.h748
-rw-r--r--ydb/core/sys_view/common/utils.h80
-rw-r--r--ydb/core/sys_view/common/ya.make44
-rw-r--r--ydb/core/sys_view/nodes/nodes.cpp452
-rw-r--r--ydb/core/sys_view/nodes/nodes.h20
-rw-r--r--ydb/core/sys_view/nodes/ya.make32
-rw-r--r--ydb/core/sys_view/partition_stats/partition_stats.cpp1018
-rw-r--r--ydb/core/sys_view/partition_stats/partition_stats.h34
-rw-r--r--ydb/core/sys_view/partition_stats/partition_stats_ut.cpp390
-rw-r--r--ydb/core/sys_view/partition_stats/ut/ya.make40
-rw-r--r--ydb/core/sys_view/partition_stats/ya.make30
-rw-r--r--ydb/core/sys_view/processor/db_counters.cpp954
-rw-r--r--ydb/core/sys_view/processor/processor.cpp32
-rw-r--r--ydb/core/sys_view/processor/processor.h20
-rw-r--r--ydb/core/sys_view/processor/processor_impl.cpp1360
-rw-r--r--ydb/core/sys_view/processor/processor_impl.h656
-rw-r--r--ydb/core/sys_view/processor/schema.cpp2
-rw-r--r--ydb/core/sys_view/processor/schema.h230
-rw-r--r--ydb/core/sys_view/processor/tx_aggregate.cpp256
-rw-r--r--ydb/core/sys_view/processor/tx_collect.cpp78
-rw-r--r--ydb/core/sys_view/processor/tx_configure.cpp84
-rw-r--r--ydb/core/sys_view/processor/tx_init.cpp756
-rw-r--r--ydb/core/sys_view/processor/tx_init_schema.cpp112
-rw-r--r--ydb/core/sys_view/processor/tx_interval_metrics.cpp274
-rw-r--r--ydb/core/sys_view/processor/tx_interval_summary.cpp456
-rw-r--r--ydb/core/sys_view/processor/ya.make58
-rw-r--r--ydb/core/sys_view/query_stats/query_metrics.cpp326
-rw-r--r--ydb/core/sys_view/query_stats/query_metrics.h22
-rw-r--r--ydb/core/sys_view/query_stats/query_stats.cpp1056
-rw-r--r--ydb/core/sys_view/query_stats/query_stats.h46
-rw-r--r--ydb/core/sys_view/query_stats/query_stats_ut.cpp200
-rw-r--r--ydb/core/sys_view/query_stats/ut/ya.make40
-rw-r--r--ydb/core/sys_view/query_stats/ya.make34
-rw-r--r--ydb/core/sys_view/scan.cpp104
-rw-r--r--ydb/core/sys_view/scan.h20
-rw-r--r--ydb/core/sys_view/service/db_counters.cpp228
-rw-r--r--ydb/core/sys_view/service/db_counters.h28
-rw-r--r--ydb/core/sys_view/service/ext_counters.cpp214
-rw-r--r--ydb/core/sys_view/service/ext_counters.h38
-rw-r--r--ydb/core/sys_view/service/query_history.h1020
-rw-r--r--ydb/core/sys_view/service/query_history_ut.cpp950
-rw-r--r--ydb/core/sys_view/service/query_interval.cpp310
-rw-r--r--ydb/core/sys_view/service/query_interval.h68
-rw-r--r--ydb/core/sys_view/service/sysview_service.cpp2248
-rw-r--r--ydb/core/sys_view/service/sysview_service.h46
-rw-r--r--ydb/core/sys_view/service/ut/ya.make36
-rw-r--r--ydb/core/sys_view/service/ya.make46
-rw-r--r--ydb/core/sys_view/storage/groups.cpp34
-rw-r--r--ydb/core/sys_view/storage/groups.h20
-rw-r--r--ydb/core/sys_view/storage/pdisks.cpp36
-rw-r--r--ydb/core/sys_view/storage/pdisks.h20
-rw-r--r--ydb/core/sys_view/storage/storage_pools.cpp34
-rw-r--r--ydb/core/sys_view/storage/storage_pools.h20
-rw-r--r--ydb/core/sys_view/storage/vslots.cpp36
-rw-r--r--ydb/core/sys_view/storage/vslots.h20
-rw-r--r--ydb/core/sys_view/storage/ya.make44
-rw-r--r--ydb/core/sys_view/tablets/tablets.cpp492
-rw-r--r--ydb/core/sys_view/tablets/tablets.h20
-rw-r--r--ydb/core/sys_view/tablets/ya.make34
-rw-r--r--ydb/core/sys_view/ut_common.cpp184
-rw-r--r--ydb/core/sys_view/ut_common.h116
-rw-r--r--ydb/core/sys_view/ut_counters.cpp358
-rw-r--r--ydb/core/sys_view/ut_kqp.cpp2998
-rw-r--r--ydb/core/sys_view/ut_kqp/ya.make54
-rw-r--r--ydb/core/sys_view/ya.make30
-rw-r--r--ydb/core/tablet/node_whiteboard.cpp2
-rw-r--r--ydb/core/tablet/tablet_counters_aggregator.cpp1244
-rw-r--r--ydb/core/tablet/tablet_counters_aggregator.h46
-rw-r--r--ydb/core/tablet/tablet_counters_app.cpp50
-rw-r--r--ydb/core/tablet/tablet_counters_app.h20
-rw-r--r--ydb/core/tablet/tablet_resolver.cpp2
-rw-r--r--ydb/core/tablet/ya.make4
-rw-r--r--ydb/core/tablet_flat/flat_executor.cpp14
-rw-r--r--ydb/core/tablet_flat/flat_executor.h2
-rw-r--r--ydb/core/tablet_flat/flat_range_cache.h34
-rw-r--r--ydb/core/tablet_flat/tablet_flat_executed.h2
-rw-r--r--ydb/core/testlib/basics/helpers.h2
-rw-r--r--ydb/core/testlib/tablet_helpers.cpp4
-rw-r--r--ydb/core/testlib/tenant_runtime.cpp4
-rw-r--r--ydb/core/testlib/test_client.cpp20
-rw-r--r--ydb/core/testlib/test_client.h2
-rw-r--r--ydb/core/tx/coordinator/coordinator__plan_step.cpp12
-rw-r--r--ydb/core/tx/coordinator/coordinator_impl.cpp12
-rw-r--r--ydb/core/tx/coordinator/coordinator_impl.h10
-rw-r--r--ydb/core/tx/datashard/datashard.cpp6
-rw-r--r--ydb/core/tx/datashard/datashard__engine_host.cpp16
-rw-r--r--ydb/core/tx/datashard/datashard__read_columns.cpp4
-rw-r--r--ydb/core/tx/datashard/datashard_active_transaction.cpp80
-rw-r--r--ydb/core/tx/datashard/datashard_active_transaction.h48
-rw-r--r--ydb/core/tx/datashard/datashard_impl.h8
-rw-r--r--ydb/core/tx/datashard/datashard_ut_locks.cpp4
-rw-r--r--ydb/core/tx/datashard/sys_tables.h6
-rw-r--r--ydb/core/tx/scheme_board/cache.cpp426
-rw-r--r--ydb/core/tx/scheme_board/cache_ut.cpp182
-rw-r--r--ydb/core/tx/scheme_board/replica.cpp194
-rw-r--r--ydb/core/tx/scheme_cache/scheme_cache.h14
-rw-r--r--ydb/core/tx/schemeshard/schemeshard.h16
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__delete_tablet_reply.cpp4
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__init.cpp10
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__init_root.cpp6
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__operation.cpp54
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__operation.h12
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__operation_alter_extsubdomain.cpp8
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__operation_common.cpp6
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__operation_common.h26
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp2
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__operation_split_merge.cpp2
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__sync_update_tenants.cpp52
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__table_stats.cpp88
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_domain_links.cpp8
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_domain_links.h2
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_impl.cpp64
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_impl.h14
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_info_types.h28
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_path.cpp12
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_path_element.h8
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_user_attr_limits.h28
-rw-r--r--ydb/core/tx/schemeshard/ut_base.cpp24
-rw-r--r--ydb/core/tx/schemeshard/ut_extsubdomain.cpp124
-rw-r--r--ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp18
-rw-r--r--ydb/core/tx/schemeshard/ut_helpers/ls_checks.h2
-rw-r--r--ydb/core/tx/schemeshard/ut_helpers/test_env.cpp2
-rw-r--r--ydb/core/tx/schemeshard/ut_helpers/test_env.h4
-rw-r--r--ydb/core/tx/schemeshard/ya.make2
-rw-r--r--ydb/core/tx/tx_proxy/datareq.cpp34
-rw-r--r--ydb/core/tx/tx_proxy/describe.cpp310
-rw-r--r--ydb/core/tx/tx_proxy/mon.cpp46
-rw-r--r--ydb/core/tx/tx_proxy/proxy.h4
-rw-r--r--ydb/core/tx/tx_proxy/proxy_ut_helpers.h2
-rw-r--r--ydb/core/tx/tx_proxy/upload_rows_common_impl.h8
-rw-r--r--ydb/core/util/concurrent_rw_hash.cpp2
-rw-r--r--ydb/core/util/concurrent_rw_hash.h234
-rw-r--r--ydb/core/util/concurrent_rw_hash_ut.cpp308
-rw-r--r--ydb/core/util/memory_tracker.cpp608
-rw-r--r--ydb/core/util/memory_tracker.h34
-rw-r--r--ydb/core/util/ut/ya.make2
-rw-r--r--ydb/core/util/ya.make10
-rw-r--r--ydb/library/yql/core/common_opt/yql_co_flow1.cpp4
-rw-r--r--ydb/library/yql/core/common_opt/yql_co_simple2.cpp6
-rw-r--r--ydb/library/yql/core/expr_nodes/yql_expr_nodes.h22
-rw-r--r--ydb/library/yql/core/expr_nodes/yql_expr_nodes.json58
-rw-r--r--ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.jnj6
-rw-r--r--ydb/library/yql/core/facade/yql_facade.cpp88
-rw-r--r--ydb/library/yql/core/facade/yql_facade.h8
-rw-r--r--ydb/library/yql/core/services/yql_plan.cpp6
-rw-r--r--ydb/library/yql/core/services/yql_plan.h2
-rw-r--r--ydb/library/yql/core/type_ann/type_ann_core.cpp28
-rw-r--r--ydb/library/yql/core/type_ann/type_ann_list.cpp490
-rw-r--r--ydb/library/yql/core/type_ann/type_ann_list.h4
-rw-r--r--ydb/library/yql/core/yql_data_provider.h12
-rw-r--r--ydb/library/yql/core/yql_execution.cpp36
-rw-r--r--ydb/library/yql/core/yql_expr_type_annotation.cpp52
-rw-r--r--ydb/library/yql/core/yql_opt_proposed_by_data.cpp18
-rw-r--r--ydb/library/yql/core/yql_opt_proposed_by_data.h2
-rw-r--r--ydb/library/yql/core/yql_opt_utils.cpp22
-rw-r--r--ydb/library/yql/core/yql_opt_utils.h6
-rw-r--r--ydb/library/yql/dq/actors/compute/dq_compute_actor.h2
-rw-r--r--ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h6
-rw-r--r--ydb/library/yql/dq/actors/dq_events_ids.h8
-rw-r--r--ydb/library/yql/dq/actors/protos/dq_events.proto8
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_condense.cpp70
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_condense1.cpp70
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp2
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_filter.cpp14
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_flatmap.cpp36
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_hopping.cpp488
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_hopping.h18
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_map.cpp10
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_saveload.h106
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_while.cpp18
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ya.make6
-rw-r--r--ydb/library/yql/minikql/computation/mkql_computation_node_codegen.h2
-rw-r--r--ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.cpp158
-rw-r--r--ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.h26
-rw-r--r--ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload_ut.cpp608
-rw-r--r--ydb/library/yql/minikql/computation/mkql_computation_node_holders.h4
-rw-r--r--ydb/library/yql/minikql/computation/mkql_computation_node_impl.h38
-rw-r--r--ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp8
-rw-r--r--ydb/library/yql/minikql/computation/mkql_computation_node_pack.h18
-rw-r--r--ydb/library/yql/minikql/computation/mkql_value_builder.cpp52
-rw-r--r--ydb/library/yql/minikql/computation/mkql_value_builder.h18
-rw-r--r--ydb/library/yql/minikql/computation/presort.cpp1106
-rw-r--r--ydb/library/yql/minikql/computation/presort.h104
-rw-r--r--ydb/library/yql/minikql/computation/presort_ut.cpp754
-rw-r--r--ydb/library/yql/minikql/computation/ut/ya.make2
-rw-r--r--ydb/library/yql/minikql/computation/ya.make4
-rw-r--r--ydb/library/yql/minikql/mkql_program_builder.cpp160
-rw-r--r--ydb/library/yql/minikql/mkql_program_builder.h4
-rw-r--r--ydb/library/yql/minikql/mkql_type_builder.cpp484
-rw-r--r--ydb/library/yql/minikql/mkql_type_builder.h8
-rw-r--r--ydb/library/yql/minikql/mkql_type_ops.cpp244
-rw-r--r--ydb/library/yql/minikql/mkql_type_ops.h12
-rw-r--r--ydb/library/yql/minikql/perf/presort/presort.cpp566
-rw-r--r--ydb/library/yql/minikql/perf/presort/ya.make22
-rw-r--r--ydb/library/yql/mount/lib/yql/aggregate.yql260
-rw-r--r--ydb/library/yql/mount/lib/yql/window.yql218
-rw-r--r--ydb/library/yql/providers/common/codec/yql_codec.cpp12
-rw-r--r--ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp20
-rw-r--r--ydb/library/yql/providers/common/proto/gateways_config.proto78
-rw-r--r--ydb/library/yql/providers/common/provider/yql_data_provider_impl.cpp12
-rw-r--r--ydb/library/yql/providers/common/provider/yql_data_provider_impl.h2
-rw-r--r--ydb/library/yql/providers/common/provider/yql_provider.cpp20
-rw-r--r--ydb/library/yql/providers/common/provider/yql_provider.h4
-rw-r--r--ydb/library/yql/providers/config/yql_config_provider.cpp6
-rw-r--r--ydb/library/yql/providers/result/provider/yql_result_provider.cpp2
-rw-r--r--ydb/library/yql/providers/solomon/expr_nodes/ya.make66
-rw-r--r--ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.cpp2
-rw-r--r--ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.h122
-rw-r--r--ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.json136
-rw-r--r--ydb/library/yql/providers/solomon/gateway/ya.make30
-rw-r--r--ydb/library/yql/providers/solomon/gateway/yql_solomon_gateway.cpp114
-rw-r--r--ydb/library/yql/providers/solomon/gateway/yql_solomon_gateway.h20
-rw-r--r--ydb/library/yql/providers/solomon/provider/ya.make34
-rw-r--r--ydb/library/yql/providers/solomon/provider/yql_solomon_config.cpp28
-rw-r--r--ydb/library/yql/providers/solomon/provider/yql_solomon_config.h74
-rw-r--r--ydb/library/yql/providers/solomon/provider/yql_solomon_datasink.cpp292
-rw-r--r--ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_execution.cpp44
-rw-r--r--ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_type_ann.cpp102
-rw-r--r--ydb/library/yql/providers/solomon/provider/yql_solomon_datasource.cpp200
-rw-r--r--ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_execution.cpp42
-rw-r--r--ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_type_ann.cpp46
-rw-r--r--ydb/library/yql/providers/solomon/provider/yql_solomon_gateway.h72
-rw-r--r--ydb/library/yql/providers/solomon/provider/yql_solomon_io_discovery.cpp150
-rw-r--r--ydb/library/yql/providers/solomon/provider/yql_solomon_load_meta.cpp88
-rw-r--r--ydb/library/yql/providers/solomon/provider/yql_solomon_provider.cpp76
-rw-r--r--ydb/library/yql/providers/solomon/provider/yql_solomon_provider.h44
-rw-r--r--ydb/library/yql/providers/solomon/provider/yql_solomon_provider_impl.h38
-rw-r--r--ydb/library/yql/providers/solomon/ya.make10
-rw-r--r--ydb/library/yql/public/udf/udf_helpers.h82
-rw-r--r--ydb/library/yql/public/udf/udf_type_builder.h148
-rw-r--r--ydb/library/yql/public/udf/udf_type_ops.h856
-rw-r--r--ydb/library/yql/public/udf/udf_value.h148
-rw-r--r--ydb/library/yql/public/udf/udf_value_builder.h24
-rw-r--r--ydb/library/yql/public/udf/udf_value_inl.h88
-rw-r--r--ydb/library/yql/public/udf/udf_value_ut.cpp12
-rw-r--r--ydb/library/yql/sql/v0/SQL.g30
-rw-r--r--ydb/library/yql/sql/v0/aggregation.cpp224
-rw-r--r--ydb/library/yql/sql/v0/builtin.cpp132
-rw-r--r--ydb/library/yql/sql/v0/node.cpp62
-rw-r--r--ydb/library/yql/sql/v0/node.h44
-rw-r--r--ydb/library/yql/sql/v0/query.cpp122
-rw-r--r--ydb/library/yql/sql/v0/select.cpp54
-rw-r--r--ydb/library/yql/sql/v0/sql.cpp394
-rw-r--r--ydb/library/yql/sql/v0/sql_ut.cpp26
-rw-r--r--ydb/library/yql/sql/v1/SQLv1.g.in176
-rw-r--r--ydb/library/yql/sql/v1/aggregation.cpp216
-rw-r--r--ydb/library/yql/sql/v1/builtin.cpp132
-rw-r--r--ydb/library/yql/sql/v1/node.cpp54
-rw-r--r--ydb/library/yql/sql/v1/node.h32
-rw-r--r--ydb/library/yql/sql/v1/query.cpp62
-rw-r--r--ydb/library/yql/sql/v1/select.cpp34
-rw-r--r--ydb/library/yql/sql/v1/sql.cpp1394
-rw-r--r--ydb/library/yql/sql/v1/sql_ut.cpp24
-rw-r--r--ydb/library/yql/udfs/common/datetime2/datetime_udf.cpp2054
-rw-r--r--ydb/library/yql/udfs/common/datetime2/ya.make34
-rw-r--r--ydb/library/yql/udfs/common/set/set_udf.cpp1006
-rw-r--r--ydb/library/yql/udfs/common/set/ya.make26
-rw-r--r--ydb/library/yql/udfs/common/top/top_udf.cpp1680
-rw-r--r--ydb/library/yql/udfs/common/top/ya.make34
-rw-r--r--ydb/library/yql/udfs/common/topfreq/static/topfreq.cpp252
-rw-r--r--ydb/library/yql/udfs/common/topfreq/static/topfreq.h136
-rw-r--r--ydb/library/yql/udfs/common/topfreq/static/topfreq_udf.h516
-rw-r--r--ydb/library/yql/udfs/common/topfreq/static/ya.make2
-rw-r--r--ydb/library/yql/udfs/common/topfreq/topfreq_udf_ut.cpp22
-rw-r--r--ydb/library/yql/udfs/common/topfreq/ya.make2
-rw-r--r--ydb/library/yql/utils/log/log_component.h2
-rw-r--r--ydb/public/lib/deprecated/kicli/cpp_ut.cpp4
-rw-r--r--ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp16
-rw-r--r--ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp68
-rw-r--r--ydb/public/lib/ydb_cli/commands/ydb_service_table.h22
-rw-r--r--ydb/public/lib/ydb_cli/common/format.cpp8
-rw-r--r--ydb/public/lib/ydb_cli/common/format.h4
-rw-r--r--ydb/public/sdk/python/examples/basic_example_v1/basic_example.py12
-rw-r--r--ydb/public/sdk/python/examples/basic_example_v1/basic_example_data.py6
-rw-r--r--ydb/services/datastreams/grpc_service.cpp2
-rw-r--r--ydb/services/persqueue_v1/persqueue.cpp6
-rw-r--r--ydb/services/ydb/ydb_common_ut.h12
-rw-r--r--ydb/services/ydb/ydb_table_ut.cpp2
-rw-r--r--ydb/services/ydb/ydb_ut.cpp4
-rw-r--r--ydb/tests/functional/tenants/test_db_counters.py244
-rw-r--r--ydb/tests/functional/tenants/test_system_views.py180
-rw-r--r--ydb/tests/functional/tenants/ya.make2
498 files changed, 30855 insertions, 30855 deletions
diff --git a/contrib/libs/tcmalloc/patches/userdata.patch b/contrib/libs/tcmalloc/patches/userdata.patch
index 83373cebfe0..51012c7ef9d 100644
--- a/contrib/libs/tcmalloc/patches/userdata.patch
+++ b/contrib/libs/tcmalloc/patches/userdata.patch
@@ -1,220 +1,220 @@
---- contrib/libs/tcmalloc/tcmalloc/internal/logging.h (index)
-+++ contrib/libs/tcmalloc/tcmalloc/internal/logging.h (working tree)
-@@ -67,6 +67,8 @@ struct StackTrace {
- // between the previous sample and this one
- size_t weight;
-
-+ void* user_data;
-+
- template <typename H>
- friend H AbslHashValue(H h, const StackTrace& t) {
- // As we use StackTrace as a key-value node in StackTraceTable, we only
---- contrib/libs/tcmalloc/tcmalloc/internal_malloc_extension.h (index)
-+++ contrib/libs/tcmalloc/tcmalloc/internal_malloc_extension.h (working tree)
-@@ -120,6 +120,12 @@ ABSL_ATTRIBUTE_WEAK void MallocExtension_Internal_SetMaxTotalThreadCacheBytes(
- ABSL_ATTRIBUTE_WEAK void
- MallocExtension_EnableForkSupport();
-
-+ABSL_ATTRIBUTE_WEAK void
-+MallocExtension_SetSampleUserDataCallbacks(
-+ tcmalloc::MallocExtension::CreateSampleUserDataCallback create,
-+ tcmalloc::MallocExtension::CopySampleUserDataCallback copy,
-+ tcmalloc::MallocExtension::DestroySampleUserDataCallback destroy);
-+
- }
-
- #endif
---- contrib/libs/tcmalloc/tcmalloc/malloc_extension.cc (index)
-+++ contrib/libs/tcmalloc/tcmalloc/malloc_extension.cc (working tree)
-@@ -468,6 +468,21 @@ void MallocExtension::EnableForkSupport() {
- #endif
- }
-
-+void MallocExtension::SetSampleUserDataCallbacks(
-+ CreateSampleUserDataCallback create,
-+ CopySampleUserDataCallback copy,
-+ DestroySampleUserDataCallback destroy) {
-+#if ABSL_INTERNAL_HAVE_WEAK_MALLOCEXTENSION_STUBS
-+ if (&MallocExtension_SetSampleUserDataCallbacks != nullptr) {
-+ MallocExtension_SetSampleUserDataCallbacks(create, copy, destroy);
-+ }
-+#else
-+ (void)create;
-+ (void)copy;
-+ (void)destroy;
-+#endif
-+}
-+
- } // namespace tcmalloc
-
- // Default implementation just returns size. The expectation is that
---- contrib/libs/tcmalloc/tcmalloc/malloc_extension.h (index)
-+++ contrib/libs/tcmalloc/tcmalloc/malloc_extension.h (working tree)
-@@ -94,6 +94,8 @@ class Profile final {
-
- int depth;
- void* stack[kMaxStackDepth];
-+
-+ void* user_data;
- };
-
- void Iterate(absl::FunctionRef<void(const Sample&)> f) const;
-@@ -472,6 +474,16 @@ class MallocExtension final {
- // Enables fork support.
- // Allocator will continue to function correctly in the child, after calling fork().
- static void EnableForkSupport();
-+
-+ using CreateSampleUserDataCallback = void*();
-+ using CopySampleUserDataCallback = void*(void*);
-+ using DestroySampleUserDataCallback = void(void*);
-+
-+ // Sets callbacks for lifetime control of custom user data attached to allocation samples
-+ static void SetSampleUserDataCallbacks(
-+ CreateSampleUserDataCallback create,
-+ CopySampleUserDataCallback copy,
-+ DestroySampleUserDataCallback destroy);
- };
-
- } // namespace tcmalloc
---- contrib/libs/tcmalloc/tcmalloc/peak_heap_tracker.cc (index)
-+++ contrib/libs/tcmalloc/tcmalloc/peak_heap_tracker.cc (working tree)
-@@ -55,6 +55,7 @@ void PeakHeapTracker::MaybeSaveSample() {
- StackTrace *t = peak_sampled_span_stacks_, *next = nullptr;
- while (t != nullptr) {
- next = reinterpret_cast<StackTrace*>(t->stack[kMaxStackDepth - 1]);
-+ Static::DestroySampleUserData(t->user_data);
- Static::stacktrace_allocator().Delete(t);
- t = next;
- }
-@@ -63,7 +64,9 @@ void PeakHeapTracker::MaybeSaveSample() {
- for (Span* s : Static::sampled_objects_) {
- t = Static::stacktrace_allocator().New();
-
-- *t = *s->sampled_stack();
-+ StackTrace* sampled_stack = s->sampled_stack();
-+ *t = *sampled_stack;
-+ t->user_data = Static::CopySampleUserData(sampled_stack->user_data);
- if (t->depth == kMaxStackDepth) {
- t->depth = kMaxStackDepth - 1;
- }
---- contrib/libs/tcmalloc/tcmalloc/stack_trace_table.cc (index)
-+++ contrib/libs/tcmalloc/tcmalloc/stack_trace_table.cc (working tree)
-@@ -73,6 +73,7 @@ StackTraceTable::~StackTraceTable() {
- Bucket* b = table_[i];
- while (b != nullptr) {
- Bucket* next = b->next;
-+ Static::DestroySampleUserData(b->trace.user_data);
- Static::bucket_allocator().Delete(b);
- b = next;
- }
-@@ -104,6 +105,7 @@ void StackTraceTable::AddTrace(double count, const StackTrace& t) {
- b = Static::bucket_allocator().New();
- b->hash = h;
- b->trace = t;
-+ b->trace.user_data = Static::CopySampleUserData(t.user_data);
- b->count = count;
- b->total_weight = t.weight * count;
- b->next = table_[idx];
-@@ -135,6 +137,8 @@ void StackTraceTable::Iterate(
- e.requested_alignment = b->trace.requested_alignment;
- e.allocated_size = allocated_size;
-
-+ e.user_data = b->trace.user_data;
-+
- e.depth = b->trace.depth;
- static_assert(kMaxStackDepth <= Profile::Sample::kMaxStackDepth,
- "Profile stack size smaller than internal stack sizes");
---- contrib/libs/tcmalloc/tcmalloc/static_vars.cc (index)
-+++ contrib/libs/tcmalloc/tcmalloc/static_vars.cc (working tree)
-@@ -60,6 +60,12 @@ ABSL_CONST_INIT PageHeapAllocator<StackTraceTable::Bucket>
- ABSL_CONST_INIT std::atomic<bool> Static::inited_{false};
- ABSL_CONST_INIT bool Static::cpu_cache_active_ = false;
- ABSL_CONST_INIT bool Static::fork_support_enabled_ = false;
-+ABSL_CONST_INIT Static::CreateSampleUserDataCallback*
-+ Static::create_sample_user_data_callback_ = nullptr;
-+ABSL_CONST_INIT Static::CopySampleUserDataCallback*
-+ Static::copy_sample_user_data_callback_ = nullptr;
-+ABSL_CONST_INIT Static::DestroySampleUserDataCallback*
-+ Static::destroy_sample_user_data_callback_ = nullptr;
- ABSL_CONST_INIT Static::PageAllocatorStorage Static::page_allocator_;
- ABSL_CONST_INIT PageMap Static::pagemap_;
- ABSL_CONST_INIT absl::base_internal::SpinLock guarded_page_lock(
---- contrib/libs/tcmalloc/tcmalloc/static_vars.h (index)
-+++ contrib/libs/tcmalloc/tcmalloc/static_vars.h (working tree)
-@@ -130,6 +130,34 @@ class Static {
- static bool ForkSupportEnabled() { return fork_support_enabled_; }
- static void EnableForkSupport() { fork_support_enabled_ = true; }
-
-+ using CreateSampleUserDataCallback = void*();
-+ using CopySampleUserDataCallback = void*(void*);
-+ using DestroySampleUserDataCallback = void(void*);
-+
-+ static void SetSampleUserDataCallbacks(
-+ CreateSampleUserDataCallback create,
-+ CopySampleUserDataCallback copy,
-+ DestroySampleUserDataCallback destroy) {
-+ create_sample_user_data_callback_ = create;
-+ copy_sample_user_data_callback_ = copy;
-+ destroy_sample_user_data_callback_ = destroy;
-+ }
-+
-+ static void* CreateSampleUserData() {
-+ if (create_sample_user_data_callback_)
-+ return create_sample_user_data_callback_();
-+ return nullptr;
-+ }
-+ static void* CopySampleUserData(void* user_data) {
-+ if (copy_sample_user_data_callback_)
-+ return copy_sample_user_data_callback_(user_data);
-+ return nullptr;
-+ }
-+ static void DestroySampleUserData(void* user_data) {
-+ if (destroy_sample_user_data_callback_)
-+ destroy_sample_user_data_callback_(user_data);
-+ }
-+
- static bool ABSL_ATTRIBUTE_ALWAYS_INLINE IsOnFastPath() {
- return
- #ifndef TCMALLOC_DEPRECATED_PERTHREAD
-@@ -176,6 +204,9 @@ class Static {
- ABSL_CONST_INIT static std::atomic<bool> inited_;
- static bool cpu_cache_active_;
- static bool fork_support_enabled_;
-+ static CreateSampleUserDataCallback* create_sample_user_data_callback_;
-+ static CopySampleUserDataCallback* copy_sample_user_data_callback_;
-+ static DestroySampleUserDataCallback* destroy_sample_user_data_callback_;
- ABSL_CONST_INIT static PeakHeapTracker peak_heap_tracker_;
- ABSL_CONST_INIT static NumaTopology<kNumaPartitions, kNumBaseClasses>
- numa_topology_;
---- contrib/libs/tcmalloc/tcmalloc/tcmalloc.cc (index)
-+++ contrib/libs/tcmalloc/tcmalloc/tcmalloc.cc (working tree)
-@@ -1151,6 +1151,13 @@ void TCMallocPostFork() {
- }
- }
-
-+extern "C" void MallocExtension_SetSampleUserDataCallbacks(
-+ MallocExtension::CreateSampleUserDataCallback create,
-+ MallocExtension::CopySampleUserDataCallback copy,
-+ MallocExtension::DestroySampleUserDataCallback destroy) {
-+ Static::SetSampleUserDataCallbacks(create, copy, destroy);
-+}
-+
- // nallocx slow path.
- // Moved to a separate function because size_class_with_alignment is not inlined
- // which would cause nallocx to become non-leaf function with stack frame and
-@@ -1500,6 +1507,7 @@ static void* SampleifyAllocation(size_t requested_size, size_t weight,
- tmp.requested_alignment = requested_alignment;
- tmp.allocated_size = allocated_size;
- tmp.weight = weight;
-+ tmp.user_data = Static::CreateSampleUserData();
-
- {
- absl::base_internal::SpinLockHolder h(&pageheap_lock);
-@@ -1629,6 +1637,7 @@ static void do_free_pages(void* ptr, const PageId p) {
- 1);
- }
- notify_sampled_alloc = true;
-+ Static::DestroySampleUserData(st->user_data);
- Static::stacktrace_allocator().Delete(st);
- }
- if (IsSampledMemory(ptr)) {
+--- contrib/libs/tcmalloc/tcmalloc/internal/logging.h (index)
++++ contrib/libs/tcmalloc/tcmalloc/internal/logging.h (working tree)
+@@ -67,6 +67,8 @@ struct StackTrace {
+ // between the previous sample and this one
+ size_t weight;
+
++ void* user_data;
++
+ template <typename H>
+ friend H AbslHashValue(H h, const StackTrace& t) {
+ // As we use StackTrace as a key-value node in StackTraceTable, we only
+--- contrib/libs/tcmalloc/tcmalloc/internal_malloc_extension.h (index)
++++ contrib/libs/tcmalloc/tcmalloc/internal_malloc_extension.h (working tree)
+@@ -120,6 +120,12 @@ ABSL_ATTRIBUTE_WEAK void MallocExtension_Internal_SetMaxTotalThreadCacheBytes(
+ ABSL_ATTRIBUTE_WEAK void
+ MallocExtension_EnableForkSupport();
+
++ABSL_ATTRIBUTE_WEAK void
++MallocExtension_SetSampleUserDataCallbacks(
++ tcmalloc::MallocExtension::CreateSampleUserDataCallback create,
++ tcmalloc::MallocExtension::CopySampleUserDataCallback copy,
++ tcmalloc::MallocExtension::DestroySampleUserDataCallback destroy);
++
+ }
+
+ #endif
+--- contrib/libs/tcmalloc/tcmalloc/malloc_extension.cc (index)
++++ contrib/libs/tcmalloc/tcmalloc/malloc_extension.cc (working tree)
+@@ -468,6 +468,21 @@ void MallocExtension::EnableForkSupport() {
+ #endif
+ }
+
++void MallocExtension::SetSampleUserDataCallbacks(
++ CreateSampleUserDataCallback create,
++ CopySampleUserDataCallback copy,
++ DestroySampleUserDataCallback destroy) {
++#if ABSL_INTERNAL_HAVE_WEAK_MALLOCEXTENSION_STUBS
++ if (&MallocExtension_SetSampleUserDataCallbacks != nullptr) {
++ MallocExtension_SetSampleUserDataCallbacks(create, copy, destroy);
++ }
++#else
++ (void)create;
++ (void)copy;
++ (void)destroy;
++#endif
++}
++
+ } // namespace tcmalloc
+
+ // Default implementation just returns size. The expectation is that
+--- contrib/libs/tcmalloc/tcmalloc/malloc_extension.h (index)
++++ contrib/libs/tcmalloc/tcmalloc/malloc_extension.h (working tree)
+@@ -94,6 +94,8 @@ class Profile final {
+
+ int depth;
+ void* stack[kMaxStackDepth];
++
++ void* user_data;
+ };
+
+ void Iterate(absl::FunctionRef<void(const Sample&)> f) const;
+@@ -472,6 +474,16 @@ class MallocExtension final {
+ // Enables fork support.
+ // Allocator will continue to function correctly in the child, after calling fork().
+ static void EnableForkSupport();
++
++ using CreateSampleUserDataCallback = void*();
++ using CopySampleUserDataCallback = void*(void*);
++ using DestroySampleUserDataCallback = void(void*);
++
++ // Sets callbacks for lifetime control of custom user data attached to allocation samples
++ static void SetSampleUserDataCallbacks(
++ CreateSampleUserDataCallback create,
++ CopySampleUserDataCallback copy,
++ DestroySampleUserDataCallback destroy);
+ };
+
+ } // namespace tcmalloc
+--- contrib/libs/tcmalloc/tcmalloc/peak_heap_tracker.cc (index)
++++ contrib/libs/tcmalloc/tcmalloc/peak_heap_tracker.cc (working tree)
+@@ -55,6 +55,7 @@ void PeakHeapTracker::MaybeSaveSample() {
+ StackTrace *t = peak_sampled_span_stacks_, *next = nullptr;
+ while (t != nullptr) {
+ next = reinterpret_cast<StackTrace*>(t->stack[kMaxStackDepth - 1]);
++ Static::DestroySampleUserData(t->user_data);
+ Static::stacktrace_allocator().Delete(t);
+ t = next;
+ }
+@@ -63,7 +64,9 @@ void PeakHeapTracker::MaybeSaveSample() {
+ for (Span* s : Static::sampled_objects_) {
+ t = Static::stacktrace_allocator().New();
+
+- *t = *s->sampled_stack();
++ StackTrace* sampled_stack = s->sampled_stack();
++ *t = *sampled_stack;
++ t->user_data = Static::CopySampleUserData(sampled_stack->user_data);
+ if (t->depth == kMaxStackDepth) {
+ t->depth = kMaxStackDepth - 1;
+ }
+--- contrib/libs/tcmalloc/tcmalloc/stack_trace_table.cc (index)
++++ contrib/libs/tcmalloc/tcmalloc/stack_trace_table.cc (working tree)
+@@ -73,6 +73,7 @@ StackTraceTable::~StackTraceTable() {
+ Bucket* b = table_[i];
+ while (b != nullptr) {
+ Bucket* next = b->next;
++ Static::DestroySampleUserData(b->trace.user_data);
+ Static::bucket_allocator().Delete(b);
+ b = next;
+ }
+@@ -104,6 +105,7 @@ void StackTraceTable::AddTrace(double count, const StackTrace& t) {
+ b = Static::bucket_allocator().New();
+ b->hash = h;
+ b->trace = t;
++ b->trace.user_data = Static::CopySampleUserData(t.user_data);
+ b->count = count;
+ b->total_weight = t.weight * count;
+ b->next = table_[idx];
+@@ -135,6 +137,8 @@ void StackTraceTable::Iterate(
+ e.requested_alignment = b->trace.requested_alignment;
+ e.allocated_size = allocated_size;
+
++ e.user_data = b->trace.user_data;
++
+ e.depth = b->trace.depth;
+ static_assert(kMaxStackDepth <= Profile::Sample::kMaxStackDepth,
+ "Profile stack size smaller than internal stack sizes");
+--- contrib/libs/tcmalloc/tcmalloc/static_vars.cc (index)
++++ contrib/libs/tcmalloc/tcmalloc/static_vars.cc (working tree)
+@@ -60,6 +60,12 @@ ABSL_CONST_INIT PageHeapAllocator<StackTraceTable::Bucket>
+ ABSL_CONST_INIT std::atomic<bool> Static::inited_{false};
+ ABSL_CONST_INIT bool Static::cpu_cache_active_ = false;
+ ABSL_CONST_INIT bool Static::fork_support_enabled_ = false;
++ABSL_CONST_INIT Static::CreateSampleUserDataCallback*
++ Static::create_sample_user_data_callback_ = nullptr;
++ABSL_CONST_INIT Static::CopySampleUserDataCallback*
++ Static::copy_sample_user_data_callback_ = nullptr;
++ABSL_CONST_INIT Static::DestroySampleUserDataCallback*
++ Static::destroy_sample_user_data_callback_ = nullptr;
+ ABSL_CONST_INIT Static::PageAllocatorStorage Static::page_allocator_;
+ ABSL_CONST_INIT PageMap Static::pagemap_;
+ ABSL_CONST_INIT absl::base_internal::SpinLock guarded_page_lock(
+--- contrib/libs/tcmalloc/tcmalloc/static_vars.h (index)
++++ contrib/libs/tcmalloc/tcmalloc/static_vars.h (working tree)
+@@ -130,6 +130,34 @@ class Static {
+ static bool ForkSupportEnabled() { return fork_support_enabled_; }
+ static void EnableForkSupport() { fork_support_enabled_ = true; }
+
++ using CreateSampleUserDataCallback = void*();
++ using CopySampleUserDataCallback = void*(void*);
++ using DestroySampleUserDataCallback = void(void*);
++
++ static void SetSampleUserDataCallbacks(
++ CreateSampleUserDataCallback create,
++ CopySampleUserDataCallback copy,
++ DestroySampleUserDataCallback destroy) {
++ create_sample_user_data_callback_ = create;
++ copy_sample_user_data_callback_ = copy;
++ destroy_sample_user_data_callback_ = destroy;
++ }
++
++ static void* CreateSampleUserData() {
++ if (create_sample_user_data_callback_)
++ return create_sample_user_data_callback_();
++ return nullptr;
++ }
++ static void* CopySampleUserData(void* user_data) {
++ if (copy_sample_user_data_callback_)
++ return copy_sample_user_data_callback_(user_data);
++ return nullptr;
++ }
++ static void DestroySampleUserData(void* user_data) {
++ if (destroy_sample_user_data_callback_)
++ destroy_sample_user_data_callback_(user_data);
++ }
++
+ static bool ABSL_ATTRIBUTE_ALWAYS_INLINE IsOnFastPath() {
+ return
+ #ifndef TCMALLOC_DEPRECATED_PERTHREAD
+@@ -176,6 +204,9 @@ class Static {
+ ABSL_CONST_INIT static std::atomic<bool> inited_;
+ static bool cpu_cache_active_;
+ static bool fork_support_enabled_;
++ static CreateSampleUserDataCallback* create_sample_user_data_callback_;
++ static CopySampleUserDataCallback* copy_sample_user_data_callback_;
++ static DestroySampleUserDataCallback* destroy_sample_user_data_callback_;
+ ABSL_CONST_INIT static PeakHeapTracker peak_heap_tracker_;
+ ABSL_CONST_INIT static NumaTopology<kNumaPartitions, kNumBaseClasses>
+ numa_topology_;
+--- contrib/libs/tcmalloc/tcmalloc/tcmalloc.cc (index)
++++ contrib/libs/tcmalloc/tcmalloc/tcmalloc.cc (working tree)
+@@ -1151,6 +1151,13 @@ void TCMallocPostFork() {
+ }
+ }
+
++extern "C" void MallocExtension_SetSampleUserDataCallbacks(
++ MallocExtension::CreateSampleUserDataCallback create,
++ MallocExtension::CopySampleUserDataCallback copy,
++ MallocExtension::DestroySampleUserDataCallback destroy) {
++ Static::SetSampleUserDataCallbacks(create, copy, destroy);
++}
++
+ // nallocx slow path.
+ // Moved to a separate function because size_class_with_alignment is not inlined
+ // which would cause nallocx to become non-leaf function with stack frame and
+@@ -1500,6 +1507,7 @@ static void* SampleifyAllocation(size_t requested_size, size_t weight,
+ tmp.requested_alignment = requested_alignment;
+ tmp.allocated_size = allocated_size;
+ tmp.weight = weight;
++ tmp.user_data = Static::CreateSampleUserData();
+
+ {
+ absl::base_internal::SpinLockHolder h(&pageheap_lock);
+@@ -1629,6 +1637,7 @@ static void do_free_pages(void* ptr, const PageId p) {
+ 1);
+ }
+ notify_sampled_alloc = true;
++ Static::DestroySampleUserData(st->user_data);
+ Static::stacktrace_allocator().Delete(st);
+ }
+ if (IsSampledMemory(ptr)) {
diff --git a/contrib/libs/tcmalloc/tcmalloc/internal/logging.h b/contrib/libs/tcmalloc/tcmalloc/internal/logging.h
index 4d42aa40a9f..2722c2fccc0 100644
--- a/contrib/libs/tcmalloc/tcmalloc/internal/logging.h
+++ b/contrib/libs/tcmalloc/tcmalloc/internal/logging.h
@@ -67,8 +67,8 @@ struct StackTrace {
// between the previous sample and this one
size_t weight;
- void* user_data;
-
+ void* user_data;
+
template <typename H>
friend H AbslHashValue(H h, const StackTrace& t) {
// As we use StackTrace as a key-value node in StackTraceTable, we only
diff --git a/contrib/libs/tcmalloc/tcmalloc/internal_malloc_extension.h b/contrib/libs/tcmalloc/tcmalloc/internal_malloc_extension.h
index 66027418ed2..8654588aa97 100644
--- a/contrib/libs/tcmalloc/tcmalloc/internal_malloc_extension.h
+++ b/contrib/libs/tcmalloc/tcmalloc/internal_malloc_extension.h
@@ -120,12 +120,12 @@ ABSL_ATTRIBUTE_WEAK void MallocExtension_Internal_SetMaxTotalThreadCacheBytes(
ABSL_ATTRIBUTE_WEAK void
MallocExtension_EnableForkSupport();
-ABSL_ATTRIBUTE_WEAK void
-MallocExtension_SetSampleUserDataCallbacks(
- tcmalloc::MallocExtension::CreateSampleUserDataCallback create,
- tcmalloc::MallocExtension::CopySampleUserDataCallback copy,
- tcmalloc::MallocExtension::DestroySampleUserDataCallback destroy);
-
+ABSL_ATTRIBUTE_WEAK void
+MallocExtension_SetSampleUserDataCallbacks(
+ tcmalloc::MallocExtension::CreateSampleUserDataCallback create,
+ tcmalloc::MallocExtension::CopySampleUserDataCallback copy,
+ tcmalloc::MallocExtension::DestroySampleUserDataCallback destroy);
+
}
#endif
diff --git a/contrib/libs/tcmalloc/tcmalloc/malloc_extension.cc b/contrib/libs/tcmalloc/tcmalloc/malloc_extension.cc
index ad3205fcdc4..ae958f5bb7b 100644
--- a/contrib/libs/tcmalloc/tcmalloc/malloc_extension.cc
+++ b/contrib/libs/tcmalloc/tcmalloc/malloc_extension.cc
@@ -468,21 +468,21 @@ void MallocExtension::EnableForkSupport() {
#endif
}
-void MallocExtension::SetSampleUserDataCallbacks(
- CreateSampleUserDataCallback create,
- CopySampleUserDataCallback copy,
- DestroySampleUserDataCallback destroy) {
-#if ABSL_INTERNAL_HAVE_WEAK_MALLOCEXTENSION_STUBS
- if (&MallocExtension_SetSampleUserDataCallbacks != nullptr) {
- MallocExtension_SetSampleUserDataCallbacks(create, copy, destroy);
- }
-#else
- (void)create;
- (void)copy;
- (void)destroy;
-#endif
-}
-
+void MallocExtension::SetSampleUserDataCallbacks(
+ CreateSampleUserDataCallback create,
+ CopySampleUserDataCallback copy,
+ DestroySampleUserDataCallback destroy) {
+#if ABSL_INTERNAL_HAVE_WEAK_MALLOCEXTENSION_STUBS
+ if (&MallocExtension_SetSampleUserDataCallbacks != nullptr) {
+ MallocExtension_SetSampleUserDataCallbacks(create, copy, destroy);
+ }
+#else
+ (void)create;
+ (void)copy;
+ (void)destroy;
+#endif
+}
+
} // namespace tcmalloc
// Default implementation just returns size. The expectation is that
diff --git a/contrib/libs/tcmalloc/tcmalloc/malloc_extension.h b/contrib/libs/tcmalloc/tcmalloc/malloc_extension.h
index fcbd347ca10..77255a8772d 100644
--- a/contrib/libs/tcmalloc/tcmalloc/malloc_extension.h
+++ b/contrib/libs/tcmalloc/tcmalloc/malloc_extension.h
@@ -94,8 +94,8 @@ class Profile final {
int depth;
void* stack[kMaxStackDepth];
-
- void* user_data;
+
+ void* user_data;
};
void Iterate(absl::FunctionRef<void(const Sample&)> f) const;
@@ -474,16 +474,16 @@ class MallocExtension final {
// Enables fork support.
// Allocator will continue to function correctly in the child, after calling fork().
static void EnableForkSupport();
-
- using CreateSampleUserDataCallback = void*();
- using CopySampleUserDataCallback = void*(void*);
- using DestroySampleUserDataCallback = void(void*);
-
- // Sets callbacks for lifetime control of custom user data attached to allocation samples
- static void SetSampleUserDataCallbacks(
- CreateSampleUserDataCallback create,
- CopySampleUserDataCallback copy,
- DestroySampleUserDataCallback destroy);
+
+ using CreateSampleUserDataCallback = void*();
+ using CopySampleUserDataCallback = void*(void*);
+ using DestroySampleUserDataCallback = void(void*);
+
+ // Sets callbacks for lifetime control of custom user data attached to allocation samples
+ static void SetSampleUserDataCallbacks(
+ CreateSampleUserDataCallback create,
+ CopySampleUserDataCallback copy,
+ DestroySampleUserDataCallback destroy);
};
} // namespace tcmalloc
diff --git a/contrib/libs/tcmalloc/tcmalloc/peak_heap_tracker.cc b/contrib/libs/tcmalloc/tcmalloc/peak_heap_tracker.cc
index 0dcc0df5360..7bb756885ba 100644
--- a/contrib/libs/tcmalloc/tcmalloc/peak_heap_tracker.cc
+++ b/contrib/libs/tcmalloc/tcmalloc/peak_heap_tracker.cc
@@ -55,7 +55,7 @@ void PeakHeapTracker::MaybeSaveSample() {
StackTrace *t = peak_sampled_span_stacks_, *next = nullptr;
while (t != nullptr) {
next = reinterpret_cast<StackTrace*>(t->stack[kMaxStackDepth - 1]);
- Static::DestroySampleUserData(t->user_data);
+ Static::DestroySampleUserData(t->user_data);
Static::stacktrace_allocator().Delete(t);
t = next;
}
@@ -64,9 +64,9 @@ void PeakHeapTracker::MaybeSaveSample() {
for (Span* s : Static::sampled_objects_) {
t = Static::stacktrace_allocator().New();
- StackTrace* sampled_stack = s->sampled_stack();
- *t = *sampled_stack;
- t->user_data = Static::CopySampleUserData(sampled_stack->user_data);
+ StackTrace* sampled_stack = s->sampled_stack();
+ *t = *sampled_stack;
+ t->user_data = Static::CopySampleUserData(sampled_stack->user_data);
if (t->depth == kMaxStackDepth) {
t->depth = kMaxStackDepth - 1;
}
diff --git a/contrib/libs/tcmalloc/tcmalloc/stack_trace_table.cc b/contrib/libs/tcmalloc/tcmalloc/stack_trace_table.cc
index 5b5741b6a8a..da3cbc4d8fd 100644
--- a/contrib/libs/tcmalloc/tcmalloc/stack_trace_table.cc
+++ b/contrib/libs/tcmalloc/tcmalloc/stack_trace_table.cc
@@ -73,7 +73,7 @@ StackTraceTable::~StackTraceTable() {
Bucket* b = table_[i];
while (b != nullptr) {
Bucket* next = b->next;
- Static::DestroySampleUserData(b->trace.user_data);
+ Static::DestroySampleUserData(b->trace.user_data);
Static::bucket_allocator().Delete(b);
b = next;
}
@@ -105,7 +105,7 @@ void StackTraceTable::AddTrace(double count, const StackTrace& t) {
b = Static::bucket_allocator().New();
b->hash = h;
b->trace = t;
- b->trace.user_data = Static::CopySampleUserData(t.user_data);
+ b->trace.user_data = Static::CopySampleUserData(t.user_data);
b->count = count;
b->total_weight = t.weight * count;
b->next = table_[idx];
@@ -137,8 +137,8 @@ void StackTraceTable::Iterate(
e.requested_alignment = b->trace.requested_alignment;
e.allocated_size = allocated_size;
- e.user_data = b->trace.user_data;
-
+ e.user_data = b->trace.user_data;
+
e.depth = b->trace.depth;
static_assert(kMaxStackDepth <= Profile::Sample::kMaxStackDepth,
"Profile stack size smaller than internal stack sizes");
diff --git a/contrib/libs/tcmalloc/tcmalloc/static_vars.cc b/contrib/libs/tcmalloc/tcmalloc/static_vars.cc
index 08a70de4935..c517daa3939 100644
--- a/contrib/libs/tcmalloc/tcmalloc/static_vars.cc
+++ b/contrib/libs/tcmalloc/tcmalloc/static_vars.cc
@@ -60,12 +60,12 @@ ABSL_CONST_INIT PageHeapAllocator<StackTraceTable::Bucket>
ABSL_CONST_INIT std::atomic<bool> Static::inited_{false};
ABSL_CONST_INIT bool Static::cpu_cache_active_ = false;
ABSL_CONST_INIT bool Static::fork_support_enabled_ = false;
-ABSL_CONST_INIT Static::CreateSampleUserDataCallback*
- Static::create_sample_user_data_callback_ = nullptr;
-ABSL_CONST_INIT Static::CopySampleUserDataCallback*
- Static::copy_sample_user_data_callback_ = nullptr;
-ABSL_CONST_INIT Static::DestroySampleUserDataCallback*
- Static::destroy_sample_user_data_callback_ = nullptr;
+ABSL_CONST_INIT Static::CreateSampleUserDataCallback*
+ Static::create_sample_user_data_callback_ = nullptr;
+ABSL_CONST_INIT Static::CopySampleUserDataCallback*
+ Static::copy_sample_user_data_callback_ = nullptr;
+ABSL_CONST_INIT Static::DestroySampleUserDataCallback*
+ Static::destroy_sample_user_data_callback_ = nullptr;
ABSL_CONST_INIT Static::PageAllocatorStorage Static::page_allocator_;
ABSL_CONST_INIT PageMap Static::pagemap_;
ABSL_CONST_INIT absl::base_internal::SpinLock guarded_page_lock(
diff --git a/contrib/libs/tcmalloc/tcmalloc/static_vars.h b/contrib/libs/tcmalloc/tcmalloc/static_vars.h
index be68edc1894..9985c3a18c5 100644
--- a/contrib/libs/tcmalloc/tcmalloc/static_vars.h
+++ b/contrib/libs/tcmalloc/tcmalloc/static_vars.h
@@ -130,34 +130,34 @@ class Static {
static bool ForkSupportEnabled() { return fork_support_enabled_; }
static void EnableForkSupport() { fork_support_enabled_ = true; }
- using CreateSampleUserDataCallback = void*();
- using CopySampleUserDataCallback = void*(void*);
- using DestroySampleUserDataCallback = void(void*);
-
- static void SetSampleUserDataCallbacks(
- CreateSampleUserDataCallback create,
- CopySampleUserDataCallback copy,
- DestroySampleUserDataCallback destroy) {
- create_sample_user_data_callback_ = create;
- copy_sample_user_data_callback_ = copy;
- destroy_sample_user_data_callback_ = destroy;
- }
-
- static void* CreateSampleUserData() {
- if (create_sample_user_data_callback_)
- return create_sample_user_data_callback_();
- return nullptr;
- }
- static void* CopySampleUserData(void* user_data) {
- if (copy_sample_user_data_callback_)
- return copy_sample_user_data_callback_(user_data);
- return nullptr;
- }
- static void DestroySampleUserData(void* user_data) {
- if (destroy_sample_user_data_callback_)
- destroy_sample_user_data_callback_(user_data);
- }
-
+ using CreateSampleUserDataCallback = void*();
+ using CopySampleUserDataCallback = void*(void*);
+ using DestroySampleUserDataCallback = void(void*);
+
+ static void SetSampleUserDataCallbacks(
+ CreateSampleUserDataCallback create,
+ CopySampleUserDataCallback copy,
+ DestroySampleUserDataCallback destroy) {
+ create_sample_user_data_callback_ = create;
+ copy_sample_user_data_callback_ = copy;
+ destroy_sample_user_data_callback_ = destroy;
+ }
+
+ static void* CreateSampleUserData() {
+ if (create_sample_user_data_callback_)
+ return create_sample_user_data_callback_();
+ return nullptr;
+ }
+ static void* CopySampleUserData(void* user_data) {
+ if (copy_sample_user_data_callback_)
+ return copy_sample_user_data_callback_(user_data);
+ return nullptr;
+ }
+ static void DestroySampleUserData(void* user_data) {
+ if (destroy_sample_user_data_callback_)
+ destroy_sample_user_data_callback_(user_data);
+ }
+
static bool ABSL_ATTRIBUTE_ALWAYS_INLINE IsOnFastPath() {
return
#ifndef TCMALLOC_DEPRECATED_PERTHREAD
@@ -204,9 +204,9 @@ class Static {
ABSL_CONST_INIT static std::atomic<bool> inited_;
static bool cpu_cache_active_;
static bool fork_support_enabled_;
- static CreateSampleUserDataCallback* create_sample_user_data_callback_;
- static CopySampleUserDataCallback* copy_sample_user_data_callback_;
- static DestroySampleUserDataCallback* destroy_sample_user_data_callback_;
+ static CreateSampleUserDataCallback* create_sample_user_data_callback_;
+ static CopySampleUserDataCallback* copy_sample_user_data_callback_;
+ static DestroySampleUserDataCallback* destroy_sample_user_data_callback_;
ABSL_CONST_INIT static PeakHeapTracker peak_heap_tracker_;
ABSL_CONST_INIT static NumaTopology<kNumaPartitions, kNumBaseClasses>
numa_topology_;
diff --git a/contrib/libs/tcmalloc/tcmalloc/tcmalloc.cc b/contrib/libs/tcmalloc/tcmalloc/tcmalloc.cc
index 8e62ba91b91..8c3a03f6101 100644
--- a/contrib/libs/tcmalloc/tcmalloc/tcmalloc.cc
+++ b/contrib/libs/tcmalloc/tcmalloc/tcmalloc.cc
@@ -1151,13 +1151,13 @@ void TCMallocPostFork() {
}
}
-extern "C" void MallocExtension_SetSampleUserDataCallbacks(
- MallocExtension::CreateSampleUserDataCallback create,
- MallocExtension::CopySampleUserDataCallback copy,
- MallocExtension::DestroySampleUserDataCallback destroy) {
- Static::SetSampleUserDataCallbacks(create, copy, destroy);
-}
-
+extern "C" void MallocExtension_SetSampleUserDataCallbacks(
+ MallocExtension::CreateSampleUserDataCallback create,
+ MallocExtension::CopySampleUserDataCallback copy,
+ MallocExtension::DestroySampleUserDataCallback destroy) {
+ Static::SetSampleUserDataCallbacks(create, copy, destroy);
+}
+
// nallocx slow path.
// Moved to a separate function because size_class_with_alignment is not inlined
// which would cause nallocx to become non-leaf function with stack frame and
@@ -1507,7 +1507,7 @@ static void* SampleifyAllocation(size_t requested_size, size_t weight,
tmp.requested_alignment = requested_alignment;
tmp.allocated_size = allocated_size;
tmp.weight = weight;
- tmp.user_data = Static::CreateSampleUserData();
+ tmp.user_data = Static::CreateSampleUserData();
{
absl::base_internal::SpinLockHolder h(&pageheap_lock);
@@ -1637,7 +1637,7 @@ static void do_free_pages(void* ptr, const PageId p) {
1);
}
notify_sampled_alloc = true;
- Static::DestroySampleUserData(st->user_data);
+ Static::DestroySampleUserData(st->user_data);
Static::stacktrace_allocator().Delete(st);
}
if (IsSampledMemory(ptr)) {
diff --git a/library/cpp/actors/core/README.md b/library/cpp/actors/core/README.md
index 439a8dd459a..3cd4354c6d2 100644
--- a/library/cpp/actors/core/README.md
+++ b/library/cpp/actors/core/README.md
@@ -1,99 +1,99 @@
-## Memory tracker
-
-https://a.yandex-team.ru/arc/trunk/arcadia/library/cpp/actors/core/memory_track.h
-
-Использование:
-
-* отслеживание аллокаций экземпляров конкретного класса через new/delete и new[]/delete[]
-* отслеживание аллокаций в контейнерах
-* ручное отслеживание моментов аллокации/деаллокации
-
-----
-
-### Отслеживание аллокаций класса через new/delete
-
-Использование с автоматически генерируемой меткой:
-
-```cpp
-#include <library/cpp/actors/core/memory_track.h>
-
-struct TTypeLabeled
- : public NActors::NMemory::TTrack<TTypeLabeled>
-{
- char payload[16];
-};
-```
-
-Использование с пользовательским именем метки:
-
-```cpp
-#include <library/cpp/actors/core/memory_track.h>
-
-static const char NamedLabel[] = "NamedLabel";
-
-struct TNameLabeled
- : public NActors::NMemory::TTrack<TNameLabeled, NamedLabel>
-{
- char payload[32];
-};
-```
-
-----
-
-### Отслеживание аллокаций в контейнерах
-
-```cpp
-#include <library/cpp/actors/core/memory_track.h>
-
-static const char InContainerLabel[] = "InContainerLabel";
-
-struct TInContainer {
- char payload[16];
-};
-
-std::vector<TInContainer, NActors::NMemory::TAlloc<TInContainer>> vecT;
-
-std::vector<TInContainer, NActors::NMemory::TAlloc<TInContainer, InContainerLabel>> vecN;
-
-using TKey = int;
-
-std::map<TKey, TInContainer, std::less<TKey>,
- NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>>> mapT;
-
-std::map<TKey, TInContainer, std::less<TKey>,
- NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>, InContainerLabel>> mapN;
-
-std::unordered_map<TKey, TInContainer, std::hash<TKey>, std::equal_to<TKey>,
- NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>>> umapT;
-
-std::unordered_map<TKey, TInContainer, std::hash<TKey>, std::equal_to<TKey>,
- NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>, InContainerLabel>> umapN;
-```
-
-----
-
-### Ручное отслеживание аллокаций/деаллокаций
-
-```cpp
-#include <library/cpp/actors/core/memory_track.h>
-
-static const char ManualLabel[] = "ManualLabel";
-
-...
-NActors::NMemory::TLabel<ManualLabel>::Add(size);
-
-...
-NActors::NMemory::TLabel<ManualLabel>::Sub(size);
-```
-
-----
-
-### Собираемые метрики
-
-Сервис **utils**, пользовательская метка **label**, сенсоры:
-
-- MT/Count: количество аллокаций в моменте
-- MT/Memory: аллоцированная память в моменте
-- MT/PeakCount: пиковое значение количества аллокаций (сэмплится с фиксированной частотой)
-- MT/PeakMemory: пиковое значение аллоцированной памяти
-
+## Memory tracker
+
+https://a.yandex-team.ru/arc/trunk/arcadia/library/cpp/actors/core/memory_track.h
+
+Использование:
+
+* отслеживание аллокаций экземпляров конкретного класса через new/delete и new[]/delete[]
+* отслеживание аллокаций в контейнерах
+* ручное отслеживание моментов аллокации/деаллокации
+
+----
+
+### Отслеживание аллокаций класса через new/delete
+
+Использование с автоматически генерируемой меткой:
+
+```cpp
+#include <library/cpp/actors/core/memory_track.h>
+
+struct TTypeLabeled
+ : public NActors::NMemory::TTrack<TTypeLabeled>
+{
+ char payload[16];
+};
+```
+
+Использование с пользовательским именем метки:
+
+```cpp
+#include <library/cpp/actors/core/memory_track.h>
+
+static const char NamedLabel[] = "NamedLabel";
+
+struct TNameLabeled
+ : public NActors::NMemory::TTrack<TNameLabeled, NamedLabel>
+{
+ char payload[32];
+};
+```
+
+----
+
+### Отслеживание аллокаций в контейнерах
+
+```cpp
+#include <library/cpp/actors/core/memory_track.h>
+
+static const char InContainerLabel[] = "InContainerLabel";
+
+struct TInContainer {
+ char payload[16];
+};
+
+std::vector<TInContainer, NActors::NMemory::TAlloc<TInContainer>> vecT;
+
+std::vector<TInContainer, NActors::NMemory::TAlloc<TInContainer, InContainerLabel>> vecN;
+
+using TKey = int;
+
+std::map<TKey, TInContainer, std::less<TKey>,
+ NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>>> mapT;
+
+std::map<TKey, TInContainer, std::less<TKey>,
+ NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>, InContainerLabel>> mapN;
+
+std::unordered_map<TKey, TInContainer, std::hash<TKey>, std::equal_to<TKey>,
+ NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>>> umapT;
+
+std::unordered_map<TKey, TInContainer, std::hash<TKey>, std::equal_to<TKey>,
+ NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>, InContainerLabel>> umapN;
+```
+
+----
+
+### Ручное отслеживание аллокаций/деаллокаций
+
+```cpp
+#include <library/cpp/actors/core/memory_track.h>
+
+static const char ManualLabel[] = "ManualLabel";
+
+...
+NActors::NMemory::TLabel<ManualLabel>::Add(size);
+
+...
+NActors::NMemory::TLabel<ManualLabel>::Sub(size);
+```
+
+----
+
+### Собираемые метрики
+
+Сервис **utils**, пользовательская метка **label**, сенсоры:
+
+- MT/Count: количество аллокаций в моменте
+- MT/Memory: аллоцированная память в моменте
+- MT/PeakCount: пиковое значение количества аллокаций (сэмплится с фиксированной частотой)
+- MT/PeakMemory: пиковое значение аллоцированной памяти
+
diff --git a/library/cpp/actors/core/actorsystem.h b/library/cpp/actors/core/actorsystem.h
index 40499d7586f..7df0a9f633a 100644
--- a/library/cpp/actors/core/actorsystem.h
+++ b/library/cpp/actors/core/actorsystem.h
@@ -332,8 +332,8 @@ namespace NActors {
TActorId LookupLocalService(const TActorId& x) const;
TActorId RegisterLocalService(const TActorId& serviceId, const TActorId& actorId);
- ui32 GetMaxActivityType() const {
- return SystemSetup ? SystemSetup->MaxActivityType : 1;
+ ui32 GetMaxActivityType() const {
+ return SystemSetup ? SystemSetup->MaxActivityType : 1;
}
TInstant Timestamp() const {
diff --git a/library/cpp/actors/core/event.h b/library/cpp/actors/core/event.h
index 6ff02aaf943..dbdb45daace 100644
--- a/library/cpp/actors/core/event.h
+++ b/library/cpp/actors/core/event.h
@@ -46,8 +46,8 @@ namespace NActors {
};
// fat handle
- class IEventHandle : TNonCopyable {
- struct TOnNondelivery {
+ class IEventHandle : TNonCopyable {
+ struct TOnNondelivery {
TActorId Recipient;
TOnNondelivery(const TActorId& recipient)
diff --git a/library/cpp/actors/core/executor_thread.cpp b/library/cpp/actors/core/executor_thread.cpp
index 446b651efd2..ae5f111dff6 100644
--- a/library/cpp/actors/core/executor_thread.cpp
+++ b/library/cpp/actors/core/executor_thread.cpp
@@ -137,7 +137,7 @@ namespace NActors {
NHPTimer::STime hpprev = hpstart;
IActor* actor = nullptr;
- ui32 prevActivityType = std::numeric_limits<ui32>::max();
+ ui32 prevActivityType = std::numeric_limits<ui32>::max();
TActorId recipient;
for (ui32 executed = 0; executed < Ctx.EventsPerMailbox; ++executed) {
TAutoPtr<IEventHandle> ev(mailbox->Pop());
@@ -170,14 +170,14 @@ namespace NActors {
ui32 evTypeForTracing = ev->Type;
- ui32 activityType = actor->GetActivityType();
- if (activityType != prevActivityType) {
- prevActivityType = activityType;
- NProfiling::TMemoryTagScope::Reset(ActorSystem->MemProfActivityBase + activityType);
+ ui32 activityType = actor->GetActivityType();
+ if (activityType != prevActivityType) {
+ prevActivityType = activityType;
+ NProfiling::TMemoryTagScope::Reset(ActorSystem->MemProfActivityBase + activityType);
}
- actor->Receive(ev, ctx);
-
+ actor->Receive(ev, ctx);
+
size_t dyingActorsCnt = DyingActors.size();
Ctx.UpdateActorsStats(dyingActorsCnt);
if (dyingActorsCnt) {
@@ -273,7 +273,7 @@ namespace NActors {
}
}
- NProfiling::TMemoryTagScope::Reset(0);
+ NProfiling::TMemoryTagScope::Reset(0);
TlsActivationContext = nullptr;
UnlockFromExecution(mailbox, Ctx.Executor, reclaimAsFree, hint, Ctx.WorkerId, RevolvingWriteCounter);
}
diff --git a/library/cpp/actors/core/mailbox.h b/library/cpp/actors/core/mailbox.h
index 0bd9c4d314e..277a259932e 100644
--- a/library/cpp/actors/core/mailbox.h
+++ b/library/cpp/actors/core/mailbox.h
@@ -397,7 +397,7 @@ namespace NActors {
static_assert(sizeof(TRevolvingMailbox) == 128, "expect sizeof(TRevolvingMailbox) == 128");
struct THTSwapMailbox: public TMailboxHeader {
- using TQueueType = NThreading::THTSwapQueue<IEventHandle*>;
+ using TQueueType = NThreading::THTSwapQueue<IEventHandle*>;
TQueueType Queue;
NHPTimer::STime ScheduleMoment;
@@ -447,7 +447,7 @@ namespace NActors {
"expect sizeof(THTSwapMailbox) == 64");
struct TReadAsFilledMailbox: public TMailboxHeader {
- using TQueueType = NThreading::TReadAsFilledQueue<IEventHandle>;
+ using TQueueType = NThreading::TReadAsFilledQueue<IEventHandle>;
TQueueType Queue;
NHPTimer::STime ScheduleMoment;
diff --git a/library/cpp/actors/core/memory_track.cpp b/library/cpp/actors/core/memory_track.cpp
index 5f422116bee..79392ef6863 100644
--- a/library/cpp/actors/core/memory_track.cpp
+++ b/library/cpp/actors/core/memory_track.cpp
@@ -1,38 +1,38 @@
-#include "memory_track.h"
-#include "memory_tracker.h"
-
-namespace NActors {
-namespace NMemory {
-
-namespace NPrivate {
-
-TThreadLocalInfo::TThreadLocalInfo()
- : Metrics(TMemoryTracker::Instance()->GetCount())
-{
- TMemoryTracker::Instance()->OnCreateThread(this);
-}
-
-TThreadLocalInfo::~TThreadLocalInfo() {
- TMemoryTracker::Instance()->OnDestroyThread(this);
-}
-
-TMetric* TThreadLocalInfo::GetMetric(size_t index) {
- if (Y_UNLIKELY(index >= Metrics.size())) {
- return &Null;
- }
- return &Metrics[index];
-}
-
-const std::vector<TMetric>& TThreadLocalInfo::GetMetrics() const {
- return Metrics;
-}
-
-size_t TBaseLabel::RegisterStaticMemoryLabel(const char* name, bool hasSensor) {
- return TMemoryTracker::Instance()->RegisterStaticMemoryLabel(name, hasSensor);
-}
-
-}
-
-}
-}
-
+#include "memory_track.h"
+#include "memory_tracker.h"
+
+namespace NActors {
+namespace NMemory {
+
+namespace NPrivate {
+
+TThreadLocalInfo::TThreadLocalInfo()
+ : Metrics(TMemoryTracker::Instance()->GetCount())
+{
+ TMemoryTracker::Instance()->OnCreateThread(this);
+}
+
+TThreadLocalInfo::~TThreadLocalInfo() {
+ TMemoryTracker::Instance()->OnDestroyThread(this);
+}
+
+TMetric* TThreadLocalInfo::GetMetric(size_t index) {
+ if (Y_UNLIKELY(index >= Metrics.size())) {
+ return &Null;
+ }
+ return &Metrics[index];
+}
+
+const std::vector<TMetric>& TThreadLocalInfo::GetMetrics() const {
+ return Metrics;
+}
+
+size_t TBaseLabel::RegisterStaticMemoryLabel(const char* name, bool hasSensor) {
+ return TMemoryTracker::Instance()->RegisterStaticMemoryLabel(name, hasSensor);
+}
+
+}
+
+}
+}
+
diff --git a/library/cpp/actors/core/memory_track.h b/library/cpp/actors/core/memory_track.h
index 6035333eeb4..97a7c5dfae6 100644
--- a/library/cpp/actors/core/memory_track.h
+++ b/library/cpp/actors/core/memory_track.h
@@ -1,293 +1,293 @@
-#pragma once
-
-#include <vector>
-
+#pragma once
+
+#include <vector>
+
#include <util/system/type_name.h>
-#include <util/thread/singleton.h>
-
-#define ENABLE_MEMORY_TRACKING
-
-namespace NActors {
-namespace NMemory {
-
-namespace NPrivate {
-
-class TMetric {
- std::atomic<ssize_t> Memory;
- std::atomic<ssize_t> Count;
-
- void Copy(const TMetric& other) {
- Memory.store(other.GetMemory(), std::memory_order_relaxed);
- Count.store(other.GetCount(), std::memory_order_relaxed);
- }
-
-public:
- TMetric()
- : Memory(0)
- , Count(0)
- {}
-
- inline TMetric(const TMetric& other) {
- Copy(other);
- }
-
- inline TMetric(TMetric&& other) {
- Copy(other);
- }
-
- inline TMetric& operator=(const TMetric& other) {
- Copy(other);
- return *this;
- }
-
- inline TMetric& operator=(TMetric&& other) {
- Copy(other);
- return *this;
- }
-
- inline ssize_t GetMemory() const {
- return Memory.load(std::memory_order_relaxed);
- }
- inline void SetMemory(ssize_t value) {
- Memory.store(value, std::memory_order_relaxed);
- }
-
- inline ssize_t GetCount() const {
- return Count.load(std::memory_order_relaxed);
- }
- inline void SetCount(ssize_t value) {
- Count.store(value, std::memory_order_relaxed);
- }
-
- inline void operator+=(const TMetric& other) {
- SetMemory(GetMemory() + other.GetMemory());
- SetCount(GetCount() + other.GetCount());
- }
-
- inline void CalculatePeak(const TMetric& other) {
- SetMemory(Max(GetMemory(), other.GetMemory()));
- SetCount(Max(GetCount(), other.GetCount()));
- }
-
- inline void Add(size_t size) {
- SetMemory(GetMemory() + size);
- SetCount(GetCount() + 1);
- }
-
- inline void Sub(size_t size) {
- SetMemory(GetMemory() - size);
- SetCount(GetCount() - 1);
- }
-};
-
-
-class TThreadLocalInfo {
-public:
- TThreadLocalInfo();
- ~TThreadLocalInfo();
-
- TMetric* GetMetric(size_t index);
- const std::vector<TMetric>& GetMetrics() const;
-
-private:
- std::vector<TMetric> Metrics;
-
- inline static TMetric Null = {};
-};
-
-
-class TBaseLabel {
-protected:
- static size_t RegisterStaticMemoryLabel(const char* name, bool hasSensor);
-
- inline static TMetric* GetLocalMetric(size_t index) {
- return FastTlsSingleton<TThreadLocalInfo>()->GetMetric(index);
- }
-};
-
-
-template <const char* Name>
-class TNameLabel
- : TBaseLabel
-{
-public:
- static void Add(size_t size) {
-#if defined(ENABLE_MEMORY_TRACKING)
- Y_UNUSED(MetricInit);
-
- if (Y_UNLIKELY(!Metric)) {
- Metric = GetLocalMetric(Index);
- }
-
- Metric->Add(size);
-#else
- Y_UNUSED(size);
-#endif
- }
-
- static void Sub(size_t size) {
-#if defined(ENABLE_MEMORY_TRACKING)
- Y_UNUSED(MetricInit);
-
- if (Y_UNLIKELY(!Metric)) {
- Metric = GetLocalMetric(Index);
- }
-
- Metric->Sub(size);
-#else
- Y_UNUSED(size);
-#endif
- }
-
-private:
-#if defined(ENABLE_MEMORY_TRACKING)
- inline static size_t Index = Max<size_t>();
- inline static struct TMetricInit {
- TMetricInit() {
- Index = RegisterStaticMemoryLabel(Name, true);
- }
- } MetricInit;
-
- inline static thread_local TMetric* Metric = nullptr;
-#endif
-};
-
-
-template <typename TType>
-class TTypeLabel
- : TBaseLabel
-{
-public:
- static void Add(size_t size) {
-#if defined(ENABLE_MEMORY_TRACKING)
- Y_UNUSED(MetricInit);
-
- if (Y_UNLIKELY(!Metric)) {
- Metric = GetLocalMetric(Index);
- }
-
- Metric->Add(size);
-#else
- Y_UNUSED(size);
-#endif
- }
-
- static void Sub(size_t size) {
-#if defined(ENABLE_MEMORY_TRACKING)
- Y_UNUSED(MetricInit);
-
- if (Y_UNLIKELY(!Metric)) {
- Metric = GetLocalMetric(Index);
- }
-
- Metric->Sub(size);
-#else
- Y_UNUSED(size);
-#endif
- }
-
-private:
-#if defined(ENABLE_MEMORY_TRACKING)
- inline static size_t Index = Max<size_t>();
- inline static struct TMetricInit {
- TMetricInit() {
- Index = RegisterStaticMemoryLabel(TypeName<TType>().c_str(), false);
- }
- } MetricInit;
-
- inline static thread_local TMetric* Metric = nullptr;
-#endif
-};
-
-
-template <typename T>
-struct TTrackHelper {
-#if defined(ENABLE_MEMORY_TRACKING)
- void* operator new(size_t size) {
- T::Add(size);
- return malloc(size);
- }
-
- void* operator new[](size_t size) {
- T::Add(size);
- return malloc(size);
- }
-
- void operator delete(void* ptr, size_t size) {
- T::Sub(size);
- free(ptr);
- }
-
- void operator delete[](void* ptr, size_t size) {
- T::Sub(size);
- free(ptr);
- }
-#endif
-};
-
-template <typename TType, typename T>
-struct TAllocHelper {
- typedef size_t size_type;
- typedef TType value_type;
- typedef TType* pointer;
- typedef const TType* const_pointer;
-
- struct propagate_on_container_copy_assignment : public std::false_type {};
- struct propagate_on_container_move_assignment : public std::false_type {};
- struct propagate_on_container_swap : public std::false_type {};
-
- pointer allocate(size_type n, const void* hint = nullptr) {
- Y_UNUSED(hint);
- auto size = n * sizeof(TType);
- T::Add(size);
- return (pointer)malloc(size);
- }
-
- void deallocate(pointer ptr, size_t n) {
- auto size = n * sizeof(TType);
- T::Sub(size);
- free((void*)ptr);
- }
-};
-
-} // NPrivate
-
-
-template <const char* Name>
-using TLabel = NPrivate::TNameLabel<Name>;
-
-template <typename TType, const char* Name = nullptr>
-struct TTrack
- : public NPrivate::TTrackHelper<NPrivate::TNameLabel<Name>>
-{
-};
-
-template <typename TType>
-struct TTrack<TType, nullptr>
- : public NPrivate::TTrackHelper<NPrivate::TTypeLabel<TType>>
-{
-};
-
-template <typename TType, const char* Name = nullptr>
-struct TAlloc
- : public NPrivate::TAllocHelper<TType, NPrivate::TNameLabel<Name>>
-{
- template<typename U>
- struct rebind {
- typedef TAlloc<U, Name> other;
- };
-};
-
-template <typename TType>
-struct TAlloc<TType, nullptr>
- : public NPrivate::TAllocHelper<TType, NPrivate::TTypeLabel<TType>>
-{
- template<typename U>
- struct rebind {
- typedef TAlloc<U> other;
- };
-};
-
-}
-}
-
+#include <util/thread/singleton.h>
+
+#define ENABLE_MEMORY_TRACKING
+
+namespace NActors {
+namespace NMemory {
+
+namespace NPrivate {
+
+class TMetric {
+ std::atomic<ssize_t> Memory;
+ std::atomic<ssize_t> Count;
+
+ void Copy(const TMetric& other) {
+ Memory.store(other.GetMemory(), std::memory_order_relaxed);
+ Count.store(other.GetCount(), std::memory_order_relaxed);
+ }
+
+public:
+ TMetric()
+ : Memory(0)
+ , Count(0)
+ {}
+
+ inline TMetric(const TMetric& other) {
+ Copy(other);
+ }
+
+ inline TMetric(TMetric&& other) {
+ Copy(other);
+ }
+
+ inline TMetric& operator=(const TMetric& other) {
+ Copy(other);
+ return *this;
+ }
+
+ inline TMetric& operator=(TMetric&& other) {
+ Copy(other);
+ return *this;
+ }
+
+ inline ssize_t GetMemory() const {
+ return Memory.load(std::memory_order_relaxed);
+ }
+ inline void SetMemory(ssize_t value) {
+ Memory.store(value, std::memory_order_relaxed);
+ }
+
+ inline ssize_t GetCount() const {
+ return Count.load(std::memory_order_relaxed);
+ }
+ inline void SetCount(ssize_t value) {
+ Count.store(value, std::memory_order_relaxed);
+ }
+
+ inline void operator+=(const TMetric& other) {
+ SetMemory(GetMemory() + other.GetMemory());
+ SetCount(GetCount() + other.GetCount());
+ }
+
+ inline void CalculatePeak(const TMetric& other) {
+ SetMemory(Max(GetMemory(), other.GetMemory()));
+ SetCount(Max(GetCount(), other.GetCount()));
+ }
+
+ inline void Add(size_t size) {
+ SetMemory(GetMemory() + size);
+ SetCount(GetCount() + 1);
+ }
+
+ inline void Sub(size_t size) {
+ SetMemory(GetMemory() - size);
+ SetCount(GetCount() - 1);
+ }
+};
+
+
+class TThreadLocalInfo {
+public:
+ TThreadLocalInfo();
+ ~TThreadLocalInfo();
+
+ TMetric* GetMetric(size_t index);
+ const std::vector<TMetric>& GetMetrics() const;
+
+private:
+ std::vector<TMetric> Metrics;
+
+ inline static TMetric Null = {};
+};
+
+
+class TBaseLabel {
+protected:
+ static size_t RegisterStaticMemoryLabel(const char* name, bool hasSensor);
+
+ inline static TMetric* GetLocalMetric(size_t index) {
+ return FastTlsSingleton<TThreadLocalInfo>()->GetMetric(index);
+ }
+};
+
+
+template <const char* Name>
+class TNameLabel
+ : TBaseLabel
+{
+public:
+ static void Add(size_t size) {
+#if defined(ENABLE_MEMORY_TRACKING)
+ Y_UNUSED(MetricInit);
+
+ if (Y_UNLIKELY(!Metric)) {
+ Metric = GetLocalMetric(Index);
+ }
+
+ Metric->Add(size);
+#else
+ Y_UNUSED(size);
+#endif
+ }
+
+ static void Sub(size_t size) {
+#if defined(ENABLE_MEMORY_TRACKING)
+ Y_UNUSED(MetricInit);
+
+ if (Y_UNLIKELY(!Metric)) {
+ Metric = GetLocalMetric(Index);
+ }
+
+ Metric->Sub(size);
+#else
+ Y_UNUSED(size);
+#endif
+ }
+
+private:
+#if defined(ENABLE_MEMORY_TRACKING)
+ inline static size_t Index = Max<size_t>();
+ inline static struct TMetricInit {
+ TMetricInit() {
+ Index = RegisterStaticMemoryLabel(Name, true);
+ }
+ } MetricInit;
+
+ inline static thread_local TMetric* Metric = nullptr;
+#endif
+};
+
+
+template <typename TType>
+class TTypeLabel
+ : TBaseLabel
+{
+public:
+ static void Add(size_t size) {
+#if defined(ENABLE_MEMORY_TRACKING)
+ Y_UNUSED(MetricInit);
+
+ if (Y_UNLIKELY(!Metric)) {
+ Metric = GetLocalMetric(Index);
+ }
+
+ Metric->Add(size);
+#else
+ Y_UNUSED(size);
+#endif
+ }
+
+ static void Sub(size_t size) {
+#if defined(ENABLE_MEMORY_TRACKING)
+ Y_UNUSED(MetricInit);
+
+ if (Y_UNLIKELY(!Metric)) {
+ Metric = GetLocalMetric(Index);
+ }
+
+ Metric->Sub(size);
+#else
+ Y_UNUSED(size);
+#endif
+ }
+
+private:
+#if defined(ENABLE_MEMORY_TRACKING)
+ inline static size_t Index = Max<size_t>();
+ inline static struct TMetricInit {
+ TMetricInit() {
+ Index = RegisterStaticMemoryLabel(TypeName<TType>().c_str(), false);
+ }
+ } MetricInit;
+
+ inline static thread_local TMetric* Metric = nullptr;
+#endif
+};
+
+
+template <typename T>
+struct TTrackHelper {
+#if defined(ENABLE_MEMORY_TRACKING)
+ void* operator new(size_t size) {
+ T::Add(size);
+ return malloc(size);
+ }
+
+ void* operator new[](size_t size) {
+ T::Add(size);
+ return malloc(size);
+ }
+
+ void operator delete(void* ptr, size_t size) {
+ T::Sub(size);
+ free(ptr);
+ }
+
+ void operator delete[](void* ptr, size_t size) {
+ T::Sub(size);
+ free(ptr);
+ }
+#endif
+};
+
+template <typename TType, typename T>
+struct TAllocHelper {
+ typedef size_t size_type;
+ typedef TType value_type;
+ typedef TType* pointer;
+ typedef const TType* const_pointer;
+
+ struct propagate_on_container_copy_assignment : public std::false_type {};
+ struct propagate_on_container_move_assignment : public std::false_type {};
+ struct propagate_on_container_swap : public std::false_type {};
+
+ pointer allocate(size_type n, const void* hint = nullptr) {
+ Y_UNUSED(hint);
+ auto size = n * sizeof(TType);
+ T::Add(size);
+ return (pointer)malloc(size);
+ }
+
+ void deallocate(pointer ptr, size_t n) {
+ auto size = n * sizeof(TType);
+ T::Sub(size);
+ free((void*)ptr);
+ }
+};
+
+} // NPrivate
+
+
+template <const char* Name>
+using TLabel = NPrivate::TNameLabel<Name>;
+
+template <typename TType, const char* Name = nullptr>
+struct TTrack
+ : public NPrivate::TTrackHelper<NPrivate::TNameLabel<Name>>
+{
+};
+
+template <typename TType>
+struct TTrack<TType, nullptr>
+ : public NPrivate::TTrackHelper<NPrivate::TTypeLabel<TType>>
+{
+};
+
+template <typename TType, const char* Name = nullptr>
+struct TAlloc
+ : public NPrivate::TAllocHelper<TType, NPrivate::TNameLabel<Name>>
+{
+ template<typename U>
+ struct rebind {
+ typedef TAlloc<U, Name> other;
+ };
+};
+
+template <typename TType>
+struct TAlloc<TType, nullptr>
+ : public NPrivate::TAllocHelper<TType, NPrivate::TTypeLabel<TType>>
+{
+ template<typename U>
+ struct rebind {
+ typedef TAlloc<U> other;
+ };
+};
+
+}
+}
+
diff --git a/library/cpp/actors/core/memory_tracker.cpp b/library/cpp/actors/core/memory_tracker.cpp
index 8a12452c71c..42b93a2868b 100644
--- a/library/cpp/actors/core/memory_tracker.cpp
+++ b/library/cpp/actors/core/memory_tracker.cpp
@@ -1,103 +1,103 @@
-#include "memory_tracker.h"
-
-#include <util/generic/xrange.h>
-
-namespace NActors {
-namespace NMemory {
-
-namespace NPrivate {
-
-TMemoryTracker* TMemoryTracker::Instance() {
- return SingletonWithPriority<TMemoryTracker, 0>();
-}
-
-void TMemoryTracker::Initialize() {
- GlobalMetrics.resize(Indices.size());
-}
-
-const std::map<TString, size_t>& TMemoryTracker::GetMetricIndices() const {
- return Indices;
-}
-
-const std::unordered_set<size_t>& TMemoryTracker::GetSensors() const {
- return Sensors;
-}
-
-TString TMemoryTracker::GetName(size_t index) const {
- return Names[index];
-}
-
-size_t TMemoryTracker::GetCount() const {
- return Indices.size();
-}
-
-void TMemoryTracker::GatherMetrics(std::vector<TMetric>& metrics) const {
- metrics.resize(0);
- auto count = GetCount();
-
- if (!count || GlobalMetrics.size() != count) {
- return;
- }
-
- TReadGuard guard(LockThreadInfo);
-
- metrics.resize(count);
- for (size_t i : xrange(count)) {
- metrics[i] += GlobalMetrics[i];
- }
-
- for (auto info : ThreadInfo) {
- auto& localMetrics = info->GetMetrics();
- if (localMetrics.size() == count) {
- for (size_t i : xrange(count)) {
- metrics[i] += localMetrics[i];
- }
- }
- }
-}
-
-size_t TMemoryTracker::RegisterStaticMemoryLabel(const char* name, bool hasSensor) {
- size_t index = 0;
- auto found = Indices.find(name);
- if (found == Indices.end()) {
- TString str(name);
- auto next = Names.size();
- Indices.emplace(str, next);
- Names.push_back(str);
- index = next;
- } else {
- index = found->second;
- }
-
- if (hasSensor) {
- Sensors.emplace(index);
- }
- return index;
-}
-
-void TMemoryTracker::OnCreateThread(TThreadLocalInfo* info) {
- TWriteGuard guard(LockThreadInfo);
- ThreadInfo.insert(info);
-}
-
-void TMemoryTracker::OnDestroyThread(TThreadLocalInfo* info) {
- TWriteGuard guard(LockThreadInfo);
-
- auto count = GetCount();
- if (count && GlobalMetrics.size() == count) {
- const auto& localMetrics = info->GetMetrics();
- if (localMetrics.size() == count) {
- for (size_t i : xrange(count)) {
- GlobalMetrics[i] += localMetrics[i];
- }
- }
- }
-
- ThreadInfo.erase(info);
-}
-
-}
-
-}
-}
-
+#include "memory_tracker.h"
+
+#include <util/generic/xrange.h>
+
+namespace NActors {
+namespace NMemory {
+
+namespace NPrivate {
+
+TMemoryTracker* TMemoryTracker::Instance() {
+ return SingletonWithPriority<TMemoryTracker, 0>();
+}
+
+void TMemoryTracker::Initialize() {
+ GlobalMetrics.resize(Indices.size());
+}
+
+const std::map<TString, size_t>& TMemoryTracker::GetMetricIndices() const {
+ return Indices;
+}
+
+const std::unordered_set<size_t>& TMemoryTracker::GetSensors() const {
+ return Sensors;
+}
+
+TString TMemoryTracker::GetName(size_t index) const {
+ return Names[index];
+}
+
+size_t TMemoryTracker::GetCount() const {
+ return Indices.size();
+}
+
+void TMemoryTracker::GatherMetrics(std::vector<TMetric>& metrics) const {
+ metrics.resize(0);
+ auto count = GetCount();
+
+ if (!count || GlobalMetrics.size() != count) {
+ return;
+ }
+
+ TReadGuard guard(LockThreadInfo);
+
+ metrics.resize(count);
+ for (size_t i : xrange(count)) {
+ metrics[i] += GlobalMetrics[i];
+ }
+
+ for (auto info : ThreadInfo) {
+ auto& localMetrics = info->GetMetrics();
+ if (localMetrics.size() == count) {
+ for (size_t i : xrange(count)) {
+ metrics[i] += localMetrics[i];
+ }
+ }
+ }
+}
+
+size_t TMemoryTracker::RegisterStaticMemoryLabel(const char* name, bool hasSensor) {
+ size_t index = 0;
+ auto found = Indices.find(name);
+ if (found == Indices.end()) {
+ TString str(name);
+ auto next = Names.size();
+ Indices.emplace(str, next);
+ Names.push_back(str);
+ index = next;
+ } else {
+ index = found->second;
+ }
+
+ if (hasSensor) {
+ Sensors.emplace(index);
+ }
+ return index;
+}
+
+void TMemoryTracker::OnCreateThread(TThreadLocalInfo* info) {
+ TWriteGuard guard(LockThreadInfo);
+ ThreadInfo.insert(info);
+}
+
+void TMemoryTracker::OnDestroyThread(TThreadLocalInfo* info) {
+ TWriteGuard guard(LockThreadInfo);
+
+ auto count = GetCount();
+ if (count && GlobalMetrics.size() == count) {
+ const auto& localMetrics = info->GetMetrics();
+ if (localMetrics.size() == count) {
+ for (size_t i : xrange(count)) {
+ GlobalMetrics[i] += localMetrics[i];
+ }
+ }
+ }
+
+ ThreadInfo.erase(info);
+}
+
+}
+
+}
+}
+
diff --git a/library/cpp/actors/core/memory_tracker.h b/library/cpp/actors/core/memory_tracker.h
index e74508191b1..7b29c673cfc 100644
--- a/library/cpp/actors/core/memory_tracker.h
+++ b/library/cpp/actors/core/memory_tracker.h
@@ -1,53 +1,53 @@
-#pragma once
-
-#include "memory_track.h"
-
-#include <map>
-#include <unordered_map>
-#include <unordered_set>
-
-#include <util/system/rwlock.h>
-
-namespace NActors {
-namespace NMemory {
-
-namespace NPrivate {
-
-class TMemoryTracker {
-public:
- static TMemoryTracker* Instance();
-
- void Initialize();
-
- const std::map<TString, size_t>& GetMetricIndices() const;
- const std::unordered_set<size_t>& GetSensors() const;
- TString GetName(size_t index) const;
- size_t GetCount() const;
-
- void GatherMetrics(std::vector<TMetric>& metrics) const;
-
-private:
- size_t RegisterStaticMemoryLabel(const char* name, bool hasSensor);
-
- void OnCreateThread(TThreadLocalInfo* info);
- void OnDestroyThread(TThreadLocalInfo* info);
-
-private:
- std::map<TString, size_t> Indices;
- std::vector<TString> Names;
-
- std::vector<TMetric> GlobalMetrics;
-
- std::unordered_set<size_t> Sensors;
-
- std::unordered_set<TThreadLocalInfo*> ThreadInfo;
- TRWMutex LockThreadInfo;
-
- friend class TThreadLocalInfo;
- friend class TBaseLabel;
-};
-
-}
-
-}
-}
+#pragma once
+
+#include "memory_track.h"
+
+#include <map>
+#include <unordered_map>
+#include <unordered_set>
+
+#include <util/system/rwlock.h>
+
+namespace NActors {
+namespace NMemory {
+
+namespace NPrivate {
+
+class TMemoryTracker {
+public:
+ static TMemoryTracker* Instance();
+
+ void Initialize();
+
+ const std::map<TString, size_t>& GetMetricIndices() const;
+ const std::unordered_set<size_t>& GetSensors() const;
+ TString GetName(size_t index) const;
+ size_t GetCount() const;
+
+ void GatherMetrics(std::vector<TMetric>& metrics) const;
+
+private:
+ size_t RegisterStaticMemoryLabel(const char* name, bool hasSensor);
+
+ void OnCreateThread(TThreadLocalInfo* info);
+ void OnDestroyThread(TThreadLocalInfo* info);
+
+private:
+ std::map<TString, size_t> Indices;
+ std::vector<TString> Names;
+
+ std::vector<TMetric> GlobalMetrics;
+
+ std::unordered_set<size_t> Sensors;
+
+ std::unordered_set<TThreadLocalInfo*> ThreadInfo;
+ TRWMutex LockThreadInfo;
+
+ friend class TThreadLocalInfo;
+ friend class TBaseLabel;
+};
+
+}
+
+}
+}
diff --git a/library/cpp/actors/core/memory_tracker_ut.cpp b/library/cpp/actors/core/memory_tracker_ut.cpp
index d168214da63..a9d3f502f14 100644
--- a/library/cpp/actors/core/memory_tracker_ut.cpp
+++ b/library/cpp/actors/core/memory_tracker_ut.cpp
@@ -1,262 +1,262 @@
-#include "memory_tracker.h"
-
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <util/system/hp_timer.h>
-#include <util/system/thread.h>
-
-namespace NActors {
-namespace NMemory {
-
-Y_UNIT_TEST_SUITE(TMemoryTrackerTest) {
-
-#if defined(ENABLE_MEMORY_TRACKING)
-
-using namespace NPrivate;
-
-size_t FindLabelIndex(const char* label) {
- auto indices = TMemoryTracker::Instance()->GetMetricIndices();
- auto it = indices.find(label);
- UNIT_ASSERT(it != indices.end());
- return it->second;
-}
-
-
-struct TTypeLabeled
- : public NActors::NMemory::TTrack<TTypeLabeled>
-{
- char payload[16];
-};
-
-static constexpr char NamedLabel[] = "NamedLabel";
-
-struct TNameLabeled
- : public NActors::NMemory::TTrack<TNameLabeled, NamedLabel>
-{
- char payload[32];
-};
-
-Y_UNIT_TEST(Gathering)
-{
- TMemoryTracker::Instance()->Initialize();
-
- auto* typed = new TTypeLabeled;
- auto* typedArray = new TTypeLabeled[3];
-
- auto* named = new TNameLabeled;
- auto* namedArray = new TNameLabeled[5];
- NActors::NMemory::TLabel<NamedLabel>::Add(100);
-
- std::vector<TMetric> metrics;
- TMemoryTracker::Instance()->GatherMetrics(metrics);
-
- auto typeIndex = FindLabelIndex(TypeName<TTypeLabeled>().c_str());
- UNIT_ASSERT(typeIndex < metrics.size());
- UNIT_ASSERT(metrics[typeIndex].GetMemory() == sizeof(TTypeLabeled) * 4 + sizeof(size_t));
- UNIT_ASSERT(metrics[typeIndex].GetCount() == 2);
-
- auto nameIndex = FindLabelIndex(NamedLabel);
- UNIT_ASSERT(nameIndex < metrics.size());
- UNIT_ASSERT(metrics[nameIndex].GetMemory() == sizeof(TNameLabeled) * 6 + sizeof(size_t) + 100);
- UNIT_ASSERT(metrics[nameIndex].GetCount() == 3);
-
- NActors::NMemory::TLabel<NamedLabel>::Sub(100);
- delete [] namedArray;
- delete named;
-
- delete [] typedArray;
- delete typed;
-
- TMemoryTracker::Instance()->GatherMetrics(metrics);
-
- UNIT_ASSERT(metrics[typeIndex].GetMemory() == 0);
- UNIT_ASSERT(metrics[typeIndex].GetCount() == 0);
-
- UNIT_ASSERT(metrics[nameIndex].GetMemory() == 0);
- UNIT_ASSERT(metrics[nameIndex].GetCount() == 0);
-}
-
-
-static constexpr char InContainerLabel[] = "InContainerLabel";
-
-struct TInContainer {
- char payload[16];
-};
-
-Y_UNIT_TEST(Containers) {
- TMemoryTracker::Instance()->Initialize();
-
- std::vector<TInContainer, NActors::NMemory::TAlloc<TInContainer>> vecT;
- vecT.resize(5);
-
- std::vector<TInContainer, NActors::NMemory::TAlloc<TInContainer, InContainerLabel>> vecN;
- vecN.resize(7);
-
- using TKey = int;
-
- std::map<TKey, TInContainer, std::less<TKey>,
- NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>>> mapT;
- mapT.emplace(0, TInContainer());
- mapT.emplace(1, TInContainer());
-
- std::map<TKey, TInContainer, std::less<TKey>,
- NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>, InContainerLabel>> mapN;
- mapN.emplace(0, TInContainer());
-
- std::unordered_map<TKey, TInContainer, std::hash<TKey>, std::equal_to<TKey>,
- NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>>> umapT;
- umapT.emplace(0, TInContainer());
-
- std::unordered_map<TKey, TInContainer, std::hash<TKey>, std::equal_to<TKey>,
- NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>, InContainerLabel>> umapN;
- umapN.emplace(0, TInContainer());
- umapN.emplace(1, TInContainer());
-
- std::vector<TMetric> metrics;
- TMemoryTracker::Instance()->GatherMetrics(metrics);
-
- auto indices = TMemoryTracker::Instance()->GetMetricIndices();
- for (auto& [name, index] : indices) {
- Cerr << "---- " << name
- << ": memory = " << metrics[index].GetMemory()
- << ", count = " << metrics[index].GetCount() << Endl;
- }
-
- auto vecTIndex = FindLabelIndex(TypeName<TInContainer>().c_str());
- UNIT_ASSERT(metrics[vecTIndex].GetMemory() >= ssize_t(sizeof(TInContainer) * 5));
- UNIT_ASSERT(metrics[vecTIndex].GetCount() == 1);
-
- auto labelIndex = FindLabelIndex(InContainerLabel);
- UNIT_ASSERT(metrics[labelIndex].GetCount() == 5);
- UNIT_ASSERT(metrics[labelIndex].GetMemory() >= ssize_t(
- sizeof(TInContainer) * 7 +
- sizeof(decltype(mapN)::value_type) +
- sizeof(decltype(umapN)::value_type) * 2));
-}
-
-
-static constexpr char InThreadLabel[] = "InThreadLabel";
-
-struct TInThread
- : public NActors::NMemory::TTrack<TInThread, InThreadLabel>
-{
- char payload[16];
-};
-
-void* ThreadProc(void*) {
- return new TInThread;
-}
-
-Y_UNIT_TEST(Threads) {
- TMemoryTracker::Instance()->Initialize();
-
- auto index = FindLabelIndex(InThreadLabel);
-
- auto* object1 = new TInThread;
-
- std::vector<TMetric> metrics;
- TMemoryTracker::Instance()->GatherMetrics(metrics);
- UNIT_ASSERT(metrics[index].GetMemory() == sizeof(TInThread));
- UNIT_ASSERT(metrics[index].GetCount() == 1);
-
- TThread thread(&ThreadProc, nullptr);
- thread.Start();
- auto* object2 = static_cast<TInThread*>(thread.Join());
-
- TMemoryTracker::Instance()->GatherMetrics(metrics);
- UNIT_ASSERT(metrics[index].GetMemory() == sizeof(TInThread) * 2);
- UNIT_ASSERT(metrics[index].GetCount() == 2);
-
- delete object2;
-
- TMemoryTracker::Instance()->GatherMetrics(metrics);
- UNIT_ASSERT(metrics[index].GetMemory() == sizeof(TInThread));
- UNIT_ASSERT(metrics[index].GetCount() == 1);
-
- delete object1;
-}
-
-
-struct TNotTracked {
- char payload[16];
-};
-
-struct TTracked
- : public NActors::NMemory::TTrack<TTracked>
-{
- char payload[16];
-};
-
-template <typename T>
-double MeasureAllocations() {
- constexpr size_t objectsCount = 4 << 20;
-
- std::vector<T*> objects;
- objects.resize(objectsCount);
-
- THPTimer timer;
-
- for (size_t i = 0; i < objectsCount; ++i) {
- objects[i] = new T;
- }
-
- for (size_t i = 0; i < objectsCount; ++i) {
- delete objects[i];
- }
-
- auto seconds = timer.Passed();
- Cerr << "---- objects: " << objectsCount << ", time: " << seconds << Endl;
- return seconds;
-}
-
-Y_UNIT_TEST(Performance) {
- TMemoryTracker::Instance()->Initialize();
-
- constexpr size_t Runs = 16;
-
- Cerr << "---- warmup" << Endl;
- MeasureAllocations<TNotTracked>();
- MeasureAllocations<TTracked>();
-
- std::vector<double> noTrack;
- std::vector<double> track;
-
- for (size_t run = 0; run < Runs; ++run) {
- Cerr << "---- no track" << Endl;
- auto time = MeasureAllocations<TNotTracked>();
- noTrack.push_back(time);
-
- Cerr << "---- track" << Endl;
- time = MeasureAllocations<TTracked>();
- track.push_back(time);
- }
-
- double meanNoTrack = 0, stddevNoTrack = 0;
- double meanTrack = 0, stddevTrack = 0;
- for (size_t i = 0; i < Runs; ++i) {
- meanNoTrack += noTrack[i];
- meanTrack += track[i];
- }
- meanNoTrack /= Runs;
- meanTrack /= Runs;
-
- auto sqr = [](double val) { return val * val; };
-
- for (size_t i = 0; i < Runs; ++i) {
- stddevNoTrack += sqr(noTrack[i] - meanNoTrack);
- stddevTrack += sqr(track[i] - meanTrack);
- }
- stddevNoTrack = sqrt(stddevNoTrack / (Runs - 1));
- stddevTrack = sqrt(stddevTrack / (Runs - 1));
-
- Cerr << "---- no track - mean: " << meanNoTrack << ", stddev: " << stddevNoTrack << Endl;
- Cerr << "---- track - mean: " << meanTrack << ", stddev: " << stddevTrack << Endl;
- Cerr << "---- tracking is slower by " << int((meanTrack / meanNoTrack - 1.0) * 100) << "%" << Endl;
-}
-
-#endif
-
-}
-
-}
-}
+#include "memory_tracker.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/system/hp_timer.h>
+#include <util/system/thread.h>
+
+namespace NActors {
+namespace NMemory {
+
+Y_UNIT_TEST_SUITE(TMemoryTrackerTest) {
+
+#if defined(ENABLE_MEMORY_TRACKING)
+
+using namespace NPrivate;
+
+size_t FindLabelIndex(const char* label) {
+ auto indices = TMemoryTracker::Instance()->GetMetricIndices();
+ auto it = indices.find(label);
+ UNIT_ASSERT(it != indices.end());
+ return it->second;
+}
+
+
+struct TTypeLabeled
+ : public NActors::NMemory::TTrack<TTypeLabeled>
+{
+ char payload[16];
+};
+
+static constexpr char NamedLabel[] = "NamedLabel";
+
+struct TNameLabeled
+ : public NActors::NMemory::TTrack<TNameLabeled, NamedLabel>
+{
+ char payload[32];
+};
+
+Y_UNIT_TEST(Gathering)
+{
+ TMemoryTracker::Instance()->Initialize();
+
+ auto* typed = new TTypeLabeled;
+ auto* typedArray = new TTypeLabeled[3];
+
+ auto* named = new TNameLabeled;
+ auto* namedArray = new TNameLabeled[5];
+ NActors::NMemory::TLabel<NamedLabel>::Add(100);
+
+ std::vector<TMetric> metrics;
+ TMemoryTracker::Instance()->GatherMetrics(metrics);
+
+ auto typeIndex = FindLabelIndex(TypeName<TTypeLabeled>().c_str());
+ UNIT_ASSERT(typeIndex < metrics.size());
+ UNIT_ASSERT(metrics[typeIndex].GetMemory() == sizeof(TTypeLabeled) * 4 + sizeof(size_t));
+ UNIT_ASSERT(metrics[typeIndex].GetCount() == 2);
+
+ auto nameIndex = FindLabelIndex(NamedLabel);
+ UNIT_ASSERT(nameIndex < metrics.size());
+ UNIT_ASSERT(metrics[nameIndex].GetMemory() == sizeof(TNameLabeled) * 6 + sizeof(size_t) + 100);
+ UNIT_ASSERT(metrics[nameIndex].GetCount() == 3);
+
+ NActors::NMemory::TLabel<NamedLabel>::Sub(100);
+ delete [] namedArray;
+ delete named;
+
+ delete [] typedArray;
+ delete typed;
+
+ TMemoryTracker::Instance()->GatherMetrics(metrics);
+
+ UNIT_ASSERT(metrics[typeIndex].GetMemory() == 0);
+ UNIT_ASSERT(metrics[typeIndex].GetCount() == 0);
+
+ UNIT_ASSERT(metrics[nameIndex].GetMemory() == 0);
+ UNIT_ASSERT(metrics[nameIndex].GetCount() == 0);
+}
+
+
+static constexpr char InContainerLabel[] = "InContainerLabel";
+
+struct TInContainer {
+ char payload[16];
+};
+
+Y_UNIT_TEST(Containers) {
+ TMemoryTracker::Instance()->Initialize();
+
+ std::vector<TInContainer, NActors::NMemory::TAlloc<TInContainer>> vecT;
+ vecT.resize(5);
+
+ std::vector<TInContainer, NActors::NMemory::TAlloc<TInContainer, InContainerLabel>> vecN;
+ vecN.resize(7);
+
+ using TKey = int;
+
+ std::map<TKey, TInContainer, std::less<TKey>,
+ NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>>> mapT;
+ mapT.emplace(0, TInContainer());
+ mapT.emplace(1, TInContainer());
+
+ std::map<TKey, TInContainer, std::less<TKey>,
+ NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>, InContainerLabel>> mapN;
+ mapN.emplace(0, TInContainer());
+
+ std::unordered_map<TKey, TInContainer, std::hash<TKey>, std::equal_to<TKey>,
+ NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>>> umapT;
+ umapT.emplace(0, TInContainer());
+
+ std::unordered_map<TKey, TInContainer, std::hash<TKey>, std::equal_to<TKey>,
+ NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>, InContainerLabel>> umapN;
+ umapN.emplace(0, TInContainer());
+ umapN.emplace(1, TInContainer());
+
+ std::vector<TMetric> metrics;
+ TMemoryTracker::Instance()->GatherMetrics(metrics);
+
+ auto indices = TMemoryTracker::Instance()->GetMetricIndices();
+ for (auto& [name, index] : indices) {
+ Cerr << "---- " << name
+ << ": memory = " << metrics[index].GetMemory()
+ << ", count = " << metrics[index].GetCount() << Endl;
+ }
+
+ auto vecTIndex = FindLabelIndex(TypeName<TInContainer>().c_str());
+ UNIT_ASSERT(metrics[vecTIndex].GetMemory() >= ssize_t(sizeof(TInContainer) * 5));
+ UNIT_ASSERT(metrics[vecTIndex].GetCount() == 1);
+
+ auto labelIndex = FindLabelIndex(InContainerLabel);
+ UNIT_ASSERT(metrics[labelIndex].GetCount() == 5);
+ UNIT_ASSERT(metrics[labelIndex].GetMemory() >= ssize_t(
+ sizeof(TInContainer) * 7 +
+ sizeof(decltype(mapN)::value_type) +
+ sizeof(decltype(umapN)::value_type) * 2));
+}
+
+
+static constexpr char InThreadLabel[] = "InThreadLabel";
+
+struct TInThread
+ : public NActors::NMemory::TTrack<TInThread, InThreadLabel>
+{
+ char payload[16];
+};
+
+void* ThreadProc(void*) {
+ return new TInThread;
+}
+
+Y_UNIT_TEST(Threads) {
+ TMemoryTracker::Instance()->Initialize();
+
+ auto index = FindLabelIndex(InThreadLabel);
+
+ auto* object1 = new TInThread;
+
+ std::vector<TMetric> metrics;
+ TMemoryTracker::Instance()->GatherMetrics(metrics);
+ UNIT_ASSERT(metrics[index].GetMemory() == sizeof(TInThread));
+ UNIT_ASSERT(metrics[index].GetCount() == 1);
+
+ TThread thread(&ThreadProc, nullptr);
+ thread.Start();
+ auto* object2 = static_cast<TInThread*>(thread.Join());
+
+ TMemoryTracker::Instance()->GatherMetrics(metrics);
+ UNIT_ASSERT(metrics[index].GetMemory() == sizeof(TInThread) * 2);
+ UNIT_ASSERT(metrics[index].GetCount() == 2);
+
+ delete object2;
+
+ TMemoryTracker::Instance()->GatherMetrics(metrics);
+ UNIT_ASSERT(metrics[index].GetMemory() == sizeof(TInThread));
+ UNIT_ASSERT(metrics[index].GetCount() == 1);
+
+ delete object1;
+}
+
+
+struct TNotTracked {
+ char payload[16];
+};
+
+struct TTracked
+ : public NActors::NMemory::TTrack<TTracked>
+{
+ char payload[16];
+};
+
+template <typename T>
+double MeasureAllocations() {
+ constexpr size_t objectsCount = 4 << 20;
+
+ std::vector<T*> objects;
+ objects.resize(objectsCount);
+
+ THPTimer timer;
+
+ for (size_t i = 0; i < objectsCount; ++i) {
+ objects[i] = new T;
+ }
+
+ for (size_t i = 0; i < objectsCount; ++i) {
+ delete objects[i];
+ }
+
+ auto seconds = timer.Passed();
+ Cerr << "---- objects: " << objectsCount << ", time: " << seconds << Endl;
+ return seconds;
+}
+
+Y_UNIT_TEST(Performance) {
+ TMemoryTracker::Instance()->Initialize();
+
+ constexpr size_t Runs = 16;
+
+ Cerr << "---- warmup" << Endl;
+ MeasureAllocations<TNotTracked>();
+ MeasureAllocations<TTracked>();
+
+ std::vector<double> noTrack;
+ std::vector<double> track;
+
+ for (size_t run = 0; run < Runs; ++run) {
+ Cerr << "---- no track" << Endl;
+ auto time = MeasureAllocations<TNotTracked>();
+ noTrack.push_back(time);
+
+ Cerr << "---- track" << Endl;
+ time = MeasureAllocations<TTracked>();
+ track.push_back(time);
+ }
+
+ double meanNoTrack = 0, stddevNoTrack = 0;
+ double meanTrack = 0, stddevTrack = 0;
+ for (size_t i = 0; i < Runs; ++i) {
+ meanNoTrack += noTrack[i];
+ meanTrack += track[i];
+ }
+ meanNoTrack /= Runs;
+ meanTrack /= Runs;
+
+ auto sqr = [](double val) { return val * val; };
+
+ for (size_t i = 0; i < Runs; ++i) {
+ stddevNoTrack += sqr(noTrack[i] - meanNoTrack);
+ stddevTrack += sqr(track[i] - meanTrack);
+ }
+ stddevNoTrack = sqrt(stddevNoTrack / (Runs - 1));
+ stddevTrack = sqrt(stddevTrack / (Runs - 1));
+
+ Cerr << "---- no track - mean: " << meanNoTrack << ", stddev: " << stddevNoTrack << Endl;
+ Cerr << "---- track - mean: " << meanTrack << ", stddev: " << stddevTrack << Endl;
+ Cerr << "---- tracking is slower by " << int((meanTrack / meanNoTrack - 1.0) * 100) << "%" << Endl;
+}
+
+#endif
+
+}
+
+}
+}
diff --git a/library/cpp/actors/core/mon_stats.h b/library/cpp/actors/core/mon_stats.h
index d55552af0cb..e58429ad0a8 100644
--- a/library/cpp/actors/core/mon_stats.h
+++ b/library/cpp/actors/core/mon_stats.h
@@ -2,13 +2,13 @@
#include "defs.h"
#include "actor.h"
-#include <library/cpp/monlib/metrics/histogram_snapshot.h>
+#include <library/cpp/monlib/metrics/histogram_snapshot.h>
#include <util/system/hp_timer.h>
namespace NActors {
- struct TLogHistogram : public NMonitoring::IHistogramSnapshot {
+ struct TLogHistogram : public NMonitoring::IHistogramSnapshot {
TLogHistogram() {
- memset(Buckets, 0, sizeof(Buckets));
+ memset(Buckets, 0, sizeof(Buckets));
}
inline void Add(ui64 val, ui64 inc = 1) {
@@ -17,8 +17,8 @@ namespace NActors {
asm volatile("" ::
: "memory");
#endif
- if (val > 1) {
- ind = GetValueBitCount(val - 1);
+ if (val > 1) {
+ ind = GetValueBitCount(val - 1);
}
#if defined(__clang__) && __clang_major__ == 3 && __clang_minor__ == 7
asm volatile("" ::
@@ -36,25 +36,25 @@ namespace NActors {
}
}
- // IHistogramSnapshot
- ui32 Count() const override {
- return Y_ARRAY_SIZE(Buckets);
- }
-
- NMonitoring::TBucketBound UpperBound(ui32 index) const override {
- Y_ASSERT(index < Y_ARRAY_SIZE(Buckets));
- if (index == 0) {
- return 1;
- }
- return NMonitoring::TBucketBound(1ull << (index - 1)) * 2.0;
- }
-
- NMonitoring::TBucketValue Value(ui32 index) const override {
- Y_ASSERT(index < Y_ARRAY_SIZE(Buckets));
- return Buckets[index];
- }
-
- ui64 TotalSamples = 0;
+ // IHistogramSnapshot
+ ui32 Count() const override {
+ return Y_ARRAY_SIZE(Buckets);
+ }
+
+ NMonitoring::TBucketBound UpperBound(ui32 index) const override {
+ Y_ASSERT(index < Y_ARRAY_SIZE(Buckets));
+ if (index == 0) {
+ return 1;
+ }
+ return NMonitoring::TBucketBound(1ull << (index - 1)) * 2.0;
+ }
+
+ NMonitoring::TBucketValue Value(ui32 index) const override {
+ Y_ASSERT(index < Y_ARRAY_SIZE(Buckets));
+ return Buckets[index];
+ }
+
+ ui64 TotalSamples = 0;
ui64 Buckets[65];
};
diff --git a/library/cpp/actors/core/process_stats.cpp b/library/cpp/actors/core/process_stats.cpp
index 0e1dbd00314..2cc73216b42 100644
--- a/library/cpp/actors/core/process_stats.cpp
+++ b/library/cpp/actors/core/process_stats.cpp
@@ -181,9 +181,9 @@ namespace {
class TDynamicCounterCollector: public TProcStatCollectingActor<TDynamicCounterCollector> {
using TBase = TProcStatCollectingActor<TDynamicCounterCollector>;
public:
- TDynamicCounterCollector(
- ui32 intervalSeconds,
- NMonitoring::TDynamicCounterPtr counters)
+ TDynamicCounterCollector(
+ ui32 intervalSeconds,
+ NMonitoring::TDynamicCounterPtr counters)
: TBase{TDuration::Seconds(intervalSeconds)}
{
ProcStatGroup = counters->GetSubgroup("counters", "utils");
@@ -218,7 +218,7 @@ namespace {
}
private:
- NMonitoring::TDynamicCounterPtr ProcStatGroup;
+ NMonitoring::TDynamicCounterPtr ProcStatGroup;
NMonitoring::TDynamicCounters::TCounterPtr VmSize;
NMonitoring::TDynamicCounters::TCounterPtr AnonRssSize;
NMonitoring::TDynamicCounters::TCounterPtr FileRssSize;
@@ -293,8 +293,8 @@ namespace {
};
} // namespace
- IActor* CreateProcStatCollector(ui32 intervalSec, NMonitoring::TDynamicCounterPtr counters) {
- return new TDynamicCounterCollector(intervalSec, counters);
+ IActor* CreateProcStatCollector(ui32 intervalSec, NMonitoring::TDynamicCounterPtr counters) {
+ return new TDynamicCounterCollector(intervalSec, counters);
}
IActor* CreateProcStatCollector(TDuration interval, NMonitoring::TMetricRegistry& registry) {
diff --git a/library/cpp/actors/core/process_stats.h b/library/cpp/actors/core/process_stats.h
index 66346d0b5aa..236ea829ee5 100644
--- a/library/cpp/actors/core/process_stats.h
+++ b/library/cpp/actors/core/process_stats.h
@@ -61,6 +61,6 @@ namespace NActors {
long ObtainPageSize();
};
- IActor* CreateProcStatCollector(ui32 intervalSec, NMonitoring::TDynamicCounterPtr counters);
+ IActor* CreateProcStatCollector(ui32 intervalSec, NMonitoring::TDynamicCounterPtr counters);
IActor* CreateProcStatCollector(TDuration interval, NMonitoring::TMetricRegistry& registry);
}
diff --git a/library/cpp/actors/core/scheduler_queue.h b/library/cpp/actors/core/scheduler_queue.h
index 3b8fac28f0d..000321e7d32 100644
--- a/library/cpp/actors/core/scheduler_queue.h
+++ b/library/cpp/actors/core/scheduler_queue.h
@@ -13,7 +13,7 @@ namespace NActors {
ISchedulerCookie* Cookie;
};
- struct TChunk : TQueueChunkDerived<TEntry, 512, TChunk> {};
+ struct TChunk : TQueueChunkDerived<TEntry, 512, TChunk> {};
class TReader;
class TWriter;
diff --git a/library/cpp/actors/core/ut/ya.make b/library/cpp/actors/core/ut/ya.make
index 3ee28d58503..fc3195905b7 100644
--- a/library/cpp/actors/core/ut/ya.make
+++ b/library/cpp/actors/core/ut/ya.make
@@ -39,7 +39,7 @@ SRCS(
executor_pool_basic_ut.cpp
executor_pool_united_ut.cpp
log_ut.cpp
- memory_tracker_ut.cpp
+ memory_tracker_ut.cpp
scheduler_actor_ut.cpp
)
diff --git a/library/cpp/actors/core/ya.make b/library/cpp/actors/core/ya.make
index 880a9d00dba..633de9edc1d 100644
--- a/library/cpp/actors/core/ya.make
+++ b/library/cpp/actors/core/ya.make
@@ -75,10 +75,10 @@ SRCS(
mailbox.h
mailbox_queue_revolving.h
mailbox_queue_simple.h
- memory_track.cpp
- memory_track.h
- memory_tracker.cpp
- memory_tracker.h
+ memory_track.cpp
+ memory_track.h
+ memory_tracker.cpp
+ memory_tracker.h
mon.h
mon_stats.h
monotonic.cpp
diff --git a/library/cpp/actors/helpers/mon_histogram_helper.h b/library/cpp/actors/helpers/mon_histogram_helper.h
index a9a57e38238..1da0eec1db9 100644
--- a/library/cpp/actors/helpers/mon_histogram_helper.h
+++ b/library/cpp/actors/helpers/mon_histogram_helper.h
@@ -16,9 +16,9 @@ namespace NActors {
THistogramCounterHelper(const THistogramCounterHelper&) = default;
- void Init(NMonitoring::TDynamicCounters* group, const TString& baseName, const TString& unit,
- ui64 firstBucket, ui64 bucketCnt, bool useSensorLabelName = true)
- {
+ void Init(NMonitoring::TDynamicCounters* group, const TString& baseName, const TString& unit,
+ ui64 firstBucket, ui64 bucketCnt, bool useSensorLabelName = true)
+ {
Y_ASSERT(FirstBucketVal == 0);
Y_ASSERT(BucketCount == 0);
@@ -28,8 +28,8 @@ namespace NActors {
Buckets.reserve(BucketCount);
for (size_t i = 0; i < BucketCount; ++i) {
TString bucketName = GetBucketName(i) + " " + unit;
- auto labelName = useSensorLabelName ? "sensor" : "name";
- BucketsHolder.push_back(group->GetSubgroup(labelName, baseName)->GetNamedCounter("range", bucketName, true));
+ auto labelName = useSensorLabelName ? "sensor" : "name";
+ BucketsHolder.push_back(group->GetSubgroup(labelName, baseName)->GetNamedCounter("range", bucketName, true));
Buckets.push_back(BucketsHolder.back().Get());
}
}
@@ -48,20 +48,20 @@ namespace NActors {
Buckets[ind]->Inc();
}
- ui64 GetBucketCount() const {
- return BucketCount;
- }
-
- ui64 GetBucketValue(size_t index) const {
- Y_ASSERT(index < BucketCount);
- return Buckets[index]->Val();
- }
-
- void SetBucketValue(ui64 index, ui64 value) {
- Y_ASSERT(index < BucketCount);
- *Buckets[index] = value;
- }
-
+ ui64 GetBucketCount() const {
+ return BucketCount;
+ }
+
+ ui64 GetBucketValue(size_t index) const {
+ Y_ASSERT(index < BucketCount);
+ return Buckets[index]->Val();
+ }
+
+ void SetBucketValue(ui64 index, ui64 value) {
+ Y_ASSERT(index < BucketCount);
+ *Buckets[index] = value;
+ }
+
private:
TString GetBucketName(size_t ind) const {
Y_ASSERT(FirstBucketVal != 0);
diff --git a/library/cpp/actors/interconnect/interconnect_counters.cpp b/library/cpp/actors/interconnect/interconnect_counters.cpp
index 224160d4b4f..e4b1412f625 100644
--- a/library/cpp/actors/interconnect/interconnect_counters.cpp
+++ b/library/cpp/actors/interconnect/interconnect_counters.cpp
@@ -278,11 +278,11 @@ namespace {
InflyLimitReach = AdaptiveCounters->GetCounter("InflyLimitReach", true);
InflightDataAmount = AdaptiveCounters->GetCounter("Inflight_Data");
- LegacyPingTimeHist = {};
- LegacyPingTimeHist.Init(AdaptiveCounters.Get(), "PingTimeHist", "mks", 125, 18);
-
- PingTimeHistogram = AdaptiveCounters->GetHistogram(
- "PingTimeUs", NMonitoring::ExponentialHistogram(18, 2, 125));
+ LegacyPingTimeHist = {};
+ LegacyPingTimeHist.Init(AdaptiveCounters.Get(), "PingTimeHist", "mks", 125, 18);
+
+ PingTimeHistogram = AdaptiveCounters->GetHistogram(
+ "PingTimeUs", NMonitoring::ExponentialHistogram(18, 2, 125));
}
if (updateGlobal) {
@@ -299,7 +299,7 @@ namespace {
auto disconnectReasonGroup = Counters->GetSubgroup("subsystem", "disconnectReason");
for (const char *reason : TDisconnectReason::Reasons) {
- DisconnectByReason[reason] = disconnectReasonGroup->GetCounter(reason, true);
+ DisconnectByReason[reason] = disconnectReasonGroup->GetCounter(reason, true);
}
}
@@ -329,9 +329,9 @@ namespace {
NMonitoring::TDynamicCounters::TCounterPtr SpuriousReadWakeups;
NMonitoring::TDynamicCounters::TCounterPtr UsefulWriteWakeups;
NMonitoring::TDynamicCounters::TCounterPtr SpuriousWriteWakeups;
- NMon::THistogramCounterHelper LegacyPingTimeHist;
- NMonitoring::THistogramPtr PingTimeHistogram;
-
+ NMon::THistogramCounterHelper LegacyPingTimeHist;
+ NMonitoring::THistogramPtr PingTimeHistogram;
+
std::unordered_map<ui16, TOutputChannel> OutputChannels;
TOutputChannel OtherOutputChannel;
TInputChannels InputChannels;
diff --git a/library/cpp/actors/interconnect/interconnect_tcp_input_session.cpp b/library/cpp/actors/interconnect/interconnect_tcp_input_session.cpp
index 0abe9fe6598..16f02235024 100644
--- a/library/cpp/actors/interconnect/interconnect_tcp_input_session.cpp
+++ b/library/cpp/actors/interconnect/interconnect_tcp_input_session.cpp
@@ -196,7 +196,7 @@ namespace NActors {
if (AtomicGet(Context->ControlPacketId) <= HeaderConfirm && !NewPingProtocol) {
ui64 sendTime = AtomicGet(Context->ControlPacketSendTimer);
TDuration duration = CyclesToDuration(GetCycleCountFast() - sendTime);
- const auto durationUs = duration.MicroSeconds();
+ const auto durationUs = duration.MicroSeconds();
Metrics->UpdateLegacyPingTimeHist(durationUs);
PingQ.push_back(duration);
if (PingQ.size() > 16) {
@@ -448,8 +448,8 @@ namespace NActors {
PingQ.pop_front();
}
const TDuration ping = *std::min_element(PingQ.begin(), PingQ.end());
- const auto pingUs = ping.MicroSeconds();
- Context->PingRTT_us = pingUs;
+ const auto pingUs = ping.MicroSeconds();
+ Context->PingRTT_us = pingUs;
NewPingProtocol = true;
Metrics->UpdateLegacyPingTimeHist(pingUs);
}
diff --git a/library/cpp/actors/prof/tag.cpp b/library/cpp/actors/prof/tag.cpp
index 9ccf03e1a97..caf516812bd 100644
--- a/library/cpp/actors/prof/tag.cpp
+++ b/library/cpp/actors/prof/tag.cpp
@@ -1,9 +1,9 @@
#include "tag.h"
-#include "tcmalloc.h"
+#include "tcmalloc.h"
#include <library/cpp/charset/ci_string.h>
#include <library/cpp/containers/atomizer/atomizer.h>
-#include <library/cpp/malloc/api/malloc.h>
+#include <library/cpp/malloc/api/malloc.h>
#if defined(PROFILE_MEMORY_ALLOCATIONS)
#include <library/cpp/lfalloc/dbg_info/dbg_info.h>
@@ -76,11 +76,11 @@ namespace NProfiling {
return TStringAtoms::Instance().GetTagsCount();
}
- static ui32 SetThreadAllocTag_Default(ui32 tag) {
- Y_UNUSED(tag);
- return 0;
- }
-
+ static ui32 SetThreadAllocTag_Default(ui32 tag) {
+ Y_UNUSED(tag);
+ return 0;
+ }
+
#if defined(PROFILE_MEMORY_ALLOCATIONS)
static ui32 SetThreadAllocTag_YT(ui32 tag) {
auto prev = NYT::NYTAlloc::GetCurrentMemoryTag();
@@ -96,24 +96,24 @@ namespace NProfiling {
return (TSetThreadAllocTag*)NAllocDbg::SetThreadAllocTag;
} else if (name.StartsWith("yt")) {
return SetThreadAllocTag_YT;
- } else if (name.StartsWith("tc")) {
- return SetTCMallocThreadAllocTag;
- } else {
- return SetThreadAllocTag_Default;
- }
- }
-#else
- static TSetThreadAllocTag* SetThreadAllocTagFn() {
- const auto& info = NMalloc::MallocInfo();
-
- TStringBuf name(info.Name);
- if (name.StartsWith("tc")) {
- return SetTCMallocThreadAllocTag;
+ } else if (name.StartsWith("tc")) {
+ return SetTCMallocThreadAllocTag;
} else {
return SetThreadAllocTag_Default;
}
}
-#endif
-
+#else
+ static TSetThreadAllocTag* SetThreadAllocTagFn() {
+ const auto& info = NMalloc::MallocInfo();
+
+ TStringBuf name(info.Name);
+ if (name.StartsWith("tc")) {
+ return SetTCMallocThreadAllocTag;
+ } else {
+ return SetThreadAllocTag_Default;
+ }
+ }
+#endif
+
TSetThreadAllocTag* SetThreadAllocTag = SetThreadAllocTagFn();
}
diff --git a/library/cpp/actors/prof/tag.h b/library/cpp/actors/prof/tag.h
index 357e264a229..81815b7289e 100644
--- a/library/cpp/actors/prof/tag.h
+++ b/library/cpp/actors/prof/tag.h
@@ -5,7 +5,7 @@
/*
Common registry for tagging memory profiler.
Register a new tag with MakeTag using a unique string.
- Use registered tags with SetThreadAllocTag function in allocator API.
+ Use registered tags with SetThreadAllocTag function in allocator API.
*/
namespace NProfiling {
@@ -22,43 +22,43 @@ namespace NProfiling {
class TMemoryTagScope {
public:
- explicit TMemoryTagScope(ui32 tag)
+ explicit TMemoryTagScope(ui32 tag)
: RestoreTag(SetThreadAllocTag(tag))
{
}
- explicit TMemoryTagScope(const char* tagName) {
+ explicit TMemoryTagScope(const char* tagName) {
ui32 newTag = MakeTag(tagName);
RestoreTag = SetThreadAllocTag(newTag);
}
TMemoryTagScope(TMemoryTagScope&& move)
: RestoreTag(move.RestoreTag)
- , Released(move.Released)
+ , Released(move.Released)
{
- move.Released = true;
+ move.Released = true;
}
TMemoryTagScope& operator=(TMemoryTagScope&& move) {
RestoreTag = move.RestoreTag;
- Released = move.Released;
- move.Released = true;
+ Released = move.Released;
+ move.Released = true;
return *this;
}
- static void Reset(ui32 tag) {
+ static void Reset(ui32 tag) {
SetThreadAllocTag(tag);
}
void Release() {
- if (!Released) {
+ if (!Released) {
SetThreadAllocTag(RestoreTag);
- Released = true;
+ Released = true;
}
}
~TMemoryTagScope() {
- if (!Released) {
+ if (!Released) {
SetThreadAllocTag(RestoreTag);
}
}
@@ -67,7 +67,7 @@ namespace NProfiling {
TMemoryTagScope(const TMemoryTagScope&) = delete;
void operator=(const TMemoryTagScope&) = delete;
- ui32 RestoreTag = 0;
- bool Released = false;
+ ui32 RestoreTag = 0;
+ bool Released = false;
};
}
diff --git a/library/cpp/actors/prof/tcmalloc.cpp b/library/cpp/actors/prof/tcmalloc.cpp
index 3d4f203dbb4..331181e80b7 100644
--- a/library/cpp/actors/prof/tcmalloc.cpp
+++ b/library/cpp/actors/prof/tcmalloc.cpp
@@ -1,32 +1,32 @@
-#include "tcmalloc.h"
-
-#include <contrib/libs/tcmalloc/tcmalloc/malloc_extension.h>
-
-namespace NProfiling {
-
-static thread_local ui32 AllocationTag = 0;
-
-static struct TInitTCMallocCallbacks {
- static void* CreateTag() {
- return reinterpret_cast<void*>(AllocationTag);
- }
- static void* CopyTag(void* tag) {
- return tag;
- }
- static void DestroyTag(void* tag) {
- Y_UNUSED(tag);
- }
-
- TInitTCMallocCallbacks() {
- tcmalloc::MallocExtension::SetSampleUserDataCallbacks(
- CreateTag, CopyTag, DestroyTag);
- }
-} InitTCMallocCallbacks;
-
-ui32 SetTCMallocThreadAllocTag(ui32 tag) {
- ui32 prev = AllocationTag;
- AllocationTag = tag;
- return prev;
-}
-
-}
+#include "tcmalloc.h"
+
+#include <contrib/libs/tcmalloc/tcmalloc/malloc_extension.h>
+
+namespace NProfiling {
+
+static thread_local ui32 AllocationTag = 0;
+
+static struct TInitTCMallocCallbacks {
+ static void* CreateTag() {
+ return reinterpret_cast<void*>(AllocationTag);
+ }
+ static void* CopyTag(void* tag) {
+ return tag;
+ }
+ static void DestroyTag(void* tag) {
+ Y_UNUSED(tag);
+ }
+
+ TInitTCMallocCallbacks() {
+ tcmalloc::MallocExtension::SetSampleUserDataCallbacks(
+ CreateTag, CopyTag, DestroyTag);
+ }
+} InitTCMallocCallbacks;
+
+ui32 SetTCMallocThreadAllocTag(ui32 tag) {
+ ui32 prev = AllocationTag;
+ AllocationTag = tag;
+ return prev;
+}
+
+}
diff --git a/library/cpp/actors/prof/tcmalloc.h b/library/cpp/actors/prof/tcmalloc.h
index 659fb4eaf33..0da31a2024f 100644
--- a/library/cpp/actors/prof/tcmalloc.h
+++ b/library/cpp/actors/prof/tcmalloc.h
@@ -1,9 +1,9 @@
-#pragma once
-
-#include <util/generic/fwd.h>
-
-namespace NProfiling {
-
-ui32 SetTCMallocThreadAllocTag(ui32 tag);
-
-}
+#pragma once
+
+#include <util/generic/fwd.h>
+
+namespace NProfiling {
+
+ui32 SetTCMallocThreadAllocTag(ui32 tag);
+
+}
diff --git a/library/cpp/actors/prof/tcmalloc_null.cpp b/library/cpp/actors/prof/tcmalloc_null.cpp
index 75c00131542..26e6f61bb99 100644
--- a/library/cpp/actors/prof/tcmalloc_null.cpp
+++ b/library/cpp/actors/prof/tcmalloc_null.cpp
@@ -1,10 +1,10 @@
-#include "tcmalloc.h"
-
-namespace NProfiling {
-
-ui32 SetTCMallocThreadAllocTag(ui32 tag) {
- Y_UNUSED(tag);
- return 0;
-}
-
-}
+#include "tcmalloc.h"
+
+namespace NProfiling {
+
+ui32 SetTCMallocThreadAllocTag(ui32 tag) {
+ Y_UNUSED(tag);
+ return 0;
+}
+
+}
diff --git a/library/cpp/actors/prof/ya.make b/library/cpp/actors/prof/ya.make
index b5e24975632..d20ee798c4e 100644
--- a/library/cpp/actors/prof/ya.make
+++ b/library/cpp/actors/prof/ya.make
@@ -23,11 +23,11 @@ IF (PROFILE_MEMORY_ALLOCATIONS)
)
ENDIF()
-IF(ALLOCATOR == "TCMALLOC_256K")
- SRCS(tcmalloc.cpp)
- PEERDIR(contrib/libs/tcmalloc)
-ELSE()
- SRCS(tcmalloc_null.cpp)
-ENDIF()
-
+IF(ALLOCATOR == "TCMALLOC_256K")
+ SRCS(tcmalloc.cpp)
+ PEERDIR(contrib/libs/tcmalloc)
+ELSE()
+ SRCS(tcmalloc_null.cpp)
+ENDIF()
+
END()
diff --git a/library/cpp/binsaver/bin_saver.cpp b/library/cpp/binsaver/bin_saver.cpp
index fe0775af9f2..ca215fa8095 100644
--- a/library/cpp/binsaver/bin_saver.cpp
+++ b/library/cpp/binsaver/bin_saver.cpp
@@ -42,10 +42,10 @@ void IBinSaver::StoreObject(IObjectBase* pObject) {
ObjectQueue.push_back(pObject);
(*Objects)[ptrId];
int typeId = pSaverClasses->GetObjectTypeID(pObject);
- if (typeId == -1) {
- fprintf(stderr, "IBinSaver: trying to save unregistered object\n");
- abort();
- }
+ if (typeId == -1) {
+ fprintf(stderr, "IBinSaver: trying to save unregistered object\n");
+ abort();
+ }
DataChunk(&typeId, sizeof(typeId));
}
}
@@ -64,9 +64,9 @@ IObjectBase* IBinSaver::LoadObject() {
IObjectBase* pObj = pSaverClasses->CreateObject(typeId);
Y_ASSERT(pObj != nullptr);
if (pObj == nullptr) {
- fprintf(stderr, "IBinSaver: trying to load unregistered object\n");
- abort();
- }
+ fprintf(stderr, "IBinSaver: trying to load unregistered object\n");
+ abort();
+ }
(*Objects)[ptrId] = pObj;
ObjectQueue.push_back(pObj);
return pObj;
diff --git a/library/cpp/binsaver/bin_saver.h b/library/cpp/binsaver/bin_saver.h
index 412424889f3..0e9e46a2e53 100644
--- a/library/cpp/binsaver/bin_saver.h
+++ b/library/cpp/binsaver/bin_saver.h
@@ -156,13 +156,13 @@ private:
}
}
- // hash_multimap
+ // hash_multimap
template <class AMM>
void DoAnyMultiMap(AMM& data) {
- if (IsReading()) {
- data.clear();
+ if (IsReading()) {
+ data.clear();
TStoredSize nSize;
- Add(3, &nSize);
+ Add(3, &nSize);
TVector<typename AMM::key_type, typename std::allocator_traits<typename AMM::allocator_type>::template rebind_alloc<typename AMM::key_type>> indices;
indices.resize(nSize);
for (TStoredSize i = 0; i < nSize; ++i)
@@ -170,19 +170,19 @@ private:
for (TStoredSize i = 0; i < nSize; ++i) {
std::pair<typename AMM::key_type, typename AMM::mapped_type> valToInsert;
valToInsert.first = indices[i];
- Add(2, &valToInsert.second);
- data.insert(valToInsert);
- }
- } else {
+ Add(2, &valToInsert.second);
+ data.insert(valToInsert);
+ }
+ } else {
TStoredSize nSize = data.size();
CheckOverflow(nSize, data.size());
- Add(3, &nSize);
+ Add(3, &nSize);
for (auto pos = data.begin(); pos != data.end(); ++pos)
Add(1, (typename AMM::key_type*)(&pos->first));
for (auto pos = data.begin(); pos != data.end(); ++pos)
- Add(2, &pos->second);
- }
- }
+ Add(2, &pos->second);
+ }
+ }
template <class T>
void DoAnySet(T& data) {
@@ -270,7 +270,7 @@ private:
IObjectBase* LoadObject();
bool bRead;
- TBufferedStream<> File;
+ TBufferedStream<> File;
// maps objects addresses during save(first) to addresses during load(second) - during loading
// or serves as a sign that some object has been already stored - during storing
bool StableOutput;
@@ -357,7 +357,7 @@ public:
int Add(const chunk_id, THashMultiMap<T1, T2, T3, T4, T5>* pHash) {
DoAnyMultiMap(*pHash);
return 0;
- }
+ }
template <class K, class L, class A>
int Add(const chunk_id, TSet<K, L, A>* pSet) {
DoAnySet(*pSet);
diff --git a/library/cpp/binsaver/buffered_io.h b/library/cpp/binsaver/buffered_io.h
index 75465c9c5ca..659bc9f62b1 100644
--- a/library/cpp/binsaver/buffered_io.h
+++ b/library/cpp/binsaver/buffered_io.h
@@ -26,7 +26,7 @@ struct IBinaryStream {
}
virtual bool IsValid() const = 0;
- virtual bool IsFailed() const = 0;
+ virtual bool IsFailed() const = 0;
private:
virtual int WriteImpl(const void* userBuffer, int size) = 0;
@@ -36,49 +36,49 @@ private:
i64 LongWrite(const void* userBuffer, i64 size);
};
-template <int N_SIZE = 16384>
+template <int N_SIZE = 16384>
class TBufferedStream {
char Buf[N_SIZE];
i64 Pos, BufSize;
IBinaryStream& Stream;
- bool bIsReading, bIsEof, bFailed;
+ bool bIsReading, bIsEof, bFailed;
void ReadComplex(void* userBuffer, i64 size) {
- if (bIsEof) {
- memset(userBuffer, 0, size);
- return;
- }
+ if (bIsEof) {
+ memset(userBuffer, 0, size);
+ return;
+ }
char* dst = (char*)userBuffer;
i64 leftBytes = BufSize - Pos;
- memcpy(dst, Buf + Pos, leftBytes);
- dst += leftBytes;
- size -= leftBytes;
- Pos = BufSize = 0;
- if (size > N_SIZE) {
+ memcpy(dst, Buf + Pos, leftBytes);
+ dst += leftBytes;
+ size -= leftBytes;
+ Pos = BufSize = 0;
+ if (size > N_SIZE) {
i64 n = Stream.Read(dst, size);
- bFailed = Stream.IsFailed();
- if (n != size) {
- bIsEof = true;
- memset(dst + n, 0, size - n);
- }
- } else {
- BufSize = Stream.Read(Buf, N_SIZE);
- bFailed = Stream.IsFailed();
- if (BufSize == 0)
- bIsEof = true;
- Read(dst, size);
- }
- }
-
+ bFailed = Stream.IsFailed();
+ if (n != size) {
+ bIsEof = true;
+ memset(dst + n, 0, size - n);
+ }
+ } else {
+ BufSize = Stream.Read(Buf, N_SIZE);
+ bFailed = Stream.IsFailed();
+ if (BufSize == 0)
+ bIsEof = true;
+ Read(dst, size);
+ }
+ }
+
void WriteComplex(const void* userBuffer, i64 size) {
- Flush();
- if (size >= N_SIZE) {
- Stream.Write(userBuffer, size);
- bFailed = Stream.IsFailed();
+ Flush();
+ if (size >= N_SIZE) {
+ Stream.Write(userBuffer, size);
+ bFailed = Stream.IsFailed();
} else
- Write(userBuffer, size);
- }
-
+ Write(userBuffer, size);
+ }
+
void operator=(const TBufferedStream&) {
}
@@ -94,16 +94,16 @@ public:
}
~TBufferedStream() {
if (!bIsReading)
- Flush();
+ Flush();
}
- void Flush() {
+ void Flush() {
Y_ASSERT(!bIsReading);
- if (bIsReading)
- return;
- Stream.Write(Buf, Pos);
- bFailed = Stream.IsFailed();
- Pos = 0;
- }
+ if (bIsReading)
+ return;
+ Stream.Write(Buf, Pos);
+ bFailed = Stream.IsFailed();
+ Pos = 0;
+ }
bool IsEof() const {
return bIsEof;
}
diff --git a/library/cpp/binsaver/class_factory.h b/library/cpp/binsaver/class_factory.h
index e83512331ba..4bfb225b289 100644
--- a/library/cpp/binsaver/class_factory.h
+++ b/library/cpp/binsaver/class_factory.h
@@ -15,12 +15,12 @@ public:
typedef const std::type_info* VFT;
private:
- typedef T* (*newFunc)();
+ typedef T* (*newFunc)();
typedef THashMap<int, newFunc> CTypeNewHash; // typeID->newFunc()
typedef THashMap<VFT, int> CTypeIndexHash; // vftable->typeID
- CTypeIndexHash typeIndex;
- CTypeNewHash typeInfo;
+ CTypeIndexHash typeIndex;
+ CTypeNewHash typeInfo;
void RegisterTypeBase(int nTypeID, newFunc func, VFT vft);
static VFT GetObjectType(T* pObject) {
@@ -29,15 +29,15 @@ private:
int VFT2TypeID(VFT t) {
CTypeIndexHash::iterator i = typeIndex.find(t);
if (i != typeIndex.end())
- return i->second;
+ return i->second;
for (i = typeIndex.begin(); i != typeIndex.end(); ++i) {
if (*i->first == *t) {
- typeIndex[t] = i->second;
- return i->second;
- }
- }
- return -1;
- }
+ typeIndex[t] = i->second;
+ return i->second;
+ }
+ }
+ return -1;
+ }
public:
template <class TT>
@@ -45,10 +45,10 @@ public:
RegisterTypeBase(nTypeID, func, &typeid(TT));
}
void RegisterTypeSafe(int nTypeID, newFunc func) {
- TPtr<T> pObj = func();
+ TPtr<T> pObj = func();
VFT vft = GetObjectType(pObj);
RegisterTypeBase(nTypeID, func, vft);
- }
+ }
T* CreateObject(int nTypeID) {
newFunc f = typeInfo[nTypeID];
if (f)
diff --git a/library/cpp/binsaver/mem_io.h b/library/cpp/binsaver/mem_io.h
index 2a9e36fe681..ca5cf0598a7 100644
--- a/library/cpp/binsaver/mem_io.h
+++ b/library/cpp/binsaver/mem_io.h
@@ -6,7 +6,7 @@ namespace NMemIoInternals {
class TMemoryStream: public IBinaryStream {
TVector<char>& Data;
ui64 Pos;
-
+
public:
TMemoryStream(TVector<char>* data, ui64 pos = 0)
: Data(*data)
@@ -15,7 +15,7 @@ namespace NMemIoInternals {
}
~TMemoryStream() override {
} // keep gcc happy
-
+
bool IsValid() const override {
return true;
}
diff --git a/library/cpp/digest/sfh/sfh.h b/library/cpp/digest/sfh/sfh.h
index 372938654cb..65350132011 100644
--- a/library/cpp/digest/sfh/sfh.h
+++ b/library/cpp/digest/sfh/sfh.h
@@ -1,51 +1,51 @@
#pragma once
-
+
#include <util/system/defaults.h>
#include <util/system/unaligned_mem.h>
-
+
inline ui32 SuperFastHash(const void* d, size_t l) noexcept {
ui32 hash = (ui32)l;
ui32 tmp;
if (!l || !d)
return 0;
-
+
TUnalignedMemoryIterator<ui16, 4> iter(d, l);
-
+
while (!iter.AtEnd()) {
hash += (ui32)iter.Next();
tmp = ((ui32)iter.Next() << 11) ^ hash;
hash = (hash << 16) ^ tmp;
hash += hash >> 11;
- }
-
+ }
+
switch (iter.Left()) {
case 3:
hash += (ui32)iter.Next();
- hash ^= hash << 16;
+ hash ^= hash << 16;
hash ^= ((ui32)(i32) * (const i8*)iter.Last()) << 18;
- hash += hash >> 11;
- break;
+ hash += hash >> 11;
+ break;
case 2:
hash += (ui32)iter.Cur();
- hash ^= hash << 11;
- hash += hash >> 17;
- break;
+ hash ^= hash << 11;
+ hash += hash >> 17;
+ break;
case 1:
hash += *((const i8*)iter.Last());
- hash ^= hash << 10;
- hash += hash >> 1;
- }
-
- /* Force "avalanching" of final 127 bits */
- hash ^= hash << 3;
- hash += hash >> 5;
- hash ^= hash << 4;
- hash += hash >> 17;
- hash ^= hash << 25;
- hash += hash >> 6;
-
- return hash;
-}
+ hash ^= hash << 10;
+ hash += hash >> 1;
+ }
+
+ /* Force "avalanching" of final 127 bits */
+ hash ^= hash << 3;
+ hash += hash >> 5;
+ hash ^= hash << 4;
+ hash += hash >> 17;
+ hash ^= hash << 25;
+ hash += hash >> 6;
+
+ return hash;
+}
diff --git a/library/cpp/grpc/server/grpc_counters.h b/library/cpp/grpc/server/grpc_counters.h
index 0b6c36c84cc..29600c7d963 100644
--- a/library/cpp/grpc/server/grpc_counters.h
+++ b/library/cpp/grpc/server/grpc_counters.h
@@ -18,13 +18,13 @@ struct ICounterBlock : public TThrRefBase {
virtual void CountRequestsWithoutDatabase() {}
virtual void CountRequestsWithoutToken() {}
virtual void CountRequestWithoutTls() {}
-
- virtual TIntrusivePtr<ICounterBlock> Clone() { return this; }
- virtual void UseDatabase(const TString& database) { Y_UNUSED(database); }
+
+ virtual TIntrusivePtr<ICounterBlock> Clone() { return this; }
+ virtual void UseDatabase(const TString& database) { Y_UNUSED(database); }
};
-using ICounterBlockPtr = TIntrusivePtr<ICounterBlock>;
-
+using ICounterBlockPtr = TIntrusivePtr<ICounterBlock>;
+
class TCounterBlock final : public ICounterBlock {
NMonitoring::TDynamicCounters::TCounterPtr TotalCounter;
NMonitoring::TDynamicCounters::TCounterPtr InflyCounter;
@@ -113,10 +113,10 @@ public:
}
}
- ICounterBlockPtr Clone() override {
- return this;
- }
-
+ ICounterBlockPtr Clone() override {
+ return this;
+ }
+
void Update() {
if (Percentile) {
RequestHistMs.Update();
diff --git a/library/cpp/grpc/server/grpc_request.h b/library/cpp/grpc/server/grpc_request.h
index 5bd8d3902b5..aeb8d31f299 100644
--- a/library/cpp/grpc/server/grpc_request.h
+++ b/library/cpp/grpc/server/grpc_request.h
@@ -224,10 +224,10 @@ public:
return &Arena_;
}
- void UseDatabase(const TString& database) override {
- Counters_->UseDatabase(database);
- }
-
+ void UseDatabase(const TString& database) override {
+ Counters_->UseDatabase(database);
+ }
+
private:
void Clone() {
if (!Server_->IsShuttingDown()) {
diff --git a/library/cpp/grpc/server/grpc_request_base.h b/library/cpp/grpc/server/grpc_request_base.h
index fcfce1c181a..c6d64ed648b 100644
--- a/library/cpp/grpc/server/grpc_request_base.h
+++ b/library/cpp/grpc/server/grpc_request_base.h
@@ -90,9 +90,9 @@ public:
//! The metadata will be send at the time of rpc finish
virtual void AddTrailingMetadata(const TString& key, const TString& value) = 0;
- //! Use validated database name for counters
- virtual void UseDatabase(const TString& database) = 0;
-
+ //! Use validated database name for counters
+ virtual void UseDatabase(const TString& database) = 0;
+
// Streaming part
//! Set callback. The callback will be called when response deliverid to the client
diff --git a/library/cpp/grpc/server/grpc_server.cpp b/library/cpp/grpc/server/grpc_server.cpp
index 7437b7a8f5e..579ee24f06f 100644
--- a/library/cpp/grpc/server/grpc_server.cpp
+++ b/library/cpp/grpc/server/grpc_server.cpp
@@ -147,7 +147,7 @@ void TGRpcServer::Start() {
Server_ = builder.BuildAndStart();
if (!Server_) {
- ythrow yexception() << "can't start grpc server on " << server_address;
+ ythrow yexception() << "can't start grpc server on " << server_address;
}
size_t index = 0;
diff --git a/library/cpp/monlib/dynamic_counters/contention_ut.cpp b/library/cpp/monlib/dynamic_counters/contention_ut.cpp
index 8798044ee3c..d142397e1a9 100644
--- a/library/cpp/monlib/dynamic_counters/contention_ut.cpp
+++ b/library/cpp/monlib/dynamic_counters/contention_ut.cpp
@@ -33,7 +33,7 @@ Y_UNIT_TEST_SUITE(TDynamicCountersContentionTest) {
Response.Wait();
}
- void OnHistogram(const TString& /*labelName*/, const TString& /*labelValue*/, IHistogramSnapshotPtr /*snapshot*/, bool /*derivative*/) override {
+ void OnHistogram(const TString& /*labelName*/, const TString& /*labelValue*/, IHistogramSnapshotPtr /*snapshot*/, bool /*derivative*/) override {
}
void OnGroupBegin(const TString& /*labelName*/, const TString& /*labelValue*/, const TDynamicCounters* /*group*/) override {
diff --git a/library/cpp/monlib/dynamic_counters/counters.cpp b/library/cpp/monlib/dynamic_counters/counters.cpp
index 3635d87d0d2..977c8ea7b0e 100644
--- a/library/cpp/monlib/dynamic_counters/counters.cpp
+++ b/library/cpp/monlib/dynamic_counters/counters.cpp
@@ -70,38 +70,38 @@ TDynamicCounters::TCounterPtr TDynamicCounters::GetNamedCounter(const TString& n
return AsCounterRef(GetNamedCounterImpl<false, TCounterForPtr>(name, value, derivative, vis));
}
-THistogramPtr TDynamicCounters::GetHistogram(const TString& value, IHistogramCollectorPtr collector, bool derivative, EVisibility vis) {
- return GetNamedHistogram("sensor", value, std::move(collector), derivative, vis);
+THistogramPtr TDynamicCounters::GetHistogram(const TString& value, IHistogramCollectorPtr collector, bool derivative, EVisibility vis) {
+ return GetNamedHistogram("sensor", value, std::move(collector), derivative, vis);
}
-THistogramPtr TDynamicCounters::GetNamedHistogram(const TString& name, const TString& value, IHistogramCollectorPtr collector, bool derivative, EVisibility vis) {
- return AsHistogramRef(GetNamedCounterImpl<false, THistogramCounter>(name, value, std::move(collector), derivative, vis));
+THistogramPtr TDynamicCounters::GetNamedHistogram(const TString& name, const TString& value, IHistogramCollectorPtr collector, bool derivative, EVisibility vis) {
+ return AsHistogramRef(GetNamedCounterImpl<false, THistogramCounter>(name, value, std::move(collector), derivative, vis));
}
-THistogramPtr TDynamicCounters::GetExpiringHistogram(const TString& value, IHistogramCollectorPtr collector, bool derivative, EVisibility vis) {
- return GetExpiringNamedHistogram("sensor", value, std::move(collector), derivative, vis);
+THistogramPtr TDynamicCounters::GetExpiringHistogram(const TString& value, IHistogramCollectorPtr collector, bool derivative, EVisibility vis) {
+ return GetExpiringNamedHistogram("sensor", value, std::move(collector), derivative, vis);
}
-THistogramPtr TDynamicCounters::GetExpiringNamedHistogram(const TString& name, const TString& value, IHistogramCollectorPtr collector, bool derivative, EVisibility vis) {
- return AsHistogramRef(GetNamedCounterImpl<true, TExpiringHistogramCounter>(name, value, std::move(collector), derivative, vis));
-}
-
-TDynamicCounters::TCounterPtr TDynamicCounters::FindCounter(const TString& value) const {
- return FindNamedCounter("sensor", value);
-}
-
-TDynamicCounters::TCounterPtr TDynamicCounters::FindNamedCounter(const TString& name, const TString& value) const {
- return AsCounterRef(FindNamedCounterImpl<TCounterForPtr>(name, value));
-}
-
-THistogramPtr TDynamicCounters::FindHistogram(const TString& value) const {
- return FindNamedHistogram("sensor", value);
-}
-
-THistogramPtr TDynamicCounters::FindNamedHistogram(const TString& name,const TString& value) const {
- return AsHistogramRef(FindNamedCounterImpl<THistogramCounter>(name, value));
+THistogramPtr TDynamicCounters::GetExpiringNamedHistogram(const TString& name, const TString& value, IHistogramCollectorPtr collector, bool derivative, EVisibility vis) {
+ return AsHistogramRef(GetNamedCounterImpl<true, TExpiringHistogramCounter>(name, value, std::move(collector), derivative, vis));
}
+TDynamicCounters::TCounterPtr TDynamicCounters::FindCounter(const TString& value) const {
+ return FindNamedCounter("sensor", value);
+}
+
+TDynamicCounters::TCounterPtr TDynamicCounters::FindNamedCounter(const TString& name, const TString& value) const {
+ return AsCounterRef(FindNamedCounterImpl<TCounterForPtr>(name, value));
+}
+
+THistogramPtr TDynamicCounters::FindHistogram(const TString& value) const {
+ return FindNamedHistogram("sensor", value);
+}
+
+THistogramPtr TDynamicCounters::FindNamedHistogram(const TString& name,const TString& value) const {
+ return AsHistogramRef(FindNamedCounterImpl<THistogramCounter>(name, value));
+}
+
void TDynamicCounters::RemoveCounter(const TString &value) {
RemoveNamedCounter("sensor", value);
}
@@ -298,11 +298,11 @@ TDynamicCounters::TCountablePtr TDynamicCounters::GetNamedCounterImpl(const TStr
}
return it->second;
}
-
-template <class TCounterType>
-TDynamicCounters::TCountablePtr TDynamicCounters::FindNamedCounterImpl(const TString& name, const TString& value) const {
- TReadGuard g(Lock);
- auto it = Counters.find({name, value});
- return it != Counters.end() ? it->second : nullptr;
-}
-
+
+template <class TCounterType>
+TDynamicCounters::TCountablePtr TDynamicCounters::FindNamedCounterImpl(const TString& name, const TString& value) const {
+ TReadGuard g(Lock);
+ auto it = Counters.find({name, value});
+ return it != Counters.end() ? it->second : nullptr;
+}
+
diff --git a/library/cpp/monlib/dynamic_counters/counters.h b/library/cpp/monlib/dynamic_counters/counters.h
index dc178cfbe01..9882001de5f 100644
--- a/library/cpp/monlib/dynamic_counters/counters.h
+++ b/library/cpp/monlib/dynamic_counters/counters.h
@@ -66,7 +66,7 @@ namespace NMonitoring {
virtual void OnHistogram(
const TString& labelName, const TString& labelValue,
- IHistogramSnapshotPtr snapshot, bool derivative) = 0;
+ IHistogramSnapshotPtr snapshot, bool derivative) = 0;
virtual void OnGroupBegin(
const TString& labelName, const TString& labelValue,
@@ -93,8 +93,8 @@ namespace NMonitoring {
Visibility_ = vis;
}
- TCounterForPtr(const TCounterForPtr&) = delete;
- TCounterForPtr& operator=(const TCounterForPtr& other) = delete;
+ TCounterForPtr(const TCounterForPtr&) = delete;
+ TCounterForPtr& operator=(const TCounterForPtr& other) = delete;
void Accept(
const TString& labelName, const TString& labelValue,
@@ -129,10 +129,10 @@ namespace NMonitoring {
};
struct THistogramCounter: public TCountableBase {
- explicit THistogramCounter(
- IHistogramCollectorPtr collector, bool derivative = true, EVisibility vis = EVisibility::Public)
+ explicit THistogramCounter(
+ IHistogramCollectorPtr collector, bool derivative = true, EVisibility vis = EVisibility::Public)
: Collector_(std::move(collector))
- , Derivative_(derivative)
+ , Derivative_(derivative)
{
Visibility_ = vis;
}
@@ -145,34 +145,34 @@ namespace NMonitoring {
Collector_->Collect(value, count);
}
- void Collect(double value, ui32 count) {
- Collector_->Collect(value, count);
- }
-
- void Collect(const IHistogramSnapshot& snapshot) {
- Collector_->Collect(snapshot);
- }
-
+ void Collect(double value, ui32 count) {
+ Collector_->Collect(value, count);
+ }
+
+ void Collect(const IHistogramSnapshot& snapshot) {
+ Collector_->Collect(snapshot);
+ }
+
void Accept(
const TString& labelName, const TString& labelValue,
ICountableConsumer& consumer) const override
{
if (IsVisible(Visibility(), consumer.Visibility())) {
- consumer.OnHistogram(labelName, labelValue, Collector_->Snapshot(), Derivative_);
+ consumer.OnHistogram(labelName, labelValue, Collector_->Snapshot(), Derivative_);
}
}
- void Reset() {
- Collector_->Reset();
- }
-
+ void Reset() {
+ Collector_->Reset();
+ }
+
IHistogramSnapshotPtr Snapshot() const {
return Collector_->Snapshot();
}
private:
IHistogramCollectorPtr Collector_;
- bool Derivative_;
+ bool Derivative_;
};
struct TExpiringHistogramCounter: public THistogramCounter {
@@ -284,14 +284,14 @@ namespace NMonitoring {
THistogramPtr GetHistogram(
const TString& value,
IHistogramCollectorPtr collector,
- bool derivative = true,
+ bool derivative = true,
TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public);
THistogramPtr GetNamedHistogram(
const TString& name,
const TString& value,
IHistogramCollectorPtr collector,
- bool derivative = true,
+ bool derivative = true,
TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public);
// These counters will be automatically removed from the registry
@@ -310,22 +310,22 @@ namespace NMonitoring {
THistogramPtr GetExpiringHistogram(
const TString& value,
IHistogramCollectorPtr collector,
- bool derivative = true,
+ bool derivative = true,
TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public);
THistogramPtr GetExpiringNamedHistogram(
const TString& name,
const TString& value,
IHistogramCollectorPtr collector,
- bool derivative = true,
+ bool derivative = true,
TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public);
- TCounterPtr FindCounter(const TString& value) const;
- TCounterPtr FindNamedCounter(const TString& name, const TString& value) const;
-
- THistogramPtr FindHistogram(const TString& value) const;
- THistogramPtr FindNamedHistogram(const TString& name,const TString& value) const;
-
+ TCounterPtr FindCounter(const TString& value) const;
+ TCounterPtr FindNamedCounter(const TString& name, const TString& value) const;
+
+ THistogramPtr FindHistogram(const TString& value) const;
+ THistogramPtr FindNamedHistogram(const TString& name,const TString& value) const;
+
void RemoveCounter(const TString &value);
void RemoveNamedCounter(const TString& name, const TString &value);
@@ -366,9 +366,9 @@ namespace NMonitoring {
template <bool expiring, class TCounterType, class... TArgs>
TCountablePtr GetNamedCounterImpl(const TString& name, const TString& value, TArgs&&... args);
-
- template <class TCounterType>
- TCountablePtr FindNamedCounterImpl(const TString& name, const TString& value) const;
+
+ template <class TCounterType>
+ TCountablePtr FindNamedCounterImpl(const TString& name, const TString& value) const;
};
}
diff --git a/library/cpp/monlib/dynamic_counters/counters_ut.cpp b/library/cpp/monlib/dynamic_counters/counters_ut.cpp
index 3591037e0a7..04285ae5e3b 100644
--- a/library/cpp/monlib/dynamic_counters/counters_ut.cpp
+++ b/library/cpp/monlib/dynamic_counters/counters_ut.cpp
@@ -23,7 +23,7 @@ private:
void OnHistogram(
const TString& labelName, const TString& labelValue,
- IHistogramSnapshotPtr snapshot, bool /*derivative*/) override {
+ IHistogramSnapshotPtr snapshot, bool /*derivative*/) override {
Indent(Out_, Level_)
<< labelName << ':' << labelValue
<< " = " << *snapshot << '\n';
@@ -311,32 +311,32 @@ Y_UNIT_TEST_SUITE(TDynamicCountersTest) {
UNIT_ASSERT_VALUES_EQUAL(subGroupLookups->Val(), 1);
UNIT_ASSERT_VALUES_EQUAL(lookups->Val(), 5);
}
-
- Y_UNIT_TEST(FindCounters) {
- TDynamicCounterPtr rootGroup(new TDynamicCounters());
-
- auto counter = rootGroup->FindCounter("counter1");
- UNIT_ASSERT(!counter);
- rootGroup->GetCounter("counter1");
- counter = rootGroup->FindCounter("counter1");
- UNIT_ASSERT(counter);
-
- counter = rootGroup->FindNamedCounter("name", "counter2");
- UNIT_ASSERT(!counter);
- rootGroup->GetNamedCounter("name", "counter2");
- counter = rootGroup->FindNamedCounter("name", "counter2");
- UNIT_ASSERT(counter);
-
- auto histogram = rootGroup->FindHistogram("histogram1");
- UNIT_ASSERT(!histogram);
- rootGroup->GetHistogram("histogram1", ExponentialHistogram(4, 2));
- histogram = rootGroup->FindHistogram("histogram1");
- UNIT_ASSERT(histogram);
-
- histogram = rootGroup->FindNamedHistogram("name", "histogram2");
- UNIT_ASSERT(!histogram);
- rootGroup->GetNamedHistogram("name", "histogram2", ExponentialHistogram(4, 2));
- histogram = rootGroup->FindNamedHistogram("name", "histogram2");
- UNIT_ASSERT(histogram);
- }
+
+ Y_UNIT_TEST(FindCounters) {
+ TDynamicCounterPtr rootGroup(new TDynamicCounters());
+
+ auto counter = rootGroup->FindCounter("counter1");
+ UNIT_ASSERT(!counter);
+ rootGroup->GetCounter("counter1");
+ counter = rootGroup->FindCounter("counter1");
+ UNIT_ASSERT(counter);
+
+ counter = rootGroup->FindNamedCounter("name", "counter2");
+ UNIT_ASSERT(!counter);
+ rootGroup->GetNamedCounter("name", "counter2");
+ counter = rootGroup->FindNamedCounter("name", "counter2");
+ UNIT_ASSERT(counter);
+
+ auto histogram = rootGroup->FindHistogram("histogram1");
+ UNIT_ASSERT(!histogram);
+ rootGroup->GetHistogram("histogram1", ExponentialHistogram(4, 2));
+ histogram = rootGroup->FindHistogram("histogram1");
+ UNIT_ASSERT(histogram);
+
+ histogram = rootGroup->FindNamedHistogram("name", "histogram2");
+ UNIT_ASSERT(!histogram);
+ rootGroup->GetNamedHistogram("name", "histogram2", ExponentialHistogram(4, 2));
+ histogram = rootGroup->FindNamedHistogram("name", "histogram2");
+ UNIT_ASSERT(histogram);
+ }
}
diff --git a/library/cpp/monlib/dynamic_counters/encode.cpp b/library/cpp/monlib/dynamic_counters/encode.cpp
index ffa48d276e8..6f5005f71a1 100644
--- a/library/cpp/monlib/dynamic_counters/encode.cpp
+++ b/library/cpp/monlib/dynamic_counters/encode.cpp
@@ -3,7 +3,7 @@
#include <library/cpp/monlib/encode/encoder.h>
#include <library/cpp/monlib/encode/json/json.h>
#include <library/cpp/monlib/encode/spack/spack_v1.h>
-#include <library/cpp/monlib/encode/prometheus/prometheus.h>
+#include <library/cpp/monlib/encode/prometheus/prometheus.h>
#include <util/stream/str.h>
@@ -41,10 +41,10 @@ namespace NMonitoring {
void OnHistogram(
const TString& labelName, const TString& labelValue,
- IHistogramSnapshotPtr snapshot, bool derivative) override {
- NMonitoring::EMetricType metricType = derivative ? EMetricType::HIST_RATE : EMetricType::HIST;
-
- EncoderImpl_->OnMetricBegin(metricType);
+ IHistogramSnapshotPtr snapshot, bool derivative) override {
+ NMonitoring::EMetricType metricType = derivative ? EMetricType::HIST_RATE : EMetricType::HIST;
+
+ EncoderImpl_->OnMetricBegin(metricType);
EncodeLabels(labelName, labelValue);
EncoderImpl_->OnHistogram(ZERO_TIME, snapshot);
EncoderImpl_->OnMetricEnd();
@@ -104,9 +104,9 @@ namespace NMonitoring {
out,
NMonitoring::ETimePrecision::SECONDS,
NMonitoring::ECompression::ZSTD), vis);
- case EFormat::PROMETHEUS:
- return MakeHolder<TConsumer>(NMonitoring::EncoderPrometheus(
- out), vis);
+ case EFormat::PROMETHEUS:
+ return MakeHolder<TConsumer>(NMonitoring::EncoderPrometheus(
+ out), vis);
default:
ythrow yexception() << "unsupported metric encoding format: " << format;
break;
diff --git a/library/cpp/monlib/dynamic_counters/golovan_page.cpp b/library/cpp/monlib/dynamic_counters/golovan_page.cpp
index 49cf2d39bb6..6d76956234b 100644
--- a/library/cpp/monlib/dynamic_counters/golovan_page.cpp
+++ b/library/cpp/monlib/dynamic_counters/golovan_page.cpp
@@ -39,7 +39,7 @@ public:
out << "\"," << counter->Val() << "]";
}
- void OnHistogram(const TString&, const TString&, IHistogramSnapshotPtr, bool) override {
+ void OnHistogram(const TString&, const TString&, IHistogramSnapshotPtr, bool) override {
}
void OnGroupBegin(const TString&, const TString& value, const TDynamicCounters*) override {
diff --git a/library/cpp/monlib/dynamic_counters/page.cpp b/library/cpp/monlib/dynamic_counters/page.cpp
index 5124a47bb33..812ee6b4ca2 100644
--- a/library/cpp/monlib/dynamic_counters/page.cpp
+++ b/library/cpp/monlib/dynamic_counters/page.cpp
@@ -19,8 +19,8 @@ TMaybe<EFormat> ParseFormat(TStringBuf str) {
return EFormat::JSON;
} else if (str == TStringBuf("spack")) {
return EFormat::SPACK;
- } else if (str == TStringBuf("prometheus")) {
- return EFormat::PROMETHEUS;
+ } else if (str == TStringBuf("prometheus")) {
+ return EFormat::PROMETHEUS;
} else {
return Nothing();
}
@@ -84,8 +84,8 @@ void TDynamicCountersPage::Output(NMonitoring::IMonHttpRequest& request) {
out << HTTPOKJSON;
} else if (*format == EFormat::SPACK) {
out << HTTPOKSPACK;
- } else if (*format == EFormat::PROMETHEUS) {
- out << HTTPOKPROMETHEUS;
+ } else if (*format == EFormat::PROMETHEUS) {
+ out << HTTPOKPROMETHEUS;
} else {
ythrow yexception() << "unsupported metric encoding format: " << *format;
}
diff --git a/library/cpp/monlib/dynamic_counters/ya.make b/library/cpp/monlib/dynamic_counters/ya.make
index aafe1c34bed..0b1e97fa4eb 100644
--- a/library/cpp/monlib/dynamic_counters/ya.make
+++ b/library/cpp/monlib/dynamic_counters/ya.make
@@ -18,7 +18,7 @@ PEERDIR(
library/cpp/containers/stack_vector
library/cpp/monlib/encode/json
library/cpp/monlib/encode/spack
- library/cpp/monlib/encode/prometheus
+ library/cpp/monlib/encode/prometheus
library/cpp/monlib/service/pages
library/cpp/string_utils/quote
library/cpp/threading/light_rw_lock
diff --git a/library/cpp/monlib/service/pages/mon_page.h b/library/cpp/monlib/service/pages/mon_page.h
index e396612bb0d..ad7afb867db 100644
--- a/library/cpp/monlib/service/pages/mon_page.h
+++ b/library/cpp/monlib/service/pages/mon_page.h
@@ -12,7 +12,7 @@ namespace NMonitoring {
static const char HTTPOKHTML[] = "HTTP/1.1 200 Ok\r\nContent-Type: text/html\r\nConnection: Close\r\n\r\n";
static const char HTTPOKJSON[] = "HTTP/1.1 200 Ok\r\nContent-Type: application/json\r\nConnection: Close\r\n\r\n";
static const char HTTPOKSPACK[] = "HTTP/1.1 200 Ok\r\nContent-Type: application/x-solomon-spack\r\nConnection: Close\r\n\r\n";
- static const char HTTPOKPROMETHEUS[] = "HTTP/1.1 200 Ok\r\nContent-Type: text/plain\r\nConnection: Close\r\n\r\n";
+ static const char HTTPOKPROMETHEUS[] = "HTTP/1.1 200 Ok\r\nContent-Type: text/plain\r\nConnection: Close\r\n\r\n";
static const char HTTPOKJAVASCRIPT[] = "HTTP/1.1 200 Ok\r\nContent-Type: text/javascript\r\nConnection: Close\r\n\r\n";
static const char HTTPOKCSS[] = "HTTP/1.1 200 Ok\r\nContent-Type: text/css\r\nConnection: Close\r\n\r\n";
static const char HTTPNOCONTENT[] = "HTTP/1.1 204 No content\r\nConnection: Close\r\n\r\n";
diff --git a/library/cpp/yson/consumer.cpp b/library/cpp/yson/consumer.cpp
index 40ae452978c..33e9ee63271 100644
--- a/library/cpp/yson/consumer.cpp
+++ b/library/cpp/yson/consumer.cpp
@@ -1,15 +1,15 @@
-#include "consumer.h"
+#include "consumer.h"
#include "string.h"
-#include "parser.h"
-
+#include "parser.h"
+
namespace NYson {
////////////////////////////////////////////////////////////////////////////////
-
+
void TYsonConsumerBase::OnRaw(TStringBuf str, NYT::NYson::EYsonType type) {
ParseYsonStringBuffer(str, this, type);
}
-
+
////////////////////////////////////////////////////////////////////////////////
} // namespace NYson
diff --git a/library/cpp/yson/consumer.h b/library/cpp/yson/consumer.h
index d5a9d663355..fe1902c8f06 100644
--- a/library/cpp/yson/consumer.h
+++ b/library/cpp/yson/consumer.h
@@ -1,10 +1,10 @@
-#pragma once
-
+#pragma once
+
#include <library/cpp/yt/yson/consumer.h>
-
-#include <util/generic/strbuf.h>
-#include <util/system/defaults.h>
-
+
+#include <util/generic/strbuf.h>
+#include <util/system/defaults.h>
+
namespace NYson {
struct TYsonConsumerBase
: public virtual NYT::NYson::IYsonConsumer {
diff --git a/library/cpp/yson/detail.h b/library/cpp/yson/detail.h
index 27f5e8ffff2..d2a4a6308ef 100644
--- a/library/cpp/yson/detail.h
+++ b/library/cpp/yson/detail.h
@@ -1,47 +1,47 @@
-#pragma once
-
-#include "public.h"
-#include "zigzag.h"
-
-#include <util/generic/vector.h>
-#include <util/generic/maybe.h>
-#include <util/generic/buffer.h>
-#include <util/string/escape.h>
-#include <util/string/cast.h>
-#include <util/stream/input.h>
-
+#pragma once
+
+#include "public.h"
+#include "zigzag.h"
+
+#include <util/generic/vector.h>
+#include <util/generic/maybe.h>
+#include <util/generic/buffer.h>
+#include <util/string/escape.h>
+#include <util/string/cast.h>
+#include <util/stream/input.h>
+
namespace NYson {
namespace NDetail {
////////////////////////////////////////////////////////////////////////////////
-
+
//! Indicates the beginning of a list.
const char BeginListSymbol = '[';
//! Indicates the end of a list.
const char EndListSymbol = ']';
-
+
//! Indicates the beginning of a map.
const char BeginMapSymbol = '{';
//! Indicates the end of a map.
const char EndMapSymbol = '}';
-
+
//! Indicates the beginning of an attribute map.
const char BeginAttributesSymbol = '<';
//! Indicates the end of an attribute map.
const char EndAttributesSymbol = '>';
-
+
//! Separates items in lists.
const char ListItemSeparatorSymbol = ';';
//! Separates items in maps, attributes.
const char KeyedItemSeparatorSymbol = ';';
//! Separates keys from values in maps.
const char KeyValueSeparatorSymbol = '=';
-
+
//! Indicates an entity.
const char EntitySymbol = '#';
-
+
//! Indicates end of stream.
const char EndSymbol = '\0';
-
+
//! Marks the beginning of a binary string literal.
const char StringMarker = '\x01';
//! Marks the beginning of a binary i64 literal.
@@ -53,19 +53,19 @@ namespace NYson {
const char TrueMarker = '\x05';
//! Marks the beginning of a binary ui64 literal.
const char Uint64Marker = '\x06';
-
+
////////////////////////////////////////////////////////////////////////////////
-
+
template <bool EnableLinePositionInfo>
class TPositionInfo;
-
+
template <>
class TPositionInfo<true> {
private:
int Offset;
int Line;
int Column;
-
+
public:
TPositionInfo()
: Offset(0)
@@ -73,7 +73,7 @@ namespace NYson {
, Column(1)
{
}
-
+
void OnRangeConsumed(const char* begin, const char* end) {
Offset += end - begin;
for (auto current = begin; current != end; ++current) {
@@ -85,7 +85,7 @@ namespace NYson {
}
}
};
-
+
template <>
class TPositionInfo<false> {
private:
@@ -95,13 +95,13 @@ namespace NYson {
TPositionInfo()
: Offset(0)
{
- }
-
+ }
+
void OnRangeConsumed(const char* begin, const char* end) {
Offset += end - begin;
}
};
-
+
template <class TBlockStream, class TPositionBase>
class TCharStream
: public TBlockStream,
@@ -111,11 +111,11 @@ namespace NYson {
: TBlockStream(blockStream)
{
}
-
+
bool IsEmpty() const {
return TBlockStream::Begin() == TBlockStream::End();
}
-
+
template <bool AllowFinish>
void Refresh() {
while (IsEmpty() && !TBlockStream::IsFinished()) {
@@ -125,46 +125,46 @@ namespace NYson {
ythrow TYsonException() << "Premature end of yson stream";
}
}
-
+
void Refresh() {
return Refresh<false>();
}
-
+
template <bool AllowFinish>
char GetChar() {
Refresh<AllowFinish>();
return !IsEmpty() ? *TBlockStream::Begin() : '\0';
}
-
+
char GetChar() {
return GetChar<false>();
}
-
+
void Advance(size_t bytes) {
TPositionBase::OnRangeConsumed(TBlockStream::Begin(), TBlockStream::Begin() + bytes);
TBlockStream::Advance(bytes);
}
-
+
size_t Length() const {
return TBlockStream::End() - TBlockStream::Begin();
}
};
-
+
template <class TBaseStream>
class TCodedStream
: public TBaseStream {
private:
static const int MaxVarintBytes = 10;
static const int MaxVarint32Bytes = 5;
-
+
const ui8* BeginByte() const {
return reinterpret_cast<const ui8*>(TBaseStream::Begin());
}
-
+
const ui8* EndByte() const {
return reinterpret_cast<const ui8*>(TBaseStream::End());
}
-
+
// Following functions is an adaptation Protobuf code from coded_stream.cc
bool ReadVarint32FromArray(ui32* value) {
// Fast path: We have enough bytes left in the buffer to guarantee that
@@ -172,7 +172,7 @@ namespace NYson {
const ui8* ptr = BeginByte();
ui32 b;
ui32 result;
-
+
b = *(ptr++);
result = (b & 0x7F);
if (!(b & 0x80))
@@ -193,26 +193,26 @@ namespace NYson {
result |= b << 28;
if (!(b & 0x80))
goto done;
-
+
// If the input is larger than 32 bits, we still need to read it all
// and discard the high-order bits.
-
+
for (int i = 0; i < MaxVarintBytes - MaxVarint32Bytes; i++) {
b = *(ptr++);
if (!(b & 0x80))
goto done;
}
-
+
// We have overrun the maximum size of a Varint (10 bytes). Assume
// the data is corrupt.
return false;
-
+
done:
TBaseStream::Advance(ptr - BeginByte());
*value = result;
return true;
}
-
+
bool ReadVarint32Fallback(ui32* value) {
if (BeginByte() + MaxVarint32Bytes <= EndByte() ||
// Optimization: If the Varint ends at exactly the end of the buffer,
@@ -227,7 +227,7 @@ namespace NYson {
return ReadVarint32Slow(value);
}
}
-
+
bool ReadVarint32Slow(ui32* value) {
ui64 result;
// Directly invoke ReadVarint64Fallback, since we already tried to optimize
@@ -239,15 +239,15 @@ namespace NYson {
return false;
}
}
-
+
bool ReadVarint64Slow(ui64* value) {
// Slow path: This read might cross the end of the buffer, so we
// need to check and refresh the buffer if and when it does.
-
+
ui64 result = 0;
int count = 0;
ui32 b;
-
+
do {
if (count == MaxVarintBytes) {
return false;
@@ -260,11 +260,11 @@ namespace NYson {
TBaseStream::Advance(1);
++count;
} while (b & 0x80);
-
+
*value = result;
return true;
- }
-
+ }
+
bool ReadVarint64Fallback(ui64* value) {
if (BeginByte() + MaxVarintBytes <= EndByte() ||
// Optimization: If the Varint ends at exactly the end of the buffer,
@@ -273,14 +273,14 @@ namespace NYson {
{
// Fast path: We have enough bytes left in the buffer to guarantee that
// this read won't cross the end, so we can skip the checks.
-
+
const ui8* ptr = BeginByte();
ui32 b;
-
+
// Splitting into 32-bit pieces gives better performance on 32-bit
// processors.
ui32 part0 = 0, part1 = 0, part2 = 0;
-
+
b = *(ptr++);
part0 = (b & 0x7F);
if (!(b & 0x80))
@@ -321,11 +321,11 @@ namespace NYson {
part2 |= (b & 0x7F) << 7;
if (!(b & 0x80))
goto done;
-
+
// We have overrun the maximum size of a Varint (10 bytes). The data
// must be corrupt.
return false;
-
+
done:
TBaseStream::Advance(ptr - BeginByte());
*value = (static_cast<ui64>(part0)) |
@@ -336,13 +336,13 @@ namespace NYson {
return ReadVarint64Slow(value);
}
}
-
+
public:
TCodedStream(const TBaseStream& baseStream)
: TBaseStream(baseStream)
{
}
-
+
bool ReadVarint64(ui64* value) {
if (BeginByte() < EndByte() && *BeginByte() < 0x80) {
*value = *BeginByte();
@@ -352,7 +352,7 @@ namespace NYson {
return ReadVarint64Fallback(value);
}
}
-
+
bool ReadVarint32(ui32* value) {
if (BeginByte() < EndByte() && *BeginByte() < 0x80) {
*value = *BeginByte();
@@ -363,13 +363,13 @@ namespace NYson {
}
}
};
-
+
enum ENumericResult {
Int64 = 0,
Uint64 = 1,
Double = 2
};
-
+
template <class TBlockStream, bool EnableLinePositionInfo>
class TLexerBase
: public TCodedStream<TCharStream<TBlockStream, TPositionInfo<EnableLinePositionInfo>>> {
@@ -377,7 +377,7 @@ namespace NYson {
using TBaseStream = TCodedStream<TCharStream<TBlockStream, TPositionInfo<EnableLinePositionInfo>>>;
TVector<char> Buffer_;
TMaybe<ui64> MemoryLimit_;
-
+
void CheckMemoryLimit() {
if (MemoryLimit_ && Buffer_.capacity() > *MemoryLimit_) {
ythrow TYsonException()
@@ -385,17 +385,17 @@ namespace NYson {
<< Buffer_.capacity() << ", limit " << (*MemoryLimit_);
}
}
-
+
public:
TLexerBase(const TBlockStream& blockStream, TMaybe<ui64> memoryLimit)
: TBaseStream(blockStream)
, MemoryLimit_(memoryLimit)
{
}
-
+
protected:
/// Lexer routines
-
+
template <bool AllowFinish>
ENumericResult ReadNumeric(TStringBuf* value) {
Buffer_.clear();
@@ -418,18 +418,18 @@ namespace NYson {
CheckMemoryLimit();
TBaseStream::Advance(1);
}
-
+
*value = TStringBuf(Buffer_.data(), Buffer_.size());
return result;
- }
-
+ }
+
template <bool AllowFinish>
double ReadNanOrInf() {
static const TStringBuf nanString = "nan";
static const TStringBuf infString = "inf";
static const TStringBuf plusInfString = "+inf";
static const TStringBuf minusInfString = "-inf";
-
+
TStringBuf expectedString;
double expectedValue;
char ch = TBaseStream::template GetChar<AllowFinish>();
@@ -515,25 +515,25 @@ namespace NYson {
}
CheckMemoryLimit();
TBaseStream::Advance(1);
- }
+ }
*value = TStringBuf(Buffer_.data(), Buffer_.size());
- }
-
+ }
+
void ReadUnquotedString(TStringBuf* value) {
return ReadUnquotedString<false>(value);
- }
-
+ }
+
void ReadBinaryString(TStringBuf* value) {
ui32 ulength = 0;
if (!TBaseStream::ReadVarint32(&ulength)) {
ythrow TYsonException() << "Error parsing varint value";
}
-
+
i32 length = ZigZagDecode32(ulength);
if (length < 0) {
ythrow TYsonException() << "Negative binary string literal length " << length;
}
-
+
if (TBaseStream::Begin() + length <= TBaseStream::End()) {
*value = TStringBuf(TBaseStream::Begin(), length);
TBaseStream::Advance(length);
@@ -546,29 +546,29 @@ namespace NYson {
continue;
}
size_t readingBytes = Min(needToRead, TBaseStream::Length());
-
+
Buffer_.insert(Buffer_.end(), TBaseStream::Begin(), TBaseStream::Begin() + readingBytes);
CheckMemoryLimit();
needToRead -= readingBytes;
TBaseStream::Advance(readingBytes);
}
*value = TStringBuf(Buffer_.data(), Buffer_.size());
- }
- }
-
+ }
+ }
+
template <bool AllowFinish>
bool ReadBoolean() {
Buffer_.clear();
-
+
static TStringBuf trueString = "true";
static TStringBuf falseString = "false";
-
+
auto throwIncorrectBoolean = [&]() {
ythrow TYsonException() << "Incorrect boolean string " << TString(Buffer_.data(), Buffer_.size());
};
-
- Buffer_.push_back(TBaseStream::template GetChar<AllowFinish>());
- TBaseStream::Advance(1);
+
+ Buffer_.push_back(TBaseStream::template GetChar<AllowFinish>());
+ TBaseStream::Advance(1);
if (Buffer_[0] == trueString[0]) {
for (size_t i = 1; i < trueString.size(); ++i) {
Buffer_.push_back(TBaseStream::template GetChar<AllowFinish>());
@@ -588,21 +588,21 @@ namespace NYson {
}
return false;
} else {
- throwIncorrectBoolean();
- }
+ throwIncorrectBoolean();
+ }
Y_FAIL("unreachable");
;
- }
+ }
void ReadBinaryInt64(i64* result) {
ui64 uvalue;
if (!TBaseStream::ReadVarint64(&uvalue)) {
ythrow TYsonException() << "Error parsing varint value";
- }
+ }
*result = ZigZagDecode64(uvalue);
- }
-
+ }
+
void ReadBinaryUint64(ui64* result) {
ui64 uvalue;
if (!TBaseStream::ReadVarint64(&uvalue)) {
@@ -610,16 +610,16 @@ namespace NYson {
}
*result = uvalue;
}
-
+
void ReadBinaryDouble(double* value) {
size_t needToRead = sizeof(double);
-
+
while (needToRead != 0) {
if (TBaseStream::IsEmpty()) {
TBaseStream::Refresh();
continue;
}
-
+
size_t chunkSize = Min(needToRead, TBaseStream::Length());
if (chunkSize == 0) {
ythrow TYsonException() << "Error parsing binary double literal";
@@ -631,8 +631,8 @@ namespace NYson {
needToRead -= chunkSize;
TBaseStream::Advance(chunkSize);
}
- }
-
+ }
+
/// Helpers
void SkipCharToken(char symbol) {
char ch = SkipSpaceAndGetChar();
@@ -641,8 +641,8 @@ namespace NYson {
}
TBaseStream::Advance(1);
- }
-
+ }
+
static bool IsSpaceFast(char ch) {
static const ui8 lookupTable[] =
{
@@ -650,24 +650,24 @@ namespace NYson {
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-
+
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-
+
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-
+
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
return lookupTable[static_cast<ui8>(ch)];
}
-
+
template <bool AllowFinish>
char SkipSpaceAndGetChar() {
if (!TBaseStream::IsEmpty()) {
@@ -678,11 +678,11 @@ namespace NYson {
}
return SkipSpaceAndGetCharFallback<AllowFinish>();
}
-
+
char SkipSpaceAndGetChar() {
return SkipSpaceAndGetChar<false>();
- }
-
+ }
+
template <bool AllowFinish>
char SkipSpaceAndGetCharFallback() {
while (true) {
@@ -697,63 +697,63 @@ namespace NYson {
break;
}
TBaseStream::Advance(1);
- }
+ }
return TBaseStream::template GetChar<AllowFinish>();
- }
+ }
};
-
+
////////////////////////////////////////////////////////////////////////////////
-
+
}
-
+
////////////////////////////////////////////////////////////////////////////////
-
+
class TStringReader {
private:
const char* BeginPtr;
const char* EndPtr;
-
+
public:
TStringReader()
: BeginPtr(nullptr)
, EndPtr(nullptr)
{
}
-
+
TStringReader(const char* begin, const char* end)
: BeginPtr(begin)
, EndPtr(end)
{
}
-
+
const char* Begin() const {
return BeginPtr;
}
-
+
const char* End() const {
return EndPtr;
}
-
+
void RefreshBlock() {
Y_FAIL("unreachable");
}
-
+
void Advance(size_t bytes) {
BeginPtr += bytes;
}
-
+
bool IsFinished() const {
return true;
}
-
+
void SetBuffer(const char* begin, const char* end) {
BeginPtr = begin;
EndPtr = end;
}
};
-
+
////////////////////////////////////////////////////////////////////////////////
-
+
class TStreamReader {
public:
TStreamReader(
@@ -767,40 +767,40 @@ namespace NYson {
BeginPtr = EndPtr = Buffer;
FinishFlag = false;
}
-
+
const char* Begin() const {
return BeginPtr;
}
-
+
const char* End() const {
return EndPtr;
}
-
+
void RefreshBlock() {
size_t bytes = Stream->Read(Buffer, BufferSize);
BeginPtr = Buffer;
EndPtr = Buffer + bytes;
FinishFlag = (bytes == 0);
}
-
+
void Advance(size_t bytes) {
BeginPtr += bytes;
}
-
+
bool IsFinished() const {
return FinishFlag;
}
-
+
private:
IInputStream* Stream;
char* Buffer;
size_t BufferSize;
-
+
const char* BeginPtr;
const char* EndPtr;
bool FinishFlag;
};
-
+
////////////////////////////////////////////////////////////////////////////////
-
+
} // namespace NYson
diff --git a/library/cpp/yson/format.h b/library/cpp/yson/format.h
index 2ff6dc9f6e2..c2bad91a6a6 100644
--- a/library/cpp/yson/format.h
+++ b/library/cpp/yson/format.h
@@ -1,25 +1,25 @@
-#pragma once
-
-#include "token.h"
-
+#pragma once
+
+#include "token.h"
+
namespace NYson {
////////////////////////////////////////////////////////////////////////////////
-
+
const ETokenType BeginListToken = LeftBracket;
const ETokenType EndListToken = RightBracket;
-
+
const ETokenType BeginMapToken = LeftBrace;
const ETokenType EndMapToken = RightBrace;
-
+
const ETokenType BeginAttributesToken = LeftAngle;
const ETokenType EndAttributesToken = RightAngle;
-
+
const ETokenType ListItemSeparatorToken = Semicolon;
const ETokenType KeyedItemSeparatorToken = Semicolon;
const ETokenType KeyValueSeparatorToken = Equals;
-
+
const ETokenType EntityToken = Hash;
-
+
////////////////////////////////////////////////////////////////////////////////
-
+
} // namespace NYson
diff --git a/library/cpp/yson/json/json_writer.cpp b/library/cpp/yson/json/json_writer.cpp
index 87481256ecd..2c961a02fa8 100644
--- a/library/cpp/yson/json/json_writer.cpp
+++ b/library/cpp/yson/json/json_writer.cpp
@@ -1,16 +1,16 @@
-#include "json_writer.h"
-
+#include "json_writer.h"
+
#include <library/cpp/json/json_writer.h>
-
-namespace NYT {
+
+namespace NYT {
////////////////////////////////////////////////////////////////////////////////
-
+
static bool IsSpecialJsonKey(const TStringBuf& key) {
return key.size() > 0 && key[0] == '$';
}
-
+
////////////////////////////////////////////////////////////////////////////////
-
+
TJsonWriter::TJsonWriter(
IOutputStream* output,
::NYson::EYsonType type,
@@ -41,15 +41,15 @@ namespace NYT {
if (Type == ::NYson::EYsonType::MapFragment) {
ythrow ::NYson::TYsonException() << ("Map fragments are not supported by Json");
}
-
+
UnderlyingJsonWriter.Reset(new NJson::TJsonWriter(
output,
config));
JsonWriter = UnderlyingJsonWriter.Get();
HasAttributes = false;
InAttributesBalance = 0;
- }
-
+ }
+
void TJsonWriter::EnterNode() {
if (AttributesMode == JAM_NEVER) {
HasAttributes = false;
@@ -65,15 +65,15 @@ namespace NYT {
HasAttributes = true;
}
HasUnfoldedStructureStack.push_back(HasAttributes);
-
+
if (HasAttributes) {
JsonWriter->Write("$value");
HasAttributes = false;
- }
-
+ }
+
Depth += 1;
- }
-
+ }
+
void TJsonWriter::LeaveNode() {
Y_ASSERT(!HasUnfoldedStructureStack.empty());
if (HasUnfoldedStructureStack.back()) {
@@ -81,54 +81,54 @@ namespace NYT {
JsonWriter->CloseMap();
}
HasUnfoldedStructureStack.pop_back();
-
+
Depth -= 1;
if (Depth == 0 && Type == ::NYson::EYsonType::ListFragment && InAttributesBalance == 0) {
JsonWriter->Flush();
Output->Write("\n");
}
- }
-
+ }
+
bool TJsonWriter::IsWriteAllowed() {
if (AttributesMode == JAM_NEVER) {
return InAttributesBalance == 0;
}
return true;
- }
-
+ }
+
void TJsonWriter::OnStringScalar(TStringBuf value) {
if (IsWriteAllowed()) {
EnterNode();
WriteStringScalar(value);
LeaveNode();
}
- }
-
+ }
+
void TJsonWriter::OnInt64Scalar(i64 value) {
if (IsWriteAllowed()) {
EnterNode();
JsonWriter->Write(value);
LeaveNode();
}
- }
-
+ }
+
void TJsonWriter::OnUint64Scalar(ui64 value) {
if (IsWriteAllowed()) {
EnterNode();
JsonWriter->Write(value);
LeaveNode();
}
- }
-
+ }
+
void TJsonWriter::OnDoubleScalar(double value) {
if (IsWriteAllowed()) {
EnterNode();
JsonWriter->Write(value);
LeaveNode();
}
- }
-
+ }
+
void TJsonWriter::OnBooleanScalar(bool value) {
if (IsWriteAllowed()) {
if (BooleanFormat == SBF_STRING) {
@@ -139,40 +139,40 @@ namespace NYT {
LeaveNode();
}
}
- }
-
+ }
+
void TJsonWriter::OnEntity() {
if (IsWriteAllowed()) {
EnterNode();
JsonWriter->WriteNull();
LeaveNode();
}
- }
-
+ }
+
void TJsonWriter::OnBeginList() {
if (IsWriteAllowed()) {
EnterNode();
JsonWriter->OpenArray();
}
- }
-
+ }
+
void TJsonWriter::OnListItem() {
- }
-
+ }
+
void TJsonWriter::OnEndList() {
if (IsWriteAllowed()) {
JsonWriter->CloseArray();
LeaveNode();
}
}
-
+
void TJsonWriter::OnBeginMap() {
if (IsWriteAllowed()) {
EnterNode();
JsonWriter->OpenMap();
}
- }
-
+ }
+
void TJsonWriter::OnKeyedItem(TStringBuf name) {
if (IsWriteAllowed()) {
if (IsSpecialJsonKey(name)) {
@@ -181,15 +181,15 @@ namespace NYT {
WriteStringScalar(name);
}
}
- }
-
+ }
+
void TJsonWriter::OnEndMap() {
if (IsWriteAllowed()) {
JsonWriter->CloseMap();
LeaveNode();
- }
- }
-
+ }
+ }
+
void TJsonWriter::OnBeginAttributes() {
InAttributesBalance += 1;
if (AttributesMode != JAM_NEVER) {
@@ -197,24 +197,24 @@ namespace NYT {
JsonWriter->Write("$attributes");
JsonWriter->OpenMap();
}
- }
-
+ }
+
void TJsonWriter::OnEndAttributes() {
InAttributesBalance -= 1;
if (AttributesMode != JAM_NEVER) {
HasAttributes = true;
JsonWriter->CloseMap();
}
- }
-
+ }
+
void TJsonWriter::WriteStringScalar(const TStringBuf& value) {
JsonWriter->Write(value);
- }
-
+ }
+
void TJsonWriter::Flush() {
JsonWriter->Flush();
}
-
+
////////////////////////////////////////////////////////////////////////////////
-}
+}
diff --git a/library/cpp/yson/json/json_writer.h b/library/cpp/yson/json/json_writer.h
index d84ac0de530..43e5293b0d4 100644
--- a/library/cpp/yson/json/json_writer.h
+++ b/library/cpp/yson/json/json_writer.h
@@ -1,31 +1,31 @@
-#pragma once
-
+#pragma once
+
#include <library/cpp/yson/public.h>
#include <library/cpp/yson/consumer.h>
-
+
#include <library/cpp/json/json_writer.h>
-
-#include <util/generic/vector.h>
-
-namespace NYT {
+
+#include <util/generic/vector.h>
+
+namespace NYT {
////////////////////////////////////////////////////////////////////////////////
-
+
enum EJsonFormat {
JF_TEXT,
JF_PRETTY
};
-
+
enum EJsonAttributesMode {
JAM_NEVER,
JAM_ON_DEMAND,
JAM_ALWAYS
};
-
+
enum ESerializedBoolFormat {
SBF_BOOLEAN,
SBF_STRING
};
-
+
class TJsonWriter
: public ::NYson::TYsonConsumerBase {
public:
@@ -44,26 +44,26 @@ namespace NYT {
ESerializedBoolFormat booleanFormat = SBF_STRING);
void Flush();
-
+
void OnStringScalar(TStringBuf value) override;
void OnInt64Scalar(i64 value) override;
void OnUint64Scalar(ui64 value) override;
void OnDoubleScalar(double value) override;
void OnBooleanScalar(bool value) override;
-
+
void OnEntity() override;
-
+
void OnBeginList() override;
void OnListItem() override;
void OnEndList() override;
-
+
void OnBeginMap() override;
void OnKeyedItem(TStringBuf key) override;
void OnEndMap() override;
-
+
void OnBeginAttributes() override;
void OnEndAttributes() override;
-
+
private:
THolder<NJson::TJsonWriter> UnderlyingJsonWriter;
NJson::TJsonWriter* JsonWriter;
@@ -71,19 +71,19 @@ namespace NYT {
::NYson::EYsonType Type;
EJsonAttributesMode AttributesMode;
ESerializedBoolFormat BooleanFormat;
-
+
void WriteStringScalar(const TStringBuf& value);
-
+
void EnterNode();
void LeaveNode();
bool IsWriteAllowed();
-
+
TVector<bool> HasUnfoldedStructureStack;
int InAttributesBalance;
bool HasAttributes;
int Depth;
};
-
+
////////////////////////////////////////////////////////////////////////////////
-
+
}
diff --git a/library/cpp/yson/lexer.cpp b/library/cpp/yson/lexer.cpp
index 5eae94273bf..6864991c248 100644
--- a/library/cpp/yson/lexer.cpp
+++ b/library/cpp/yson/lexer.cpp
@@ -1,16 +1,16 @@
-#include "lexer.h"
-#include "lexer_detail.h"
-#include "token.h"
-
-#include <util/generic/ptr.h>
-
+#include "lexer.h"
+#include "lexer_detail.h"
+#include "token.h"
+
+#include <util/generic/ptr.h>
+
namespace NYson {
////////////////////////////////////////////////////////////////////////////////
-
+
class TStatelessLexer::TImpl {
private:
THolder<TStatelessYsonLexerImplBase> Impl;
-
+
public:
TImpl(bool enableLinePositionInfo = false)
: Impl(enableLinePositionInfo
@@ -18,26 +18,26 @@ namespace NYson {
: static_cast<TStatelessYsonLexerImplBase*>(new TStatelesYsonLexerImpl<false>()))
{
}
-
+
size_t GetToken(const TStringBuf& data, TToken* token) {
return Impl->GetToken(data, token);
}
};
-
+
////////////////////////////////////////////////////////////////////////////////
TStatelessLexer::TStatelessLexer()
: Impl(new TImpl())
- {
- }
-
+ {
+ }
+
TStatelessLexer::~TStatelessLexer() {
}
-
+
size_t TStatelessLexer::GetToken(const TStringBuf& data, TToken* token) {
return Impl->GetToken(data, token);
}
-
+
////////////////////////////////////////////////////////////////////////////////
-
+
} // namespace NYson
diff --git a/library/cpp/yson/lexer.h b/library/cpp/yson/lexer.h
index d9d701874d4..17cb87ec6dd 100644
--- a/library/cpp/yson/lexer.h
+++ b/library/cpp/yson/lexer.h
@@ -1,26 +1,26 @@
-#pragma once
-
-#include "public.h"
-#include "token.h"
-
-#include <util/generic/ptr.h>
-
+#pragma once
+
+#include "public.h"
+#include "token.h"
+
+#include <util/generic/ptr.h>
+
namespace NYson {
////////////////////////////////////////////////////////////////////////////////
-
+
class TStatelessLexer {
public:
TStatelessLexer();
-
+
~TStatelessLexer();
-
+
size_t GetToken(const TStringBuf& data, TToken* token);
-
+
private:
class TImpl;
THolder<TImpl> Impl;
};
-
+
////////////////////////////////////////////////////////////////////////////////
-
+
} // namespace NYson
diff --git a/library/cpp/yson/lexer_detail.h b/library/cpp/yson/lexer_detail.h
index 0bba30acdd2..5bb5dccc065 100644
--- a/library/cpp/yson/lexer_detail.h
+++ b/library/cpp/yson/lexer_detail.h
@@ -1,15 +1,15 @@
-#pragma once
-
-#include "detail.h"
-#include "token.h"
-
+#pragma once
+
+#include "detail.h"
+#include "token.h"
+
namespace NYson {
////////////////////////////////////////////////////////////////////////////////
-
+
namespace NDetail {
/*! \internal */
////////////////////////////////////////////////////////////////////////////////
-
+
// EReadStartCase tree representation:
// Root = xb
// BinaryStringOrOtherSpecialToken = x0b
@@ -29,17 +29,17 @@ namespace NYson {
// Plus = 10011b
// None = 10111b
// Percent = 11011b
-
+
enum EReadStartCase : unsigned {
BinaryString = 0, // = 00b
OtherSpecialToken = 2, // = 10b
-
+
BinaryInt64 = 1, // = 001b
BinaryDouble = 5, // = 101b
BinaryFalse = 9, // = 1001b
BinaryTrue = 13, // = 1101b
BinaryUint64 = 17, // = 10001b
-
+
Quote = 3, // = 00011b
DigitOrMinus = 7, // = 00111b
String = 11, // = 01011b
@@ -48,34 +48,34 @@ namespace NYson {
None = 23, // = 10111b
Percent = 27 // = 11011b
};
-
+
template <class TBlockStream, bool EnableLinePositionInfo>
class TLexer
: public TLexerBase<TBlockStream, EnableLinePositionInfo> {
private:
using TBase = TLexerBase<TBlockStream, EnableLinePositionInfo>;
-
+
static EReadStartCase GetStartState(char ch) {
-#define NN EReadStartCase::None
-#define BS EReadStartCase::BinaryString
-#define BI EReadStartCase::BinaryInt64
-#define BD EReadStartCase::BinaryDouble
-#define BF EReadStartCase::BinaryFalse
-#define BT EReadStartCase::BinaryTrue
-#define BU EReadStartCase::BinaryUint64
-#define SP NN // EReadStartCase::Space
-#define DM EReadStartCase::DigitOrMinus
-#define ST EReadStartCase::String
-#define PL EReadStartCase::Plus
-#define QU EReadStartCase::Quote
-#define PC EReadStartCase::Percent
-#define TT(name) (EReadStartCase(static_cast<ui8>(ETokenType::name) << 2) | EReadStartCase::OtherSpecialToken)
-
+#define NN EReadStartCase::None
+#define BS EReadStartCase::BinaryString
+#define BI EReadStartCase::BinaryInt64
+#define BD EReadStartCase::BinaryDouble
+#define BF EReadStartCase::BinaryFalse
+#define BT EReadStartCase::BinaryTrue
+#define BU EReadStartCase::BinaryUint64
+#define SP NN // EReadStartCase::Space
+#define DM EReadStartCase::DigitOrMinus
+#define ST EReadStartCase::String
+#define PL EReadStartCase::Plus
+#define QU EReadStartCase::Quote
+#define PC EReadStartCase::Percent
+#define TT(name) (EReadStartCase(static_cast<ui8>(ETokenType::name) << 2) | EReadStartCase::OtherSpecialToken)
+
static const ui8 lookupTable[] =
{
NN, BS, BI, BD, BF, BT, BU, NN, NN, SP, SP, SP, SP, SP, NN, NN,
NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN,
-
+
// 32
SP, // ' '
NN, // '!'
@@ -93,7 +93,7 @@ namespace NYson {
DM, // '-'
NN, // '.'
NN, // '/'
-
+
// 48
DM, DM, DM, DM, DM, DM, DM, DM, DM, DM, // '0' - '9'
TT(Colon), // ':'
@@ -102,7 +102,7 @@ namespace NYson {
TT(Equals), // '='
TT(RightAngle), // '>'
NN, // '?'
-
+
// 64
NN, // '@'
ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'A' - 'M'
@@ -112,10 +112,10 @@ namespace NYson {
TT(RightBracket), // ']'
NN, // '^'
ST, // '_'
-
+
// 96
NN, // '`'
-
+
ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'a' - 'm'
ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'n' - 'z'
TT(LeftBrace), // '{'
@@ -128,41 +128,41 @@ namespace NYson {
NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN,
NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN,
NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN,
-
+
NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN,
NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN,
NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN,
NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN};
-
-#undef NN
-#undef BS
-#undef BI
-#undef BD
-#undef SP
-#undef DM
-#undef ST
-#undef PL
-#undef QU
-#undef TT
+
+#undef NN
+#undef BS
+#undef BI
+#undef BD
+#undef SP
+#undef DM
+#undef ST
+#undef PL
+#undef QU
+#undef TT
return static_cast<EReadStartCase>(lookupTable[static_cast<ui8>(ch)]);
}
-
+
public:
TLexer(const TBlockStream& blockStream, TMaybe<ui64> memoryLimit)
: TBase(blockStream, memoryLimit)
{
}
-
+
void GetToken(TToken* token) {
char ch1 = TBase::SkipSpaceAndGetChar();
auto state = GetStartState(ch1);
auto stateBits = static_cast<unsigned>(state);
-
+
if (ch1 == '\0') {
*token = TToken::EndOfStream;
return;
}
-
+
if (stateBits & 1) { // Other = x1b
if (stateBits & 1 << 1) { // Other = xxx11b
if (state == EReadStartCase::Quote) {
@@ -174,9 +174,9 @@ namespace NYson {
ReadNumeric<true>(token);
} else if (state == EReadStartCase::Plus) {
TBase::Advance(1);
-
+
char ch2 = TBase::template GetChar<true>();
-
+
if (!isdigit(ch2)) {
*token = TToken(ETokenType::Plus);
} else {
@@ -219,9 +219,9 @@ namespace NYson {
} else {
Y_FAIL("unreachable");
}
- }
+ }
} else { // BinaryStringOrOtherSpecialToken = x0b
- TBase::Advance(1);
+ TBase::Advance(1);
if (stateBits & 1 << 1) { // OtherSpecialToken = 10b
Y_ASSERT((stateBits & 3) == static_cast<unsigned>(EReadStartCase::OtherSpecialToken));
*token = TToken(ETokenType(stateBits >> 2));
@@ -231,14 +231,14 @@ namespace NYson {
TBase::ReadBinaryString(&value);
*token = TToken(value);
}
- }
- }
-
+ }
+ }
+
template <bool AllowFinish>
void ReadNumeric(TToken* token) {
TStringBuf valueBuffer;
ENumericResult numericResult = TBase::template ReadNumeric<AllowFinish>(&valueBuffer);
-
+
if (numericResult == ENumericResult::Double) {
try {
*token = TToken(FromString<double>(valueBuffer));
@@ -258,39 +258,39 @@ namespace NYson {
ythrow TYsonException() << "Error parsing uint64 literal " << valueBuffer;
}
}
- }
+ }
};
////////////////////////////////////////////////////////////////////////////////
/*! \endinternal */
- }
-
+ }
+
class TStatelessYsonLexerImplBase {
public:
virtual size_t GetToken(const TStringBuf& data, TToken* token) = 0;
-
+
virtual ~TStatelessYsonLexerImplBase() {
}
};
-
+
template <bool EnableLinePositionInfo>
class TStatelesYsonLexerImpl: public TStatelessYsonLexerImplBase {
private:
using TLexer = NDetail::TLexer<TStringReader, EnableLinePositionInfo>;
TLexer Lexer;
-
+
public:
TStatelesYsonLexerImpl()
: Lexer(TStringReader(), Nothing())
{
}
-
+
size_t GetToken(const TStringBuf& data, TToken* token) override {
Lexer.SetBuffer(data.begin(), data.end());
Lexer.GetToken(token);
return Lexer.Begin() - data.begin();
}
};
-
+
////////////////////////////////////////////////////////////////////////////////
-
+
} // namespace NYson
diff --git a/library/cpp/yson/node/node.cpp b/library/cpp/yson/node/node.cpp
index b39e0707187..a533e34fa62 100644
--- a/library/cpp/yson/node/node.cpp
+++ b/library/cpp/yson/node/node.cpp
@@ -1,5 +1,5 @@
#include "node.h"
-
+
#include "node_io.h"
#include <library/cpp/yson/writer.h>
@@ -13,15 +13,15 @@ namespace NYT {
bool TNode::TNull::operator==(const TNull&) const {
return true;
}
-
-////////////////////////////////////////////////////////////////////////////////
-
+
+////////////////////////////////////////////////////////////////////////////////
+
bool TNode::TUndefined::operator==(const TUndefined&) const {
return true;
}
-
+
////////////////////////////////////////////////////////////////////////////////
-
+
namespace NNodeCmp {
bool IsComparableType(const TNode::EType type) {
@@ -94,15 +94,15 @@ bool operator>=(const TNode& lhs, const TNode& rhs)
TNode::TNode()
: Value_(TUndefined{})
{ }
-
+
TNode::TNode(const char* s)
: Value_(TString(s))
{ }
-
+
TNode::TNode(TStringBuf s)
: Value_(TString(s))
{ }
-
+
TNode::TNode(std::string_view s)
: Value_(TString(s))
{ }
@@ -114,12 +114,12 @@ TNode::TNode(const std::string& s)
TNode::TNode(TString s)
: Value_(std::move(s))
{ }
-
+
TNode::TNode(int i)
: Value_(static_cast<i64>(i))
{ }
-
-
+
+
TNode::TNode(unsigned int ui)
: Value_(static_cast<ui64>(ui))
{ }
@@ -127,27 +127,27 @@ TNode::TNode(unsigned int ui)
TNode::TNode(long i)
: Value_(static_cast<i64>(i))
{ }
-
+
TNode::TNode(unsigned long ui)
: Value_(static_cast<ui64>(ui))
{ }
-
+
TNode::TNode(long long i)
: Value_(static_cast<i64>(i))
{ }
-
+
TNode::TNode(unsigned long long ui)
: Value_(static_cast<ui64>(ui))
{ }
-
+
TNode::TNode(double d)
: Value_(d)
{ }
-
+
TNode::TNode(bool b)
: Value_(b)
{ }
-
+
TNode::TNode(TMapType map)
: Value_(std::move(map))
{ }
@@ -161,39 +161,39 @@ TNode::TNode(const TNode& rhs)
}
Value_ = rhs.Value_;
}
-
+
TNode& TNode::operator=(const TNode& rhs)
{
if (this != &rhs) {
TNode tmp = rhs;
Move(std::move(tmp));
- }
+ }
return *this;
}
-
+
TNode::TNode(TNode&& rhs) noexcept
: TNode()
{
Move(std::move(rhs));
}
-
+
TNode& TNode::operator=(TNode&& rhs) noexcept
{
if (this != &rhs) {
TNode tmp = std::move(rhs);
Move(std::move(tmp));
- }
+ }
return *this;
}
-
+
TNode::~TNode() = default;
-
+
void TNode::Clear()
{
ClearAttributes();
Value_ = TUndefined();
}
-
+
bool TNode::IsString() const
{
return std::holds_alternative<TString>(Value_);
@@ -203,37 +203,37 @@ bool TNode::IsInt64() const
{
return std::holds_alternative<i64>(Value_);
}
-
+
bool TNode::IsUint64() const
{
return std::holds_alternative<ui64>(Value_);
}
-
+
bool TNode::IsDouble() const
{
return std::holds_alternative<double>(Value_);
}
-
+
bool TNode::IsBool() const
{
return std::holds_alternative<bool>(Value_);
}
-
+
bool TNode::IsList() const
{
return std::holds_alternative<TListType>(Value_);
}
-
+
bool TNode::IsMap() const
{
return std::holds_alternative<TMapType>(Value_);
}
-
+
bool TNode::IsEntity() const
{
return IsNull();
}
-
+
bool TNode::IsNull() const
{
return std::holds_alternative<TNull>(Value_);
@@ -260,9 +260,9 @@ bool TNode::Empty() const
return std::get<TMapType>(Value_).empty();
default:
ythrow TTypeError() << "Empty() called for type " << GetType();
- }
+ }
}
-
+
size_t TNode::Size() const
{
switch (GetType()) {
@@ -274,9 +274,9 @@ size_t TNode::Size() const
return std::get<TMapType>(Value_).size();
default:
ythrow TTypeError() << "Size() called for type " << GetType();
- }
+ }
}
-
+
TNode::EType TNode::GetType() const
{
return std::visit(TOverloaded{
@@ -291,61 +291,61 @@ TNode::EType TNode::GetType() const
[](const TNull&) { return Null; }
}, Value_);
}
-
+
const TString& TNode::AsString() const
{
CheckType(String);
return std::get<TString>(Value_);
}
-
+
i64 TNode::AsInt64() const
{
CheckType(Int64);
return std::get<i64>(Value_);
}
-
+
ui64 TNode::AsUint64() const
{
CheckType(Uint64);
return std::get<ui64>(Value_);
}
-
+
double TNode::AsDouble() const
{
CheckType(Double);
return std::get<double>(Value_);
}
-
+
bool TNode::AsBool() const
{
CheckType(Bool);
return std::get<bool>(Value_);
}
-
+
const TNode::TListType& TNode::AsList() const
{
CheckType(List);
return std::get<TListType>(Value_);
}
-
+
const TNode::TMapType& TNode::AsMap() const
{
CheckType(Map);
return std::get<TMapType>(Value_);
}
-
+
TNode::TListType& TNode::AsList()
{
CheckType(List);
return std::get<TListType>(Value_);
}
-
+
TNode::TMapType& TNode::AsMap()
{
CheckType(Map);
return std::get<TMapType>(Value_);
}
-
+
const TString& TNode::UncheckedAsString() const noexcept
{
return std::get<TString>(Value_);
@@ -397,7 +397,7 @@ TNode TNode::CreateList()
node.Value_ = TListType{};
return node;
}
-
+
TNode TNode::CreateList(TListType list)
{
TNode node;
@@ -411,7 +411,7 @@ TNode TNode::CreateMap()
node.Value_ = TMapType{};
return node;
}
-
+
TNode TNode::CreateMap(TMapType map)
{
TNode node;
@@ -425,19 +425,19 @@ TNode TNode::CreateEntity()
node.Value_ = TNull{};
return node;
}
-
+
const TNode& TNode::operator[](size_t index) const
{
CheckType(List);
return std::get<TListType>(Value_)[index];
}
-
+
TNode& TNode::operator[](size_t index)
{
CheckType(List);
return std::get<TListType>(Value_)[index];
}
-
+
const TNode& TNode::At(size_t index) const {
CheckType(List);
const auto& list = std::get<TListType>(Value_);
@@ -461,7 +461,7 @@ TNode& TNode::Add() &
AssureList();
return std::get<TListType>(Value_).emplace_back();
}
-
+
TNode TNode::Add() &&
{
return std::move(Add());
@@ -473,7 +473,7 @@ TNode& TNode::Add(const TNode& node) &
std::get<TListType>(Value_).emplace_back(node);
return *this;
}
-
+
TNode TNode::Add(const TNode& node) &&
{
return std::move(Add(node));
@@ -485,7 +485,7 @@ TNode& TNode::Add(TNode&& node) &
std::get<TListType>(Value_).emplace_back(std::move(node));
return *this;
}
-
+
TNode TNode::Add(TNode&& node) &&
{
return std::move(Add(std::move(node)));
@@ -496,14 +496,14 @@ bool TNode::HasKey(const TStringBuf key) const
CheckType(Map);
return std::get<TMapType>(Value_).contains(key);
}
-
+
TNode& TNode::operator()(const TString& key, const TNode& value) &
{
AssureMap();
std::get<TMapType>(Value_)[key] = value;
return *this;
}
-
+
TNode TNode::operator()(const TString& key, const TNode& value) &&
{
return std::move(operator()(key, value));
@@ -515,7 +515,7 @@ TNode& TNode::operator()(const TString& key, TNode&& value) &
std::get<TMapType>(Value_)[key] = std::move(value);
return *this;
}
-
+
TNode TNode::operator()(const TString& key, TNode&& value) &&
{
return std::move(operator()(key, std::move(value)));
@@ -531,15 +531,15 @@ const TNode& TNode::operator[](const TStringBuf key) const
return notFound;
} else {
return i->second;
- }
+ }
}
-
+
TNode& TNode::operator[](const TStringBuf key)
{
AssureMap();
return std::get<TMapType>(Value_)[key];
}
-
+
const TNode& TNode::At(const TStringBuf key) const {
CheckType(Map);
const auto& map = std::get<TMapType>(Value_);
@@ -782,68 +782,68 @@ bool TNode::HasAttributes() const
{
return Attributes_ && !Attributes_->Empty();
}
-
+
void TNode::ClearAttributes()
{
if (Attributes_) {
Attributes_.Destroy();
- }
+ }
}
-
+
const TNode& TNode::GetAttributes() const
{
static TNode notFound = TNode::CreateMap();
if (!Attributes_) {
return notFound;
- }
+ }
return *Attributes_;
}
-
+
TNode& TNode::Attributes()
{
if (!Attributes_) {
CreateAttributes();
- }
+ }
return *Attributes_;
}
-
+
void TNode::MoveWithoutAttributes(TNode&& rhs)
{
Value_ = std::move(rhs.Value_);
rhs.Clear();
}
-
+
void TNode::Move(TNode&& rhs)
{
Value_ = std::move(rhs.Value_);
Attributes_ = std::move(rhs.Attributes_);
}
-
+
void TNode::CheckType(EType type) const
{
Y_ENSURE_EX(GetType() == type,
TTypeError() << "TNode type " << type << " expected, actual type " << GetType();
);
}
-
+
void TNode::AssureMap()
{
if (std::holds_alternative<TUndefined>(Value_)) {
Value_ = TMapType();
} else {
CheckType(Map);
- }
+ }
}
-
+
void TNode::AssureList()
{
if (std::holds_alternative<TUndefined>(Value_)) {
Value_ = TListType();
} else {
CheckType(List);
- }
+ }
}
-
+
void TNode::CreateAttributes()
{
Attributes_ = MakeHolder<TNode>();
@@ -862,54 +862,54 @@ void TNode::Load(IInputStream* in)
}
////////////////////////////////////////////////////////////////////////////////
-
+
bool operator==(const TNode& lhs, const TNode& rhs)
{
if (std::holds_alternative<TNode::TUndefined>(lhs.Value_) ||
std::holds_alternative<TNode::TUndefined>(rhs.Value_))
- {
+ {
// TODO: should try to remove this behaviour if nobody uses it.
return false;
- }
-
+ }
+
if (lhs.GetType() != rhs.GetType()) {
- return false;
- }
-
- if (lhs.Attributes_) {
- if (rhs.Attributes_) {
+ return false;
+ }
+
+ if (lhs.Attributes_) {
+ if (rhs.Attributes_) {
if (*lhs.Attributes_ != *rhs.Attributes_) {
return false;
}
- } else {
- return false;
- }
- } else {
- if (rhs.Attributes_) {
- return false;
- }
- }
-
+ } else {
+ return false;
+ }
+ } else {
+ if (rhs.Attributes_) {
+ return false;
+ }
+ }
+
return rhs.Value_ == lhs.Value_;
-}
-
+}
+
bool operator!=(const TNode& lhs, const TNode& rhs)
-{
- return !(lhs == rhs);
-}
-
+{
+ return !(lhs == rhs);
+}
+
bool GetBool(const TNode& node)
-{
- if (node.IsBool()) {
- return node.AsBool();
- } else if (node.IsString()) {
- return node.AsString() == "true";
- } else {
+{
+ if (node.IsBool()) {
+ return node.AsBool();
+ } else if (node.IsString()) {
+ return node.AsString() == "true";
+ } else {
ythrow TNode::TTypeError()
<< "GetBool(): not a boolean or string type";
- }
-}
-
-////////////////////////////////////////////////////////////////////////////////
-
-} // namespace NYT
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/library/cpp/yson/node/node.h b/library/cpp/yson/node/node.h
index 5f90f95df07..bc7609e6da9 100644
--- a/library/cpp/yson/node/node.h
+++ b/library/cpp/yson/node/node.h
@@ -1,11 +1,11 @@
-#pragma once
-
+#pragma once
+
#include <util/generic/bt_exception.h>
#include <util/generic/cast.h>
-#include <util/generic/hash.h>
+#include <util/generic/hash.h>
#include <util/generic/variant.h>
-#include <util/generic/vector.h>
-#include <util/generic/yexception.h>
+#include <util/generic/vector.h>
+#include <util/generic/yexception.h>
#include <util/generic/ylimits.h>
#include <util/string/cast.h>
@@ -14,23 +14,23 @@
class IInputStream;
class IOutputStream;
-
-namespace NYT {
-
-////////////////////////////////////////////////////////////////////////////////
-
-class TNode
-{
-public:
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TNode
+{
+public:
class TLookupError
: public TWithBackTrace<yexception>
{ };
- class TTypeError
+ class TTypeError
: public TWithBackTrace<yexception>
- { };
-
- enum EType {
+ { };
+
+ enum EType {
Undefined = 0 /*"undefined"*/,
// NOTE: string representation of all node types
@@ -43,20 +43,20 @@ public:
List = 6 /*"list_node"*/,
Map = 7 /*"map_node"*/,
Null = 8 /*"null"*/,
- };
-
+ };
+
using TListType = TVector<TNode>;
using TMapType = THashMap<TString, TNode>;
-
+
private:
struct TNull {
bool operator==(const TNull&) const;
};
-
+
struct TUndefined {
bool operator==(const TUndefined&) const;
};
-
+
using TValue = std::variant<
bool,
i64,
@@ -68,9 +68,9 @@ private:
TNull,
TUndefined
>;
-
+
public:
-
+
TNode();
TNode(const char* s);
TNode(TStringBuf s);
@@ -78,7 +78,7 @@ public:
explicit TNode(const std::string& s);
TNode(TString s);
TNode(int i);
-
+
//this case made speccially for prevent mess cast of EType into TNode through TNode(int) constructor
//usual case of error SomeNode == TNode::Undefined <-- SomeNode indeed will be compared with TNode(0) without this method
//correct way is SomeNode.GetType() == TNode::Undefined
@@ -102,17 +102,17 @@ public:
TNode(double d);
TNode(bool b);
TNode(TMapType map);
-
+
TNode(const TNode& rhs);
TNode& operator=(const TNode& rhs);
-
+
TNode(TNode&& rhs) noexcept;
TNode& operator=(TNode&& rhs) noexcept;
-
+
~TNode();
-
+
void Clear();
-
+
bool IsString() const;
bool IsInt64() const;
bool IsUint64() const;
@@ -127,7 +127,7 @@ public:
bool IsUndefined() const;
// Returns true if TNode is neither Null, nor Undefined
bool HasValue() const;
-
+
template<typename T>
bool IsOfType() const noexcept;
@@ -136,9 +136,9 @@ public:
bool Empty() const;
size_t Size() const;
-
+
EType GetType() const;
-
+
const TString& AsString() const;
i64 AsInt64() const;
ui64 AsUint64() const;
@@ -148,7 +148,7 @@ public:
const TMapType& AsMap() const;
TListType& AsList();
TMapType& AsMap();
-
+
const TString& UncheckedAsString() const noexcept;
i64 UncheckedAsInt64() const noexcept;
ui64 UncheckedAsUint64() const noexcept;
@@ -180,31 +180,31 @@ public:
static TNode CreateMap();
static TNode CreateMap(TMapType map);
static TNode CreateEntity();
-
+
const TNode& operator[](size_t index) const;
TNode& operator[](size_t index);
const TNode& At(size_t index) const;
TNode& At(size_t index);
-
+
TNode& Add() &;
TNode Add() &&;
TNode& Add(const TNode& node) &;
TNode Add(const TNode& node) &&;
TNode& Add(TNode&& node) &;
TNode Add(TNode&& node) &&;
-
+
bool HasKey(const TStringBuf key) const;
TNode& operator()(const TString& key, const TNode& value) &;
TNode operator()(const TString& key, const TNode& value) &&;
TNode& operator()(const TString& key, TNode&& value) &;
TNode operator()(const TString& key, TNode&& value) &&;
-
+
const TNode& operator[](const TStringBuf key) const;
TNode& operator[](const TStringBuf key);
const TNode& At(const TStringBuf key) const;
TNode& At(const TStringBuf key);
-
+
// map getters
// works the same way like simple getters
const TString& ChildAsString(const TStringBuf key) const;
@@ -254,42 +254,42 @@ public:
T& ChildAs(size_t index);
- // attributes
+ // attributes
bool HasAttributes() const;
void ClearAttributes();
const TNode& GetAttributes() const;
TNode& Attributes();
-
+
void MoveWithoutAttributes(TNode&& rhs);
-
+
// Serialize TNode using binary yson format.
// Methods for ysaveload.
void Save(IOutputStream* output) const;
void Load(IInputStream* input);
-private:
+private:
void Move(TNode&& rhs);
-
+
void CheckType(EType type) const;
-
+
void AssureMap();
void AssureList();
-
+
void CreateAttributes();
-
-private:
+
+private:
TValue Value_;
THolder<TNode> Attributes_;
-
- friend bool operator==(const TNode& lhs, const TNode& rhs);
- friend bool operator!=(const TNode& lhs, const TNode& rhs);
-};
-
+
+ friend bool operator==(const TNode& lhs, const TNode& rhs);
+ friend bool operator!=(const TNode& lhs, const TNode& rhs);
+};
+
bool operator==(const TNode& lhs, const TNode& rhs);
bool operator!=(const TNode& lhs, const TNode& rhs);
-
+
bool GetBool(const TNode& node);
-
+
inline bool TNode::IsArithmetic() const {
return IsInt64() || IsUint64() || IsDouble() || IsBool();
}
@@ -508,8 +508,8 @@ inline const T& TNode::As() const {
return std::get<T>(Value_);
}
-////////////////////////////////////////////////////////////////////////////////
-
+////////////////////////////////////////////////////////////////////////////////
+
namespace NNodeCmp {
bool operator<(const TNode& lhs, const TNode& rhs);
bool operator<=(const TNode& lhs, const TNode& rhs);
@@ -520,4 +520,4 @@ namespace NNodeCmp {
////////////////////////////////////////////////////////////////////////////////
-} // namespace NYT
+} // namespace NYT
diff --git a/library/cpp/yson/node/node_builder.cpp b/library/cpp/yson/node/node_builder.cpp
index b4431bc77af..b21bc4502e9 100644
--- a/library/cpp/yson/node/node_builder.cpp
+++ b/library/cpp/yson/node/node_builder.cpp
@@ -1,96 +1,96 @@
-#include "node_builder.h"
-
-namespace NYT {
-
-////////////////////////////////////////////////////////////////////////////////
-
-TNodeBuilder::TNodeBuilder(TNode* node)
-{
- Stack_.push(node);
-}
-
+#include "node_builder.h"
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+TNodeBuilder::TNodeBuilder(TNode* node)
+{
+ Stack_.push(node);
+}
+
void TNodeBuilder::OnStringScalar(TStringBuf value)
-{
- AddNode(value, true);
-}
-
-void TNodeBuilder::OnInt64Scalar(i64 value)
-{
- AddNode(value, true);
-}
-
-void TNodeBuilder::OnUint64Scalar(ui64 value)
-{
- AddNode(value, true);
-}
-
-void TNodeBuilder::OnDoubleScalar(double value)
-{
- AddNode(value, true);
-}
-
-void TNodeBuilder::OnBooleanScalar(bool value)
-{
- AddNode(value, true);
-}
-
-void TNodeBuilder::OnEntity()
-{
- AddNode(TNode::CreateEntity(), true);
-}
-
-void TNodeBuilder::OnBeginList()
-{
- AddNode(TNode::CreateList(), false);
-}
-
-void TNodeBuilder::OnListItem()
-{
- Stack_.push(&Stack_.top()->Add());
-}
-
-void TNodeBuilder::OnEndList()
-{
- Stack_.pop();
-}
-
-void TNodeBuilder::OnBeginMap()
-{
- AddNode(TNode::CreateMap(), false);
-}
-
+{
+ AddNode(value, true);
+}
+
+void TNodeBuilder::OnInt64Scalar(i64 value)
+{
+ AddNode(value, true);
+}
+
+void TNodeBuilder::OnUint64Scalar(ui64 value)
+{
+ AddNode(value, true);
+}
+
+void TNodeBuilder::OnDoubleScalar(double value)
+{
+ AddNode(value, true);
+}
+
+void TNodeBuilder::OnBooleanScalar(bool value)
+{
+ AddNode(value, true);
+}
+
+void TNodeBuilder::OnEntity()
+{
+ AddNode(TNode::CreateEntity(), true);
+}
+
+void TNodeBuilder::OnBeginList()
+{
+ AddNode(TNode::CreateList(), false);
+}
+
+void TNodeBuilder::OnListItem()
+{
+ Stack_.push(&Stack_.top()->Add());
+}
+
+void TNodeBuilder::OnEndList()
+{
+ Stack_.pop();
+}
+
+void TNodeBuilder::OnBeginMap()
+{
+ AddNode(TNode::CreateMap(), false);
+}
+
void TNodeBuilder::OnKeyedItem(TStringBuf key)
-{
+{
Stack_.push(&(*Stack_.top())[TString(key)]);
-}
-
-void TNodeBuilder::OnEndMap()
-{
- Stack_.pop();
-}
-
-void TNodeBuilder::OnBeginAttributes()
-{
- Stack_.push(&Stack_.top()->Attributes());
-}
-
-void TNodeBuilder::OnEndAttributes()
-{
- Stack_.pop();
-}
-
+}
+
+void TNodeBuilder::OnEndMap()
+{
+ Stack_.pop();
+}
+
+void TNodeBuilder::OnBeginAttributes()
+{
+ Stack_.push(&Stack_.top()->Attributes());
+}
+
+void TNodeBuilder::OnEndAttributes()
+{
+ Stack_.pop();
+}
+
void TNodeBuilder::OnNode(TNode node)
{
AddNode(std::move(node), true);
}
-void TNodeBuilder::AddNode(TNode value, bool pop)
-{
+void TNodeBuilder::AddNode(TNode value, bool pop)
+{
Stack_.top()->MoveWithoutAttributes(std::move(value));
- if (pop)
- Stack_.pop();
-}
-
-////////////////////////////////////////////////////////////////////////////////
-
-} // namespace NYT
+ if (pop)
+ Stack_.pop();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/library/cpp/yson/node/node_builder.h b/library/cpp/yson/node/node_builder.h
index 69800016e09..35bb256a5e4 100644
--- a/library/cpp/yson/node/node_builder.h
+++ b/library/cpp/yson/node/node_builder.h
@@ -1,23 +1,23 @@
-#pragma once
-
+#pragma once
+
#include "node.h"
-
+
#include <library/cpp/json/json_reader.h>
#include <library/cpp/yson/consumer.h>
-#include <util/generic/stack.h>
-
-namespace NYT {
-
-////////////////////////////////////////////////////////////////////////////////
-
-class TNodeBuilder
+#include <util/generic/stack.h>
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TNodeBuilder
: public ::NYson::TYsonConsumerBase
-{
-public:
- TNodeBuilder(TNode* node);
-
+{
+public:
+ TNodeBuilder(TNode* node);
+
void OnStringScalar(TStringBuf) override;
void OnInt64Scalar(i64) override;
void OnUint64Scalar(ui64) override;
@@ -33,14 +33,14 @@ public:
void OnBeginAttributes() override;
void OnEndAttributes() override;
void OnNode(TNode node);
-
-private:
+
+private:
TStack<TNode*> Stack_;
-
-private:
- inline void AddNode(TNode node, bool pop);
-};
-
-////////////////////////////////////////////////////////////////////////////////
-
-} // namespace NYT
+
+private:
+ inline void AddNode(TNode node, bool pop);
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/library/cpp/yson/node/node_io.cpp b/library/cpp/yson/node/node_io.cpp
index 294a7f7217f..ea53e4d8b7f 100644
--- a/library/cpp/yson/node/node_io.cpp
+++ b/library/cpp/yson/node/node_io.cpp
@@ -1,8 +1,8 @@
#include "node_io.h"
-
-#include "node_builder.h"
-#include "node_visitor.h"
-
+
+#include "node_builder.h"
+#include "node_visitor.h"
+
#include <library/cpp/yson/json/json_writer.h>
#include <library/cpp/yson/parser.h>
#include <library/cpp/yson/writer.h>
@@ -11,13 +11,13 @@
#include <library/cpp/json/json_reader.h>
#include <library/cpp/json/json_value.h>
-#include <util/stream/input.h>
-#include <util/stream/output.h>
-#include <util/stream/str.h>
+#include <util/stream/input.h>
+#include <util/stream/output.h>
+#include <util/stream/str.h>
#include <util/stream/mem.h>
-
-namespace NYT {
-
+
+namespace NYT {
+
static void WalkJsonTree(const NJson::TJsonValue& jsonValue, NJson::TJsonCallbacks* callbacks)
{
using namespace NJson;
@@ -68,20 +68,20 @@ static void WalkJsonTree(const NJson::TJsonValue& jsonValue, NJson::TJsonCallbac
static TNode CreateEmptyNodeByType(::NYson::EYsonType type)
{
- TNode result;
- switch (type) {
+ TNode result;
+ switch (type) {
case ::NYson::EYsonType::ListFragment:
- result = TNode::CreateList();
- break;
+ result = TNode::CreateList();
+ break;
case ::NYson::EYsonType::MapFragment:
- result = TNode::CreateMap();
- break;
- default:
- break;
- }
+ result = TNode::CreateMap();
+ break;
+ default:
+ break;
+ }
return result;
}
-
+
TNode NodeFromYsonString(const TStringBuf input, ::NYson::EYsonType type)
{
TMemoryInput stream(input);
@@ -106,19 +106,19 @@ TNode NodeFromYsonStream(IInputStream* input, ::NYson::EYsonType type)
{
TNode result = CreateEmptyNodeByType(type);
- TNodeBuilder builder(&result);
+ TNodeBuilder builder(&result);
::NYson::TYsonParser parser(&builder, input, type);
- parser.Parse();
- return result;
-}
-
+ parser.Parse();
+ return result;
+}
+
void NodeToYsonStream(const TNode& node, IOutputStream* output, NYson::EYsonFormat format)
-{
+{
::NYson::TYsonWriter writer(output, format);
- TNodeVisitor visitor(&writer);
- visitor.Visit(node);
-}
-
+ TNodeVisitor visitor(&writer);
+ visitor.Visit(node);
+}
+
void NodeToCanonicalYsonStream(const TNode& node, IOutputStream* output, NYson::EYsonFormat format)
{
::NYson::TYsonWriter writer(output, format);
@@ -149,6 +149,6 @@ TNode NodeFromJsonValue(const NJson::TJsonValue& input)
return result;
}
-////////////////////////////////////////////////////////////////////////////////
-
-} // namespace NYT
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/library/cpp/yson/node/node_io.h b/library/cpp/yson/node/node_io.h
index 2ad23b658f2..5b632433175 100644
--- a/library/cpp/yson/node/node_io.h
+++ b/library/cpp/yson/node/node_io.h
@@ -1,22 +1,22 @@
-#pragma once
-
+#pragma once
+
#include "node.h"
#include <library/cpp/yson/public.h>
-
+
namespace NJson {
class TJsonValue;
} // namespace NJson
-namespace NYT {
-
-////////////////////////////////////////////////////////////////////////////////
-
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
// Parse TNode from string in YSON format
TNode NodeFromYsonString(const TStringBuf input, ::NYson::EYsonType type = ::NYson::EYsonType::Node);
// Serialize TNode to string in one of YSON formats with random order of maps' keys (don't use in tests)
TString NodeToYsonString(const TNode& node, ::NYson::EYsonFormat format = ::NYson::EYsonFormat::Text);
-
+
// Same as the latter, but maps' keys are sorted lexicographically (to be used in tests)
TString NodeToCanonicalYsonString(const TNode& node, ::NYson::EYsonFormat format = ::NYson::EYsonFormat::Text);
@@ -35,6 +35,6 @@ TNode NodeFromJsonString(const TStringBuf input);
// Convert TJsonValue to TNode
TNode NodeFromJsonValue(const NJson::TJsonValue& input);
-////////////////////////////////////////////////////////////////////////////////
-
-} // namespace NYT
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/library/cpp/yson/node/node_visitor.cpp b/library/cpp/yson/node/node_visitor.cpp
index 899fbfa02aa..824d9348677 100644
--- a/library/cpp/yson/node/node_visitor.cpp
+++ b/library/cpp/yson/node/node_visitor.cpp
@@ -1,12 +1,12 @@
-#include "node_visitor.h"
-
+#include "node_visitor.h"
+
#include <util/generic/algorithm.h>
#include <util/string/printf.h>
-namespace NYT {
-
-////////////////////////////////////////////////////////////////////////////////
-
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
namespace {
template <typename Fun>
@@ -31,122 +31,122 @@ void Iterate(const TNode::TMapType& nodeMap, bool sortByKey, Fun action)
////////////////////////////////////////////////////////////////////////////////
TNodeVisitor::TNodeVisitor(NYson::IYsonConsumer* consumer, bool sortMapKeys)
- : Consumer_(consumer)
+ : Consumer_(consumer)
, SortMapKeys_(sortMapKeys)
-{ }
-
-void TNodeVisitor::Visit(const TNode& node)
-{
- VisitAny(node);
-}
-
-void TNodeVisitor::VisitAny(const TNode& node)
-{
- if (node.HasAttributes()) {
- Consumer_->OnBeginAttributes();
+{ }
+
+void TNodeVisitor::Visit(const TNode& node)
+{
+ VisitAny(node);
+}
+
+void TNodeVisitor::VisitAny(const TNode& node)
+{
+ if (node.HasAttributes()) {
+ Consumer_->OnBeginAttributes();
Iterate(node.GetAttributes().AsMap(), SortMapKeys_, [&](const std::pair<TString, TNode>& item) {
- Consumer_->OnKeyedItem(item.first);
+ Consumer_->OnKeyedItem(item.first);
if (item.second.IsUndefined()) {
ythrow TNode::TTypeError() << "unable to visit attribute value of type "
<< TNode::EType::Undefined << "; attribute name: `" << item.first << '\'' ;
}
- VisitAny(item.second);
+ VisitAny(item.second);
});
- Consumer_->OnEndAttributes();
- }
-
- switch (node.GetType()) {
+ Consumer_->OnEndAttributes();
+ }
+
+ switch (node.GetType()) {
case TNode::String:
- VisitString(node);
- break;
+ VisitString(node);
+ break;
case TNode::Int64:
- VisitInt64(node);
- break;
+ VisitInt64(node);
+ break;
case TNode::Uint64:
- VisitUint64(node);
- break;
+ VisitUint64(node);
+ break;
case TNode::Double:
- VisitDouble(node);
- break;
+ VisitDouble(node);
+ break;
case TNode::Bool:
- VisitBool(node);
- break;
+ VisitBool(node);
+ break;
case TNode::List:
VisitList(node.AsList());
- break;
+ break;
case TNode::Map:
VisitMap(node.AsMap());
- break;
+ break;
case TNode::Null:
- VisitEntity();
- break;
+ VisitEntity();
+ break;
case TNode::Undefined:
ythrow TNode::TTypeError() << "unable to visit TNode of type " << node.GetType();
default:
Y_FAIL("Unexpected type: %d", node.GetType());
- }
-}
-
-void TNodeVisitor::VisitString(const TNode& node)
-{
- Consumer_->OnStringScalar(node.AsString());
-}
-
-void TNodeVisitor::VisitInt64(const TNode& node)
-{
- Consumer_->OnInt64Scalar(node.AsInt64());
-}
-
-void TNodeVisitor::VisitUint64(const TNode& node)
-{
- Consumer_->OnUint64Scalar(node.AsUint64());
-}
-
-void TNodeVisitor::VisitDouble(const TNode& node)
-{
- Consumer_->OnDoubleScalar(node.AsDouble());
-}
-
-void TNodeVisitor::VisitBool(const TNode& node)
-{
- Consumer_->OnBooleanScalar(node.AsBool());
-}
-
+ }
+}
+
+void TNodeVisitor::VisitString(const TNode& node)
+{
+ Consumer_->OnStringScalar(node.AsString());
+}
+
+void TNodeVisitor::VisitInt64(const TNode& node)
+{
+ Consumer_->OnInt64Scalar(node.AsInt64());
+}
+
+void TNodeVisitor::VisitUint64(const TNode& node)
+{
+ Consumer_->OnUint64Scalar(node.AsUint64());
+}
+
+void TNodeVisitor::VisitDouble(const TNode& node)
+{
+ Consumer_->OnDoubleScalar(node.AsDouble());
+}
+
+void TNodeVisitor::VisitBool(const TNode& node)
+{
+ Consumer_->OnBooleanScalar(node.AsBool());
+}
+
void TNodeVisitor::VisitList(const TNode::TListType& nodeList)
-{
- Consumer_->OnBeginList();
+{
+ Consumer_->OnBeginList();
size_t index = 0;
for (const auto& item : nodeList) {
- Consumer_->OnListItem();
+ Consumer_->OnListItem();
if (item.IsUndefined()) {
ythrow TNode::TTypeError() << "unable to visit list node child of type "
<< TNode::EType::Undefined << "; list index: " << index;
}
- VisitAny(item);
+ VisitAny(item);
++index;
- }
- Consumer_->OnEndList();
-}
-
+ }
+ Consumer_->OnEndList();
+}
+
void TNodeVisitor::VisitMap(const TNode::TMapType& nodeMap)
-{
- Consumer_->OnBeginMap();
+{
+ Consumer_->OnBeginMap();
Iterate(nodeMap, SortMapKeys_, [&](const std::pair<TString, TNode>& item) {
- Consumer_->OnKeyedItem(item.first);
+ Consumer_->OnKeyedItem(item.first);
if (item.second.IsUndefined()) {
ythrow TNode::TTypeError() << "unable to visit map node child of type "
<< TNode::EType::Undefined << "; map key: `" << item.first << '\'' ;
}
- VisitAny(item.second);
+ VisitAny(item.second);
});
- Consumer_->OnEndMap();
-}
-
-void TNodeVisitor::VisitEntity()
-{
- Consumer_->OnEntity();
-}
-
-////////////////////////////////////////////////////////////////////////////////
-
-} // namespace NYT
+ Consumer_->OnEndMap();
+}
+
+void TNodeVisitor::VisitEntity()
+{
+ Consumer_->OnEntity();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/library/cpp/yson/node/node_visitor.h b/library/cpp/yson/node/node_visitor.h
index db258323092..d4eb8e56675 100644
--- a/library/cpp/yson/node/node_visitor.h
+++ b/library/cpp/yson/node/node_visitor.h
@@ -1,37 +1,37 @@
-#pragma once
-
+#pragma once
+
#include "node.h"
#include <library/cpp/yson/consumer.h>
-
-namespace NYT {
-
-////////////////////////////////////////////////////////////////////////////////
-
-class TNodeVisitor
-{
-public:
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TNodeVisitor
+{
+public:
TNodeVisitor(NYson::IYsonConsumer* consumer, bool sortMapKeys = false);
-
- void Visit(const TNode& node);
+
+ void Visit(const TNode& node);
void VisitMap(const TNode::TMapType& nodeMap);
void VisitList(const TNode::TListType& nodeMap);
-
-private:
+
+private:
NYson::IYsonConsumer* Consumer_;
bool SortMapKeys_;
-
-private:
- void VisitAny(const TNode& node);
-
- void VisitString(const TNode& node);
- void VisitInt64(const TNode& node);
- void VisitUint64(const TNode& node);
- void VisitDouble(const TNode& node);
- void VisitBool(const TNode& node);
- void VisitEntity();
-};
-
-////////////////////////////////////////////////////////////////////////////////
-
-} // namespace NYT
+
+private:
+ void VisitAny(const TNode& node);
+
+ void VisitString(const TNode& node);
+ void VisitInt64(const TNode& node);
+ void VisitUint64(const TNode& node);
+ void VisitDouble(const TNode& node);
+ void VisitBool(const TNode& node);
+ void VisitEntity();
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT
diff --git a/library/cpp/yson/parser.cpp b/library/cpp/yson/parser.cpp
index 783f9b90479..3af222ed4e7 100644
--- a/library/cpp/yson/parser.cpp
+++ b/library/cpp/yson/parser.cpp
@@ -1,14 +1,14 @@
-#include "parser.h"
-#include "consumer.h"
-#include "format.h"
-#include "parser_detail.h"
-
-#include <util/stream/input.h>
-#include <util/generic/buffer.h>
-
+#include "parser.h"
+#include "consumer.h"
+#include "format.h"
+#include "parser_detail.h"
+
+#include <util/stream/input.h>
+#include <util/generic/buffer.h>
+
namespace NYson {
////////////////////////////////////////////////////////////////////////////////
-
+
class TYsonParser::TImpl {
public:
TImpl(
@@ -24,7 +24,7 @@ namespace NYson {
, MemoryLimit_(memoryLimit)
{
}
-
+
void Parse() {
TBuffer buffer(64 << 10);
ParseYsonStreamImpl<NYT::NYson::IYsonConsumer, TStreamReader>(
@@ -48,26 +48,26 @@ namespace NYson {
TYsonParser::TYsonParser(
NYT::NYson::IYsonConsumer* consumer,
IInputStream* stream,
- EYsonType type,
- bool enableLinePositionInfo,
+ EYsonType type,
+ bool enableLinePositionInfo,
TMaybe<ui64> memoryLimit)
: Impl(new TImpl(consumer, stream, type, enableLinePositionInfo, memoryLimit))
- {
- }
-
+ {
+ }
+
TYsonParser::~TYsonParser() {
}
-
+
void TYsonParser::Parse() {
Impl->Parse();
}
-
+
////////////////////////////////////////////////////////////////////////////////
-
+
class TStatelessYsonParser::TImpl {
private:
THolder<TStatelessYsonParserImplBase> Impl;
-
+
public:
TImpl(
NYT::NYson::IYsonConsumer* consumer,
@@ -79,31 +79,31 @@ namespace NYson {
: static_cast<TStatelessYsonParserImplBase*>(new TStatelessYsonParserImpl<NYT::NYson::IYsonConsumer, false>(consumer, memoryLimit)))
{
}
-
+
void Parse(const TStringBuf& data, EYsonType type = ::NYson::EYsonType::Node) {
Impl->Parse(data, type);
}
};
-
+
////////////////////////////////////////////////////////////////////////////////
-
+
TStatelessYsonParser::TStatelessYsonParser(
NYT::NYson::IYsonConsumer* consumer,
- bool enableLinePositionInfo,
- TMaybe<ui64> memoryLimit)
+ bool enableLinePositionInfo,
+ TMaybe<ui64> memoryLimit)
: Impl(new TImpl(consumer, enableLinePositionInfo, memoryLimit))
{
}
-
+
TStatelessYsonParser::~TStatelessYsonParser() {
}
void TStatelessYsonParser::Parse(const TStringBuf& data, EYsonType type) {
- Impl->Parse(data, type);
- }
-
+ Impl->Parse(data, type);
+ }
+
////////////////////////////////////////////////////////////////////////////////
-
+
void ParseYsonStringBuffer(
const TStringBuf& buffer,
NYT::NYson::IYsonConsumer* consumer,
@@ -117,9 +117,9 @@ namespace NYson {
enableLinePositionInfo,
memoryLimit);
}
-
+
////////////////////////////////////////////////////////////////////////////////
-
+
class TYsonListParser::TImpl {
public:
TImpl(
@@ -135,7 +135,7 @@ namespace NYson {
, Reader_(Stream_, Buffer_.Data(), Buffer_.Capacity())
{
}
-
+
bool Parse() {
if (!Impl_) {
Impl_.Reset(
@@ -145,7 +145,7 @@ namespace NYson {
}
return Impl_->Parse();
}
-
+
private:
NYT::NYson::IYsonConsumer* Consumer_;
IInputStream* Stream_;
@@ -155,9 +155,9 @@ namespace NYson {
TStreamReader Reader_;
THolder<TYsonListParserImplBase> Impl_;
};
-
+
////////////////////////////////////////////////////////////////////////////////
-
+
TYsonListParser::TYsonListParser(
NYT::NYson::IYsonConsumer* consumer,
IInputStream* stream,
diff --git a/library/cpp/yson/parser.h b/library/cpp/yson/parser.h
index dce35a8cd40..715341d5b57 100644
--- a/library/cpp/yson/parser.h
+++ b/library/cpp/yson/parser.h
@@ -1,19 +1,19 @@
-#pragma once
-
-#include "public.h"
-
-#include <util/generic/maybe.h>
-#include <util/generic/ptr.h>
-
+#pragma once
+
+#include "public.h"
+
+#include <util/generic/maybe.h>
+#include <util/generic/ptr.h>
+
class IInputStream;
-
+
namespace NYT::NYson {
struct IYsonConsumer;
} // namespace NYT::NYson
namespace NYson {
////////////////////////////////////////////////////////////////////////////////
-
+
class TYsonParser {
public:
TYsonParser(
@@ -22,36 +22,36 @@ namespace NYson {
EYsonType type = ::NYson::EYsonType::Node,
bool enableLinePositionInfo = false,
TMaybe<ui64> memoryLimit = Nothing());
-
+
~TYsonParser();
-
+
void Parse();
-
+
private:
class TImpl;
THolder<TImpl> Impl;
};
-
+
////////////////////////////////////////////////////////////////////////////////
-
+
class TStatelessYsonParser {
public:
TStatelessYsonParser(
NYT::NYson::IYsonConsumer* consumer,
bool enableLinePositionInfo = false,
TMaybe<ui64> memoryLimit = Nothing());
-
+
~TStatelessYsonParser();
-
+
void Parse(const TStringBuf& data, EYsonType type = ::NYson::EYsonType::Node);
-
+
private:
class TImpl;
THolder<TImpl> Impl;
};
-
+
////////////////////////////////////////////////////////////////////////////////
-
+
class TYsonListParser {
public:
TYsonListParser(
@@ -59,7 +59,7 @@ namespace NYson {
IInputStream* stream,
bool enableLinePositionInfo = false,
TMaybe<ui64> memoryLimit = Nothing());
-
+
~TYsonListParser();
bool Parse(); // Returns false, if there is no more list items
@@ -79,5 +79,5 @@ namespace NYson {
TMaybe<ui64> memoryLimit = Nothing());
////////////////////////////////////////////////////////////////////////////////
-
+
} // namespace NYson
diff --git a/library/cpp/yson/parser_detail.h b/library/cpp/yson/parser_detail.h
index 44223caf125..b6eee05664a 100644
--- a/library/cpp/yson/parser_detail.h
+++ b/library/cpp/yson/parser_detail.h
@@ -1,43 +1,43 @@
-#pragma once
-
-#include "detail.h"
-
+#pragma once
+
+#include "detail.h"
+
namespace NYson {
namespace NDetail {
////////////////////////////////////////////////////////////////////////////////
-
+
template <class TConsumer, class TBlockStream, bool EnableLinePositionInfo>
class TParser
: public TLexerBase<TBlockStream, EnableLinePositionInfo> {
private:
using TBase = TLexerBase<TBlockStream, EnableLinePositionInfo>;
TConsumer* Consumer;
-
+
public:
TParser(const TBlockStream& blockStream, TConsumer* consumer, TMaybe<ui64> memoryLimit)
: TBase(blockStream, memoryLimit)
, Consumer(consumer)
{
}
-
+
void DoParse(EYsonType ysonType) {
switch (ysonType) {
case ::NYson::EYsonType::Node:
ParseNode<true>();
break;
-
+
case ::NYson::EYsonType::ListFragment:
ParseListFragment<true>(EndSymbol);
break;
-
+
case ::NYson::EYsonType::MapFragment:
ParseMapFragment<true>(EndSymbol);
break;
-
+
default:
Y_FAIL("unreachable");
}
-
+
while (!(TBase::IsFinished() && TBase::IsEmpty())) {
if (TBase::template SkipSpaceAndGetChar<true>() != EndSymbol) {
ythrow TYsonException() << "Stray '" << (*TBase::Begin()) << "' found";
@@ -46,12 +46,12 @@ namespace NYson {
}
}
}
-
+
bool DoParseListFragment(bool first) {
bool ret = first ? first : ParseListSeparator<true>(EndSymbol);
return ret && ParseListItem<true>(EndSymbol);
- }
-
+ }
+
void ParseAttributes() {
Consumer->OnBeginAttributes();
ParseMapFragment(EndAttributesSymbol);
@@ -65,19 +65,19 @@ namespace NYson {
TBase::SkipCharToken(EndMapSymbol);
Consumer->OnEndMap();
}
-
+
void ParseList() {
Consumer->OnBeginList();
ParseListFragment(EndListSymbol);
TBase::SkipCharToken(EndListSymbol);
Consumer->OnEndList();
}
-
+
template <bool AllowFinish>
void ParseNode() {
return ParseNode<AllowFinish>(TBase::SkipSpaceAndGetChar());
}
-
+
template <bool AllowFinish>
void ParseNode(char ch) {
if (ch == BeginAttributesSymbol) {
@@ -85,18 +85,18 @@ namespace NYson {
ParseAttributes();
ch = TBase::SkipSpaceAndGetChar();
}
-
+
switch (ch) {
case BeginMapSymbol:
TBase::Advance(1);
ParseMap();
break;
-
+
case BeginListSymbol:
TBase::Advance(1);
ParseList();
break;
-
+
case '"': {
TBase::Advance(1);
TStringBuf value;
@@ -146,7 +146,7 @@ namespace NYson {
TBase::Advance(1);
Consumer->OnEntity();
break;
-
+
default: {
if (isdigit((unsigned char)ch) || ch == '-' || ch == '+') { // case of '+' is handled in AfterPlus state
ReadNumeric<AllowFinish>();
@@ -167,11 +167,11 @@ namespace NYson {
}
}
}
- }
+ }
void ParseKey() {
return ParseKey(TBase::SkipSpaceAndGetChar());
- }
+ }
void ParseKey(char ch) {
switch (ch) {
@@ -199,8 +199,8 @@ namespace NYson {
}
}
}
- }
-
+ }
+
template <bool AllowFinish>
void ParseMapFragment(char endSymbol) {
char ch = TBase::template SkipSpaceAndGetChar<AllowFinish>();
@@ -221,13 +221,13 @@ namespace NYson {
ythrow TYsonException() << "Expected '" << KeyedItemSeparatorSymbol
<< "' or '" << endSymbol << "' but '" << ch << "' found";
}
- }
- }
-
+ }
+ }
+
void ParseMapFragment(char endSymbol) {
ParseMapFragment<false>(endSymbol);
}
-
+
template <bool AllowFinish>
bool ParseListItem(char endSymbol) {
char ch = TBase::template SkipSpaceAndGetChar<AllowFinish>();
@@ -237,7 +237,7 @@ namespace NYson {
return true;
}
return false;
- }
+ }
template <bool AllowFinish>
bool ParseListSeparator(char endSymbol) {
@@ -250,17 +250,17 @@ namespace NYson {
<< "' or '" << endSymbol << "' but '" << ch << "' found";
}
return false;
- }
+ }
template <bool AllowFinish>
void ParseListFragment(char endSymbol) {
while (ParseListItem<AllowFinish>(endSymbol) && ParseListSeparator<AllowFinish>(endSymbol)) {
- }
- }
-
+ }
+ }
+
void ParseListFragment(char endSymbol) {
ParseListFragment<false>(endSymbol);
- }
+ }
template <bool AllowFinish>
void ReadNumeric() {
@@ -295,13 +295,13 @@ namespace NYson {
}
Consumer->OnUint64Scalar(value);
}
- }
+ }
};
-
+
////////////////////////////////////////////////////////////////////////////////
-
- }
-
+
+ }
+
template <class TConsumer, class TBlockStream>
void ParseYsonStreamImpl(
const TBlockStream& blockStream,
@@ -317,9 +317,9 @@ namespace NYson {
using TImpl = NDetail::TParser<TConsumer, TBlockStream, false>;
TImpl impl(blockStream, consumer, memoryLimit);
impl.DoParse(parsingMode);
- }
- }
-
+ }
+ }
+
class TStatelessYsonParserImplBase {
public:
virtual void Parse(const TStringBuf& data, EYsonType type = ::NYson::EYsonType::Node) = 0;
@@ -334,27 +334,27 @@ namespace NYson {
private:
using TParser = NDetail::TParser<TConsumer, TStringReader, EnableLinePositionInfo>;
TParser Parser;
-
+
public:
TStatelessYsonParserImpl(TConsumer* consumer, TMaybe<ui64> memoryLimit)
: Parser(TStringReader(), consumer, memoryLimit)
{
}
-
+
void Parse(const TStringBuf& data, EYsonType type = ::NYson::EYsonType::Node) override {
Parser.SetBuffer(data.begin(), data.end());
Parser.DoParse(type);
- }
+ }
};
-
+
class TYsonListParserImplBase {
public:
virtual bool Parse() = 0;
-
+
virtual ~TYsonListParserImplBase() {
}
};
-
+
template <class TConsumer, class TBlockStream, bool EnableLinePositionInfo>
class TYsonListParserImpl
: public TYsonListParserImplBase {
@@ -362,20 +362,20 @@ namespace NYson {
using TParser = NDetail::TParser<TConsumer, TBlockStream, EnableLinePositionInfo>;
TParser Parser;
bool First = true;
-
+
public:
TYsonListParserImpl(const TBlockStream& blockStream, TConsumer* consumer, TMaybe<ui64> memoryLimit)
: Parser(blockStream, consumer, memoryLimit)
{
}
-
+
bool Parse() override {
bool ret = Parser.DoParseListFragment(First);
First = false;
return ret;
}
};
-
+
////////////////////////////////////////////////////////////////////////////////
-
+
} // namespace NYson
diff --git a/library/cpp/yson/public.h b/library/cpp/yson/public.h
index 1ed793592ba..217fff9c8bc 100644
--- a/library/cpp/yson/public.h
+++ b/library/cpp/yson/public.h
@@ -1,22 +1,22 @@
-#pragma once
-
+#pragma once
+
#include <library/cpp/yt/misc/enum.h>
-#include <util/generic/yexception.h>
-
+#include <util/generic/yexception.h>
+
#include <library/cpp/yt/yson_string/public.h>
#include <library/cpp/yt/yson/public.h>
namespace NYson {
////////////////////////////////////////////////////////////////////////////////
-
+
using NYT::NYson::EYsonFormat;
using NYT::NYson::EYsonType;
-
+
class TYsonStringBuf;
struct TYsonConsumerBase;
-
+
class TYsonWriter;
class TYsonParser;
class TStatelessYsonParser;
@@ -24,7 +24,7 @@ namespace NYson {
class TYsonException
: public yexception {};
-
+
////////////////////////////////////////////////////////////////////////////////
-
+
} // namespace NYson
diff --git a/library/cpp/yson/token.cpp b/library/cpp/yson/token.cpp
index c8584c8c2ea..ae9fd741b80 100644
--- a/library/cpp/yson/token.cpp
+++ b/library/cpp/yson/token.cpp
@@ -1,11 +1,11 @@
-#include "token.h"
-
-#include <util/string/vector.h>
-#include <util/string/printf.h>
-
+#include "token.h"
+
+#include <util/string/vector.h>
+#include <util/string/printf.h>
+
namespace NYson {
////////////////////////////////////////////////////////////////////////////////
-
+
ETokenType CharToTokenType(char ch) {
switch (ch) {
case ';':
@@ -40,7 +40,7 @@ namespace NYson {
return ETokenType::EndOfStream;
}
}
-
+
char TokenTypeToChar(ETokenType type) {
switch (type) {
case ETokenType::Semicolon:
@@ -74,16 +74,16 @@ namespace NYson {
default:
Y_FAIL("unreachable");
}
- }
-
+ }
+
TString TokenTypeToString(ETokenType type) {
return TString(1, TokenTypeToChar(type));
- }
-
+ }
+
////////////////////////////////////////////////////////////////////////////////
-
+
const TToken TToken::EndOfStream;
-
+
TToken::TToken()
: Type_(ETokenType::EndOfStream)
, Int64Value(0)
@@ -92,7 +92,7 @@ namespace NYson {
, BooleanValue(false)
{
}
-
+
TToken::TToken(ETokenType type)
: Type_(type)
, Int64Value(0)
@@ -111,7 +111,7 @@ namespace NYson {
break;
}
}
-
+
TToken::TToken(const TStringBuf& stringValue)
: Type_(ETokenType::String)
, StringValue(stringValue)
@@ -120,8 +120,8 @@ namespace NYson {
, DoubleValue(0.0)
, BooleanValue(false)
{
- }
-
+ }
+
TToken::TToken(i64 int64Value)
: Type_(ETokenType::Int64)
, Int64Value(int64Value)
@@ -129,7 +129,7 @@ namespace NYson {
, DoubleValue(0.0)
{
}
-
+
TToken::TToken(ui64 uint64Value)
: Type_(ETokenType::Uint64)
, Int64Value(0)
@@ -138,7 +138,7 @@ namespace NYson {
, BooleanValue(false)
{
}
-
+
TToken::TToken(double doubleValue)
: Type_(ETokenType::Double)
, Int64Value(0)
@@ -147,7 +147,7 @@ namespace NYson {
, BooleanValue(false)
{
}
-
+
TToken::TToken(bool booleanValue)
: Type_(ETokenType::Boolean)
, Int64Value(0)
@@ -155,36 +155,36 @@ namespace NYson {
, BooleanValue(booleanValue)
{
}
-
+
bool TToken::IsEmpty() const {
return Type_ == ETokenType::EndOfStream;
}
-
+
const TStringBuf& TToken::GetStringValue() const {
CheckType(ETokenType::String);
return StringValue;
}
-
+
i64 TToken::GetInt64Value() const {
CheckType(ETokenType::Int64);
return Int64Value;
}
-
+
ui64 TToken::GetUint64Value() const {
CheckType(ETokenType::Uint64);
return Uint64Value;
}
-
+
double TToken::GetDoubleValue() const {
CheckType(ETokenType::Double);
return DoubleValue;
}
-
+
bool TToken::GetBooleanValue() const {
CheckType(ETokenType::Boolean);
return BooleanValue;
}
-
+
void TToken::CheckType(ETokenType expectedType) const {
if (Type_ != expectedType) {
if (Type_ == ETokenType::EndOfStream) {
@@ -194,9 +194,9 @@ namespace NYson {
<< "', Type: " << TokenTypeToString(Type_)
<< ", ExpectedType: " << TokenTypeToString(expectedType) << ")";
}
- }
- }
-
+ }
+ }
+
void TToken::Reset() {
Type_ = ETokenType::EndOfStream;
Int64Value = 0;
@@ -205,32 +205,32 @@ namespace NYson {
StringValue = TStringBuf();
BooleanValue = false;
}
-
+
TString ToString(const TToken& token) {
switch (token.GetType()) {
case ETokenType::EndOfStream:
return TString();
-
+
case ETokenType::String:
return TString(token.GetStringValue());
-
+
case ETokenType::Int64:
return ::ToString(token.GetInt64Value());
-
+
case ETokenType::Uint64:
return ::ToString(token.GetUint64Value());
-
+
case ETokenType::Double:
return ::ToString(token.GetDoubleValue());
-
+
case ETokenType::Boolean:
return token.GetBooleanValue() ? "true" : "false";
-
+
default:
return TokenTypeToString(token.GetType());
}
- }
-
+ }
+
////////////////////////////////////////////////////////////////////////////////
-
+
} // namespace NYson
diff --git a/library/cpp/yson/token.h b/library/cpp/yson/token.h
index 7283e569504..ee455a29ca6 100644
--- a/library/cpp/yson/token.h
+++ b/library/cpp/yson/token.h
@@ -1,21 +1,21 @@
-#pragma once
-
-#include "public.h"
-
-#include <util/generic/strbuf.h>
-
+#pragma once
+
+#include "public.h"
+
+#include <util/generic/strbuf.h>
+
namespace NYson {
////////////////////////////////////////////////////////////////////////////////
-
+
enum ETokenType {
EndOfStream,
-
+
String,
Int64,
Uint64,
Double,
Boolean,
-
+
// Special values:
// YSON
Semicolon, // ;
@@ -27,7 +27,7 @@ namespace NYson {
RightBrace, // }
LeftAngle, // <
RightAngle, // >
-
+
// Table ranges
LeftParenthesis, // (
RightParenthesis, // )
@@ -35,23 +35,23 @@ namespace NYson {
Colon, // :
Comma, // ,
};
-
+
////////////////////////////////////////////////////////////////////////////////
-
+
ETokenType CharToTokenType(char ch);
char TokenTypeToChar(ETokenType type);
TString TokenTypeToString(ETokenType type);
-
+
////////////////////////////////////////////////////////////////////////////////
-
+
class TLexerImpl;
-
+
////////////////////////////////////////////////////////////////////////////////
-
+
class TToken {
public:
static const TToken EndOfStream;
-
+
TToken();
TToken(ETokenType type);
explicit TToken(const TStringBuf& stringValue);
@@ -59,35 +59,35 @@ namespace NYson {
explicit TToken(ui64 int64Value);
explicit TToken(double doubleValue);
explicit TToken(bool booleanValue);
-
+
ETokenType GetType() const {
return Type_;
}
-
+
bool IsEmpty() const;
const TStringBuf& GetStringValue() const;
i64 GetInt64Value() const;
ui64 GetUint64Value() const;
double GetDoubleValue() const;
bool GetBooleanValue() const;
-
+
void CheckType(ETokenType expectedType) const;
void Reset();
-
+
private:
friend class TLexerImpl;
-
+
ETokenType Type_;
-
+
TStringBuf StringValue;
i64 Int64Value;
ui64 Uint64Value;
double DoubleValue;
bool BooleanValue;
};
-
+
TString ToString(const TToken& token);
-
+
////////////////////////////////////////////////////////////////////////////////
-
+
} // namespace NYson
diff --git a/library/cpp/yson/tokenizer.cpp b/library/cpp/yson/tokenizer.cpp
index 06760170d48..595700e62f7 100644
--- a/library/cpp/yson/tokenizer.cpp
+++ b/library/cpp/yson/tokenizer.cpp
@@ -1,37 +1,37 @@
-#include "tokenizer.h"
-
+#include "tokenizer.h"
+
namespace NYson {
////////////////////////////////////////////////////////////////////////////////
-
+
TTokenizer::TTokenizer(const TStringBuf& input)
: Input(input)
, Parsed(0)
{
}
-
+
bool TTokenizer::ParseNext() {
Input = Input.Tail(Parsed);
Token.Reset();
Parsed = Lexer.GetToken(Input, &Token);
return !CurrentToken().IsEmpty();
}
-
+
const TToken& TTokenizer::CurrentToken() const {
return Token;
}
-
+
ETokenType TTokenizer::GetCurrentType() const {
return CurrentToken().GetType();
}
-
+
TStringBuf TTokenizer::GetCurrentSuffix() const {
return Input.Tail(Parsed);
}
-
+
const TStringBuf& TTokenizer::CurrentInput() const {
return Input;
}
-
+
////////////////////////////////////////////////////////////////////////////////
} // namespace NYson
diff --git a/library/cpp/yson/tokenizer.h b/library/cpp/yson/tokenizer.h
index 0576aace95c..25e42ab9fc3 100644
--- a/library/cpp/yson/tokenizer.h
+++ b/library/cpp/yson/tokenizer.h
@@ -1,28 +1,28 @@
-#pragma once
-
-#include "public.h"
-#include "lexer.h"
-
+#pragma once
+
+#include "public.h"
+#include "lexer.h"
+
namespace NYson {
////////////////////////////////////////////////////////////////////////////////
-
+
class TTokenizer {
public:
explicit TTokenizer(const TStringBuf& input);
-
+
bool ParseNext();
const TToken& CurrentToken() const;
ETokenType GetCurrentType() const;
TStringBuf GetCurrentSuffix() const;
const TStringBuf& CurrentInput() const;
-
+
private:
TStringBuf Input;
TToken Token;
TStatelessLexer Lexer;
size_t Parsed;
};
-
+
////////////////////////////////////////////////////////////////////////////////
-
+
} // namespace NYson
diff --git a/library/cpp/yson/varint.cpp b/library/cpp/yson/varint.cpp
index d538ee3cffa..b962dcede7f 100644
--- a/library/cpp/yson/varint.cpp
+++ b/library/cpp/yson/varint.cpp
@@ -1,12 +1,12 @@
#include "varint.h"
-#include "zigzag.h"
-
+#include "zigzag.h"
+
#include <util/generic/yexception.h>
-
+
namespace NYson {
////////////////////////////////////////////////////////////////////////////////
-
+
int WriteVarUInt64(IOutputStream* output, ui64 value) {
bool stop = false;
int bytesWritten = 0;
@@ -19,22 +19,22 @@ namespace NYson {
byte &= 0x7F;
}
output->Write(byte);
- }
+ }
return bytesWritten;
- }
-
+ }
+
int WriteVarInt32(IOutputStream* output, i32 value) {
return WriteVarUInt64(output, static_cast<ui64>(ZigZagEncode32(value)));
}
-
+
int WriteVarInt64(IOutputStream* output, i64 value) {
return WriteVarUInt64(output, static_cast<ui64>(ZigZagEncode64(value)));
}
-
+
int ReadVarUInt64(IInputStream* input, ui64* value) {
size_t count = 0;
ui64 result = 0;
-
+
ui8 byte = 0;
do {
if (7 * count > 8 * sizeof(ui64)) {
@@ -56,16 +56,16 @@ namespace NYson {
int bytesRead = ReadVarUInt64(input, &varInt);
if (varInt > Max<ui32>()) {
ythrow yexception() << "The data is too long to read ui64";
- }
+ }
*value = ZigZagDecode32(static_cast<ui32>(varInt));
return bytesRead;
}
-
+
int ReadVarInt64(IInputStream* input, i64* value) {
ui64 varInt;
int bytesRead = ReadVarUInt64(input, &varInt);
*value = ZigZagDecode64(varInt);
return bytesRead;
- }
-
+ }
+
} // namespace NYson
diff --git a/library/cpp/yson/writer.cpp b/library/cpp/yson/writer.cpp
index 054459f9f5f..9f0386e213d 100644
--- a/library/cpp/yson/writer.cpp
+++ b/library/cpp/yson/writer.cpp
@@ -1,18 +1,18 @@
-#include "writer.h"
+#include "writer.h"
#include "detail.h"
-#include "format.h"
+#include "format.h"
#include "parser.h"
#include "varint.h"
-#include "zigzag.h"
-
-#include <util/string/cast.h>
-
+#include "zigzag.h"
+
+#include <util/string/cast.h>
+
#include <cmath>
namespace NYson {
////////////////////////////////////////////////////////////////////////////////
-
+
// Copied from <util/string/escape.cpp>
namespace {
inline char HexDigit(char value) {
@@ -22,26 +22,26 @@ namespace NYson {
else
return 'A' + value - 10;
}
-
+
inline char OctDigit(char value) {
Y_ASSERT(value < 8);
return '0' + value;
}
-
+
inline bool IsPrintable(char c) {
return c >= 32 && c <= 126;
}
-
+
inline bool IsHexDigit(char c) {
return (c >= '0' && c <= '9') || (c >= 'A' && c <= 'F') || (c >= 'a' && c <= 'f');
}
-
+
inline bool IsOctDigit(char c) {
return c >= '0' && c <= '7';
}
-
+
const size_t ESCAPE_C_BUFFER_SIZE = 4;
-
+
inline size_t EscapeC(unsigned char c, char next, char r[ESCAPE_C_BUFFER_SIZE]) {
// (1) Printable characters go as-is, except backslash and double quote.
// (2) Characters \r, \n, \t and \0 ... \7 replaced by their simple escape characters (if possible).
@@ -87,33 +87,33 @@ namespace NYson {
return 4;
}
}
-
+
void EscapeC(const char* str, size_t len, IOutputStream& output) {
char buffer[ESCAPE_C_BUFFER_SIZE];
-
+
size_t i, j;
for (i = 0, j = 0; i < len; ++i) {
size_t rlen = EscapeC(str[i], (i + 1 < len ? str[i + 1] : 0), buffer);
-
+
if (rlen > 1) {
output.Write(str + j, i - j);
j = i + 1;
output.Write(buffer, rlen);
}
}
-
+
if (j > 0) {
output.Write(str + j, len - j);
} else {
output.Write(str, len);
}
- }
-
+ }
+
TString FloatToStringWithNanInf(double value) {
if (std::isfinite(value)) {
return ::ToString(value);
}
-
+
static const TStringBuf nanLiteral = "%nan";
static const TStringBuf infLiteral = "%inf";
static const TStringBuf negativeInfLiteral = "%-inf";
@@ -130,7 +130,7 @@ namespace NYson {
}
}
-
+
////////////////////////////////////////////////////////////////////////////////
TYsonWriter::TYsonWriter(
@@ -147,17 +147,17 @@ namespace NYson {
{
Y_ASSERT(stream);
}
-
+
void TYsonWriter::WriteIndent() {
for (int i = 0; i < IndentSize * Depth; ++i) {
Stream->Write(' ');
}
}
-
+
bool TYsonWriter::IsTopLevelFragmentContext() const {
return Depth == 0 && (Type == ::NYson::EYsonType::ListFragment || Type == ::NYson::EYsonType::MapFragment);
- }
-
+ }
+
void TYsonWriter::EndNode() {
if (IsTopLevelFragmentContext()) {
ETokenType separatorToken =
@@ -168,15 +168,15 @@ namespace NYson {
if (Format == EYsonFormat::Text || Format == EYsonFormat::Pretty) {
Stream->Write('\n');
}
- }
- }
-
+ }
+ }
+
void TYsonWriter::BeginCollection(ETokenType beginToken) {
Stream->Write(TokenTypeToChar(beginToken));
++Depth;
BeforeFirstItem = true;
}
-
+
void TYsonWriter::CollectionItem(ETokenType separatorToken) {
if (!IsTopLevelFragmentContext()) {
if (!BeforeFirstItem) {
@@ -187,21 +187,21 @@ namespace NYson {
Stream->Write('\n');
WriteIndent();
}
- }
-
+ }
+
BeforeFirstItem = false;
}
void TYsonWriter::EndCollection(ETokenType endToken) {
--Depth;
if (Format == EYsonFormat::Pretty && !BeforeFirstItem) {
- Stream->Write('\n');
- WriteIndent();
- }
+ Stream->Write('\n');
+ WriteIndent();
+ }
Stream->Write(TokenTypeToChar(endToken));
BeforeFirstItem = false;
- }
-
+ }
+
void TYsonWriter::WriteStringScalar(const TStringBuf& value) {
if (Format == EYsonFormat::Binary) {
Stream->Write(NDetail::StringMarker);
@@ -213,12 +213,12 @@ namespace NYson {
Stream->Write('"');
}
}
-
+
void TYsonWriter::OnStringScalar(TStringBuf value) {
WriteStringScalar(value);
EndNode();
- }
-
+ }
+
void TYsonWriter::OnInt64Scalar(i64 value) {
if (Format == EYsonFormat::Binary) {
Stream->Write(NDetail::Int64Marker);
@@ -227,8 +227,8 @@ namespace NYson {
Stream->Write(::ToString(value));
}
EndNode();
- }
-
+ }
+
void TYsonWriter::OnUint64Scalar(ui64 value) {
if (Format == EYsonFormat::Binary) {
Stream->Write(NDetail::Uint64Marker);
@@ -238,8 +238,8 @@ namespace NYson {
Stream->Write("u");
}
EndNode();
- }
-
+ }
+
void TYsonWriter::OnDoubleScalar(double value) {
if (Format == EYsonFormat::Binary) {
Stream->Write(NDetail::DoubleMarker);
@@ -252,44 +252,44 @@ namespace NYson {
}
}
EndNode();
- }
-
+ }
+
void TYsonWriter::OnBooleanScalar(bool value) {
if (Format == EYsonFormat::Binary) {
Stream->Write(value ? NDetail::TrueMarker : NDetail::FalseMarker);
} else {
Stream->Write(value ? "%true" : "%false");
- }
+ }
EndNode();
- }
-
+ }
+
void TYsonWriter::OnEntity() {
Stream->Write(TokenTypeToChar(EntityToken));
EndNode();
- }
-
+ }
+
void TYsonWriter::OnBeginList() {
BeginCollection(BeginListToken);
}
-
+
void TYsonWriter::OnListItem() {
CollectionItem(ListItemSeparatorToken);
}
-
+
void TYsonWriter::OnEndList() {
EndCollection(EndListToken);
EndNode();
}
-
+
void TYsonWriter::OnBeginMap() {
BeginCollection(BeginMapToken);
}
-
+
void TYsonWriter::OnKeyedItem(TStringBuf key) {
CollectionItem(KeyedItemSeparatorToken);
-
+
WriteStringScalar(key);
-
+
if (Format == NYson::EYsonFormat::Pretty) {
Stream->Write(' ');
}
@@ -297,26 +297,26 @@ namespace NYson {
if (Format == NYson::EYsonFormat::Pretty) {
Stream->Write(' ');
}
-
+
BeforeFirstItem = false;
- }
+ }
void TYsonWriter::OnEndMap() {
EndCollection(EndMapToken);
EndNode();
- }
-
+ }
+
void TYsonWriter::OnBeginAttributes() {
BeginCollection(BeginAttributesToken);
}
-
+
void TYsonWriter::OnEndAttributes() {
EndCollection(EndAttributesToken);
if (Format == NYson::EYsonFormat::Pretty) {
Stream->Write(' ');
}
}
-
+
void TYsonWriter::OnRaw(TStringBuf yson, EYsonType type) {
if (EnableRaw) {
Stream->Write(yson);
@@ -325,19 +325,19 @@ namespace NYson {
TYsonConsumerBase::OnRaw(yson, type);
}
}
-
+
TYsonWriter::TState TYsonWriter::State() const {
TState state;
state.Depth = Depth;
state.BeforeFirstItem = BeforeFirstItem;
return state;
- }
-
+ }
+
void TYsonWriter::Reset(const TState& state) {
Depth = state.Depth;
BeforeFirstItem = state.BeforeFirstItem;
- }
-
+ }
+
////////////////////////////////////////////////////////////////////////////////
void ReformatYsonStream(
@@ -351,5 +351,5 @@ namespace NYson {
}
////////////////////////////////////////////////////////////////////////////////
-
+
} // namespace NYson
diff --git a/library/cpp/yson/writer.h b/library/cpp/yson/writer.h
index 40f5d7d5014..0ae04be6e84 100644
--- a/library/cpp/yson/writer.h
+++ b/library/cpp/yson/writer.h
@@ -1,17 +1,17 @@
-#pragma once
-
-#include "public.h"
-#include "token.h"
-#include "consumer.h"
-
-#include <util/generic/noncopyable.h>
-
+#pragma once
+
+#include "public.h"
+#include "token.h"
+#include "consumer.h"
+
+#include <util/generic/noncopyable.h>
+
class IOutputStream;
class IZeroCopyInput;
-
+
namespace NYson {
////////////////////////////////////////////////////////////////////////////////
-
+
class TYsonWriter
: public TYsonConsumerBase,
private TNonCopyable {
@@ -20,7 +20,7 @@ namespace NYson {
private:
int Depth;
bool BeforeFirstItem;
-
+
friend class TYsonWriter;
};
@@ -37,23 +37,23 @@ namespace NYson {
void OnDoubleScalar(double value) override;
void OnBooleanScalar(bool value) override;
void OnEntity() override;
-
+
void OnBeginList() override;
void OnListItem() override;
void OnEndList() override;
-
+
void OnBeginMap() override;
void OnKeyedItem(TStringBuf key) override;
void OnEndMap() override;
-
+
void OnBeginAttributes() override;
void OnEndAttributes() override;
-
+
void OnRaw(TStringBuf yson, EYsonType type = ::NYson::EYsonType::Node) override;
-
+
TState State() const;
void Reset(const TState& state);
-
+
protected:
IOutputStream* Stream;
EYsonFormat Format;
@@ -62,28 +62,28 @@ namespace NYson {
int Depth;
bool BeforeFirstItem;
-
+
static const int IndentSize = 4;
-
+
void WriteIndent();
void WriteStringScalar(const TStringBuf& value);
-
+
void BeginCollection(ETokenType beginToken);
void CollectionItem(ETokenType separatorToken);
void EndCollection(ETokenType endToken);
-
+
bool IsTopLevelFragmentContext() const;
void EndNode();
};
-
+
////////////////////////////////////////////////////////////////////////////////
-
+
void ReformatYsonStream(
IInputStream* input,
IOutputStream* output,
EYsonFormat format = EYsonFormat::Binary,
EYsonType type = ::NYson::EYsonType::Node);
-
+
////////////////////////////////////////////////////////////////////////////////
-
+
} // namespace NYson
diff --git a/library/cpp/yson/ya.make b/library/cpp/yson/ya.make
index c55a189b105..b30109a31fa 100644
--- a/library/cpp/yson/ya.make
+++ b/library/cpp/yson/ya.make
@@ -1,23 +1,23 @@
-LIBRARY()
-
+LIBRARY()
+
OWNER(
ermolovd
g:yt
)
-
+
PEERDIR(
library/cpp/yt/misc
library/cpp/yt/yson
)
-SRCS(
+SRCS(
consumer.cpp
- lexer.cpp
+ lexer.cpp
parser.cpp
- token.cpp
- tokenizer.cpp
+ token.cpp
+ tokenizer.cpp
varint.cpp
- writer.cpp
-)
-
-END()
+ writer.cpp
+)
+
+END()
diff --git a/library/cpp/yson/zigzag.h b/library/cpp/yson/zigzag.h
index 2f1190508fb..dc130a74109 100644
--- a/library/cpp/yson/zigzag.h
+++ b/library/cpp/yson/zigzag.h
@@ -1,31 +1,31 @@
-#pragma once
-
-#include <util/system/defaults.h>
-
+#pragma once
+
+#include <util/system/defaults.h>
+
namespace NYson {
////////////////////////////////////////////////////////////////////////////////
-
+
//! Functions that provide coding of integers with property: 0 <= f(x) <= 2 * |x|
//! Actually taken 'as is' from protobuf/wire_format_lite.h
-
+
inline ui32 ZigZagEncode32(i32 n) {
// Note: the right-shift must be arithmetic
return (ui32(n) << 1) ^ (n >> 31);
}
-
+
inline i32 ZigZagDecode32(ui32 n) {
return (n >> 1) ^ -static_cast<i32>(n & 1);
}
-
+
inline ui64 ZigZagEncode64(i64 n) {
// Note: the right-shift must be arithmetic
return (ui64(n) << 1) ^ (n >> 63);
}
-
+
inline i64 ZigZagDecode64(ui64 n) {
return (n >> 1) ^ -static_cast<i64>(n & 1);
}
-
+
////////////////////////////////////////////////////////////////////////////////
} // namespace NYson
diff --git a/util/generic/guid.cpp b/util/generic/guid.cpp
index 8b907457bc6..bda7a822449 100644
--- a/util/generic/guid.cpp
+++ b/util/generic/guid.cpp
@@ -125,7 +125,7 @@ bool GetGuid(const TStringBuf s, TGUID& result) {
}
result.dw[partId] = static_cast<ui32>(partValue);
return true;
-}
+}
// Parses GUID from s and checks that it's valid.
// In case of error returns TGUID().
diff --git a/ydb/core/base/appdata.cpp b/ydb/core/base/appdata.cpp
index f9e517fc424..757cc87b110 100644
--- a/ydb/core/base/appdata.cpp
+++ b/ydb/core/base/appdata.cpp
@@ -42,7 +42,7 @@ TAppData::TDefaultTabletTypes::TDefaultTabletTypes()
, Mediator(TTabletTypes::TX_MEDIATOR)
, Kesus(TTabletTypes::KESUS)
, Hive(TTabletTypes::Hive)
- , SysViewProcessor(TTabletTypes::SysViewProcessor)
+ , SysViewProcessor(TTabletTypes::SysViewProcessor)
, ColumnShard(TTabletTypes::COLUMNSHARD)
, TestShard(TTabletTypes::TestShard)
, SequenceShard(TTabletTypes::SequenceShard)
diff --git a/ydb/core/base/appdata.h b/ydb/core/base/appdata.h
index c666f7468c0..bb066bb173b 100644
--- a/ydb/core/base/appdata.h
+++ b/ydb/core/base/appdata.h
@@ -98,7 +98,7 @@ struct TAppData {
TTabletTypes::EType Mediator;
TTabletTypes::EType Kesus;
TTabletTypes::EType Hive;
- TTabletTypes::EType SysViewProcessor;
+ TTabletTypes::EType SysViewProcessor;
TTabletTypes::EType ColumnShard;
TTabletTypes::EType TestShard;
TTabletTypes::EType SequenceShard;
diff --git a/ydb/core/base/blobstorage.h b/ydb/core/base/blobstorage.h
index a2faee326e5..4b42f2ee866 100644
--- a/ydb/core/base/blobstorage.h
+++ b/ydb/core/base/blobstorage.h
@@ -799,8 +799,8 @@ struct TEvBlobStorage {
EvControllerMigrationBatchRequest,
EvControllerMigrationDone,
- EvControllerUpdateSystemViews,
-
+ EvControllerUpdateSystemViews,
+
// proxy - node controller interface
EvConfigureProxy = EvPut + 13 * 512,
EvProxyConfigurationRequest, // DEPRECATED
diff --git a/ydb/core/base/counters.cpp b/ydb/core/base/counters.cpp
index b192fa36a1d..16215f38fcf 100644
--- a/ydb/core/base/counters.cpp
+++ b/ydb/core/base/counters.cpp
@@ -47,8 +47,8 @@ static const THashSet<TString> DATABASE_ATTRIBUTE_SERVICES
static const THashSet<TString> DATABASE_ATTRIBUTE_LABELS
= {{ TString("cloud_id"),
- TString("folder_id"),
- TString("database_id")
+ TString("folder_id"),
+ TString("database_id")
}};
using NMonitoring::TDynamicCounters;
diff --git a/ydb/core/base/events.h b/ydb/core/base/events.h
index f5fedfe19b2..5b00d9d9740 100644
--- a/ydb/core/base/events.h
+++ b/ydb/core/base/events.h
@@ -111,7 +111,7 @@ struct TKikimrEvents : TEvents {
ES_TX_ALLOCATOR_CLIENT,
ES_PQ_CLUSTER_TRACKER,
ES_NET_CLASSIFIER,
- ES_SYSTEM_VIEW,
+ ES_SYSTEM_VIEW,
ES_TENANT_NODE_ENUMERATOR,
ES_SERVICE_ACCOUNT_SERVICE,
ES_INDEX_BUILD,
diff --git a/ydb/core/base/pool_stats_collector.cpp b/ydb/core/base/pool_stats_collector.cpp
index 7dcbf28954d..6db21278875 100644
--- a/ydb/core/base/pool_stats_collector.cpp
+++ b/ydb/core/base/pool_stats_collector.cpp
@@ -58,9 +58,9 @@ private:
IActor *CreateStatsCollector(ui32 intervalSec,
const TActorSystemSetup& setup,
- NMonitoring::TDynamicCounterPtr counters)
+ NMonitoring::TDynamicCounterPtr counters)
{
- return new TStatsCollectingActor(intervalSec, setup, counters);
+ return new TStatsCollectingActor(intervalSec, setup, counters);
}
} // namespace NKikimr
diff --git a/ydb/core/base/pool_stats_collector.h b/ydb/core/base/pool_stats_collector.h
index cbf06f9f513..045314ec668 100644
--- a/ydb/core/base/pool_stats_collector.h
+++ b/ydb/core/base/pool_stats_collector.h
@@ -12,6 +12,6 @@ namespace NKikimr {
IActor* CreateStatsCollector(ui32 intervalSec,
const TActorSystemSetup& setup,
- NMonitoring::TDynamicCounterPtr counters);
+ NMonitoring::TDynamicCounterPtr counters);
}
diff --git a/ydb/core/base/shared_data.cpp b/ydb/core/base/shared_data.cpp
index c375acc515a..632469fc981 100644
--- a/ydb/core/base/shared_data.cpp
+++ b/ydb/core/base/shared_data.cpp
@@ -1,32 +1,32 @@
#include "shared_data.h"
-#include <library/cpp/actors/core/memory_tracker.h>
-
+#include <library/cpp/actors/core/memory_tracker.h>
+
#include <util/system/sys_alloc.h>
#include <util/system/sanitizers.h>
namespace NKikimr {
- static constexpr char MemoryLabelSharedData[] = "Tablet/TSharedData/Buffers";
-
+ static constexpr char MemoryLabelSharedData[] = "Tablet/TSharedData/Buffers";
+
char* TSharedData::Allocate(size_t size) {
char* data = nullptr;
if (size > 0) {
if (size >= MaxDataSize) {
throw std::length_error("Allocate size overflow");
}
- auto allocSize = OverheadSize + size;
- char* raw = reinterpret_cast<char*>(y_allocate(allocSize));
-
- auto* privateHeader = reinterpret_cast<TPrivateHeader*>(raw);
- privateHeader->AllocSize = allocSize;
- NActors::NMemory::TLabel<MemoryLabelSharedData>::Add(allocSize);
-
- auto* header = reinterpret_cast<THeader*>(raw + PrivateHeaderSize);
+ auto allocSize = OverheadSize + size;
+ char* raw = reinterpret_cast<char*>(y_allocate(allocSize));
+
+ auto* privateHeader = reinterpret_cast<TPrivateHeader*>(raw);
+ privateHeader->AllocSize = allocSize;
+ NActors::NMemory::TLabel<MemoryLabelSharedData>::Add(allocSize);
+
+ auto* header = reinterpret_cast<THeader*>(raw + PrivateHeaderSize);
header->RefCount = 1;
header->Owner = nullptr;
-
- data = raw + OverheadSize;
+
+ data = raw + OverheadSize;
NSan::Poison(data, size);
}
return data;
@@ -34,14 +34,14 @@ namespace NKikimr {
void TSharedData::Deallocate(char* data) noexcept {
if (data) {
- char* raw = data - OverheadSize;
-
- auto* privateHeader = reinterpret_cast<TPrivateHeader*>(raw);
- NActors::NMemory::TLabel<MemoryLabelSharedData>::Sub(privateHeader->AllocSize);
-
- auto* header = reinterpret_cast<THeader*>(raw + PrivateHeaderSize);
+ char* raw = data - OverheadSize;
+
+ auto* privateHeader = reinterpret_cast<TPrivateHeader*>(raw);
+ NActors::NMemory::TLabel<MemoryLabelSharedData>::Sub(privateHeader->AllocSize);
+
+ auto* header = reinterpret_cast<THeader*>(raw + PrivateHeaderSize);
Y_VERIFY_DEBUG(header->Owner == nullptr);
-
+
y_deallocate(raw);
}
}
diff --git a/ydb/core/base/shared_data.h b/ydb/core/base/shared_data.h
index 2ccfc0e1879..7899ca6c8db 100644
--- a/ydb/core/base/shared_data.h
+++ b/ydb/core/base/shared_data.h
@@ -15,24 +15,24 @@ namespace NKikimr {
virtual void Deallocate(char*) noexcept = 0;
};
- struct TPrivateHeader {
- size_t AllocSize;
- size_t Pad;
- };
-
- static_assert(sizeof(TPrivateHeader) == 16, "TPrivateHeader has an unexpected size");
-
+ struct TPrivateHeader {
+ size_t AllocSize;
+ size_t Pad;
+ };
+
+ static_assert(sizeof(TPrivateHeader) == 16, "TPrivateHeader has an unexpected size");
+
struct THeader {
TAtomic RefCount;
IOwner* Owner;
};
- static_assert(sizeof(THeader) == 16, "THeader has an unexpected size");
-
+ static_assert(sizeof(THeader) == 16, "THeader has an unexpected size");
+
enum : size_t {
- PrivateHeaderSize = sizeof(TPrivateHeader),
- HeaderSize = sizeof(THeader),
- OverheadSize = PrivateHeaderSize + HeaderSize,
+ PrivateHeaderSize = sizeof(TPrivateHeader),
+ HeaderSize = sizeof(THeader),
+ OverheadSize = PrivateHeaderSize + HeaderSize,
MaxDataSize = (std::numeric_limits<size_t>::max() - OverheadSize)
};
diff --git a/ydb/core/base/tablet_types.h b/ydb/core/base/tablet_types.h
index f585819299c..bde322dbd92 100644
--- a/ydb/core/base/tablet_types.h
+++ b/ydb/core/base/tablet_types.h
@@ -36,7 +36,7 @@ public:
static constexpr EType TENANT_SLOT_BROKER = TenantSlotBroker;
static constexpr EType CONSOLE = Console;
static constexpr EType KESUS = Kesus;
- static constexpr EType SYS_VIEW_PROCESSOR = SysViewProcessor;
+ static constexpr EType SYS_VIEW_PROCESSOR = SysViewProcessor;
static constexpr EType FILESTORE = FileStore;
static constexpr EType TESTSHARD = TestShard;
static constexpr EType SEQUENCESHARD = SequenceShard;
diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_nodemon.cpp b/ydb/core/blobstorage/dsproxy/dsproxy_nodemon.cpp
index addca8c5330..3d974726da7 100644
--- a/ydb/core/blobstorage/dsproxy/dsproxy_nodemon.cpp
+++ b/ydb/core/blobstorage/dsproxy/dsproxy_nodemon.cpp
@@ -62,9 +62,9 @@ TDsProxyNodeMon::TDsProxyNodeMon(TIntrusivePtr<NMonitoring::TDynamicCounters> &c
{
auto group = Group->GetSubgroup("subsystem", "restart_histo");
- auto histoGroup = group->GetSubgroup("sensor", "restart_histo");
+ auto histoGroup = group->GetSubgroup("sensor", "restart_histo");
for (size_t i = 0; i < RestartHisto.size(); ++i) {
- RestartHisto[i] = histoGroup->GetNamedCounter("restartCount", ToString(i), true);
+ RestartHisto[i] = histoGroup->GetNamedCounter("restartCount", ToString(i), true);
}
}
// Accelerate counters
@@ -189,7 +189,7 @@ void TDsProxyNodeMon::CheckNodeMonCountersForDeviceType(TPDiskCategory::EDeviceT
auto getNamedHisto = [&subGroup, &type] (const TString& name) {
auto buckets = NMonitoring::ExplicitHistogram(GetCommonLatencyHistBounds(type));
- return subGroup->GetHistogram(name, std::move(buckets));
+ return subGroup->GetHistogram(name, std::move(buckets));
};
PutTabletLogResponseTimeHist256Ki[idx] = getNamedHisto("putTabletLog256KiMs");
diff --git a/ydb/core/blobstorage/incrhuge/incrhuge.h b/ydb/core/blobstorage/incrhuge/incrhuge.h
index c2f5848e72e..4b9f8190b59 100644
--- a/ydb/core/blobstorage/incrhuge/incrhuge.h
+++ b/ydb/core/blobstorage/incrhuge/incrhuge.h
@@ -54,7 +54,7 @@ namespace NKikimr {
// TWritePayload structure contains user fields needed by VDisk when receiving write confirmation. It is
// simply moved from request to response.
- struct TWritePayload {
+ struct TWritePayload {
virtual ~TWritePayload() = default;
};
diff --git a/ydb/core/blobstorage/incrhuge/incrhuge_keeper_common.h b/ydb/core/blobstorage/incrhuge/incrhuge_keeper_common.h
index bc9f706f26b..48d248e0d9f 100644
--- a/ydb/core/blobstorage/incrhuge/incrhuge_keeper_common.h
+++ b/ydb/core/blobstorage/incrhuge/incrhuge_keeper_common.h
@@ -201,7 +201,7 @@ namespace NKikimr {
// event callback base; it is a class that holds some data and has an apply function which is called when
// log entry is written or write is failed or for other event; uses NALF incremental allocator, because it
// is short-term object
- class IEventCallback {
+ class IEventCallback {
public:
virtual ~IEventCallback() = default;
virtual void Apply(NKikimrProto::EReplyStatus status, IEventBase *result, const TActorContext& ctx) = 0;
diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h
index 8e722ce42a0..0511bdd6119 100644
--- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h
+++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h
@@ -243,7 +243,7 @@ struct TEvLogResult;
// LOG
////////////////////////////////////////////////////////////////////////////
struct TEvLog : public TEventLocal<TEvLog, TEvBlobStorage::EvLog> {
- struct ICallback {
+ struct ICallback {
virtual ~ICallback() = default;
virtual void operator ()(TActorSystem *actorSystem, const TEvLogResult &ev) = 0;
};
diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_util_countedqueuemanyone.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_util_countedqueuemanyone.h
index a0ac88b2504..3351f17cbf4 100644
--- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_util_countedqueuemanyone.h
+++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_util_countedqueuemanyone.h
@@ -13,7 +13,7 @@ namespace NPDisk {
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
template<typename T, ui32 TSize>
class TCountedQueueManyOne {
- NThreading::TObstructiveConsumerAuxQueue<T> Queue;
+ NThreading::TObstructiveConsumerAuxQueue<T> Queue;
TAtomic SizeLowerEstimate;
TMutex ProducedMutex;
TCondVar ProducedCondVar;
diff --git a/ydb/core/blobstorage/testload/test_load_actor.cpp b/ydb/core/blobstorage/testload/test_load_actor.cpp
index c8788b13104..a1733721122 100644
--- a/ydb/core/blobstorage/testload/test_load_actor.cpp
+++ b/ydb/core/blobstorage/testload/test_load_actor.cpp
@@ -207,19 +207,19 @@ public:
break;
}
- case NKikimrBlobStorage::TEvTestLoadRequest::CommandCase::kMemoryLoadStart: {
- const auto& cmd = record.GetMemoryLoadStart();
- const ui64 tag = GetOrGenerateTag(cmd);
- if (LoadActors.count(tag) != 0) {
- ythrow TLoadActorException() << Sprintf("duplicate load actor with Tag# %" PRIu64, tag);
- }
-
- LOG_DEBUG_S(ctx, NKikimrServices::BS_LOAD_TEST, "Create new memory load actor with tag# " << tag);
- LoadActors.emplace(tag, ctx.Register(CreateMemoryTestLoad(
- cmd, ctx.SelfID, GetServiceCounters(Counters, "load_actor"), 0, tag)));
- break;
- }
-
+ case NKikimrBlobStorage::TEvTestLoadRequest::CommandCase::kMemoryLoadStart: {
+ const auto& cmd = record.GetMemoryLoadStart();
+ const ui64 tag = GetOrGenerateTag(cmd);
+ if (LoadActors.count(tag) != 0) {
+ ythrow TLoadActorException() << Sprintf("duplicate load actor with Tag# %" PRIu64, tag);
+ }
+
+ LOG_DEBUG_S(ctx, NKikimrServices::BS_LOAD_TEST, "Create new memory load actor with tag# " << tag);
+ LoadActors.emplace(tag, ctx.Register(CreateMemoryTestLoad(
+ cmd, ctx.SelfID, GetServiceCounters(Counters, "load_actor"), 0, tag)));
+ break;
+ }
+
default: {
TString protoTxt;
google::protobuf::TextFormat::PrintToString(record, &protoTxt);
diff --git a/ydb/core/blobstorage/testload/test_load_actor.h b/ydb/core/blobstorage/testload/test_load_actor.h
index 38b7fb3d305..f67a851861b 100644
--- a/ydb/core/blobstorage/testload/test_load_actor.h
+++ b/ydb/core/blobstorage/testload/test_load_actor.h
@@ -51,15 +51,15 @@ namespace NKikimr {
NActors::IActor *CreateKeyValueWriterTestLoad(const NKikimrBlobStorage::TEvTestLoadRequest::TKeyValueLoadStart& cmd,
const NActors::TActorId& parent, const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters,
ui64 index, ui64 tag);
-
+
NActors::IActor *CreateKqpWriterTestLoad(const NKikimrBlobStorage::TEvTestLoadRequest::TKqpLoadStart& cmd,
const NActors::TActorId& parent, const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters,
ui64 index, ui64 tag);
- NActors::IActor *CreateMemoryTestLoad(const NKikimrBlobStorage::TEvTestLoadRequest::TMemoryLoadStart& cmd,
- const NActors::TActorId& parent, const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters,
- ui64 index, ui64 tag);
-
+ NActors::IActor *CreateMemoryTestLoad(const NKikimrBlobStorage::TEvTestLoadRequest::TMemoryLoadStart& cmd,
+ const NActors::TActorId& parent, const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters,
+ ui64 index, ui64 tag);
+
struct TLoadReport : public TThrRefBase {
enum ELoadType {
LOAD_READ,
diff --git a/ydb/core/blobstorage/testload/test_load_memory.cpp b/ydb/core/blobstorage/testload/test_load_memory.cpp
index 9da565186ae..27db469055a 100644
--- a/ydb/core/blobstorage/testload/test_load_memory.cpp
+++ b/ydb/core/blobstorage/testload/test_load_memory.cpp
@@ -1,141 +1,141 @@
-#include "test_load_actor.h"
-
+#include "test_load_actor.h"
+
#include <ydb/core/base/appdata.h>
-
-#include <library/cpp/monlib/service/pages/templates.h>
-
-namespace NKikimr {
-
-class TMemoryTestLoadActor : public TActorBootstrapped<TMemoryTestLoadActor> {
- enum {
- EvAllocateBlock = EventSpaceBegin(TEvents::ES_PRIVATE),
- EvEnd
- };
-
- struct TEvAllocateBlock : public TEventLocal<TEvAllocateBlock, EvAllocateBlock> {};
-
- const TActorId Parent;
- const ui64 Tag;
-
- TDuration Duration;
- ui64 BlockSize;
- TDuration Interval;
-
- TInstant TestStartTime;
- TVector<TVector<char>> Blocks;
- ui64 AllocatedSize = 0;
-
-public:
+
+#include <library/cpp/monlib/service/pages/templates.h>
+
+namespace NKikimr {
+
+class TMemoryTestLoadActor : public TActorBootstrapped<TMemoryTestLoadActor> {
+ enum {
+ EvAllocateBlock = EventSpaceBegin(TEvents::ES_PRIVATE),
+ EvEnd
+ };
+
+ struct TEvAllocateBlock : public TEventLocal<TEvAllocateBlock, EvAllocateBlock> {};
+
+ const TActorId Parent;
+ const ui64 Tag;
+
+ TDuration Duration;
+ ui64 BlockSize;
+ TDuration Interval;
+
+ TInstant TestStartTime;
+ TVector<TVector<char>> Blocks;
+ ui64 AllocatedSize = 0;
+
+public:
static constexpr auto ActorActivityType() {
- return NKikimrServices::TActivity::BS_LOAD_PDISK_LOG_WRITE;
- }
-
- TMemoryTestLoadActor(const NKikimrBlobStorage::TEvTestLoadRequest::TMemoryLoadStart& cmd,
- const TActorId& parent, const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters, ui64 index, ui64 tag)
- : Parent(parent)
- , Tag(tag)
- {
- Y_UNUSED(counters);
- Y_UNUSED(index);
-
- VERIFY_PARAM(DurationSeconds);
- Duration = TDuration::Seconds(cmd.GetDurationSeconds());
-
- VERIFY_PARAM(BlockSize);
- BlockSize = cmd.GetBlockSize();
-
- VERIFY_PARAM(IntervalUs);
- Interval = TDuration::MicroSeconds(cmd.GetIntervalUs());
-
- Blocks.reserve(Duration.MicroSeconds() / Interval.MicroSeconds() + 1);
- }
-
- void Bootstrap(const TActorContext& ctx) {
- LOG_DEBUG_S(ctx, NKikimrServices::BS_LOAD_TEST, "Tag# " << Tag
- << " TMemoryTestLoadActor Bootstrap called");
-
- Become(&TMemoryTestLoadActor::StateFunc);
-
- LOG_INFO_S(ctx, NKikimrServices::BS_LOAD_TEST, "Tag# " << Tag
- << " Schedule PoisonPill");
-
- ctx.Schedule(Duration, new TEvents::TEvPoisonPill);
- ctx.Schedule(Interval, new TEvAllocateBlock);
- TestStartTime = TAppData::TimeProvider->Now();
- }
-
- void HandlePoisonPill(const TActorContext& ctx) {
- LOG_INFO_S(ctx, NKikimrServices::BS_LOAD_TEST, "Tag# " << Tag
- << " Handle PoisonPill");
-
- TIntrusivePtr<TLoadReport> report(new TLoadReport());
- report->Duration = Duration;
- ctx.Send(Parent, new TEvTestLoadFinished(Tag, report, "OK"));
- Die(ctx);
- }
-
- void Handle(TEvAllocateBlock::TPtr&, const TActorContext& ctx) {
- auto size = RandomNumber<ui64>(BlockSize * 2 + 1);
-
- Blocks.push_back({});
- auto& block = Blocks.back();
- block.resize(size);
- for (size_t i = 0; i < size; ++i) {
- block[i] = 0;
- }
- AllocatedSize += size;
-
- LOG_DEBUG_S(ctx, NKikimrServices::BS_LOAD_TEST, "Tag# " << Tag
- << " Handle AllocateBlock");
-
- ctx.Schedule(Interval, new TEvAllocateBlock);
- }
-
- void Handle(NMon::TEvHttpInfo::TPtr& ev, const TActorContext& ctx) {
-#define PARAM(NAME, VALUE) \
- TABLER() { \
- TABLED() { str << NAME; } \
- TABLED() { str << VALUE; } \
- }
- TStringStream str;
- HTML(str) {
- TABLE_CLASS("table table-condensed") {
- TABLEHEAD() {
- TABLER() {
- TABLEH() { str << "Parameter"; }
- TABLEH() { str << "Value"; }
- }
- }
- TABLEBODY() {
- PARAM("Elapsed time / Duration",
- (TAppData::TimeProvider->Now() - TestStartTime).Seconds() << "s / "
- << Duration.Seconds() << "s");
- PARAM("Interval", Interval.MicroSeconds() << "us");
- PARAM("Block size", BlockSize);
- PARAM("Allocated bytes", AllocatedSize);
- PARAM("Allocated blocks", Blocks.size());
- }
- }
- }
-
- ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str(), ev->Get()->SubRequestId));
- }
-
- STRICT_STFUNC(StateFunc,
- CFunc(TEvents::TSystem::PoisonPill, HandlePoisonPill)
- HFunc(TEvAllocateBlock, Handle)
- HFunc(NMon::TEvHttpInfo, Handle)
- )
-};
-
-IActor* CreateMemoryTestLoad(
- const NKikimrBlobStorage::TEvTestLoadRequest::TMemoryLoadStart& cmd,
- const TActorId& parent,
- const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters,
- ui64 index,
- ui64 tag)
-{
- return new TMemoryTestLoadActor(cmd, parent, counters, index, tag);
-}
-
-} // NKikimr
+ return NKikimrServices::TActivity::BS_LOAD_PDISK_LOG_WRITE;
+ }
+
+ TMemoryTestLoadActor(const NKikimrBlobStorage::TEvTestLoadRequest::TMemoryLoadStart& cmd,
+ const TActorId& parent, const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters, ui64 index, ui64 tag)
+ : Parent(parent)
+ , Tag(tag)
+ {
+ Y_UNUSED(counters);
+ Y_UNUSED(index);
+
+ VERIFY_PARAM(DurationSeconds);
+ Duration = TDuration::Seconds(cmd.GetDurationSeconds());
+
+ VERIFY_PARAM(BlockSize);
+ BlockSize = cmd.GetBlockSize();
+
+ VERIFY_PARAM(IntervalUs);
+ Interval = TDuration::MicroSeconds(cmd.GetIntervalUs());
+
+ Blocks.reserve(Duration.MicroSeconds() / Interval.MicroSeconds() + 1);
+ }
+
+ void Bootstrap(const TActorContext& ctx) {
+ LOG_DEBUG_S(ctx, NKikimrServices::BS_LOAD_TEST, "Tag# " << Tag
+ << " TMemoryTestLoadActor Bootstrap called");
+
+ Become(&TMemoryTestLoadActor::StateFunc);
+
+ LOG_INFO_S(ctx, NKikimrServices::BS_LOAD_TEST, "Tag# " << Tag
+ << " Schedule PoisonPill");
+
+ ctx.Schedule(Duration, new TEvents::TEvPoisonPill);
+ ctx.Schedule(Interval, new TEvAllocateBlock);
+ TestStartTime = TAppData::TimeProvider->Now();
+ }
+
+ void HandlePoisonPill(const TActorContext& ctx) {
+ LOG_INFO_S(ctx, NKikimrServices::BS_LOAD_TEST, "Tag# " << Tag
+ << " Handle PoisonPill");
+
+ TIntrusivePtr<TLoadReport> report(new TLoadReport());
+ report->Duration = Duration;
+ ctx.Send(Parent, new TEvTestLoadFinished(Tag, report, "OK"));
+ Die(ctx);
+ }
+
+ void Handle(TEvAllocateBlock::TPtr&, const TActorContext& ctx) {
+ auto size = RandomNumber<ui64>(BlockSize * 2 + 1);
+
+ Blocks.push_back({});
+ auto& block = Blocks.back();
+ block.resize(size);
+ for (size_t i = 0; i < size; ++i) {
+ block[i] = 0;
+ }
+ AllocatedSize += size;
+
+ LOG_DEBUG_S(ctx, NKikimrServices::BS_LOAD_TEST, "Tag# " << Tag
+ << " Handle AllocateBlock");
+
+ ctx.Schedule(Interval, new TEvAllocateBlock);
+ }
+
+ void Handle(NMon::TEvHttpInfo::TPtr& ev, const TActorContext& ctx) {
+#define PARAM(NAME, VALUE) \
+ TABLER() { \
+ TABLED() { str << NAME; } \
+ TABLED() { str << VALUE; } \
+ }
+ TStringStream str;
+ HTML(str) {
+ TABLE_CLASS("table table-condensed") {
+ TABLEHEAD() {
+ TABLER() {
+ TABLEH() { str << "Parameter"; }
+ TABLEH() { str << "Value"; }
+ }
+ }
+ TABLEBODY() {
+ PARAM("Elapsed time / Duration",
+ (TAppData::TimeProvider->Now() - TestStartTime).Seconds() << "s / "
+ << Duration.Seconds() << "s");
+ PARAM("Interval", Interval.MicroSeconds() << "us");
+ PARAM("Block size", BlockSize);
+ PARAM("Allocated bytes", AllocatedSize);
+ PARAM("Allocated blocks", Blocks.size());
+ }
+ }
+ }
+
+ ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str(), ev->Get()->SubRequestId));
+ }
+
+ STRICT_STFUNC(StateFunc,
+ CFunc(TEvents::TSystem::PoisonPill, HandlePoisonPill)
+ HFunc(TEvAllocateBlock, Handle)
+ HFunc(NMon::TEvHttpInfo, Handle)
+ )
+};
+
+IActor* CreateMemoryTestLoad(
+ const NKikimrBlobStorage::TEvTestLoadRequest::TMemoryLoadStart& cmd,
+ const TActorId& parent,
+ const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters,
+ ui64 index,
+ ui64 tag)
+{
+ return new TMemoryTestLoadActor(cmd, parent, counters, index, tag);
+}
+
+} // NKikimr
diff --git a/ydb/core/blobstorage/testload/ya.make b/ydb/core/blobstorage/testload/ya.make
index f1448fe2404..8015a87b231 100644
--- a/ydb/core/blobstorage/testload/ya.make
+++ b/ydb/core/blobstorage/testload/ya.make
@@ -25,7 +25,7 @@ SRCS(
test_load_gen.h
test_load_interval_gen.h
test_load_keyvalue_write.cpp
- test_load_memory.cpp
+ test_load_memory.cpp
test_load_pdisk_read.cpp
test_load_pdisk_write.cpp
test_load_pdisk_log.cpp
diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_histograms.cpp b/ydb/core/blobstorage/vdisk/common/vdisk_histograms.cpp
index d9cce39e9e1..6a4d39f1f6c 100644
--- a/ydb/core/blobstorage/vdisk/common/vdisk_histograms.cpp
+++ b/ydb/core/blobstorage/vdisk/common/vdisk_histograms.cpp
@@ -16,7 +16,7 @@ namespace NKikimr {
std::make_pair(&VPutUserDataLatencyHistogram, "PutUserData" ),
std::make_pair(&VPutAsyncBlobLatencyHistogram, "PutAsyncBlob")
}) {
- *item.first = std::make_shared<NVDiskMon::TLtcHisto>(counters, "sensor", item.second, type);
+ *item.first = std::make_shared<NVDiskMon::TLtcHisto>(counters, "sensor", item.second, type);
}
}
diff --git a/ydb/core/client/flat_ut.cpp b/ydb/core/client/flat_ut.cpp
index 362c052aa11..2aa3d5eb0a5 100644
--- a/ydb/core/client/flat_ut.cpp
+++ b/ydb/core/client/flat_ut.cpp
@@ -1114,7 +1114,7 @@ Y_UNIT_TEST_SUITE(TFlatTest) {
Y_UNIT_TEST(Ls) {
TPortManager pm;
ui16 port = pm.GetPort(2134);
- TServer cleverServer = TServer(TServerSettings(port).SetEnableSystemViews(false));
+ TServer cleverServer = TServer(TServerSettings(port).SetEnableSystemViews(false));
TFlatMsgBusClient annoyingClient(port);
annoyingClient.InitRoot();
@@ -1207,7 +1207,7 @@ Y_UNIT_TEST_SUITE(TFlatTest) {
Y_UNIT_TEST(PathSorting) {
TPortManager pm;
ui16 port = pm.GetPort(2134);
- TServer cleverServer = TServer(TServerSettings(port).SetEnableSystemViews(false));
+ TServer cleverServer = TServer(TServerSettings(port).SetEnableSystemViews(false));
TFlatMsgBusClient annoyingClient(port);
@@ -1253,7 +1253,7 @@ Y_UNIT_TEST_SUITE(TFlatTest) {
Y_UNIT_TEST(LsPathId) {
TPortManager pm;
ui16 port = pm.GetPort(2134);
- TServer cleverServer = TServer(TServerSettings(port).SetEnableSystemViews(false));
+ TServer cleverServer = TServer(TServerSettings(port).SetEnableSystemViews(false));
TFlatMsgBusClient annoyingClient(port);
diff --git a/ydb/core/client/server/msgbus_server_s3_listing.cpp b/ydb/core/client/server/msgbus_server_s3_listing.cpp
index 3626866322b..4c0abf38c1b 100644
--- a/ydb/core/client/server/msgbus_server_s3_listing.cpp
+++ b/ydb/core/client/server/msgbus_server_s3_listing.cpp
@@ -396,7 +396,7 @@ private:
ui64 shardId = KeyRange->Partitions[idx].ShardId;
THolder<TEvDataShard::TEvS3ListingRequest> ev(new TEvDataShard::TEvS3ListingRequest());
- ev->Record.SetTableId(KeyRange->TableId.PathId.LocalPathId);
+ ev->Record.SetTableId(KeyRange->TableId.PathId.LocalPathId);
ev->Record.SetSerializedKeyPrefix(PrefixColumns.GetBuffer());
ev->Record.SetPathColumnPrefix(Request->GetPathColumnPrefix());
ev->Record.SetPathColumnDelimiter(Request->GetPathColumnDelimiter());
diff --git a/ydb/core/cms/console/console__create_tenant.cpp b/ydb/core/cms/console/console__create_tenant.cpp
index 77d8166d547..e113cb3047f 100644
--- a/ydb/core/cms/console/console__create_tenant.cpp
+++ b/ydb/core/cms/console/console__create_tenant.cpp
@@ -126,7 +126,7 @@ public:
Tenant->IsExternalSubdomain = Self->FeatureFlags.GetEnableExternalSubdomains();
Tenant->IsExternalHive = Self->FeatureFlags.GetEnableExternalHive();
- Tenant->IsExternalSysViewProcessor = Self->FeatureFlags.GetEnablePersistentQueryStats();
+ Tenant->IsExternalSysViewProcessor = Self->FeatureFlags.GetEnablePersistentQueryStats();
if (rec.options().disable_external_subdomain()) {
Tenant->IsExternalSubdomain = false;
@@ -143,11 +143,11 @@ public:
Tenant->TimeCastBucketsPerMediator = 0;
Tenant->IsExternalSubdomain = false;
Tenant->IsExternalHive = false;
- Tenant->IsExternalSysViewProcessor = false;
+ Tenant->IsExternalSysViewProcessor = false;
}
Tenant->IsExternalHive &= Tenant->IsExternalSubdomain; // external hive without external sub domain is pointless
- Tenant->IsExternalSysViewProcessor &= Tenant->IsExternalSubdomain;
+ Tenant->IsExternalSysViewProcessor &= Tenant->IsExternalSubdomain;
Tenant->StorageUnitsQuota = Self->Config.DefaultStorageUnitsQuota;
Tenant->ComputationalUnitsQuota = Self->Config.DefaultComputationalUnitsQuota;
diff --git a/ydb/core/cms/console/console__scheme.h b/ydb/core/cms/console/console__scheme.h
index 29eb6c3035a..1f61019db24 100644
--- a/ydb/core/cms/console/console__scheme.h
+++ b/ydb/core/cms/console/console__scheme.h
@@ -42,7 +42,7 @@ struct Schema : NIceDb::Schema {
// SharedDomainId {
struct SharedDomainSchemeShardId : Column<21, NScheme::NTypeIds::Uint64> {};
struct SharedDomainPathId : Column<22, NScheme::NTypeIds::Uint64> {};
- struct IsExternalSysViewProcessor : Column<23, NScheme::NTypeIds::Bool> {};
+ struct IsExternalSysViewProcessor : Column<23, NScheme::NTypeIds::Bool> {};
// } // SharedDomainId
struct SchemaOperationQuotas : Column<24, NScheme::NTypeIds::String> {};
struct CreateIdempotencyKey : Column<25, NScheme::NTypeIds::Utf8> {};
diff --git a/ydb/core/cms/console/console_tenants_manager.cpp b/ydb/core/cms/console/console_tenants_manager.cpp
index 0db3f5dceda..b40b1eed02c 100644
--- a/ydb/core/cms/console/console_tenants_manager.cpp
+++ b/ydb/core/cms/console/console_tenants_manager.cpp
@@ -455,9 +455,9 @@ public:
if (Tenant->IsExternalHive) {
subdomain.SetExternalHive(true);
}
- if (Tenant->IsExternalSysViewProcessor) {
- subdomain.SetExternalSysViewProcessor(true);
- }
+ if (Tenant->IsExternalSysViewProcessor) {
+ subdomain.SetExternalSysViewProcessor(true);
+ }
}
if (SharedTenant) {
@@ -483,9 +483,9 @@ public:
if (Tenant->IsExternalHive) {
subdomain.SetExternalHive(true);
}
- if (Tenant->IsExternalSysViewProcessor) {
- subdomain.SetExternalSysViewProcessor(true);
- }
+ if (Tenant->IsExternalSysViewProcessor) {
+ subdomain.SetExternalSysViewProcessor(true);
+ }
}
for (auto &pr : (SharedTenant ? SharedTenant->StoragePools : Tenant->StoragePools)) {
@@ -1183,7 +1183,7 @@ TTenantsManager::TTenant::TTenant(const TString &path,
, Generation(0)
, IsExternalSubdomain(false)
, IsExternalHive(false)
- , IsExternalSysViewProcessor(false)
+ , IsExternalSysViewProcessor(false)
, AreResourcesShared(false)
{
}
@@ -2248,7 +2248,7 @@ void TTenantsManager::DbAddTenant(TTenant::TPtr tenant,
<< " errorcode=" << tenant->ErrorCode
<< " isExternalSubDomain=" << tenant->IsExternalSubdomain
<< " isExternalHive=" << tenant->IsExternalHive
- << " isExternalSysViewProcessor=" << tenant->IsExternalSysViewProcessor
+ << " isExternalSysViewProcessor=" << tenant->IsExternalSysViewProcessor
<< " areResourcesShared=" << tenant->AreResourcesShared
<< " sharedDomainId=" << tenant->SharedDomainId);
@@ -2269,7 +2269,7 @@ void TTenantsManager::DbAddTenant(TTenant::TPtr tenant,
NIceDb::TUpdate<Schema::Tenants::ErrorCode>(tenant->ErrorCode),
NIceDb::TUpdate<Schema::Tenants::IsExternalSubDomain>(tenant->IsExternalSubdomain),
NIceDb::TUpdate<Schema::Tenants::IsExternalHive>(tenant->IsExternalHive),
- NIceDb::TUpdate<Schema::Tenants::IsExternalSysViewProcessor>(tenant->IsExternalSysViewProcessor),
+ NIceDb::TUpdate<Schema::Tenants::IsExternalSysViewProcessor>(tenant->IsExternalSysViewProcessor),
NIceDb::TUpdate<Schema::Tenants::AreResourcesShared>(tenant->AreResourcesShared),
NIceDb::TUpdate<Schema::Tenants::CreateIdempotencyKey>(tenant->CreateIdempotencyKey));
@@ -2373,7 +2373,7 @@ bool TTenantsManager::DbLoadState(TTransactionContext &txc, const TActorContext
= static_cast<Ydb::StatusIds::StatusCode>(tenantRowset.GetValueOrDefault<Schema::Tenants::ErrorCode>(0));
bool isExternalSubDomain = tenantRowset.GetValueOrDefault<Schema::Tenants::IsExternalSubDomain>(false);
bool isExternalHive = tenantRowset.GetValueOrDefault<Schema::Tenants::IsExternalHive>(false);
- bool isExternalSysViewProcessor = tenantRowset.GetValueOrDefault<Schema::Tenants::IsExternalSysViewProcessor>(false);
+ bool isExternalSysViewProcessor = tenantRowset.GetValueOrDefault<Schema::Tenants::IsExternalSysViewProcessor>(false);
const bool areResourcesShared = tenantRowset.GetValueOrDefault<Schema::Tenants::AreResourcesShared>(false);
TTenant::TPtr tenant = new TTenant(path, state, userToken);
@@ -2392,7 +2392,7 @@ bool TTenantsManager::DbLoadState(TTransactionContext &txc, const TActorContext
tenant->Issue = issue;
tenant->IsExternalSubdomain = isExternalSubDomain;
tenant->IsExternalHive = isExternalHive;
- tenant->IsExternalSysViewProcessor = isExternalSysViewProcessor;
+ tenant->IsExternalSysViewProcessor = isExternalSysViewProcessor;
tenant->AreResourcesShared = areResourcesShared;
if (tenantRowset.HaveValue<Schema::Tenants::SchemaOperationQuotas>()) {
diff --git a/ydb/core/cms/console/console_tenants_manager.h b/ydb/core/cms/console/console_tenants_manager.h
index a21615dd622..9e6d481e23e 100644
--- a/ydb/core/cms/console/console_tenants_manager.h
+++ b/ydb/core/cms/console/console_tenants_manager.h
@@ -518,7 +518,7 @@ public:
TDomainId SharedDomainId;
bool IsExternalSubdomain;
bool IsExternalHive;
- bool IsExternalSysViewProcessor;
+ bool IsExternalSysViewProcessor;
bool AreResourcesShared;
THashSet<TTenant::TPtr> HostedTenants;
diff --git a/ydb/core/control/immediate_control_board_impl.cpp b/ydb/core/control/immediate_control_board_impl.cpp
index fa26926fae5..48bcf2c7732 100644
--- a/ydb/core/control/immediate_control_board_impl.cpp
+++ b/ydb/core/control/immediate_control_board_impl.cpp
@@ -7,49 +7,49 @@
namespace NKikimr {
bool TControlBoard::RegisterLocalControl(TControlWrapper control, TString name) {
- bool result = true;
- if (Board.Has(name)) {
- result = false;
+ bool result = true;
+ if (Board.Has(name)) {
+ result = false;
}
- Board.Insert(name, control.Control);
- return result;
+ Board.Insert(name, control.Control);
+ return result;
}
void TControlBoard::RegisterSharedControl(TControlWrapper& control, TString name) {
- control.Control = Board.InsertIfAbsent(name, control.Control);
+ control.Control = Board.InsertIfAbsent(name, control.Control);
}
void TControlBoard::RestoreDefaults() {
- for (auto& bucket : Board.Buckets) {
- TReadGuard guard(bucket.GetLock());
- for (auto &control : bucket.GetMap()) {
- control.second->RestoreDefault();
- }
+ for (auto& bucket : Board.Buckets) {
+ TReadGuard guard(bucket.GetLock());
+ for (auto &control : bucket.GetMap()) {
+ control.second->RestoreDefault();
+ }
}
}
void TControlBoard::RestoreDefault(TString name) {
- TIntrusivePtr<TControl> control;
- if (Board.Get(name, control)) {
- control->RestoreDefault();
- }
+ TIntrusivePtr<TControl> control;
+ if (Board.Get(name, control)) {
+ control->RestoreDefault();
+ }
}
bool TControlBoard::SetValue(TString name, TAtomic value, TAtomic &outPrevValue) {
- TIntrusivePtr<TControl> control;
- if (Board.Get(name, control)) {
- outPrevValue = control->SetFromHtmlRequest(value);
- return control->IsDefault();
+ TIntrusivePtr<TControl> control;
+ if (Board.Get(name, control)) {
+ outPrevValue = control->SetFromHtmlRequest(value);
+ return control->IsDefault();
}
return true;
}
// Only for tests
void TControlBoard::GetValue(TString name, TAtomic &outValue, bool &outIsControlExists) const {
- TIntrusivePtr<TControl> control;
- outIsControlExists = Board.Get(name, control);
+ TIntrusivePtr<TControl> control;
+ outIsControlExists = Board.Get(name, control);
if (outIsControlExists) {
- outValue = control->Get();
+ outValue = control->Get();
}
}
@@ -68,34 +68,34 @@ TString TControlBoard::RenderAsHtml() const {
}
}
TABLEBODY() {
- for (const auto& bucket : Board.Buckets) {
- TReadGuard guard(bucket.GetLock());
- for (const auto &item : bucket.GetMap()) {
- TABLER() {
- TABLED() { str << item.first; }
- TABLED() { str << item.second->RangeAsString(); }
- TABLED() {
- if (item.second->IsDefault()) {
- str << "<p>" << item.second->Get() << "</p>";
- } else {
- str << "<p style='color:red;'><b>" << item.second->Get() << " </b></p>";
- }
+ for (const auto& bucket : Board.Buckets) {
+ TReadGuard guard(bucket.GetLock());
+ for (const auto &item : bucket.GetMap()) {
+ TABLER() {
+ TABLED() { str << item.first; }
+ TABLED() { str << item.second->RangeAsString(); }
+ TABLED() {
+ if (item.second->IsDefault()) {
+ str << "<p>" << item.second->Get() << "</p>";
+ } else {
+ str << "<p style='color:red;'><b>" << item.second->Get() << " </b></p>";
+ }
}
- TABLED() {
- if (item.second->IsDefault()) {
- str << "<p>" << item.second->GetDefault() << "</p>";
- } else {
- str << "<p style='color:red;'><b>" << item.second->GetDefault() << " </b></p>";
- }
+ TABLED() {
+ if (item.second->IsDefault()) {
+ str << "<p>" << item.second->GetDefault() << "</p>";
+ } else {
+ str << "<p style='color:red;'><b>" << item.second->GetDefault() << " </b></p>";
+ }
}
- TABLED() {
- str << "<form class='form_horizontal' method='post'>";
- str << "<input name='" << item.first << "' type='text' value='"
- << item.second->Get() << "'/>";
- str << "<button type='submit' style='color:red;'><b>Change</b></button>";
- str << "</form>";
- }
- TABLED() { str << !item.second->IsDefault(); }
+ TABLED() {
+ str << "<form class='form_horizontal' method='post'>";
+ str << "<input name='" << item.first << "' type='text' value='"
+ << item.second->Get() << "'/>";
+ str << "<button type='submit' style='color:red;'><b>Change</b></button>";
+ str << "</form>";
+ }
+ TABLED() { str << !item.second->IsDefault(); }
}
}
}
@@ -108,4 +108,4 @@ TString TControlBoard::RenderAsHtml() const {
return str.Str();
}
-}
+}
diff --git a/ydb/core/control/immediate_control_board_impl.h b/ydb/core/control/immediate_control_board_impl.h
index a01e09f4d6c..dc79f15a93b 100644
--- a/ydb/core/control/immediate_control_board_impl.h
+++ b/ydb/core/control/immediate_control_board_impl.h
@@ -1,5 +1,5 @@
#pragma once
-
+
#include "defs.h"
#include "immediate_control_board_wrapper.h"
@@ -8,8 +8,8 @@
namespace NKikimr {
class TControlBoard : public TThrRefBase {
-private:
- TConcurrentRWHashMap<TString, TIntrusivePtr<TControl>, 16> Board;
+private:
+ TConcurrentRWHashMap<TString, TIntrusivePtr<TControl>, 16> Board;
public:
bool RegisterLocalControl(TControlWrapper control, TString name);
diff --git a/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp b/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp
index 450d4135c6c..df8cb170481 100644
--- a/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp
+++ b/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp
@@ -1127,17 +1127,17 @@ public:
}
const TString& key = items.at(0);
- if (key.size() > TUserAttributesLimits::MaxNameLen) {
+ if (key.size() > TUserAttributesLimits::MaxNameLen) {
Cerr << "Key '" << key << "' too long"
- << ", max: " << TUserAttributesLimits::MaxNameLen
+ << ", max: " << TUserAttributesLimits::MaxNameLen
<< ", actual: " << key.size() << Endl;
exit(1);
}
const TString& value = items.at(1);
- if (value.size() > TUserAttributesLimits::MaxValueLen) {
+ if (value.size() > TUserAttributesLimits::MaxValueLen) {
Cerr << "Value '" << value << "' too long"
- << ", max: " << TUserAttributesLimits::MaxValueLen
+ << ", max: " << TUserAttributesLimits::MaxValueLen
<< ", actual: " << value.size() << Endl;
exit(1);
}
diff --git a/ydb/core/driver_lib/run/config.h b/ydb/core/driver_lib/run/config.h
index faf1797413e..2d6fb9391b0 100644
--- a/ydb/core/driver_lib/run/config.h
+++ b/ydb/core/driver_lib/run/config.h
@@ -56,7 +56,7 @@ union TBasicKikimrServicesMask {
bool EnablePersQueueClusterDiscovery:1;
bool EnableNetClassifier:1;
bool EnablePersQueueClusterTracker:1;
- bool EnableSysViewService:1;
+ bool EnableSysViewService:1;
bool EnableMeteringWriter:1;
bool EnableSchemeBoardMonitoring:1;
bool EnableConfigsCache:1;
diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp
index 819c1478d19..23e70a19df4 100644
--- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp
+++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp
@@ -918,8 +918,8 @@ void TLocalServiceInitializer::InitializeServices(
new TTabletSetupInfo(&NKesus::CreateKesusTablet, TMailboxType::ReadAsFilled, appData->UserPoolId, TMailboxType::ReadAsFilled, appData->SystemPoolId));
localConfig->TabletClassInfo[appData->DefaultTabletTypes.Hive] = TLocalConfig::TTabletClassInfo(
new TTabletSetupInfo(&CreateDefaultHive, TMailboxType::ReadAsFilled, importantPoolId, TMailboxType::ReadAsFilled, appData->SystemPoolId));
- localConfig->TabletClassInfo[appData->DefaultTabletTypes.SysViewProcessor] = TLocalConfig::TTabletClassInfo(
- new TTabletSetupInfo(&NSysView::CreateSysViewProcessor, TMailboxType::ReadAsFilled, appData->UserPoolId, TMailboxType::ReadAsFilled, appData->SystemPoolId));
+ localConfig->TabletClassInfo[appData->DefaultTabletTypes.SysViewProcessor] = TLocalConfig::TTabletClassInfo(
+ new TTabletSetupInfo(&NSysView::CreateSysViewProcessor, TMailboxType::ReadAsFilled, appData->UserPoolId, TMailboxType::ReadAsFilled, appData->SystemPoolId));
localConfig->TabletClassInfo[appData->DefaultTabletTypes.TestShard] = TLocalConfig::TTabletClassInfo(
new TTabletSetupInfo(&NTestShard::CreateTestShard, TMailboxType::ReadAsFilled, appData->UserPoolId, TMailboxType::ReadAsFilled, appData->SystemPoolId));
localConfig->TabletClassInfo[appData->DefaultTabletTypes.ColumnShard] = TLocalConfig::TTabletClassInfo(
@@ -1637,7 +1637,7 @@ void TStatsCollectorInitializer::InitializeServices(
IActor* statsCollector = CreateStatsCollector(
1, // seconds
*setup,
- appData->Counters);
+ appData->Counters);
setup->LocalServices.emplace_back(
TActorId(),
TActorSetupCmd(
@@ -1657,7 +1657,7 @@ void TStatsCollectorInitializer::InitializeServices(
IActor* procStatCollector = CreateProcStatCollector(
5, // seconds
- appData->Counters);
+ appData->Counters);
setup->LocalServices.emplace_back(
TActorId(),
TActorSetupCmd(
@@ -1904,16 +1904,16 @@ TMemoryTrackerInitializer::TMemoryTrackerInitializer(const TKikimrRunConfig& run
: IKikimrServicesInitializer(runConfig)
{}
-void TMemoryTrackerInitializer::InitializeServices(
- NActors::TActorSystemSetup* setup,
- const NKikimr::TAppData* appData)
-{
- auto* actor = NMemory::CreateMemoryTrackerActor(TDuration::MilliSeconds(20), appData->Counters);
- setup->LocalServices.emplace_back(
- TActorId(),
- TActorSetupCmd(actor, TMailboxType::HTSwap, appData->UserPoolId)
- );
-}
+void TMemoryTrackerInitializer::InitializeServices(
+ NActors::TActorSystemSetup* setup,
+ const NKikimr::TAppData* appData)
+{
+ auto* actor = NMemory::CreateMemoryTrackerActor(TDuration::MilliSeconds(20), appData->Counters);
+ setup->LocalServices.emplace_back(
+ TActorId(),
+ TActorSetupCmd(actor, TMailboxType::HTSwap, appData->UserPoolId)
+ );
+}
TQuoterServiceInitializer::TQuoterServiceInitializer(const TKikimrRunConfig& runConfig)
: IKikimrServicesInitializer(runConfig)
@@ -2165,29 +2165,29 @@ void TConfigValidatorsInitializer::InitializeServices(NActors::TActorSystemSetup
NConsole::RegisterCoreValidators();
}
-TSysViewServiceInitializer::TSysViewServiceInitializer(const TKikimrRunConfig& runConfig)
- : IKikimrServicesInitializer(runConfig)
-{
-}
+TSysViewServiceInitializer::TSysViewServiceInitializer(const TKikimrRunConfig& runConfig)
+ : IKikimrServicesInitializer(runConfig)
+{
+}
-void TSysViewServiceInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) {
- NSysView::TExtCountersConfig config;
+void TSysViewServiceInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) {
+ NSysView::TExtCountersConfig config;
for (ui32 i = 0; i < setup->GetExecutorsCount(); ++i) {
- config.Pools.push_back(NSysView::TExtCountersConfig::TPool{
+ config.Pools.push_back(NSysView::TExtCountersConfig::TPool{
setup->GetPoolName(i),
setup->GetThreads(i)});
- }
-
- // external counters only for dynamic nodes
- bool hasExternalCounters = Config.GetDynamicNodeConfig().HasNodeInfo();
-
- auto actor = NSysView::CreateSysViewService(std::move(config), hasExternalCounters);
-
+ }
+
+ // external counters only for dynamic nodes
+ bool hasExternalCounters = Config.GetDynamicNodeConfig().HasNodeInfo();
+
+ auto actor = NSysView::CreateSysViewService(std::move(config), hasExternalCounters);
+
setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(
- NSysView::MakeSysViewServiceID(NodeId),
- TActorSetupCmd(actor.Release(), TMailboxType::HTSwap, appData->UserPoolId)));
-}
-
+ NSysView::MakeSysViewServiceID(NodeId),
+ TActorSetupCmd(actor.Release(), TMailboxType::HTSwap, appData->UserPoolId)));
+}
+
TMeteringWriterInitializer::TMeteringWriterInitializer(const TKikimrRunConfig &runConfig)
: IKikimrServicesInitializer(runConfig)
{
diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.h b/ydb/core/driver_lib/run/kikimr_services_initializers.h
index 407ce1bb7ba..40aa6e47d50 100644
--- a/ydb/core/driver_lib/run/kikimr_services_initializers.h
+++ b/ydb/core/driver_lib/run/kikimr_services_initializers.h
@@ -364,13 +364,13 @@ public:
void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override;
};
-class TMemoryTrackerInitializer : public IKikimrServicesInitializer {
-public:
+class TMemoryTrackerInitializer : public IKikimrServicesInitializer {
+public:
TMemoryTrackerInitializer(const TKikimrRunConfig& runConfig);
void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override;
-};
-
+};
+
class TQuoterServiceInitializer : public IKikimrServicesInitializer {
public:
TQuoterServiceInitializer(const TKikimrRunConfig& runConfig);
@@ -451,13 +451,13 @@ public:
void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override;
};
-class TSysViewServiceInitializer : public IKikimrServicesInitializer {
-public:
- TSysViewServiceInitializer(const TKikimrRunConfig& runConfig);
-
- void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override;
-};
+class TSysViewServiceInitializer : public IKikimrServicesInitializer {
+public:
+ TSysViewServiceInitializer(const TKikimrRunConfig& runConfig);
+ void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override;
+};
+
class TMeteringWriterInitializer : public IKikimrServicesInitializer {
public:
TMeteringWriterInitializer(const TKikimrRunConfig& runConfig);
diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp
index a4f74aa4e0f..ef331f47826 100644
--- a/ydb/core/driver_lib/run/run.cpp
+++ b/ydb/core/driver_lib/run/run.cpp
@@ -114,7 +114,7 @@
#include <ydb/core/node_whiteboard/node_whiteboard.h>
#include <ydb/core/tablet/node_tablet_monitor.h>
-#include <library/cpp/actors/core/memory_track.h>
+#include <library/cpp/actors/core/memory_track.h>
#include <library/cpp/actors/prof/tag.h>
#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h>
@@ -1264,10 +1264,10 @@ TIntrusivePtr<TServiceInitializersList> TKikimrRunner::CreateServiceInitializers
sil->AddServiceInitializer(new TMemProfMonitorInitializer(runConfig));
-#if defined(ENABLE_MEMORY_TRACKING)
- sil->AddServiceInitializer(new TMemoryTrackerInitializer(runConfig));
-#endif
-
+#if defined(ENABLE_MEMORY_TRACKING)
+ sil->AddServiceInitializer(new TMemoryTrackerInitializer(runConfig));
+#endif
+
if (serviceMask.EnableKqp) {
sil->AddServiceInitializer(new TKqpServiceInitializer(runConfig, ModuleFactories));
}
@@ -1299,10 +1299,10 @@ TIntrusivePtr<TServiceInitializersList> TKikimrRunner::CreateServiceInitializers
sil->AddServiceInitializer(new TQuoterServiceInitializer(runConfig));
}
- if (serviceMask.EnableSysViewService) {
- sil->AddServiceInitializer(new TSysViewServiceInitializer(runConfig));
- }
-
+ if (serviceMask.EnableSysViewService) {
+ sil->AddServiceInitializer(new TSysViewServiceInitializer(runConfig));
+ }
+
if (serviceMask.EnableMeteringWriter) {
sil->AddServiceInitializer(new TMeteringWriterInitializer(runConfig));
}
diff --git a/ydb/core/engine/minikql/flat_local_minikql_host.h b/ydb/core/engine/minikql/flat_local_minikql_host.h
index edfd1822869..da3ea2c44f2 100644
--- a/ydb/core/engine/minikql/flat_local_minikql_host.h
+++ b/ydb/core/engine/minikql/flat_local_minikql_host.h
@@ -21,7 +21,7 @@ private:
bool IsMyKey(const TTableId& tableId, const TArrayRef<const TCell>& row) const override
{
Y_UNUSED(row);
- return (tableId.PathId.OwnerId == GetShardId());
+ return (tableId.PathId.OwnerId == GetShardId());
}
TRowVersion GetWriteVersion(const TTableId& tableId) const override
diff --git a/ydb/core/engine/minikql/minikql_engine_host.cpp b/ydb/core/engine/minikql/minikql_engine_host.cpp
index 36c35f32cc4..b6802a64a19 100644
--- a/ydb/core/engine/minikql/minikql_engine_host.cpp
+++ b/ydb/core/engine/minikql/minikql_engine_host.cpp
@@ -933,7 +933,7 @@ ui64 TEngineHost::GetTableSchemaVersion(const TTableId&) const {
}
ui64 TEngineHost::LocalTableId(const TTableId& tableId) const {
- return tableId.PathId.LocalPathId;
+ return tableId.PathId.LocalPathId;
}
void TEngineHost::ConvertKeys(const TScheme::TTableInfo* tableInfo, const TArrayRef<const TCell>& row,
diff --git a/ydb/core/engine/mkql_engine_flat_ut.cpp b/ydb/core/engine/mkql_engine_flat_ut.cpp
index e2b12e2d7a5..733b016009c 100644
--- a/ydb/core/engine/mkql_engine_flat_ut.cpp
+++ b/ydb/core/engine/mkql_engine_flat_ut.cpp
@@ -99,7 +99,7 @@ namespace {
void TwoShardResolver(TKeyDesc& key) {
key.Status = TKeyDesc::EStatus::Ok;
- key.Partitions.push_back(TKeyDesc::TPartitionInfo(key.TableId.PathId.LocalPathId == Table1Id ? Shard1 : Shard2));
+ key.Partitions.push_back(TKeyDesc::TPartitionInfo(key.TableId.PathId.LocalPathId == Table1Id ? Shard1 : Shard2));
}
struct TDriver {
diff --git a/ydb/core/grpc_services/base/base.h b/ydb/core/grpc_services/base/base.h
index 44b25c4a5f9..8db29589ec1 100644
--- a/ydb/core/grpc_services/base/base.h
+++ b/ydb/core/grpc_services/base/base.h
@@ -330,7 +330,7 @@ public:
virtual void ReplyUnavaliable() = 0;
virtual bool Validate(TString& error) = 0;
- virtual void UseDatabase(const TString& database) = 0;
+ virtual void UseDatabase(const TString& database) = 0;
// This method allows to set hook for unary call.
// The hook will be called at the reply time
@@ -479,10 +479,10 @@ public:
return true;
}
- void UseDatabase(const TString& database) override {
- Y_UNUSED(database);
- }
-
+ void UseDatabase(const TString& database) override {
+ Y_UNUSED(database);
+ }
+
TActorId GetFromId() const {
return From_;
}
@@ -638,10 +638,10 @@ public:
return true;
}
- void UseDatabase(const TString& database) override {
- Ctx_->UseDatabase(database);
- }
-
+ void UseDatabase(const TString& database) override {
+ Ctx_->UseDatabase(database);
+ }
+
IStreamCtx* GetStreamCtx() {
return Ctx_.Get();
}
@@ -863,10 +863,10 @@ public:
return true;
}
- void UseDatabase(const TString& database) override {
- Ctx_->UseDatabase(database);
- }
-
+ void UseDatabase(const TString& database) override {
+ Ctx_->UseDatabase(database);
+ }
+
void ReplyUnavaliable() override {
TResponse* resp = CreateResponseMessage();
TCommonResponseFiller<TResp, TDerived::IsOp>::Fill(*resp, IssueManager.GetIssues(), CostInfo, Ydb::StatusIds::UNAVAILABLE);
diff --git a/ydb/core/grpc_services/counters/counters.cpp b/ydb/core/grpc_services/counters/counters.cpp
index 8fc8ede9174..eea74746312 100644
--- a/ydb/core/grpc_services/counters/counters.cpp
+++ b/ydb/core/grpc_services/counters/counters.cpp
@@ -1,645 +1,645 @@
-#include "counters.h"
-
+#include "counters.h"
+
#include <ydb/core/base/appdata.h>
#include <ydb/core/base/counters.h>
#include <ydb/core/sys_view/service/db_counters.h>
#include <ydb/core/sys_view/service/sysview_service.h>
#include <ydb/core/util/concurrent_rw_hash.h>
#include <ydb/public/api/protos/ydb_status_codes.pb.h>
-
-#include <library/cpp/deprecated/enum_codegen/enum_codegen.h>
-
-namespace NKikimr {
-namespace NGRpcService {
-
-using namespace NActors;
-
-struct TYdbRpcCounters {
- TYdbRpcCounters(const NMonitoring::TDynamicCounterPtr& counters, const char* serviceName,
- const char* requestName, bool forDatabase);
-
- NMonitoring::TDynamicCounters::TCounterPtr RequestCount;
- NMonitoring::TDynamicCounters::TCounterPtr RequestInflight;
- NMonitoring::TDynamicCounters::TCounterPtr RequestBytes;
- NMonitoring::TDynamicCounters::TCounterPtr RequestInflightBytes;
- NMonitoring::TDynamicCounters::TCounterPtr RequestRpcError;
-
- NMonitoring::TDynamicCounters::TCounterPtr ResponseBytes;
- NMonitoring::TDynamicCounters::TCounterPtr ResponseRpcError;
- NMonitoring::TDynamicCounters::TCounterPtr ResponseRpcNotAuthenticated;
- NMonitoring::TDynamicCounters::TCounterPtr ResponseRpcResourceExhausted;
- THashMap<ui32, NMonitoring::TDynamicCounters::TCounterPtr> ResponseByStatus;
-};
-
+
+#include <library/cpp/deprecated/enum_codegen/enum_codegen.h>
+
+namespace NKikimr {
+namespace NGRpcService {
+
+using namespace NActors;
+
+struct TYdbRpcCounters {
+ TYdbRpcCounters(const NMonitoring::TDynamicCounterPtr& counters, const char* serviceName,
+ const char* requestName, bool forDatabase);
+
+ NMonitoring::TDynamicCounters::TCounterPtr RequestCount;
+ NMonitoring::TDynamicCounters::TCounterPtr RequestInflight;
+ NMonitoring::TDynamicCounters::TCounterPtr RequestBytes;
+ NMonitoring::TDynamicCounters::TCounterPtr RequestInflightBytes;
+ NMonitoring::TDynamicCounters::TCounterPtr RequestRpcError;
+
+ NMonitoring::TDynamicCounters::TCounterPtr ResponseBytes;
+ NMonitoring::TDynamicCounters::TCounterPtr ResponseRpcError;
+ NMonitoring::TDynamicCounters::TCounterPtr ResponseRpcNotAuthenticated;
+ NMonitoring::TDynamicCounters::TCounterPtr ResponseRpcResourceExhausted;
+ THashMap<ui32, NMonitoring::TDynamicCounters::TCounterPtr> ResponseByStatus;
+};
+
class TYdbCounterBlock : public NGrpc::ICounterBlock {
-protected:
- bool Streaming = false;
- bool Percentile = false;
-
- // "Internal" counters
- // TODO: Switch to public YDB counters
- NMonitoring::TDynamicCounters::TCounterPtr TotalCounter;
- NMonitoring::TDynamicCounters::TCounterPtr InflyCounter;
- NMonitoring::TDynamicCounters::TCounterPtr NotOkRequestCounter;
- NMonitoring::TDynamicCounters::TCounterPtr NotOkResponseCounter;
- NMonitoring::TDynamicCounters::TCounterPtr RequestBytes;
- NMonitoring::TDynamicCounters::TCounterPtr InflyRequestBytes;
- NMonitoring::TDynamicCounters::TCounterPtr ResponseBytes;
- NMonitoring::TDynamicCounters::TCounterPtr NotAuthenticated;
- NMonitoring::TDynamicCounters::TCounterPtr ResourceExhausted;
- NMonitoring::TDynamicCounters::TCounterPtr RequestsWithoutDatabase;
- NMonitoring::TDynamicCounters::TCounterPtr RequestsWithoutToken;
- NMonitoring::TDynamicCounters::TCounterPtr RequestsWithoutTls;
- NMonitoring::TPercentileTracker<4, 512, 15> RequestHistMs;
- std::array<NMonitoring::TDynamicCounters::TCounterPtr, 2> GRpcStatusCounters;
-
- TYdbRpcCounters YdbCounters;
-
-public:
- TYdbCounterBlock(const NMonitoring::TDynamicCounterPtr& counters, const char* serviceName,
- const char* requestName, bool percentile, bool streaming,
- bool forDatabase = false, NMonitoring::TDynamicCounterPtr internalGroup = {});
-
- void CountNotOkRequest() override {
- NotOkRequestCounter->Inc();
- YdbCounters.RequestRpcError->Inc();
- }
-
- void CountNotOkResponse() override {
- NotOkResponseCounter->Inc();
- YdbCounters.ResponseRpcError->Inc();
- }
-
- void CountNotAuthenticated() override {
- NotAuthenticated->Inc();
- YdbCounters.ResponseRpcNotAuthenticated->Inc();
- }
-
- void CountResourceExhausted() override {
- ResourceExhausted->Inc();
- YdbCounters.ResponseRpcResourceExhausted->Inc();
- }
-
- void CountRequestsWithoutDatabase() override {
- RequestsWithoutDatabase->Inc();
- }
-
- void CountRequestsWithoutToken() override {
- RequestsWithoutToken->Inc();
- }
-
- void CountRequestWithoutTls() override {
- RequestsWithoutTls->Inc();
- }
-
- void CountRequestBytes(ui32 requestSize) override {
- *RequestBytes += requestSize;
- *YdbCounters.RequestBytes += requestSize;
- }
-
- void CountResponseBytes(ui32 responseSize) override {
- *ResponseBytes += responseSize;
- *YdbCounters.ResponseBytes += responseSize;
- }
-
- void StartProcessing(ui32 requestSize) override {
- TotalCounter->Inc();
- InflyCounter->Inc();
- *RequestBytes += requestSize;
- *InflyRequestBytes += requestSize;
-
- YdbCounters.RequestCount->Inc();
- YdbCounters.RequestInflight->Inc();
- *YdbCounters.RequestBytes += requestSize;
- *YdbCounters.RequestInflightBytes += requestSize;
- }
-
- void FinishProcessing(ui32 requestSize, ui32 responseSize, bool ok, ui32 status,
- TDuration requestDuration) override
- {
- InflyCounter->Dec();
- *InflyRequestBytes -= requestSize;
- *ResponseBytes += responseSize;
-
- YdbCounters.RequestInflight->Dec();
- *YdbCounters.RequestInflightBytes -= requestSize;
- *YdbCounters.ResponseBytes += responseSize;
-
- if (!ok) {
- NotOkResponseCounter->Inc();
- YdbCounters.ResponseRpcError->Inc();
- } else if (!Streaming) {
- auto statusCounter = YdbCounters.ResponseByStatus.FindPtr(status);
- if (statusCounter) {
- (*statusCounter)->Inc();
- } else {
- YdbCounters.ResponseByStatus[0]->Inc();
- }
- }
-
- if (Percentile) {
- RequestHistMs.Increment(requestDuration.MilliSeconds());
- }
- }
-
+protected:
+ bool Streaming = false;
+ bool Percentile = false;
+
+ // "Internal" counters
+ // TODO: Switch to public YDB counters
+ NMonitoring::TDynamicCounters::TCounterPtr TotalCounter;
+ NMonitoring::TDynamicCounters::TCounterPtr InflyCounter;
+ NMonitoring::TDynamicCounters::TCounterPtr NotOkRequestCounter;
+ NMonitoring::TDynamicCounters::TCounterPtr NotOkResponseCounter;
+ NMonitoring::TDynamicCounters::TCounterPtr RequestBytes;
+ NMonitoring::TDynamicCounters::TCounterPtr InflyRequestBytes;
+ NMonitoring::TDynamicCounters::TCounterPtr ResponseBytes;
+ NMonitoring::TDynamicCounters::TCounterPtr NotAuthenticated;
+ NMonitoring::TDynamicCounters::TCounterPtr ResourceExhausted;
+ NMonitoring::TDynamicCounters::TCounterPtr RequestsWithoutDatabase;
+ NMonitoring::TDynamicCounters::TCounterPtr RequestsWithoutToken;
+ NMonitoring::TDynamicCounters::TCounterPtr RequestsWithoutTls;
+ NMonitoring::TPercentileTracker<4, 512, 15> RequestHistMs;
+ std::array<NMonitoring::TDynamicCounters::TCounterPtr, 2> GRpcStatusCounters;
+
+ TYdbRpcCounters YdbCounters;
+
+public:
+ TYdbCounterBlock(const NMonitoring::TDynamicCounterPtr& counters, const char* serviceName,
+ const char* requestName, bool percentile, bool streaming,
+ bool forDatabase = false, NMonitoring::TDynamicCounterPtr internalGroup = {});
+
+ void CountNotOkRequest() override {
+ NotOkRequestCounter->Inc();
+ YdbCounters.RequestRpcError->Inc();
+ }
+
+ void CountNotOkResponse() override {
+ NotOkResponseCounter->Inc();
+ YdbCounters.ResponseRpcError->Inc();
+ }
+
+ void CountNotAuthenticated() override {
+ NotAuthenticated->Inc();
+ YdbCounters.ResponseRpcNotAuthenticated->Inc();
+ }
+
+ void CountResourceExhausted() override {
+ ResourceExhausted->Inc();
+ YdbCounters.ResponseRpcResourceExhausted->Inc();
+ }
+
+ void CountRequestsWithoutDatabase() override {
+ RequestsWithoutDatabase->Inc();
+ }
+
+ void CountRequestsWithoutToken() override {
+ RequestsWithoutToken->Inc();
+ }
+
+ void CountRequestWithoutTls() override {
+ RequestsWithoutTls->Inc();
+ }
+
+ void CountRequestBytes(ui32 requestSize) override {
+ *RequestBytes += requestSize;
+ *YdbCounters.RequestBytes += requestSize;
+ }
+
+ void CountResponseBytes(ui32 responseSize) override {
+ *ResponseBytes += responseSize;
+ *YdbCounters.ResponseBytes += responseSize;
+ }
+
+ void StartProcessing(ui32 requestSize) override {
+ TotalCounter->Inc();
+ InflyCounter->Inc();
+ *RequestBytes += requestSize;
+ *InflyRequestBytes += requestSize;
+
+ YdbCounters.RequestCount->Inc();
+ YdbCounters.RequestInflight->Inc();
+ *YdbCounters.RequestBytes += requestSize;
+ *YdbCounters.RequestInflightBytes += requestSize;
+ }
+
+ void FinishProcessing(ui32 requestSize, ui32 responseSize, bool ok, ui32 status,
+ TDuration requestDuration) override
+ {
+ InflyCounter->Dec();
+ *InflyRequestBytes -= requestSize;
+ *ResponseBytes += responseSize;
+
+ YdbCounters.RequestInflight->Dec();
+ *YdbCounters.RequestInflightBytes -= requestSize;
+ *YdbCounters.ResponseBytes += responseSize;
+
+ if (!ok) {
+ NotOkResponseCounter->Inc();
+ YdbCounters.ResponseRpcError->Inc();
+ } else if (!Streaming) {
+ auto statusCounter = YdbCounters.ResponseByStatus.FindPtr(status);
+ if (statusCounter) {
+ (*statusCounter)->Inc();
+ } else {
+ YdbCounters.ResponseByStatus[0]->Inc();
+ }
+ }
+
+ if (Percentile) {
+ RequestHistMs.Increment(requestDuration.MilliSeconds());
+ }
+ }
+
NGrpc::ICounterBlockPtr Clone() override {
- return this;
- }
-
- void Update() {
- if (Percentile) {
- RequestHistMs.Update();
- }
- }
-};
-
-TYdbRpcCounters::TYdbRpcCounters(const NMonitoring::TDynamicCounterPtr& counters,
- const char* serviceName, const char* requestName, bool forDatabase)
-{
- NMonitoring::TDynamicCounterPtr ydbGroup;
- if (forDatabase) {
- ydbGroup = counters;
- } else {
- ydbGroup = GetServiceCounters(counters, "ydb");
- }
-
- auto serviceGroup = ydbGroup->GetSubgroup("api_service", serviceName);
- auto typeGroup = serviceGroup->GetSubgroup("method", requestName);
-
- RequestCount = typeGroup->GetNamedCounter("name", "api.grpc.request.count", true);
- RequestInflight = typeGroup->GetNamedCounter("name", "api.grpc.request.inflight_count", false);
- RequestBytes = typeGroup->GetNamedCounter("name", "api.grpc.request.bytes", true);
- RequestInflightBytes = typeGroup->GetNamedCounter("name", "api.grpc.request.inflight_bytes", false);
- RequestRpcError = typeGroup->GetNamedCounter("name", "api.grpc.request.dropped_count", true);
-
- ResponseBytes = typeGroup->GetNamedCounter("name", "api.grpc.response.bytes", true);
- ResponseRpcError = typeGroup->GetNamedCounter("name", "api.grpc.response.dropped_count", true);
-
- auto countName = "api.grpc.response.count";
-
- ResponseRpcNotAuthenticated =
- typeGroup->GetSubgroup("status", "UNAUTHENTICATED")->GetNamedCounter("name", countName, true);
- ResponseRpcResourceExhausted =
- typeGroup->GetSubgroup("status", "RESOURCE_EXHAUSTED")->GetNamedCounter("name", countName, true);
-
- ResponseByStatus[Ydb::StatusIds::STATUS_CODE_UNSPECIFIED] =
- typeGroup->GetSubgroup("status", "UNSPECIFIED")->GetNamedCounter("name", countName, true);
- ResponseByStatus[Ydb::StatusIds::SUCCESS] =
- typeGroup->GetSubgroup("status", "SUCCESS")->GetNamedCounter("name", countName, true);
- ResponseByStatus[Ydb::StatusIds::BAD_REQUEST] =
- typeGroup->GetSubgroup("status", "BAD_REQUEST")->GetNamedCounter("name", countName, true);
- ResponseByStatus[Ydb::StatusIds::UNAUTHORIZED] =
- typeGroup->GetSubgroup("status", "UNAUTHORIZED")->GetNamedCounter("name", countName, true);
- ResponseByStatus[Ydb::StatusIds::INTERNAL_ERROR] =
- typeGroup->GetSubgroup("status", "INTERNAL_ERROR")->GetNamedCounter("name", countName, true);
- ResponseByStatus[Ydb::StatusIds::ABORTED] =
- typeGroup->GetSubgroup("status", "ABORTED")->GetNamedCounter("name", countName, true);
- ResponseByStatus[Ydb::StatusIds::UNAVAILABLE] =
- typeGroup->GetSubgroup("status", "UNAVAILABLE")->GetNamedCounter("name", countName, true);
- ResponseByStatus[Ydb::StatusIds::OVERLOADED] =
- typeGroup->GetSubgroup("status", "OVERLOADED")->GetNamedCounter("name", countName, true);
- ResponseByStatus[Ydb::StatusIds::SCHEME_ERROR] =
- typeGroup->GetSubgroup("status", "SCHEME_ERROR")->GetNamedCounter("name", countName, true);
- ResponseByStatus[Ydb::StatusIds::GENERIC_ERROR] =
- typeGroup->GetSubgroup("status", "GENERIC_ERROR")->GetNamedCounter("name", countName, true);
- ResponseByStatus[Ydb::StatusIds::TIMEOUT] =
- typeGroup->GetSubgroup("status", "TIMEOUT")->GetNamedCounter("name", countName, true);
- ResponseByStatus[Ydb::StatusIds::BAD_SESSION] =
- typeGroup->GetSubgroup("status", "BAD_SESSION")->GetNamedCounter("name", countName, true);
- ResponseByStatus[Ydb::StatusIds::PRECONDITION_FAILED] =
- typeGroup->GetSubgroup("status", "PRECONDITION_FAILED")->GetNamedCounter("name", countName, true);
- ResponseByStatus[Ydb::StatusIds::ALREADY_EXISTS] =
- typeGroup->GetSubgroup("status", "ALREADY_EXISTS")->GetNamedCounter("name", countName, true);
- ResponseByStatus[Ydb::StatusIds::NOT_FOUND] =
- typeGroup->GetSubgroup("status", "NOT_FOUND")->GetNamedCounter("name", countName, true);
- ResponseByStatus[Ydb::StatusIds::SESSION_EXPIRED] =
- typeGroup->GetSubgroup("status", "SESSION_EXPIRED")->GetNamedCounter("name", countName, true);
- ResponseByStatus[Ydb::StatusIds::CANCELLED] =
- typeGroup->GetSubgroup("status", "CANCELLED")->GetNamedCounter("name", countName, true);
- ResponseByStatus[Ydb::StatusIds::SESSION_BUSY] =
- typeGroup->GetSubgroup("status", "SESSION_BUSY")->GetNamedCounter("name", countName, true);
-}
-
-TYdbCounterBlock::TYdbCounterBlock(const NMonitoring::TDynamicCounterPtr& counters, const char* serviceName,
- const char* requestName, bool percentile, bool streaming,
- bool forDatabase, NMonitoring::TDynamicCounterPtr internalGroup)
- : Streaming(streaming)
- , Percentile(percentile)
- , YdbCounters(counters, serviceName, requestName, forDatabase)
-{
- // group for all counters
- NMonitoring::TDynamicCounterPtr group;
- if (forDatabase) {
- group = internalGroup->GetSubgroup("group", "grpc");
- } else {
- group = GetServiceCounters(counters, "grpc")->GetSubgroup("subsystem", "serverStats");
- }
-
- // aggregated (non-request-specific counters)
- NotOkRequestCounter = group->GetCounter("notOkRequest", true);
- NotOkResponseCounter = group->GetCounter("notOkResponse", true);
- RequestBytes = group->GetCounter("requestBytes", true);
- InflyRequestBytes = group->GetCounter("inflyRequestBytes", false);
- ResponseBytes = group->GetCounter("responseBytes", true);
- NotAuthenticated = group->GetCounter("notAuthenticated", true);
- ResourceExhausted = group->GetCounter("resourceExhausted", true);
- RequestsWithoutDatabase = group->GetCounter("requestsWithoutDatabase", true);
- RequestsWithoutToken = group->GetCounter("requestsWithoutToken", true);
- RequestsWithoutTls = group->GetCounter("requestsWithoutTls", true);
-
- // subgroup for request-specific counters
- auto subgroup = group->GetSubgroup(streaming ? "stream" : "request", requestName);
- TotalCounter = subgroup->GetCounter("total", true);
- InflyCounter = subgroup->GetCounter("infly", false);
-
- if (Percentile) {
- RequestHistMs.Initialize(group, "event", "request", "ms", {0.5f, 0.9f, 0.99f, 0.999f, 1.0f});
- }
-}
-
-using TYdbCounterBlockPtr = TIntrusivePtr<TYdbCounterBlock>;
-
-
-#define DB_GRPC_SIMPLE_COUNTERS_MAP(XX) \
- XX(DB_GRPC_REQ_INFLIGHT_COUNT, YdbCounters.RequestInflight) \
- XX(DB_GRPC_REQ_INFLIGHT_BYTES, YdbCounters.RequestInflightBytes)
-
-#define DB_GRPC_CUMULATIVE_COUNTERS_MAP(XX) \
- XX(DB_GRPC_REQ_COUNT, YdbCounters.RequestCount) \
- XX(DB_GRPC_REQ_BYTES, YdbCounters.RequestBytes) \
- XX(DB_GRPC_REQ_RPC_ERROR, YdbCounters.RequestRpcError) \
- XX(DB_GRPC_RSP_BYTES, YdbCounters.ResponseBytes) \
- XX(DB_GRPC_RSP_RPC_ERROR, YdbCounters.ResponseRpcError) \
- XX(DB_GRPC_RSP_RPC_NOT_AUTH, YdbCounters.ResponseRpcNotAuthenticated) \
- XX(DB_GRPC_RSP_RPC_RESOURCE_EXHAUSTED, YdbCounters.ResponseRpcResourceExhausted) \
- XX(DB_GRPC_RSP_UNSPECIFIED, YdbCounters.ResponseByStatus[Ydb::StatusIds::STATUS_CODE_UNSPECIFIED]) \
- XX(DB_GRPC_RSP_SUCCESS, YdbCounters.ResponseByStatus[Ydb::StatusIds::SUCCESS]) \
- XX(DB_GRPC_RSP_BAD_REQUEST, YdbCounters.ResponseByStatus[Ydb::StatusIds::BAD_REQUEST]) \
- XX(DB_GRPC_RSP_UNAUTHORIZED, YdbCounters.ResponseByStatus[Ydb::StatusIds::UNAUTHORIZED]) \
- XX(DB_GRPC_RSP_INTERNAL_ERROR, YdbCounters.ResponseByStatus[Ydb::StatusIds::INTERNAL_ERROR]) \
- XX(DB_GRPC_RSP_ABORTED, YdbCounters.ResponseByStatus[Ydb::StatusIds::ABORTED]) \
- XX(DB_GRPC_RSP_UNAVAILABLE, YdbCounters.ResponseByStatus[Ydb::StatusIds::UNAVAILABLE]) \
- XX(DB_GRPC_RSP_OVERLOADED, YdbCounters.ResponseByStatus[Ydb::StatusIds::OVERLOADED]) \
- XX(DB_GRPC_RSP_SCHEME_ERROR, YdbCounters.ResponseByStatus[Ydb::StatusIds::SCHEME_ERROR]) \
- XX(DB_GRPC_RSP_GENERIC_ERROR, YdbCounters.ResponseByStatus[Ydb::StatusIds::GENERIC_ERROR]) \
- XX(DB_GRPC_RSP_TIMEOUT, YdbCounters.ResponseByStatus[Ydb::StatusIds::TIMEOUT]) \
- XX(DB_GRPC_RSP_BAD_SESSION, YdbCounters.ResponseByStatus[Ydb::StatusIds::BAD_SESSION]) \
- XX(DB_GRPC_RSP_PRECONDITION_FAILED, YdbCounters.ResponseByStatus[Ydb::StatusIds::PRECONDITION_FAILED]) \
- XX(DB_GRPC_RSP_ALREADY_EXISTS, YdbCounters.ResponseByStatus[Ydb::StatusIds::ALREADY_EXISTS]) \
- XX(DB_GRPC_RSP_NOT_FOUND, YdbCounters.ResponseByStatus[Ydb::StatusIds::NOT_FOUND]) \
- XX(DB_GRPC_RSP_SESSION_EXPIRED, YdbCounters.ResponseByStatus[Ydb::StatusIds::SESSION_EXPIRED]) \
- XX(DB_GRPC_RSP_CANCELLED, YdbCounters.ResponseByStatus[Ydb::StatusIds::CANCELLED]) \
- XX(DB_GRPC_RSP_SESSION_BUSY, YdbCounters.ResponseByStatus[Ydb::StatusIds::SESSION_BUSY])
-
-class TYdbDbCounterBlock : public TYdbCounterBlock {
-public:
- TYdbDbCounterBlock(const NMonitoring::TDynamicCounterPtr& counters, const char* serviceName,
- const char* requestName, bool percentile, bool streaming,
- NMonitoring::TDynamicCounterPtr internalGroup = {})
- : TYdbCounterBlock(counters, serviceName, requestName, percentile, streaming, true, internalGroup)
- {}
-
- void ToProto(NKikimrSysView::TDbGRpcCounters& counters) {
- auto* main = counters.MutableRequestCounters();
- auto* simple = main->MutableSimple();
- auto* cumulative = main->MutableCumulative();
-
- simple->Resize(DB_GRPC_SIMPLE_COUNTER_SIZE, 0);
- cumulative->Resize(DB_GRPC_CUMULATIVE_COUNTER_SIZE, 0);
-
-#define SAVE_SIMPLE_COUNTER(INDEX, TARGET) { (*simple)[INDEX] = (TARGET)->Val(); }
-#define SAVE_CUMULATIVE_COUNTER(INDEX, TARGET) { (*cumulative)[INDEX] = (TARGET)->Val(); }
-
- DB_GRPC_SIMPLE_COUNTERS_MAP(SAVE_SIMPLE_COUNTER)
- DB_GRPC_CUMULATIVE_COUNTERS_MAP(SAVE_CUMULATIVE_COUNTER)
- }
-
- void FromProto(NKikimrSysView::TDbGRpcCounters& counters) {
- auto* main = counters.MutableRequestCounters();
- auto* simple = main->MutableSimple();
- auto* cumulative = main->MutableCumulative();
-
- simple->Resize(DB_GRPC_SIMPLE_COUNTER_SIZE, 0);
- cumulative->Resize(DB_GRPC_CUMULATIVE_COUNTER_SIZE, 0);
-
-#define LOAD_SIMPLE_COUNTER(INDEX, TARGET) { (TARGET)->Set((*simple)[INDEX]); }
-#define LOAD_CUMULATIVE_COUNTER(INDEX, TARGET) { (TARGET)->Set((*cumulative)[INDEX]); }
-
- DB_GRPC_SIMPLE_COUNTERS_MAP(LOAD_SIMPLE_COUNTER)
- DB_GRPC_CUMULATIVE_COUNTERS_MAP(LOAD_CUMULATIVE_COUNTER)
- }
-
- void AggregateFrom(TYdbDbCounterBlock& other) {
-#define COPY_SIMPLE_COUNTER(INDEX, TARGET) { *TARGET += other.TARGET->Val(); }
-#define COPY_CUMULATIVE_COUNTER(INDEX, TARGET) { *TARGET += other.TARGET->Val(); }
-
- DB_GRPC_SIMPLE_COUNTERS_MAP(COPY_SIMPLE_COUNTER)
- DB_GRPC_CUMULATIVE_COUNTERS_MAP(COPY_CUMULATIVE_COUNTER)
- }
-
-private:
- enum ESimpleCounter {
- DB_GRPC_SIMPLE_COUNTERS_MAP(ENUM_VALUE_GEN_NO_VALUE)
- DB_GRPC_SIMPLE_COUNTER_SIZE
- };
- enum ECumulativeCounter {
- DB_GRPC_CUMULATIVE_COUNTERS_MAP(ENUM_VALUE_GEN_NO_VALUE)
- DB_GRPC_CUMULATIVE_COUNTER_SIZE
- };
-};
-
-using TYdbDbCounterBlockPtr = TIntrusivePtr<TYdbDbCounterBlock>;
-
-
-class TGRpcDbCounters : public NSysView::IDbCounters {
-public:
- TGRpcDbCounters()
- : Counters(new NMonitoring::TDynamicCounters)
- , InternalGroup(new NMonitoring::TDynamicCounters)
- {}
-
- TGRpcDbCounters(NMonitoring::TDynamicCounterPtr counters, NMonitoring::TDynamicCounterPtr internalGroup)
- : Counters(counters)
- , InternalGroup(internalGroup)
- {}
-
- void ToProto(NKikimr::NSysView::TDbServiceCounters& counters) override {
- for (auto& bucket : CounterBlocks.Buckets) {
- TReadGuard guard(bucket.GetLock());
- for (auto& [key, block] : bucket.GetMap()) {
- auto* proto = counters.FindOrAddGRpcCounters(key.first, key.second);
- block->ToProto(*proto);
- }
- }
- }
-
- void FromProto(NKikimr::NSysView::TDbServiceCounters& counters) override {
- for (auto& proto : *counters.Proto().MutableGRpcCounters()) {
- auto block = GetCounterBlock(proto.GetGRpcService(), proto.GetGRpcRequest());
- block->FromProto(proto);
- }
- }
-
- TYdbDbCounterBlockPtr GetCounterBlock(const TString& serviceName, const TString& requestName) {
- auto key = std::make_pair(serviceName, requestName);
-
- TYdbDbCounterBlockPtr dbCounters;
- if (CounterBlocks.Get(key, dbCounters)) {
- return dbCounters;
- }
-
- return CounterBlocks.InsertIfAbsentWithInit(key, [&] {
- return new TYdbDbCounterBlock(Counters, serviceName.c_str(), requestName.c_str(), false, false, InternalGroup);
- });
- }
-
-private:
- NMonitoring::TDynamicCounterPtr Counters;
- NMonitoring::TDynamicCounterPtr InternalGroup;
-
- using TKey = std::pair<TString, TString>;
- TConcurrentRWHashMap<TKey, TYdbDbCounterBlockPtr, 16> CounterBlocks;
-};
-
-using TGRpcDbCountersPtr = TIntrusivePtr<TGRpcDbCounters>;
-
-
-class TGRpcDbCountersRegistry {
- class TGRpcDbWatcherCallback : public NKikimr::NSysView::TDbWatcherCallback {
- public:
- void OnDatabaseRemoved(const TString& database, TPathId) override {
- Singleton<TGRpcDbCountersRegistry>()->RemoveDbCounters(database);
- }
- };
-
-public:
- void Initialize(TActorSystem* actorSystem) {
- if (Y_LIKELY(ActorSystem)) {
- return;
- }
- ActorSystem = actorSystem;
-
- auto callback = MakeIntrusive<TGRpcDbWatcherCallback>();
- DbWatcherActorId = ActorSystem->Register(NSysView::CreateDbWatcherActor(callback));
- }
-
- TYdbDbCounterBlockPtr GetCounterBlock(
- const TString& database, const TString& serviceName, const TString& requestName)
- {
- TGRpcDbCountersPtr dbCounters;
- if (DbCounters.Get(database, dbCounters)) {
- return dbCounters->GetCounterBlock(serviceName, requestName);
- }
-
+ return this;
+ }
+
+ void Update() {
+ if (Percentile) {
+ RequestHistMs.Update();
+ }
+ }
+};
+
+TYdbRpcCounters::TYdbRpcCounters(const NMonitoring::TDynamicCounterPtr& counters,
+ const char* serviceName, const char* requestName, bool forDatabase)
+{
+ NMonitoring::TDynamicCounterPtr ydbGroup;
+ if (forDatabase) {
+ ydbGroup = counters;
+ } else {
+ ydbGroup = GetServiceCounters(counters, "ydb");
+ }
+
+ auto serviceGroup = ydbGroup->GetSubgroup("api_service", serviceName);
+ auto typeGroup = serviceGroup->GetSubgroup("method", requestName);
+
+ RequestCount = typeGroup->GetNamedCounter("name", "api.grpc.request.count", true);
+ RequestInflight = typeGroup->GetNamedCounter("name", "api.grpc.request.inflight_count", false);
+ RequestBytes = typeGroup->GetNamedCounter("name", "api.grpc.request.bytes", true);
+ RequestInflightBytes = typeGroup->GetNamedCounter("name", "api.grpc.request.inflight_bytes", false);
+ RequestRpcError = typeGroup->GetNamedCounter("name", "api.grpc.request.dropped_count", true);
+
+ ResponseBytes = typeGroup->GetNamedCounter("name", "api.grpc.response.bytes", true);
+ ResponseRpcError = typeGroup->GetNamedCounter("name", "api.grpc.response.dropped_count", true);
+
+ auto countName = "api.grpc.response.count";
+
+ ResponseRpcNotAuthenticated =
+ typeGroup->GetSubgroup("status", "UNAUTHENTICATED")->GetNamedCounter("name", countName, true);
+ ResponseRpcResourceExhausted =
+ typeGroup->GetSubgroup("status", "RESOURCE_EXHAUSTED")->GetNamedCounter("name", countName, true);
+
+ ResponseByStatus[Ydb::StatusIds::STATUS_CODE_UNSPECIFIED] =
+ typeGroup->GetSubgroup("status", "UNSPECIFIED")->GetNamedCounter("name", countName, true);
+ ResponseByStatus[Ydb::StatusIds::SUCCESS] =
+ typeGroup->GetSubgroup("status", "SUCCESS")->GetNamedCounter("name", countName, true);
+ ResponseByStatus[Ydb::StatusIds::BAD_REQUEST] =
+ typeGroup->GetSubgroup("status", "BAD_REQUEST")->GetNamedCounter("name", countName, true);
+ ResponseByStatus[Ydb::StatusIds::UNAUTHORIZED] =
+ typeGroup->GetSubgroup("status", "UNAUTHORIZED")->GetNamedCounter("name", countName, true);
+ ResponseByStatus[Ydb::StatusIds::INTERNAL_ERROR] =
+ typeGroup->GetSubgroup("status", "INTERNAL_ERROR")->GetNamedCounter("name", countName, true);
+ ResponseByStatus[Ydb::StatusIds::ABORTED] =
+ typeGroup->GetSubgroup("status", "ABORTED")->GetNamedCounter("name", countName, true);
+ ResponseByStatus[Ydb::StatusIds::UNAVAILABLE] =
+ typeGroup->GetSubgroup("status", "UNAVAILABLE")->GetNamedCounter("name", countName, true);
+ ResponseByStatus[Ydb::StatusIds::OVERLOADED] =
+ typeGroup->GetSubgroup("status", "OVERLOADED")->GetNamedCounter("name", countName, true);
+ ResponseByStatus[Ydb::StatusIds::SCHEME_ERROR] =
+ typeGroup->GetSubgroup("status", "SCHEME_ERROR")->GetNamedCounter("name", countName, true);
+ ResponseByStatus[Ydb::StatusIds::GENERIC_ERROR] =
+ typeGroup->GetSubgroup("status", "GENERIC_ERROR")->GetNamedCounter("name", countName, true);
+ ResponseByStatus[Ydb::StatusIds::TIMEOUT] =
+ typeGroup->GetSubgroup("status", "TIMEOUT")->GetNamedCounter("name", countName, true);
+ ResponseByStatus[Ydb::StatusIds::BAD_SESSION] =
+ typeGroup->GetSubgroup("status", "BAD_SESSION")->GetNamedCounter("name", countName, true);
+ ResponseByStatus[Ydb::StatusIds::PRECONDITION_FAILED] =
+ typeGroup->GetSubgroup("status", "PRECONDITION_FAILED")->GetNamedCounter("name", countName, true);
+ ResponseByStatus[Ydb::StatusIds::ALREADY_EXISTS] =
+ typeGroup->GetSubgroup("status", "ALREADY_EXISTS")->GetNamedCounter("name", countName, true);
+ ResponseByStatus[Ydb::StatusIds::NOT_FOUND] =
+ typeGroup->GetSubgroup("status", "NOT_FOUND")->GetNamedCounter("name", countName, true);
+ ResponseByStatus[Ydb::StatusIds::SESSION_EXPIRED] =
+ typeGroup->GetSubgroup("status", "SESSION_EXPIRED")->GetNamedCounter("name", countName, true);
+ ResponseByStatus[Ydb::StatusIds::CANCELLED] =
+ typeGroup->GetSubgroup("status", "CANCELLED")->GetNamedCounter("name", countName, true);
+ ResponseByStatus[Ydb::StatusIds::SESSION_BUSY] =
+ typeGroup->GetSubgroup("status", "SESSION_BUSY")->GetNamedCounter("name", countName, true);
+}
+
+TYdbCounterBlock::TYdbCounterBlock(const NMonitoring::TDynamicCounterPtr& counters, const char* serviceName,
+ const char* requestName, bool percentile, bool streaming,
+ bool forDatabase, NMonitoring::TDynamicCounterPtr internalGroup)
+ : Streaming(streaming)
+ , Percentile(percentile)
+ , YdbCounters(counters, serviceName, requestName, forDatabase)
+{
+ // group for all counters
+ NMonitoring::TDynamicCounterPtr group;
+ if (forDatabase) {
+ group = internalGroup->GetSubgroup("group", "grpc");
+ } else {
+ group = GetServiceCounters(counters, "grpc")->GetSubgroup("subsystem", "serverStats");
+ }
+
+ // aggregated (non-request-specific counters)
+ NotOkRequestCounter = group->GetCounter("notOkRequest", true);
+ NotOkResponseCounter = group->GetCounter("notOkResponse", true);
+ RequestBytes = group->GetCounter("requestBytes", true);
+ InflyRequestBytes = group->GetCounter("inflyRequestBytes", false);
+ ResponseBytes = group->GetCounter("responseBytes", true);
+ NotAuthenticated = group->GetCounter("notAuthenticated", true);
+ ResourceExhausted = group->GetCounter("resourceExhausted", true);
+ RequestsWithoutDatabase = group->GetCounter("requestsWithoutDatabase", true);
+ RequestsWithoutToken = group->GetCounter("requestsWithoutToken", true);
+ RequestsWithoutTls = group->GetCounter("requestsWithoutTls", true);
+
+ // subgroup for request-specific counters
+ auto subgroup = group->GetSubgroup(streaming ? "stream" : "request", requestName);
+ TotalCounter = subgroup->GetCounter("total", true);
+ InflyCounter = subgroup->GetCounter("infly", false);
+
+ if (Percentile) {
+ RequestHistMs.Initialize(group, "event", "request", "ms", {0.5f, 0.9f, 0.99f, 0.999f, 1.0f});
+ }
+}
+
+using TYdbCounterBlockPtr = TIntrusivePtr<TYdbCounterBlock>;
+
+
+#define DB_GRPC_SIMPLE_COUNTERS_MAP(XX) \
+ XX(DB_GRPC_REQ_INFLIGHT_COUNT, YdbCounters.RequestInflight) \
+ XX(DB_GRPC_REQ_INFLIGHT_BYTES, YdbCounters.RequestInflightBytes)
+
+#define DB_GRPC_CUMULATIVE_COUNTERS_MAP(XX) \
+ XX(DB_GRPC_REQ_COUNT, YdbCounters.RequestCount) \
+ XX(DB_GRPC_REQ_BYTES, YdbCounters.RequestBytes) \
+ XX(DB_GRPC_REQ_RPC_ERROR, YdbCounters.RequestRpcError) \
+ XX(DB_GRPC_RSP_BYTES, YdbCounters.ResponseBytes) \
+ XX(DB_GRPC_RSP_RPC_ERROR, YdbCounters.ResponseRpcError) \
+ XX(DB_GRPC_RSP_RPC_NOT_AUTH, YdbCounters.ResponseRpcNotAuthenticated) \
+ XX(DB_GRPC_RSP_RPC_RESOURCE_EXHAUSTED, YdbCounters.ResponseRpcResourceExhausted) \
+ XX(DB_GRPC_RSP_UNSPECIFIED, YdbCounters.ResponseByStatus[Ydb::StatusIds::STATUS_CODE_UNSPECIFIED]) \
+ XX(DB_GRPC_RSP_SUCCESS, YdbCounters.ResponseByStatus[Ydb::StatusIds::SUCCESS]) \
+ XX(DB_GRPC_RSP_BAD_REQUEST, YdbCounters.ResponseByStatus[Ydb::StatusIds::BAD_REQUEST]) \
+ XX(DB_GRPC_RSP_UNAUTHORIZED, YdbCounters.ResponseByStatus[Ydb::StatusIds::UNAUTHORIZED]) \
+ XX(DB_GRPC_RSP_INTERNAL_ERROR, YdbCounters.ResponseByStatus[Ydb::StatusIds::INTERNAL_ERROR]) \
+ XX(DB_GRPC_RSP_ABORTED, YdbCounters.ResponseByStatus[Ydb::StatusIds::ABORTED]) \
+ XX(DB_GRPC_RSP_UNAVAILABLE, YdbCounters.ResponseByStatus[Ydb::StatusIds::UNAVAILABLE]) \
+ XX(DB_GRPC_RSP_OVERLOADED, YdbCounters.ResponseByStatus[Ydb::StatusIds::OVERLOADED]) \
+ XX(DB_GRPC_RSP_SCHEME_ERROR, YdbCounters.ResponseByStatus[Ydb::StatusIds::SCHEME_ERROR]) \
+ XX(DB_GRPC_RSP_GENERIC_ERROR, YdbCounters.ResponseByStatus[Ydb::StatusIds::GENERIC_ERROR]) \
+ XX(DB_GRPC_RSP_TIMEOUT, YdbCounters.ResponseByStatus[Ydb::StatusIds::TIMEOUT]) \
+ XX(DB_GRPC_RSP_BAD_SESSION, YdbCounters.ResponseByStatus[Ydb::StatusIds::BAD_SESSION]) \
+ XX(DB_GRPC_RSP_PRECONDITION_FAILED, YdbCounters.ResponseByStatus[Ydb::StatusIds::PRECONDITION_FAILED]) \
+ XX(DB_GRPC_RSP_ALREADY_EXISTS, YdbCounters.ResponseByStatus[Ydb::StatusIds::ALREADY_EXISTS]) \
+ XX(DB_GRPC_RSP_NOT_FOUND, YdbCounters.ResponseByStatus[Ydb::StatusIds::NOT_FOUND]) \
+ XX(DB_GRPC_RSP_SESSION_EXPIRED, YdbCounters.ResponseByStatus[Ydb::StatusIds::SESSION_EXPIRED]) \
+ XX(DB_GRPC_RSP_CANCELLED, YdbCounters.ResponseByStatus[Ydb::StatusIds::CANCELLED]) \
+ XX(DB_GRPC_RSP_SESSION_BUSY, YdbCounters.ResponseByStatus[Ydb::StatusIds::SESSION_BUSY])
+
+class TYdbDbCounterBlock : public TYdbCounterBlock {
+public:
+ TYdbDbCounterBlock(const NMonitoring::TDynamicCounterPtr& counters, const char* serviceName,
+ const char* requestName, bool percentile, bool streaming,
+ NMonitoring::TDynamicCounterPtr internalGroup = {})
+ : TYdbCounterBlock(counters, serviceName, requestName, percentile, streaming, true, internalGroup)
+ {}
+
+ void ToProto(NKikimrSysView::TDbGRpcCounters& counters) {
+ auto* main = counters.MutableRequestCounters();
+ auto* simple = main->MutableSimple();
+ auto* cumulative = main->MutableCumulative();
+
+ simple->Resize(DB_GRPC_SIMPLE_COUNTER_SIZE, 0);
+ cumulative->Resize(DB_GRPC_CUMULATIVE_COUNTER_SIZE, 0);
+
+#define SAVE_SIMPLE_COUNTER(INDEX, TARGET) { (*simple)[INDEX] = (TARGET)->Val(); }
+#define SAVE_CUMULATIVE_COUNTER(INDEX, TARGET) { (*cumulative)[INDEX] = (TARGET)->Val(); }
+
+ DB_GRPC_SIMPLE_COUNTERS_MAP(SAVE_SIMPLE_COUNTER)
+ DB_GRPC_CUMULATIVE_COUNTERS_MAP(SAVE_CUMULATIVE_COUNTER)
+ }
+
+ void FromProto(NKikimrSysView::TDbGRpcCounters& counters) {
+ auto* main = counters.MutableRequestCounters();
+ auto* simple = main->MutableSimple();
+ auto* cumulative = main->MutableCumulative();
+
+ simple->Resize(DB_GRPC_SIMPLE_COUNTER_SIZE, 0);
+ cumulative->Resize(DB_GRPC_CUMULATIVE_COUNTER_SIZE, 0);
+
+#define LOAD_SIMPLE_COUNTER(INDEX, TARGET) { (TARGET)->Set((*simple)[INDEX]); }
+#define LOAD_CUMULATIVE_COUNTER(INDEX, TARGET) { (TARGET)->Set((*cumulative)[INDEX]); }
+
+ DB_GRPC_SIMPLE_COUNTERS_MAP(LOAD_SIMPLE_COUNTER)
+ DB_GRPC_CUMULATIVE_COUNTERS_MAP(LOAD_CUMULATIVE_COUNTER)
+ }
+
+ void AggregateFrom(TYdbDbCounterBlock& other) {
+#define COPY_SIMPLE_COUNTER(INDEX, TARGET) { *TARGET += other.TARGET->Val(); }
+#define COPY_CUMULATIVE_COUNTER(INDEX, TARGET) { *TARGET += other.TARGET->Val(); }
+
+ DB_GRPC_SIMPLE_COUNTERS_MAP(COPY_SIMPLE_COUNTER)
+ DB_GRPC_CUMULATIVE_COUNTERS_MAP(COPY_CUMULATIVE_COUNTER)
+ }
+
+private:
+ enum ESimpleCounter {
+ DB_GRPC_SIMPLE_COUNTERS_MAP(ENUM_VALUE_GEN_NO_VALUE)
+ DB_GRPC_SIMPLE_COUNTER_SIZE
+ };
+ enum ECumulativeCounter {
+ DB_GRPC_CUMULATIVE_COUNTERS_MAP(ENUM_VALUE_GEN_NO_VALUE)
+ DB_GRPC_CUMULATIVE_COUNTER_SIZE
+ };
+};
+
+using TYdbDbCounterBlockPtr = TIntrusivePtr<TYdbDbCounterBlock>;
+
+
+class TGRpcDbCounters : public NSysView::IDbCounters {
+public:
+ TGRpcDbCounters()
+ : Counters(new NMonitoring::TDynamicCounters)
+ , InternalGroup(new NMonitoring::TDynamicCounters)
+ {}
+
+ TGRpcDbCounters(NMonitoring::TDynamicCounterPtr counters, NMonitoring::TDynamicCounterPtr internalGroup)
+ : Counters(counters)
+ , InternalGroup(internalGroup)
+ {}
+
+ void ToProto(NKikimr::NSysView::TDbServiceCounters& counters) override {
+ for (auto& bucket : CounterBlocks.Buckets) {
+ TReadGuard guard(bucket.GetLock());
+ for (auto& [key, block] : bucket.GetMap()) {
+ auto* proto = counters.FindOrAddGRpcCounters(key.first, key.second);
+ block->ToProto(*proto);
+ }
+ }
+ }
+
+ void FromProto(NKikimr::NSysView::TDbServiceCounters& counters) override {
+ for (auto& proto : *counters.Proto().MutableGRpcCounters()) {
+ auto block = GetCounterBlock(proto.GetGRpcService(), proto.GetGRpcRequest());
+ block->FromProto(proto);
+ }
+ }
+
+ TYdbDbCounterBlockPtr GetCounterBlock(const TString& serviceName, const TString& requestName) {
+ auto key = std::make_pair(serviceName, requestName);
+
+ TYdbDbCounterBlockPtr dbCounters;
+ if (CounterBlocks.Get(key, dbCounters)) {
+ return dbCounters;
+ }
+
+ return CounterBlocks.InsertIfAbsentWithInit(key, [&] {
+ return new TYdbDbCounterBlock(Counters, serviceName.c_str(), requestName.c_str(), false, false, InternalGroup);
+ });
+ }
+
+private:
+ NMonitoring::TDynamicCounterPtr Counters;
+ NMonitoring::TDynamicCounterPtr InternalGroup;
+
+ using TKey = std::pair<TString, TString>;
+ TConcurrentRWHashMap<TKey, TYdbDbCounterBlockPtr, 16> CounterBlocks;
+};
+
+using TGRpcDbCountersPtr = TIntrusivePtr<TGRpcDbCounters>;
+
+
+class TGRpcDbCountersRegistry {
+ class TGRpcDbWatcherCallback : public NKikimr::NSysView::TDbWatcherCallback {
+ public:
+ void OnDatabaseRemoved(const TString& database, TPathId) override {
+ Singleton<TGRpcDbCountersRegistry>()->RemoveDbCounters(database);
+ }
+ };
+
+public:
+ void Initialize(TActorSystem* actorSystem) {
+ if (Y_LIKELY(ActorSystem)) {
+ return;
+ }
+ ActorSystem = actorSystem;
+
+ auto callback = MakeIntrusive<TGRpcDbWatcherCallback>();
+ DbWatcherActorId = ActorSystem->Register(NSysView::CreateDbWatcherActor(callback));
+ }
+
+ TYdbDbCounterBlockPtr GetCounterBlock(
+ const TString& database, const TString& serviceName, const TString& requestName)
+ {
+ TGRpcDbCountersPtr dbCounters;
+ if (DbCounters.Get(database, dbCounters)) {
+ return dbCounters->GetCounterBlock(serviceName, requestName);
+ }
+
dbCounters = DbCounters.InsertIfAbsentWithInit(database, [&database, this] {
- auto counters = MakeIntrusive<TGRpcDbCounters>();
-
- if (ActorSystem) {
- auto evRegister = MakeHolder<NSysView::TEvSysView::TEvRegisterDbCounters>(
- NKikimrSysView::GRPC, database, counters);
-
- ActorSystem->Send(NSysView::MakeSysViewServiceID(ActorSystem->NodeId), evRegister.Release());
-
- if (DbWatcherActorId) {
- auto evWatch = MakeHolder<NSysView::TEvSysView::TEvWatchDatabase>(database);
- ActorSystem->Send(DbWatcherActorId, evWatch.Release());
- }
- }
-
- return counters;
- });
-
- return dbCounters->GetCounterBlock(serviceName, requestName);
- }
-
- void RemoveDbCounters(const TString& database) {
- DbCounters.Erase(database);
- }
-
-private:
- TConcurrentRWHashMap<TString, TIntrusivePtr<TGRpcDbCounters>, 256> DbCounters;
- TActorSystem* ActorSystem = {};
- TActorId DbWatcherActorId;
-};
-
-
+ auto counters = MakeIntrusive<TGRpcDbCounters>();
+
+ if (ActorSystem) {
+ auto evRegister = MakeHolder<NSysView::TEvSysView::TEvRegisterDbCounters>(
+ NKikimrSysView::GRPC, database, counters);
+
+ ActorSystem->Send(NSysView::MakeSysViewServiceID(ActorSystem->NodeId), evRegister.Release());
+
+ if (DbWatcherActorId) {
+ auto evWatch = MakeHolder<NSysView::TEvSysView::TEvWatchDatabase>(database);
+ ActorSystem->Send(DbWatcherActorId, evWatch.Release());
+ }
+ }
+
+ return counters;
+ });
+
+ return dbCounters->GetCounterBlock(serviceName, requestName);
+ }
+
+ void RemoveDbCounters(const TString& database) {
+ DbCounters.Erase(database);
+ }
+
+private:
+ TConcurrentRWHashMap<TString, TIntrusivePtr<TGRpcDbCounters>, 256> DbCounters;
+ TActorSystem* ActorSystem = {};
+ TActorId DbWatcherActorId;
+};
+
+
class TYdbCounterBlockWrapper : public NGrpc::ICounterBlock {
- TYdbCounterBlockPtr Common;
- TString ServiceName;
- TString RequestName;
- bool Percentile = false;
- bool Streaming = false;
-
- NMonitoring::TDynamicCounterPtr Root;
- TYdbDbCounterBlockPtr Db;
-
-public:
- TYdbCounterBlockWrapper(TYdbCounterBlockPtr common, const TString& serviceName, const TString& requestName,
- bool percentile, bool streaming)
- : Common(common)
- , ServiceName(serviceName)
- , RequestName(requestName)
- , Percentile(percentile)
- , Streaming(streaming)
- , Root(new NMonitoring::TDynamicCounters)
- , Db(new TYdbDbCounterBlock(Root, serviceName.c_str(), requestName.c_str(), percentile, streaming, Root))
- {}
-
- void CountNotOkRequest() override {
- Common->CountNotOkRequest();
- Db->CountNotOkRequest();
- }
-
- void CountNotOkResponse() override {
- Common->CountNotOkResponse();
- Db->CountNotOkResponse();
- }
-
- void CountNotAuthenticated() override {
- Common->CountNotAuthenticated();
- Db->CountNotAuthenticated();
- }
-
- void CountResourceExhausted() override {
- Common->CountResourceExhausted();
- Db->CountResourceExhausted();
- }
-
- void CountRequestsWithoutDatabase() override {
- Common->CountRequestsWithoutDatabase();
- Db->CountRequestsWithoutDatabase();
- }
-
- void CountRequestsWithoutToken() override {
- Common->CountRequestsWithoutToken();
- Db->CountRequestsWithoutToken();
- }
-
- void CountRequestWithoutTls() override {
- Common->CountRequestWithoutTls();
- Db->CountRequestWithoutTls();
- }
-
- void CountRequestBytes(ui32 requestSize) override {
- Common->CountRequestBytes(requestSize);
- Db->CountRequestBytes(requestSize);
- }
-
- void CountResponseBytes(ui32 responseSize) override {
- Common->CountResponseBytes(responseSize);
- Db->CountResponseBytes(responseSize);
- }
-
- void StartProcessing(ui32 requestSize) override {
- Common->StartProcessing(requestSize);
- Db->StartProcessing(requestSize);
- }
-
- void FinishProcessing(ui32 requestSize, ui32 responseSize, bool ok, ui32 status,
- TDuration requestDuration) override
- {
- Common->FinishProcessing(requestSize, responseSize, ok, status, requestDuration);
- Db->FinishProcessing(requestSize, responseSize, ok, status, requestDuration);
- }
-
+ TYdbCounterBlockPtr Common;
+ TString ServiceName;
+ TString RequestName;
+ bool Percentile = false;
+ bool Streaming = false;
+
+ NMonitoring::TDynamicCounterPtr Root;
+ TYdbDbCounterBlockPtr Db;
+
+public:
+ TYdbCounterBlockWrapper(TYdbCounterBlockPtr common, const TString& serviceName, const TString& requestName,
+ bool percentile, bool streaming)
+ : Common(common)
+ , ServiceName(serviceName)
+ , RequestName(requestName)
+ , Percentile(percentile)
+ , Streaming(streaming)
+ , Root(new NMonitoring::TDynamicCounters)
+ , Db(new TYdbDbCounterBlock(Root, serviceName.c_str(), requestName.c_str(), percentile, streaming, Root))
+ {}
+
+ void CountNotOkRequest() override {
+ Common->CountNotOkRequest();
+ Db->CountNotOkRequest();
+ }
+
+ void CountNotOkResponse() override {
+ Common->CountNotOkResponse();
+ Db->CountNotOkResponse();
+ }
+
+ void CountNotAuthenticated() override {
+ Common->CountNotAuthenticated();
+ Db->CountNotAuthenticated();
+ }
+
+ void CountResourceExhausted() override {
+ Common->CountResourceExhausted();
+ Db->CountResourceExhausted();
+ }
+
+ void CountRequestsWithoutDatabase() override {
+ Common->CountRequestsWithoutDatabase();
+ Db->CountRequestsWithoutDatabase();
+ }
+
+ void CountRequestsWithoutToken() override {
+ Common->CountRequestsWithoutToken();
+ Db->CountRequestsWithoutToken();
+ }
+
+ void CountRequestWithoutTls() override {
+ Common->CountRequestWithoutTls();
+ Db->CountRequestWithoutTls();
+ }
+
+ void CountRequestBytes(ui32 requestSize) override {
+ Common->CountRequestBytes(requestSize);
+ Db->CountRequestBytes(requestSize);
+ }
+
+ void CountResponseBytes(ui32 responseSize) override {
+ Common->CountResponseBytes(responseSize);
+ Db->CountResponseBytes(responseSize);
+ }
+
+ void StartProcessing(ui32 requestSize) override {
+ Common->StartProcessing(requestSize);
+ Db->StartProcessing(requestSize);
+ }
+
+ void FinishProcessing(ui32 requestSize, ui32 responseSize, bool ok, ui32 status,
+ TDuration requestDuration) override
+ {
+ Common->FinishProcessing(requestSize, responseSize, ok, status, requestDuration);
+ Db->FinishProcessing(requestSize, responseSize, ok, status, requestDuration);
+ }
+
NGrpc::ICounterBlockPtr Clone() override {
- return new TYdbCounterBlockWrapper(Common, ServiceName, RequestName, Percentile, Streaming);
- }
-
- void UseDatabase(const TString& database) override {
- if (database.empty()) {
- return;
- }
-
- auto block = Singleton<TGRpcDbCountersRegistry>()->GetCounterBlock(
- database, ServiceName, RequestName);
- block->AggregateFrom(*Db);
- Db = block;
- }
-};
-
-class TUpdaterActor
- : public TActor<TUpdaterActor>
-{
- TVector<TYdbCounterBlockPtr> Counters;
-
-public:
- enum : ui32 {
- EvRegisterItem = EventSpaceBegin(TEvents::ES_PRIVATE),
- };
-
- struct TEvRegisterItem
- : TEventLocal<TEvRegisterItem, EvRegisterItem>
- {
- TYdbCounterBlockPtr Counters;
-
- TEvRegisterItem(TYdbCounterBlockPtr counters)
- : Counters(std::move(counters))
- {}
- };
-
-public:
- static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
- return NKikimrServices::TActivity::GRPC_UPDATER;
- }
-
- TUpdaterActor()
- : TActor(&TThis::StateFunc)
- {}
-
- void HandleWakeup(const TActorContext& ctx) {
- ctx.Schedule(TDuration::Seconds(1), new TEvents::TEvWakeup);
- for (const auto& counter : Counters) {
- counter->Update();
- }
- }
-
- void Handle(TEvRegisterItem::TPtr& ev, const TActorContext& ctx) {
- Counters.push_back(std::move(ev->Get()->Counters));
- if (Counters.size() == 1) {
- HandleWakeup(ctx);
- }
- }
-
- STRICT_STFUNC(StateFunc, {
- CFunc(TEvents::TSystem::Wakeup, HandleWakeup)
- HFunc(TEvRegisterItem, Handle)
- })
-};
-
-TServiceCounterCB::TServiceCounterCB(NMonitoring::TDynamicCounterPtr counters, TActorSystem *actorSystem)
- : Counters(std::move(counters))
- , ActorSystem(actorSystem)
-{
- if (ActorSystem) {
- ActorId = ActorSystem->Register(new TUpdaterActor);
- Singleton<TGRpcDbCountersRegistry>()->Initialize(ActorSystem);
- }
-}
-
+ return new TYdbCounterBlockWrapper(Common, ServiceName, RequestName, Percentile, Streaming);
+ }
+
+ void UseDatabase(const TString& database) override {
+ if (database.empty()) {
+ return;
+ }
+
+ auto block = Singleton<TGRpcDbCountersRegistry>()->GetCounterBlock(
+ database, ServiceName, RequestName);
+ block->AggregateFrom(*Db);
+ Db = block;
+ }
+};
+
+class TUpdaterActor
+ : public TActor<TUpdaterActor>
+{
+ TVector<TYdbCounterBlockPtr> Counters;
+
+public:
+ enum : ui32 {
+ EvRegisterItem = EventSpaceBegin(TEvents::ES_PRIVATE),
+ };
+
+ struct TEvRegisterItem
+ : TEventLocal<TEvRegisterItem, EvRegisterItem>
+ {
+ TYdbCounterBlockPtr Counters;
+
+ TEvRegisterItem(TYdbCounterBlockPtr counters)
+ : Counters(std::move(counters))
+ {}
+ };
+
+public:
+ static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
+ return NKikimrServices::TActivity::GRPC_UPDATER;
+ }
+
+ TUpdaterActor()
+ : TActor(&TThis::StateFunc)
+ {}
+
+ void HandleWakeup(const TActorContext& ctx) {
+ ctx.Schedule(TDuration::Seconds(1), new TEvents::TEvWakeup);
+ for (const auto& counter : Counters) {
+ counter->Update();
+ }
+ }
+
+ void Handle(TEvRegisterItem::TPtr& ev, const TActorContext& ctx) {
+ Counters.push_back(std::move(ev->Get()->Counters));
+ if (Counters.size() == 1) {
+ HandleWakeup(ctx);
+ }
+ }
+
+ STRICT_STFUNC(StateFunc, {
+ CFunc(TEvents::TSystem::Wakeup, HandleWakeup)
+ HFunc(TEvRegisterItem, Handle)
+ })
+};
+
+TServiceCounterCB::TServiceCounterCB(NMonitoring::TDynamicCounterPtr counters, TActorSystem *actorSystem)
+ : Counters(std::move(counters))
+ , ActorSystem(actorSystem)
+{
+ if (ActorSystem) {
+ ActorId = ActorSystem->Register(new TUpdaterActor);
+ Singleton<TGRpcDbCountersRegistry>()->Initialize(ActorSystem);
+ }
+}
+
NGrpc::ICounterBlockPtr TServiceCounterCB::operator()(const char* serviceName,
- const char* requestName, bool percentile, bool streaming) const
-{
- auto block = MakeIntrusive<TYdbCounterBlock>(Counters, serviceName, requestName, percentile, streaming);
-
- if (ActorSystem) {
- ActorSystem->Send(ActorId, new TUpdaterActor::TEvRegisterItem(block));
- }
-
+ const char* requestName, bool percentile, bool streaming) const
+{
+ auto block = MakeIntrusive<TYdbCounterBlock>(Counters, serviceName, requestName, percentile, streaming);
+
+ if (ActorSystem) {
+ ActorSystem->Send(ActorId, new TUpdaterActor::TEvRegisterItem(block));
+ }
+
NGrpc::ICounterBlockPtr res(block);
- if (ActorSystem && AppData(ActorSystem)->FeatureFlags.GetEnableDbCounters()) {
- res = MakeIntrusive<TYdbCounterBlockWrapper>(block, serviceName, requestName, percentile, streaming);
- }
-
- return res;
-}
-
-TIntrusivePtr<NSysView::IDbCounters> CreateGRpcDbCounters(
- NMonitoring::TDynamicCounterPtr externalGroup,
- NMonitoring::TDynamicCounterPtr internalGroup)
-{
- return new TGRpcDbCounters(externalGroup, internalGroup);
-}
-
-} // namespace NGRpcService
-} // namespace NKikimr
+ if (ActorSystem && AppData(ActorSystem)->FeatureFlags.GetEnableDbCounters()) {
+ res = MakeIntrusive<TYdbCounterBlockWrapper>(block, serviceName, requestName, percentile, streaming);
+ }
+
+ return res;
+}
+
+TIntrusivePtr<NSysView::IDbCounters> CreateGRpcDbCounters(
+ NMonitoring::TDynamicCounterPtr externalGroup,
+ NMonitoring::TDynamicCounterPtr internalGroup)
+{
+ return new TGRpcDbCounters(externalGroup, internalGroup);
+}
+
+} // namespace NGRpcService
+} // namespace NKikimr
diff --git a/ydb/core/grpc_services/counters/counters.h b/ydb/core/grpc_services/counters/counters.h
index da499dac9e0..1f01cd94c06 100644
--- a/ydb/core/grpc_services/counters/counters.h
+++ b/ydb/core/grpc_services/counters/counters.h
@@ -1,34 +1,34 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/sys_view/common/events.h>
-
+
#include <library/cpp/grpc/server/grpc_counters.h>
-
-namespace NKikimr {
-namespace NGRpcService {
-
-class TServiceCounterCB {
-public:
- TServiceCounterCB(NMonitoring::TDynamicCounterPtr counters, TActorSystem *actorSystem);
-
+
+namespace NKikimr {
+namespace NGRpcService {
+
+class TServiceCounterCB {
+public:
+ TServiceCounterCB(NMonitoring::TDynamicCounterPtr counters, TActorSystem *actorSystem);
+
NGrpc::ICounterBlockPtr operator()(const char* serviceName, const char* requestName,
- bool percentile = true, bool streaming = false) const;
-
-private:
- TIntrusivePtr<NMonitoring::TDynamicCounters> Counters;
- TActorSystem* ActorSystem;
- TActorId ActorId;
-};
-
-inline TServiceCounterCB CreateCounterCb(NMonitoring::TDynamicCounterPtr counters,
- TActorSystem *actorSystem)
-{
- return TServiceCounterCB(std::move(counters), actorSystem);
-}
-
-TIntrusivePtr<NSysView::IDbCounters> CreateGRpcDbCounters(
- NMonitoring::TDynamicCounterPtr externalGroup,
- NMonitoring::TDynamicCounterPtr internalGroup);
-
-} // namespace NGRpcService
-} // namespace NKikimr
+ bool percentile = true, bool streaming = false) const;
+
+private:
+ TIntrusivePtr<NMonitoring::TDynamicCounters> Counters;
+ TActorSystem* ActorSystem;
+ TActorId ActorId;
+};
+
+inline TServiceCounterCB CreateCounterCb(NMonitoring::TDynamicCounterPtr counters,
+ TActorSystem *actorSystem)
+{
+ return TServiceCounterCB(std::move(counters), actorSystem);
+}
+
+TIntrusivePtr<NSysView::IDbCounters> CreateGRpcDbCounters(
+ NMonitoring::TDynamicCounterPtr externalGroup,
+ NMonitoring::TDynamicCounterPtr internalGroup);
+
+} // namespace NGRpcService
+} // namespace NKikimr
diff --git a/ydb/core/grpc_services/counters/ya.make b/ydb/core/grpc_services/counters/ya.make
index 01c329a887b..9247d66487e 100644
--- a/ydb/core/grpc_services/counters/ya.make
+++ b/ydb/core/grpc_services/counters/ya.make
@@ -1,21 +1,21 @@
-LIBRARY()
-
-OWNER(
- monster
- g:kikimr
-)
-
-SRCS(
- counters.cpp
- counters.h
-)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(
+ monster
+ g:kikimr
+)
+
+SRCS(
+ counters.cpp
+ counters.h
+)
+
+PEERDIR(
ydb/core/base
ydb/core/protos
ydb/core/sys_view/service
-)
-
+)
+
YQL_LAST_ABI_VERSION()
-
-END()
+
+END()
diff --git a/ydb/core/grpc_services/grpc_helper.cpp b/ydb/core/grpc_services/grpc_helper.cpp
index b1ef484cc7d..8f0a59d78b9 100644
--- a/ydb/core/grpc_services/grpc_helper.cpp
+++ b/ydb/core/grpc_services/grpc_helper.cpp
@@ -3,7 +3,7 @@
namespace NKikimr {
namespace NGRpcService {
-//using namespace NActors;
+//using namespace NActors;
NGrpc::IGRpcRequestLimiterPtr TCreateLimiterCB::operator()(const char* serviceName, const char* requestName, i64 limit) const {
TString fullName = TString(serviceName) + "_" + requestName;
diff --git a/ydb/core/grpc_services/rpc_read_columns.cpp b/ydb/core/grpc_services/rpc_read_columns.cpp
index 4baae606a97..aa847ae47a9 100644
--- a/ydb/core/grpc_services/rpc_read_columns.cpp
+++ b/ydb/core/grpc_services/rpc_read_columns.cpp
@@ -86,7 +86,7 @@ public:
, ShardRequestCount(0)
, ShardReplyCount(0)
, SysViewMaxRows(100000)
- , SysViewMaxBytes(10*1024*1024)
+ , SysViewMaxBytes(10*1024*1024)
, SysViewRowsReceived(0)
{}
@@ -237,11 +237,11 @@ private:
"Snapshots are ignored when scanning system views"));
}
- if (auto maxRows = Request->GetProtoRequest()->max_rows(); maxRows && maxRows <= 100)
- SysViewMaxRows = maxRows;
+ if (auto maxRows = Request->GetProtoRequest()->max_rows(); maxRows && maxRows <= 100)
+ SysViewMaxRows = maxRows;
- if (auto maxBytes = Request->GetProtoRequest()->max_bytes(); maxBytes && maxBytes <= 10000)
- SysViewMaxBytes = maxBytes;
+ if (auto maxBytes = Request->GetProtoRequest()->max_bytes(); maxBytes && maxBytes <= 10000)
+ SysViewMaxBytes = maxBytes;
// List of columns requested by user
TVector<std::pair<TString, NScheme::TTypeId>> valueColumnNamesAndTypes;
@@ -315,7 +315,7 @@ private:
ctx);
}
- SysViewScanActor = ctx.Register(tableScanActor.Release());
+ SysViewScanActor = ctx.Register(tableScanActor.Release());
auto ackEv = MakeHolder<NKqp::TEvKqpCompute::TEvScanDataAck>(0);
ctx.Send(SysViewScanActor, ackEv.Release());
@@ -428,18 +428,18 @@ private:
}
}
- void Handle(NKqp::TEvKqpCompute::TEvScanError::TPtr& ev, const TActorContext& ctx) {
+ void Handle(NKqp::TEvKqpCompute::TEvScanError::TPtr& ev, const TActorContext& ctx) {
NYql::TIssues issues;
Ydb::StatusIds::StatusCode status = ev->Get()->Record.GetStatus();
NYql::IssuesFromMessage(ev->Get()->Record.GetIssues(), issues);
- ReplyWithError(status, issues, ctx);
- }
-
+ ReplyWithError(status, issues, ctx);
+ }
+
STFUNC(StateSysViewScan) {
switch (ev->GetTypeRewrite()) {
HFunc(NKqp::TEvKqpCompute::TEvScanData, Handle);
- HFunc(NKqp::TEvKqpCompute::TEvScanError, Handle);
+ HFunc(NKqp::TEvKqpCompute::TEvScanError, Handle);
CFunc(TEvents::TSystem::Wakeup, HandleTimeout);
default:
@@ -616,7 +616,7 @@ private:
// Send request to the first shard
std::unique_ptr<TEvDataShard::TEvReadColumnsRequest> ev =
std::make_unique<TEvDataShard::TEvReadColumnsRequest>();
- ev->Record.SetTableId(KeyRange->TableId.PathId.LocalPathId);
+ ev->Record.SetTableId(KeyRange->TableId.PathId.LocalPathId);
for (TString col : Request->GetProtoRequest()->columns()) {
ev->Record.AddColumns(col);
}
@@ -754,16 +754,16 @@ private:
}
}
- void ReplyWithError(StatusIds::StatusCode status, const NYql::TIssues& issues, const TActorContext& ctx) {
- Finished = true;
- Request->RaiseIssues(issues);
+ void ReplyWithError(StatusIds::StatusCode status, const NYql::TIssues& issues, const TActorContext& ctx) {
+ Finished = true;
+ Request->RaiseIssues(issues);
Request->ReplyWithYdbStatus(status);
-
- if (!WaitingResolveReply) {
- Die(ctx);
- }
- }
-
+
+ if (!WaitingResolveReply) {
+ Die(ctx);
+ }
+ }
+
void ReplyWithResult(StatusIds::StatusCode status,
const Ydb::ClickhouseInternal::ScanResult& result,
const TActorContext& ctx) {
diff --git a/ydb/core/grpc_streaming/grpc_streaming.h b/ydb/core/grpc_streaming/grpc_streaming.h
index ecac6fcf171..dd80817422f 100644
--- a/ydb/core/grpc_streaming/grpc_streaming.h
+++ b/ydb/core/grpc_streaming/grpc_streaming.h
@@ -116,7 +116,7 @@ public:
virtual TString GetPeerName() const = 0;
virtual TVector<TStringBuf> GetPeerMetaValues(TStringBuf key) const = 0;
virtual grpc_compression_level GetCompressionLevel() const = 0;
- virtual void UseDatabase(const TString& database) = 0;
+ virtual void UseDatabase(const TString& database) = 0;
};
template<class TIn, class TOut, class TServer, int LoggerServiceId>
@@ -196,7 +196,7 @@ private:
void MaybeClone() {
if (!Server->IsShuttingDown()) {
- Start(Server, this->Service, this->CQ, AcceptRequest, AcceptCallback, ActorSystem, Name, Counters->Clone(), Limiter);
+ Start(Server, this->Service, this->CQ, AcceptRequest, AcceptCallback, ActorSystem, Name, Counters->Clone(), Limiter);
}
}
@@ -605,10 +605,10 @@ private:
}
}
- void UseDatabase(const TString& database) {
- Counters->UseDatabase(database);
- }
-
+ void UseDatabase(const TString& database) {
+ Counters->UseDatabase(database);
+ }
+
private:
bool IncRequest() {
if (Limiter) {
@@ -678,10 +678,10 @@ private:
return Self->GetCompressionLevel();
}
- void UseDatabase(const TString& database) override {
- Self->UseDatabase(database);
- }
-
+ void UseDatabase(const TString& database) override {
+ Self->UseDatabase(database);
+ }
+
private:
TIntrusivePtr<TSelf> Self;
};
diff --git a/ydb/core/kqp/common/kqp_tx_info.h b/ydb/core/kqp/common/kqp_tx_info.h
index a3e57414ae6..fac4ac844b9 100644
--- a/ydb/core/kqp/common/kqp_tx_info.h
+++ b/ydb/core/kqp/common/kqp_tx_info.h
@@ -1,47 +1,47 @@
-#pragma once
-
-#include <util/datetime/base.h>
-
+#pragma once
+
+#include <util/datetime/base.h>
+
#include <optional>
-namespace NKikimr {
-namespace NKqp {
-
+namespace NKikimr {
+namespace NKqp {
+
struct TKqpForceNewEngineState {
ui32 ForceNewEnginePercent = 0;
ui32 ForceNewEngineLevel = 0;
std::optional<bool> ForcedNewEngine;
};
-struct TKqpTransactionInfo {
- enum class EKind {
- Pure,
- ReadOnly,
- WriteOnly,
- ReadWrite
- };
-
- enum class EStatus {
- Active,
- Committed,
- Aborted
- };
-
+struct TKqpTransactionInfo {
+ enum class EKind {
+ Pure,
+ ReadOnly,
+ WriteOnly,
+ ReadWrite
+ };
+
+ enum class EStatus {
+ Active,
+ Committed,
+ Aborted
+ };
+
enum class EEngine {
OldEngine,
NewEngine
};
-public:
- EStatus Status;
- EKind Kind;
+public:
+ EStatus Status;
+ EKind Kind;
std::optional<EEngine> TxEngine;
- TDuration TotalDuration;
- TDuration ServerDuration;
- ui32 QueriesCount = 0;
+ TDuration TotalDuration;
+ TDuration ServerDuration;
+ ui32 QueriesCount = 0;
TKqpForceNewEngineState ForceNewEngineState;
-};
-
-} // namespace NKqp
-} // namespace NKikimr
-
+};
+
+} // namespace NKqp
+} // namespace NKikimr
+
diff --git a/ydb/core/kqp/counters/kqp_counters.cpp b/ydb/core/kqp/counters/kqp_counters.cpp
index 26a28a0249a..41eeba73d5d 100644
--- a/ydb/core/kqp/counters/kqp_counters.cpp
+++ b/ydb/core/kqp/counters/kqp_counters.cpp
@@ -7,11 +7,11 @@
#include <ydb/core/sys_view/service/sysview_service.h>
#include <library/cpp/actors/core/log.h>
-
+
#include <util/generic/size_literals.h>
#include <ydb/library/yql/core/issue/protos/issue_id.pb.h>
-
+
namespace NKikimr {
namespace NKqp {
@@ -22,33 +22,33 @@ NMonitoring::TDynamicCounterPtr TKqpCountersBase::GetQueryReplayCounters() const
return QueryReplayGroup;
}
-void TKqpCountersBase::CreateYdbTxKindCounters(TKqpTransactionInfo::EKind kind, const TString& name) {
- auto txKindGroup = YdbGroup->GetSubgroup("tx_kind", name);
-
- auto& ydbTxCounters = YdbTxByKind[kind];
- ydbTxCounters.TotalDuration = txKindGroup->GetNamedHistogram("name",
- "table.transaction.total_duration_milliseconds", NMonitoring::ExponentialHistogram(20, 2, 1));
- ydbTxCounters.ServerDuration = txKindGroup->GetNamedHistogram("name",
- "table.transaction.server_duration_milliseconds", NMonitoring::ExponentialHistogram(20, 2, 1));
- ydbTxCounters.ClientDuration = txKindGroup->GetNamedHistogram("name",
- "table.transaction.client_duration_milliseconds", NMonitoring::ExponentialHistogram(20, 2, 1));
-}
-
-void TKqpCountersBase::UpdateYdbTxCounters(const TKqpTransactionInfo& txInfo,
- THashMap<TKqpTransactionInfo::EKind, TYdbTxByKindCounters>& txCounters)
-{
- auto byKind = txCounters.FindPtr(txInfo.Kind);
- if (!byKind) {
- return;
- }
-
- byKind->TotalDuration->Collect(txInfo.TotalDuration.MilliSeconds());
- byKind->ServerDuration->Collect(txInfo.ServerDuration.MilliSeconds());
- auto clientDuration = txInfo.TotalDuration - txInfo.ServerDuration;
- byKind->ClientDuration->Collect(clientDuration.MilliSeconds());
-}
-
-void TKqpCountersBase::Init() {
+void TKqpCountersBase::CreateYdbTxKindCounters(TKqpTransactionInfo::EKind kind, const TString& name) {
+ auto txKindGroup = YdbGroup->GetSubgroup("tx_kind", name);
+
+ auto& ydbTxCounters = YdbTxByKind[kind];
+ ydbTxCounters.TotalDuration = txKindGroup->GetNamedHistogram("name",
+ "table.transaction.total_duration_milliseconds", NMonitoring::ExponentialHistogram(20, 2, 1));
+ ydbTxCounters.ServerDuration = txKindGroup->GetNamedHistogram("name",
+ "table.transaction.server_duration_milliseconds", NMonitoring::ExponentialHistogram(20, 2, 1));
+ ydbTxCounters.ClientDuration = txKindGroup->GetNamedHistogram("name",
+ "table.transaction.client_duration_milliseconds", NMonitoring::ExponentialHistogram(20, 2, 1));
+}
+
+void TKqpCountersBase::UpdateYdbTxCounters(const TKqpTransactionInfo& txInfo,
+ THashMap<TKqpTransactionInfo::EKind, TYdbTxByKindCounters>& txCounters)
+{
+ auto byKind = txCounters.FindPtr(txInfo.Kind);
+ if (!byKind) {
+ return;
+ }
+
+ byKind->TotalDuration->Collect(txInfo.TotalDuration.MilliSeconds());
+ byKind->ServerDuration->Collect(txInfo.ServerDuration.MilliSeconds());
+ auto clientDuration = txInfo.TotalDuration - txInfo.ServerDuration;
+ byKind->ClientDuration->Collect(clientDuration.MilliSeconds());
+}
+
+void TKqpCountersBase::Init() {
/* Requests */
QueryActionRequests[NKikimrKqp::QUERY_ACTION_EXECUTE] =
KqpGroup->GetCounter("Requests/QueryExecute", true);
@@ -75,10 +75,10 @@ void TKqpCountersBase::Init() {
PingSessionRequests = KqpGroup->GetCounter("Requests/PingSession", true);
RequestBytes = KqpGroup->GetCounter("Requests/Bytes", true);
- YdbRequestBytes = YdbGroup->GetNamedCounter("name", "table.query.request.bytes", true);
+ YdbRequestBytes = YdbGroup->GetNamedCounter("name", "table.query.request.bytes", true);
QueryBytes = KqpGroup->GetCounter("Requests/QueryBytes", true);
ParametersBytes = KqpGroup->GetCounter("Requests/ParametersBytes", true);
- YdbParametersBytes = YdbGroup->GetNamedCounter("name", "table.query.request.parameters_bytes", true);
+ YdbParametersBytes = YdbGroup->GetNamedCounter("name", "table.query.request.parameters_bytes", true);
SqlV0Translations = KqpGroup->GetCounter("Requests/Sql/V0", true);
SqlV1Translations = KqpGroup->GetCounter("Requests/Sql/V1", true);
@@ -121,30 +121,30 @@ void TKqpCountersBase::Init() {
NMonitoring::ExplicitHistogram({1_MB, 9_MB, 29_MB}));
/* Request latency */
- QueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE] = KqpGroup->GetHistogram(
- "Query/ExecuteLatencyMs", NMonitoring::ExponentialHistogram(20, 2, 1));
- QueryLatencies[NKikimrKqp::QUERY_ACTION_EXPLAIN] = KqpGroup->GetHistogram(
- "Query/ExplainLatencyMs", NMonitoring::ExponentialHistogram(10, 2, 1));
- QueryLatencies[NKikimrKqp::QUERY_ACTION_VALIDATE] = KqpGroup->GetHistogram(
- "Query/ValidateLatencyMs", NMonitoring::ExponentialHistogram(10, 2, 1));
- QueryLatencies[NKikimrKqp::QUERY_ACTION_PREPARE] = KqpGroup->GetHistogram(
- "Query/PrepareLatencyMs", NMonitoring::ExponentialHistogram(16, 2, 1));
- QueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED] = KqpGroup->GetHistogram(
- "Query/ExecPreparedLatencyMs", NMonitoring::ExponentialHistogram(20, 2, 1));
- QueryLatencies[NKikimrKqp::QUERY_ACTION_BEGIN_TX] = KqpGroup->GetHistogram(
- "Query/BeginTxLatencyMs", NMonitoring::ExponentialHistogram(10, 2, 1));
- QueryLatencies[NKikimrKqp::QUERY_ACTION_COMMIT_TX] = KqpGroup->GetHistogram(
- "Query/CommitTxLatencyMs", NMonitoring::ExponentialHistogram(20, 2, 1));
- QueryLatencies[NKikimrKqp::QUERY_ACTION_ROLLBACK_TX] = KqpGroup->GetHistogram(
- "Query/RollbackTxLatencyMs", NMonitoring::ExponentialHistogram(10, 2, 1));
-
- YdbQueryExecuteLatency = YdbGroup->GetNamedHistogram("name",
- "table.query.execution.latency_milliseconds", NMonitoring::ExponentialHistogram(20, 2, 1));
-
- // TODO move to grpc
- YdbResponsesLocksInvalidated = YdbGroup->GetSubgroup("issue_type", "optimistic_locks_invalidation")
- ->GetNamedCounter("name", "api.grpc.response.issues", true);
-
+ QueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE] = KqpGroup->GetHistogram(
+ "Query/ExecuteLatencyMs", NMonitoring::ExponentialHistogram(20, 2, 1));
+ QueryLatencies[NKikimrKqp::QUERY_ACTION_EXPLAIN] = KqpGroup->GetHistogram(
+ "Query/ExplainLatencyMs", NMonitoring::ExponentialHistogram(10, 2, 1));
+ QueryLatencies[NKikimrKqp::QUERY_ACTION_VALIDATE] = KqpGroup->GetHistogram(
+ "Query/ValidateLatencyMs", NMonitoring::ExponentialHistogram(10, 2, 1));
+ QueryLatencies[NKikimrKqp::QUERY_ACTION_PREPARE] = KqpGroup->GetHistogram(
+ "Query/PrepareLatencyMs", NMonitoring::ExponentialHistogram(16, 2, 1));
+ QueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED] = KqpGroup->GetHistogram(
+ "Query/ExecPreparedLatencyMs", NMonitoring::ExponentialHistogram(20, 2, 1));
+ QueryLatencies[NKikimrKqp::QUERY_ACTION_BEGIN_TX] = KqpGroup->GetHistogram(
+ "Query/BeginTxLatencyMs", NMonitoring::ExponentialHistogram(10, 2, 1));
+ QueryLatencies[NKikimrKqp::QUERY_ACTION_COMMIT_TX] = KqpGroup->GetHistogram(
+ "Query/CommitTxLatencyMs", NMonitoring::ExponentialHistogram(20, 2, 1));
+ QueryLatencies[NKikimrKqp::QUERY_ACTION_ROLLBACK_TX] = KqpGroup->GetHistogram(
+ "Query/RollbackTxLatencyMs", NMonitoring::ExponentialHistogram(10, 2, 1));
+
+ YdbQueryExecuteLatency = YdbGroup->GetNamedHistogram("name",
+ "table.query.execution.latency_milliseconds", NMonitoring::ExponentialHistogram(20, 2, 1));
+
+ // TODO move to grpc
+ YdbResponsesLocksInvalidated = YdbGroup->GetSubgroup("issue_type", "optimistic_locks_invalidation")
+ ->GetNamedCounter("name", "api.grpc.response.issues", true);
+
/* Response statuses */
YdbResponses[Ydb::StatusIds::STATUS_CODE_UNSPECIFIED] = KqpGroup->GetCounter("YdbResponses/Unspecified", true);
YdbResponses[Ydb::StatusIds::SUCCESS] = KqpGroup->GetCounter("YdbResponses/Success", true);
@@ -169,18 +169,18 @@ void TKqpCountersBase::Init() {
OtherYdbResponses = KqpGroup->GetCounter("YdbResponses/Other", true);
ResponseBytes = KqpGroup->GetCounter("Responses/Bytes", true);
- YdbResponseBytes = YdbGroup->GetNamedCounter("name", "table.query.response.bytes", true);
+ YdbResponseBytes = YdbGroup->GetNamedCounter("name", "table.query.response.bytes", true);
QueryResultsBytes = KqpGroup->GetCounter("Responses/QueryResultBytes", true);
/* Workers */
- WorkerLifeSpan = KqpGroup->GetHistogram(
- "Workers/LifeSpanMs", NMonitoring::ExponentialHistogram(20, 2, 1));
- QueriesPerWorker = KqpGroup->GetHistogram(
- "Workers/QueriesPerWorkerQ", NMonitoring::ExponentialHistogram(16, 2, 1));
-
+ WorkerLifeSpan = KqpGroup->GetHistogram(
+ "Workers/LifeSpanMs", NMonitoring::ExponentialHistogram(20, 2, 1));
+ QueriesPerWorker = KqpGroup->GetHistogram(
+ "Workers/QueriesPerWorkerQ", NMonitoring::ExponentialHistogram(16, 2, 1));
+
WorkersCreated = KqpGroup->GetCounter("Workers/Created", true);
WorkersClosedIdle = KqpGroup->GetCounter("Workers/ClosedIdle", true);
- YdbWorkersClosedIdle = YdbGroup->GetNamedCounter("name", "table.session.closed_by_idle_count", true);
+ YdbWorkersClosedIdle = YdbGroup->GetNamedCounter("name", "table.session.closed_by_idle_count", true);
WorkersClosedError = KqpGroup->GetCounter("Workers/ClosedError", true);
WorkersClosedRequest = KqpGroup->GetCounter("Workers/ClosedRequest", true);
ActiveWorkers = KqpGroup->GetCounter("Workers/Active", false);
@@ -189,49 +189,49 @@ void TKqpCountersBase::Init() {
SessionBalancerCV = KqpGroup->GetCounter("SessionBalancer/CV", false);
SessionBalancerShutdowns = KqpGroup->GetCounter("SessionBalancer/Shutdown", true);
- YdbActiveWorkers = YdbGroup->GetNamedCounter("name", "table.session.active_count", false);
-
- WorkerCleanupLatency = KqpGroup->GetHistogram(
- "Workers/CleanupLatencyMs", NMonitoring::ExponentialHistogram(10, 2, 1));
+ YdbActiveWorkers = YdbGroup->GetNamedCounter("name", "table.session.active_count", false);
+ WorkerCleanupLatency = KqpGroup->GetHistogram(
+ "Workers/CleanupLatencyMs", NMonitoring::ExponentialHistogram(10, 2, 1));
+
/* Transactions */
TxCreated = KqpGroup->GetCounter("Transactions/Created", true);
TxAborted = KqpGroup->GetCounter("Transactions/Aborted", true);
TxCommited = KqpGroup->GetCounter("Transactions/Commited", true);
TxEvicted = KqpGroup->GetCounter("Transactions/Evicted", true);
- TxActivePerSession = KqpGroup->GetHistogram(
- "Transactions/TxActivePerSession", NMonitoring::ExponentialHistogram(16, 2, 1));
- TxAbortedPerSession = KqpGroup->GetHistogram(
- "Transactions/TxAbortedPerSession", NMonitoring::ExponentialHistogram(16, 2, 1));
-
- CompileRequestsCompile = KqpGroup->GetCounter("Compilation/Requests/Compile", true);
- CompileRequestsGet = KqpGroup->GetCounter("Compilation/Requests/Get", true);
- CompileRequestsInvalidate = KqpGroup->GetCounter("Compilation/Requests/Invalidate", true);
- CompileRequestsRejected = KqpGroup->GetCounter("Compilation/Requests/Rejected", true);
- CompileRequestsTimeout = KqpGroup->GetCounter("Compilation/Requests/Timeout", true);
+ TxActivePerSession = KqpGroup->GetHistogram(
+ "Transactions/TxActivePerSession", NMonitoring::ExponentialHistogram(16, 2, 1));
+ TxAbortedPerSession = KqpGroup->GetHistogram(
+ "Transactions/TxAbortedPerSession", NMonitoring::ExponentialHistogram(16, 2, 1));
+
+ CompileRequestsCompile = KqpGroup->GetCounter("Compilation/Requests/Compile", true);
+ CompileRequestsGet = KqpGroup->GetCounter("Compilation/Requests/Get", true);
+ CompileRequestsInvalidate = KqpGroup->GetCounter("Compilation/Requests/Invalidate", true);
+ CompileRequestsRejected = KqpGroup->GetCounter("Compilation/Requests/Rejected", true);
+ CompileRequestsTimeout = KqpGroup->GetCounter("Compilation/Requests/Timeout", true);
CompileRequestsRecompile = KqpGroup->GetCounter("Compilation/Requests/Recompile", true);
-
- CompileCpuTime = KqpGroup->GetHistogram(
- "Compilation/CPUTimeMs", NMonitoring::ExponentialHistogram(20, 2, 1));
-
- CreateYdbTxKindCounters(TKqpTransactionInfo::EKind::Pure, "pure");
- CreateYdbTxKindCounters(TKqpTransactionInfo::EKind::ReadOnly, "read_only");
- CreateYdbTxKindCounters(TKqpTransactionInfo::EKind::WriteOnly, "write_only");
- CreateYdbTxKindCounters(TKqpTransactionInfo::EKind::ReadWrite, "read_write");
-
- CompileQueryCacheHits = YdbGroup->GetNamedCounter("name", "table.query.compilation.cache_hits", true);
- CompileQueryCacheMisses = YdbGroup->GetNamedCounter("name", "table.query.compilation.cache_misses", true);
-
- CompileTotal = YdbGroup->GetNamedCounter("name", "table.query.compilation.count", true);
- CompileErrors = YdbGroup->GetNamedCounter("name", "table.query.compilation.error_count", true);
- CompileActive = YdbGroup->GetNamedCounter("name", "table.query.compilation.active_count", false);
-
- YdbCompileDuration = YdbGroup->GetNamedHistogram("name",
- "table.query.compilation.latency_milliseconds", NMonitoring::ExponentialHistogram(20, 2, 1));
-}
-
-void TKqpCountersBase::ReportQueryAction(NKikimrKqp::EQueryAction action) {
+
+ CompileCpuTime = KqpGroup->GetHistogram(
+ "Compilation/CPUTimeMs", NMonitoring::ExponentialHistogram(20, 2, 1));
+
+ CreateYdbTxKindCounters(TKqpTransactionInfo::EKind::Pure, "pure");
+ CreateYdbTxKindCounters(TKqpTransactionInfo::EKind::ReadOnly, "read_only");
+ CreateYdbTxKindCounters(TKqpTransactionInfo::EKind::WriteOnly, "write_only");
+ CreateYdbTxKindCounters(TKqpTransactionInfo::EKind::ReadWrite, "read_write");
+
+ CompileQueryCacheHits = YdbGroup->GetNamedCounter("name", "table.query.compilation.cache_hits", true);
+ CompileQueryCacheMisses = YdbGroup->GetNamedCounter("name", "table.query.compilation.cache_misses", true);
+
+ CompileTotal = YdbGroup->GetNamedCounter("name", "table.query.compilation.count", true);
+ CompileErrors = YdbGroup->GetNamedCounter("name", "table.query.compilation.error_count", true);
+ CompileActive = YdbGroup->GetNamedCounter("name", "table.query.compilation.active_count", false);
+
+ YdbCompileDuration = YdbGroup->GetNamedHistogram("name",
+ "table.query.compilation.latency_milliseconds", NMonitoring::ExponentialHistogram(20, 2, 1));
+}
+
+void TKqpCountersBase::ReportQueryAction(NKikimrKqp::EQueryAction action) {
auto counter = QueryActionRequests.FindPtr(action);
if (counter) {
(*counter)->Inc();
@@ -241,7 +241,7 @@ void TKqpCountersBase::ReportQueryAction(NKikimrKqp::EQueryAction action) {
OtherQueryRequests->Inc();
}
-void TKqpCountersBase::ReportQueryType(NKikimrKqp::EQueryType type) {
+void TKqpCountersBase::ReportQueryType(NKikimrKqp::EQueryType type) {
auto counter = QueryTypes.FindPtr(type);
if (counter) {
(*counter)->Inc();
@@ -255,40 +255,40 @@ void TKqpCountersBase::ReportSessionShutdownRequest() {
SessionBalancerShutdowns->Inc();
}
-void TKqpCountersBase::ReportCreateSession(ui64 requestSize) {
- CreateSessionRequests->Inc();
- *RequestBytes += requestSize;
- *YdbRequestBytes += requestSize;
-}
-
-void TKqpCountersBase::ReportPingSession(ui64 requestSize) {
- PingSessionRequests->Inc();
- *RequestBytes += requestSize;
- *YdbRequestBytes += requestSize;
-}
-
-void TKqpCountersBase::ReportCloseSession(ui64 requestSize) {
- CloseSessionRequests->Inc();
- *RequestBytes += requestSize;
- *YdbRequestBytes += requestSize;
-}
-
-void TKqpCountersBase::ReportQueryRequest(const NKikimrKqp::TQueryRequest& request) {
- ReportQueryAction(request.GetAction());
- ReportQueryType(request.GetType());
-
- auto requestBytes = request.ByteSize();
- auto parametersBytes = request.GetParameters().ByteSize();
-
- *RequestBytes += requestBytes;
- *YdbRequestBytes += requestBytes;
-
- *QueryBytes += request.GetQuery().size();
-
- *ParametersBytes += parametersBytes;
- *YdbParametersBytes += parametersBytes;
-}
-
+void TKqpCountersBase::ReportCreateSession(ui64 requestSize) {
+ CreateSessionRequests->Inc();
+ *RequestBytes += requestSize;
+ *YdbRequestBytes += requestSize;
+}
+
+void TKqpCountersBase::ReportPingSession(ui64 requestSize) {
+ PingSessionRequests->Inc();
+ *RequestBytes += requestSize;
+ *YdbRequestBytes += requestSize;
+}
+
+void TKqpCountersBase::ReportCloseSession(ui64 requestSize) {
+ CloseSessionRequests->Inc();
+ *RequestBytes += requestSize;
+ *YdbRequestBytes += requestSize;
+}
+
+void TKqpCountersBase::ReportQueryRequest(const NKikimrKqp::TQueryRequest& request) {
+ ReportQueryAction(request.GetAction());
+ ReportQueryType(request.GetType());
+
+ auto requestBytes = request.ByteSize();
+ auto parametersBytes = request.GetParameters().ByteSize();
+
+ *RequestBytes += requestBytes;
+ *YdbRequestBytes += requestBytes;
+
+ *QueryBytes += request.GetQuery().size();
+
+ *ParametersBytes += parametersBytes;
+ *YdbParametersBytes += parametersBytes;
+}
+
void TKqpCountersBase::ReportQueryWithRangeScan() {
QueriesWithRangeScan->Inc();
}
@@ -322,9 +322,9 @@ void TKqpCountersBase::ReportQueryMaxShardProgramSize(ui64 programSize) {
}
void TKqpCountersBase::ReportResponseStatus(ui64 responseSize, Ydb::StatusIds::StatusCode ydbStatus) {
- *ResponseBytes += responseSize;
- *YdbResponseBytes += responseSize;
-
+ *ResponseBytes += responseSize;
+ *YdbResponseBytes += responseSize;
+
auto ydbCounter = YdbResponses.FindPtr(ydbStatus);
if (ydbCounter) {
(*ydbCounter)->Inc();
@@ -333,11 +333,11 @@ void TKqpCountersBase::ReportResponseStatus(ui64 responseSize, Ydb::StatusIds::S
}
}
-void TKqpCountersBase::ReportResultsBytes(ui64 resultsBytes) {
- *QueryResultsBytes += resultsBytes;
-}
-
-TString TKqpCountersBase::GetIssueName(ui32 issueCode) {
+void TKqpCountersBase::ReportResultsBytes(ui64 resultsBytes) {
+ *QueryResultsBytes += resultsBytes;
+}
+
+TString TKqpCountersBase::GetIssueName(ui32 issueCode) {
auto kikimrIssueDescriptor = NKikimrIssues::TIssuesIds::EIssueCode_descriptor();
if (kikimrIssueDescriptor) {
auto valueDescriptor = kikimrIssueDescriptor->FindValueByNumber(issueCode);
@@ -357,7 +357,7 @@ TString TKqpCountersBase::GetIssueName(ui32 issueCode) {
return TStringBuilder() << "CODE:" << ToString(issueCode);
}
-void TKqpCountersBase::ReportIssues(const Ydb::Issue::IssueMessage& issue) {
+void TKqpCountersBase::ReportIssues(const Ydb::Issue::IssueMessage& issue) {
auto issueCounter = IssueCounters.FindPtr(issue.issue_code());
if (!issueCounter) {
auto counterName = TStringBuilder() << "Issues/" << GetIssueName(issue.issue_code());
@@ -378,23 +378,23 @@ void TKqpCountersBase::ReportIssues(const Ydb::Issue::IssueMessage& issue) {
}
}
-void TKqpCountersBase::ReportQueryLatency(NKikimrKqp::EQueryAction action, const TDuration& duration) {
+void TKqpCountersBase::ReportQueryLatency(NKikimrKqp::EQueryAction action, const TDuration& duration) {
if (action == NKikimrKqp::QUERY_ACTION_EXECUTE ||
action == NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED ||
action == NKikimrKqp::QUERY_ACTION_COMMIT_TX ||
action == NKikimrKqp::QUERY_ACTION_ROLLBACK_TX)
{
- YdbQueryExecuteLatency->Collect(duration.MilliSeconds());
- }
-
+ YdbQueryExecuteLatency->Collect(duration.MilliSeconds());
+ }
+
auto counter = QueryLatencies.FindPtr(action);
- if (counter) {
- (*counter)->Collect(duration.MilliSeconds());
+ if (counter) {
+ (*counter)->Collect(duration.MilliSeconds());
}
}
-void TKqpCountersBase::ReportTransaction(const TKqpTransactionInfo& txInfo) {
+void TKqpCountersBase::ReportTransaction(const TKqpTransactionInfo& txInfo) {
switch (txInfo.Status) {
case TKqpTransactionInfo::EStatus::Active:
return;
@@ -405,10 +405,10 @@ void TKqpCountersBase::ReportTransaction(const TKqpTransactionInfo& txInfo) {
TxCommited->Inc();
break;
}
- UpdateYdbTxCounters(txInfo, YdbTxByKind);
+ UpdateYdbTxCounters(txInfo, YdbTxByKind);
}
-void TKqpCountersBase::ReportSqlVersion(ui16 sqlVersion) {
+void TKqpCountersBase::ReportSqlVersion(ui16 sqlVersion) {
switch (sqlVersion) {
case 0:
SqlV0Translations->Inc();
@@ -422,12 +422,12 @@ void TKqpCountersBase::ReportSqlVersion(ui16 sqlVersion) {
}
}
-void TKqpCountersBase::ReportWorkerCreated() {
- WorkersCreated->Inc();
- ActiveWorkers->Inc();
- YdbActiveWorkers->Inc();
-}
-
+void TKqpCountersBase::ReportWorkerCreated() {
+ WorkersCreated->Inc();
+ ActiveWorkers->Inc();
+ YdbActiveWorkers->Inc();
+}
+
void TKqpCountersBase::ReportSessionBalancerCV(ui32 value) {
SessionBalancerCV->Set(value);
@@ -437,324 +437,324 @@ void TKqpCountersBase::ReportProxyForwardedRequest() {
ProxyForwardedRequests->Inc();
}
-void TKqpCountersBase::ReportWorkerFinished(TDuration lifeSpan) {
- WorkerLifeSpan->Collect(lifeSpan.MilliSeconds());
- ActiveWorkers->Dec();
- YdbActiveWorkers->Dec();
-}
-
-void TKqpCountersBase::ReportWorkerCleanupLatency(TDuration cleanupTime) {
- WorkerCleanupLatency->Collect(cleanupTime.MilliSeconds());
-}
-
-void TKqpCountersBase::ReportWorkerClosedIdle() {
- WorkersClosedIdle->Inc();
- YdbWorkersClosedIdle->Inc();
-}
-
-void TKqpCountersBase::ReportWorkerClosedError() {
- WorkersClosedError->Inc();
-}
-
-void TKqpCountersBase::ReportWorkerClosedRequest() {
- WorkersClosedRequest->Inc();
-}
-
-void TKqpCountersBase::ReportQueriesPerWorker(ui32 queryId) {
- QueriesPerWorker->Collect(queryId);
-}
-
-void TKqpCountersBase::ReportBeginTransaction(ui32 evictedTx, ui32 currentActiveTx, ui32 currentAbortedTx) {
- TxEvicted->Add(evictedTx);
- TxAborted->Add(evictedTx);
- TxActivePerSession->Collect(currentActiveTx);
- TxAbortedPerSession->Collect(currentAbortedTx);
-}
-
-void TKqpCountersBase::ReportTxCreated() {
- TxCreated->Inc();
-}
-
-void TKqpCountersBase::ReportTxAborted(ui32 abortedCount) {
- TxAborted->Add(abortedCount);
-}
-
-void TKqpCountersBase::ReportQueryCacheHit(bool hit) {
- if (hit) {
- CompileQueryCacheHits->Inc();
- } else {
- CompileQueryCacheMisses->Inc();
- }
-}
-
-void TKqpCountersBase::ReportCompileStart() {
- CompileTotal->Inc();
- CompileActive->Inc();
-}
-
-void TKqpCountersBase::ReportCompileFinish() {
- CompileActive->Dec();
-}
-
-void TKqpCountersBase::ReportCompileError() {
- CompileErrors->Inc();
-}
-
-void TKqpCountersBase::ReportCompileRequestCompile() {
- CompileRequestsCompile->Inc();
-}
-
-void TKqpCountersBase::ReportCompileRequestGet() {
- CompileRequestsGet->Inc();
-}
-
-void TKqpCountersBase::ReportCompileRequestInvalidate() {
- CompileRequestsInvalidate->Inc();
-}
-
-void TKqpCountersBase::ReportCompileRequestRejected() {
- CompileRequestsRejected->Inc();
-}
-
-void TKqpCountersBase::ReportCompileRequestTimeout() {
- CompileRequestsTimeout->Inc();
-}
-
-void TKqpCountersBase::ReportCompileDurations(TDuration duration, TDuration cpuTime) {
- YdbCompileDuration->Collect(duration.MilliSeconds());
- CompileCpuTime->Collect(cpuTime.MilliSeconds());
-}
-
+void TKqpCountersBase::ReportWorkerFinished(TDuration lifeSpan) {
+ WorkerLifeSpan->Collect(lifeSpan.MilliSeconds());
+ ActiveWorkers->Dec();
+ YdbActiveWorkers->Dec();
+}
+
+void TKqpCountersBase::ReportWorkerCleanupLatency(TDuration cleanupTime) {
+ WorkerCleanupLatency->Collect(cleanupTime.MilliSeconds());
+}
+
+void TKqpCountersBase::ReportWorkerClosedIdle() {
+ WorkersClosedIdle->Inc();
+ YdbWorkersClosedIdle->Inc();
+}
+
+void TKqpCountersBase::ReportWorkerClosedError() {
+ WorkersClosedError->Inc();
+}
+
+void TKqpCountersBase::ReportWorkerClosedRequest() {
+ WorkersClosedRequest->Inc();
+}
+
+void TKqpCountersBase::ReportQueriesPerWorker(ui32 queryId) {
+ QueriesPerWorker->Collect(queryId);
+}
+
+void TKqpCountersBase::ReportBeginTransaction(ui32 evictedTx, ui32 currentActiveTx, ui32 currentAbortedTx) {
+ TxEvicted->Add(evictedTx);
+ TxAborted->Add(evictedTx);
+ TxActivePerSession->Collect(currentActiveTx);
+ TxAbortedPerSession->Collect(currentAbortedTx);
+}
+
+void TKqpCountersBase::ReportTxCreated() {
+ TxCreated->Inc();
+}
+
+void TKqpCountersBase::ReportTxAborted(ui32 abortedCount) {
+ TxAborted->Add(abortedCount);
+}
+
+void TKqpCountersBase::ReportQueryCacheHit(bool hit) {
+ if (hit) {
+ CompileQueryCacheHits->Inc();
+ } else {
+ CompileQueryCacheMisses->Inc();
+ }
+}
+
+void TKqpCountersBase::ReportCompileStart() {
+ CompileTotal->Inc();
+ CompileActive->Inc();
+}
+
+void TKqpCountersBase::ReportCompileFinish() {
+ CompileActive->Dec();
+}
+
+void TKqpCountersBase::ReportCompileError() {
+ CompileErrors->Inc();
+}
+
+void TKqpCountersBase::ReportCompileRequestCompile() {
+ CompileRequestsCompile->Inc();
+}
+
+void TKqpCountersBase::ReportCompileRequestGet() {
+ CompileRequestsGet->Inc();
+}
+
+void TKqpCountersBase::ReportCompileRequestInvalidate() {
+ CompileRequestsInvalidate->Inc();
+}
+
+void TKqpCountersBase::ReportCompileRequestRejected() {
+ CompileRequestsRejected->Inc();
+}
+
+void TKqpCountersBase::ReportCompileRequestTimeout() {
+ CompileRequestsTimeout->Inc();
+}
+
+void TKqpCountersBase::ReportCompileDurations(TDuration duration, TDuration cpuTime) {
+ YdbCompileDuration->Collect(duration.MilliSeconds());
+ CompileCpuTime->Collect(cpuTime.MilliSeconds());
+}
+
void TKqpCountersBase::ReportRecompileRequestGet() {
CompileRequestsRecompile->Inc();
}
-
-
-TKqpDbCounters::TKqpDbCounters() {
- Counters = new NMonitoring::TDynamicCounters();
- KqpGroup = Counters->GetSubgroup("group", "kqp");
- YdbGroup = Counters->GetSubgroup("group", "ydb");
-
- Init();
-}
-
-TKqpDbCounters::TKqpDbCounters(const NMonitoring::TDynamicCounterPtr& externalGroup,
- const NMonitoring::TDynamicCounterPtr& internalGroup)
-{
- Counters = internalGroup;
- KqpGroup = Counters->GetSubgroup("group", "kqp");
- YdbGroup = externalGroup;
+
+
+TKqpDbCounters::TKqpDbCounters() {
+ Counters = new NMonitoring::TDynamicCounters();
+ KqpGroup = Counters->GetSubgroup("group", "kqp");
+ YdbGroup = Counters->GetSubgroup("group", "ydb");
+
+ Init();
+}
+
+TKqpDbCounters::TKqpDbCounters(const NMonitoring::TDynamicCounterPtr& externalGroup,
+ const NMonitoring::TDynamicCounterPtr& internalGroup)
+{
+ Counters = internalGroup;
+ KqpGroup = Counters->GetSubgroup("group", "kqp");
+ YdbGroup = externalGroup;
QueryReplayGroup = KqpGroup->GetSubgroup("subsystem", "unified_agent_query_replay");
-
- Init();
-}
-
-template <typename T>
-void SaveHistogram(T& histogram, int index, const NMon::THistogramCounterHelper& helper) {
- auto* buckets = histogram[index].MutableBuckets();
- auto count = helper.GetBucketCount();
- buckets->Resize(count, 0);
- for (size_t i = 0; i < count; ++i) {
- (*buckets)[i] = helper.GetBucketValue(i);
- }
-}
-
-template <typename T>
-void SaveHistogram(T& histogram, int index, const NMonitoring::THistogramPtr& hgram) {
- auto* buckets = histogram[index].MutableBuckets();
- auto snapshot = hgram->Snapshot();
- auto count = snapshot->Count();
- buckets->Resize(count, 0);
- for (size_t i = 0; i < count; ++i) {
- (*buckets)[i] = snapshot->Value(i);
- }
-}
-
-void TKqpDbCounters::ToProto(NKikimr::NSysView::TDbServiceCounters& counters) {
- auto* main = counters.Proto().MutableMain();
- auto* simple = main->MutableSimple();
- auto* cumulative = main->MutableCumulative();
- auto* histogram = main->MutableHistogram();
-
- simple->Resize(DB_KQP_SIMPLE_COUNTER_SIZE, 0);
- cumulative->Resize(DB_KQP_CUMULATIVE_COUNTER_SIZE, 0);
- if (main->HistogramSize() < DB_KQP_HISTOGRAM_COUNTER_SIZE) {
- auto missing = DB_KQP_HISTOGRAM_COUNTER_SIZE - main->HistogramSize();
- for (; missing > 0; --missing) {
- main->AddHistogram();
- }
- }
-
-#define SAVE_SIMPLE_COUNTER(INDEX, TARGET) { (*simple)[INDEX] = (TARGET)->Val(); }
-#define SAVE_CUMULATIVE_COUNTER(INDEX, TARGET) { (*cumulative)[INDEX] = (TARGET)->Val(); }
-#define SAVE_HISTOGRAM_COUNTER(INDEX, TARGET) { SaveHistogram(*histogram, INDEX, (TARGET)); }
-
- DB_KQP_SIMPLE_COUNTERS_MAP(SAVE_SIMPLE_COUNTER)
- DB_KQP_CUMULATIVE_COUNTERS_MAP(SAVE_CUMULATIVE_COUNTER)
- DB_KQP_HISTOGRAM_COUNTERS_MAP(SAVE_HISTOGRAM_COUNTER)
-}
-
-template <typename T>
-void LoadHistogram(T& histogram, int index, NMon::THistogramCounterHelper& helper) {
- auto* buckets = histogram[index].MutableBuckets();
- auto count = helper.GetBucketCount();
- buckets->Resize(count, 0);
- for (size_t i = 0; i < count; ++i) {
- helper.SetBucketValue(i, (*buckets)[i]);
- }
-}
-
-template <typename T>
-void LoadHistogram(T& histogram, int index, NMonitoring::THistogramPtr& hgram) {
- auto* buckets = histogram[index].MutableBuckets();
- auto snapshot = hgram->Snapshot();
- auto count = snapshot->Count();
- buckets->Resize(count, 0);
- hgram->Reset();
- for (ui32 i = 0; i < count; ++i) {
- hgram->Collect(snapshot->UpperBound(i), (*buckets)[i]);
- }
-}
-
-void TKqpDbCounters::FromProto(NKikimr::NSysView::TDbServiceCounters& counters) {
- auto* main = counters.Proto().MutableMain();
- auto* simple = main->MutableSimple();
- auto* cumulative = main->MutableCumulative();
- auto* histogram = main->MutableHistogram();
-
- simple->Resize(DB_KQP_SIMPLE_COUNTER_SIZE, 0);
- cumulative->Resize(DB_KQP_CUMULATIVE_COUNTER_SIZE, 0);
- if (main->HistogramSize() < DB_KQP_HISTOGRAM_COUNTER_SIZE) {
- auto missing = DB_KQP_HISTOGRAM_COUNTER_SIZE - main->HistogramSize();
- for (; missing > 0; --missing) {
- main->AddHistogram();
- }
- }
-
-#define LOAD_SIMPLE_COUNTER(INDEX, TARGET) { (TARGET)->Set((*simple)[INDEX]); }
-#define LOAD_CUMULATIVE_COUNTER(INDEX, TARGET) { (TARGET)->Set((*cumulative)[INDEX]); }
-#define LOAD_HISTOGRAM_COUNTER(INDEX, TARGET) { LoadHistogram(*histogram, INDEX, (TARGET)); }
-
- DB_KQP_SIMPLE_COUNTERS_MAP(LOAD_SIMPLE_COUNTER)
- DB_KQP_CUMULATIVE_COUNTERS_MAP(LOAD_CUMULATIVE_COUNTER)
- DB_KQP_HISTOGRAM_COUNTERS_MAP(LOAD_HISTOGRAM_COUNTER)
-}
-
-
-class TKqpDbWatcherCallback : public NKikimr::NSysView::TDbWatcherCallback {
- TIntrusivePtr<TKqpCounters> Counters;
-
-public:
- explicit TKqpDbWatcherCallback(TIntrusivePtr<TKqpCounters> counters)
- : Counters(counters)
- {}
-
- void OnDatabaseRemoved(const TString& database, TPathId) override {
- Counters->RemoveDbCounters(database);
- }
-};
-
-
-void TKqpCounters::CreateTxKindCounters(TKqpTransactionInfo::EKind kind, const TString& name) {
- auto& txKindCounters = TxByKind[kind];
-
- txKindCounters.TotalDuration = KqpGroup->GetHistogram("Transactions/" + name + "/TotalDurationMs",
- NMonitoring::ExponentialHistogram(20, 2, 1));
-
- txKindCounters.ServerDuration = KqpGroup->GetHistogram("Transactions/" + name + "/ServerDurationMs",
- NMonitoring::ExponentialHistogram(20, 2, 1));
-
- txKindCounters.ClientDuration = KqpGroup->GetHistogram("Transactions/" + name + "/ClientDurationMs",
- NMonitoring::ExponentialHistogram(20, 2, 1));
-
- txKindCounters.Queries = KqpGroup->GetHistogram("Transactions/" + name + "/Queries",
- NMonitoring::ExponentialHistogram(16, 2, 1));
-}
-
-void TKqpCounters::UpdateTxCounters(const TKqpTransactionInfo& txInfo,
- THashMap<TKqpTransactionInfo::EKind, TTxByKindCounters>& txCounters)
-{
- auto byKind = txCounters.FindPtr(txInfo.Kind);
- if (!byKind) {
- return;
- }
-
- byKind->TotalDuration->Collect(txInfo.TotalDuration.MilliSeconds());
- byKind->ServerDuration->Collect(txInfo.ServerDuration.MilliSeconds());
+
+ Init();
+}
+
+template <typename T>
+void SaveHistogram(T& histogram, int index, const NMon::THistogramCounterHelper& helper) {
+ auto* buckets = histogram[index].MutableBuckets();
+ auto count = helper.GetBucketCount();
+ buckets->Resize(count, 0);
+ for (size_t i = 0; i < count; ++i) {
+ (*buckets)[i] = helper.GetBucketValue(i);
+ }
+}
+
+template <typename T>
+void SaveHistogram(T& histogram, int index, const NMonitoring::THistogramPtr& hgram) {
+ auto* buckets = histogram[index].MutableBuckets();
+ auto snapshot = hgram->Snapshot();
+ auto count = snapshot->Count();
+ buckets->Resize(count, 0);
+ for (size_t i = 0; i < count; ++i) {
+ (*buckets)[i] = snapshot->Value(i);
+ }
+}
+
+void TKqpDbCounters::ToProto(NKikimr::NSysView::TDbServiceCounters& counters) {
+ auto* main = counters.Proto().MutableMain();
+ auto* simple = main->MutableSimple();
+ auto* cumulative = main->MutableCumulative();
+ auto* histogram = main->MutableHistogram();
+
+ simple->Resize(DB_KQP_SIMPLE_COUNTER_SIZE, 0);
+ cumulative->Resize(DB_KQP_CUMULATIVE_COUNTER_SIZE, 0);
+ if (main->HistogramSize() < DB_KQP_HISTOGRAM_COUNTER_SIZE) {
+ auto missing = DB_KQP_HISTOGRAM_COUNTER_SIZE - main->HistogramSize();
+ for (; missing > 0; --missing) {
+ main->AddHistogram();
+ }
+ }
+
+#define SAVE_SIMPLE_COUNTER(INDEX, TARGET) { (*simple)[INDEX] = (TARGET)->Val(); }
+#define SAVE_CUMULATIVE_COUNTER(INDEX, TARGET) { (*cumulative)[INDEX] = (TARGET)->Val(); }
+#define SAVE_HISTOGRAM_COUNTER(INDEX, TARGET) { SaveHistogram(*histogram, INDEX, (TARGET)); }
+
+ DB_KQP_SIMPLE_COUNTERS_MAP(SAVE_SIMPLE_COUNTER)
+ DB_KQP_CUMULATIVE_COUNTERS_MAP(SAVE_CUMULATIVE_COUNTER)
+ DB_KQP_HISTOGRAM_COUNTERS_MAP(SAVE_HISTOGRAM_COUNTER)
+}
+
+template <typename T>
+void LoadHistogram(T& histogram, int index, NMon::THistogramCounterHelper& helper) {
+ auto* buckets = histogram[index].MutableBuckets();
+ auto count = helper.GetBucketCount();
+ buckets->Resize(count, 0);
+ for (size_t i = 0; i < count; ++i) {
+ helper.SetBucketValue(i, (*buckets)[i]);
+ }
+}
+
+template <typename T>
+void LoadHistogram(T& histogram, int index, NMonitoring::THistogramPtr& hgram) {
+ auto* buckets = histogram[index].MutableBuckets();
+ auto snapshot = hgram->Snapshot();
+ auto count = snapshot->Count();
+ buckets->Resize(count, 0);
+ hgram->Reset();
+ for (ui32 i = 0; i < count; ++i) {
+ hgram->Collect(snapshot->UpperBound(i), (*buckets)[i]);
+ }
+}
+
+void TKqpDbCounters::FromProto(NKikimr::NSysView::TDbServiceCounters& counters) {
+ auto* main = counters.Proto().MutableMain();
+ auto* simple = main->MutableSimple();
+ auto* cumulative = main->MutableCumulative();
+ auto* histogram = main->MutableHistogram();
+
+ simple->Resize(DB_KQP_SIMPLE_COUNTER_SIZE, 0);
+ cumulative->Resize(DB_KQP_CUMULATIVE_COUNTER_SIZE, 0);
+ if (main->HistogramSize() < DB_KQP_HISTOGRAM_COUNTER_SIZE) {
+ auto missing = DB_KQP_HISTOGRAM_COUNTER_SIZE - main->HistogramSize();
+ for (; missing > 0; --missing) {
+ main->AddHistogram();
+ }
+ }
+
+#define LOAD_SIMPLE_COUNTER(INDEX, TARGET) { (TARGET)->Set((*simple)[INDEX]); }
+#define LOAD_CUMULATIVE_COUNTER(INDEX, TARGET) { (TARGET)->Set((*cumulative)[INDEX]); }
+#define LOAD_HISTOGRAM_COUNTER(INDEX, TARGET) { LoadHistogram(*histogram, INDEX, (TARGET)); }
+
+ DB_KQP_SIMPLE_COUNTERS_MAP(LOAD_SIMPLE_COUNTER)
+ DB_KQP_CUMULATIVE_COUNTERS_MAP(LOAD_CUMULATIVE_COUNTER)
+ DB_KQP_HISTOGRAM_COUNTERS_MAP(LOAD_HISTOGRAM_COUNTER)
+}
+
+
+class TKqpDbWatcherCallback : public NKikimr::NSysView::TDbWatcherCallback {
+ TIntrusivePtr<TKqpCounters> Counters;
+
+public:
+ explicit TKqpDbWatcherCallback(TIntrusivePtr<TKqpCounters> counters)
+ : Counters(counters)
+ {}
+
+ void OnDatabaseRemoved(const TString& database, TPathId) override {
+ Counters->RemoveDbCounters(database);
+ }
+};
+
+
+void TKqpCounters::CreateTxKindCounters(TKqpTransactionInfo::EKind kind, const TString& name) {
+ auto& txKindCounters = TxByKind[kind];
+
+ txKindCounters.TotalDuration = KqpGroup->GetHistogram("Transactions/" + name + "/TotalDurationMs",
+ NMonitoring::ExponentialHistogram(20, 2, 1));
+
+ txKindCounters.ServerDuration = KqpGroup->GetHistogram("Transactions/" + name + "/ServerDurationMs",
+ NMonitoring::ExponentialHistogram(20, 2, 1));
+
+ txKindCounters.ClientDuration = KqpGroup->GetHistogram("Transactions/" + name + "/ClientDurationMs",
+ NMonitoring::ExponentialHistogram(20, 2, 1));
+
+ txKindCounters.Queries = KqpGroup->GetHistogram("Transactions/" + name + "/Queries",
+ NMonitoring::ExponentialHistogram(16, 2, 1));
+}
+
+void TKqpCounters::UpdateTxCounters(const TKqpTransactionInfo& txInfo,
+ THashMap<TKqpTransactionInfo::EKind, TTxByKindCounters>& txCounters)
+{
+ auto byKind = txCounters.FindPtr(txInfo.Kind);
+ if (!byKind) {
+ return;
+ }
+
+ byKind->TotalDuration->Collect(txInfo.TotalDuration.MilliSeconds());
+ byKind->ServerDuration->Collect(txInfo.ServerDuration.MilliSeconds());
byKind->ClientDuration->Collect((txInfo.TotalDuration - txInfo.ServerDuration).MilliSeconds());
- byKind->Queries->Collect(txInfo.QueriesCount);
-}
-
-TKqpCounters::TKqpCounters(const NMonitoring::TDynamicCounterPtr& counters, const TActorContext* ctx)
- : AllocCounters(counters, "kqp")
-{
- Counters = counters;
- KqpGroup = GetServiceCounters(counters, "kqp");
- YdbGroup = GetServiceCounters(counters, "ydb");
+ byKind->Queries->Collect(txInfo.QueriesCount);
+}
+
+TKqpCounters::TKqpCounters(const NMonitoring::TDynamicCounterPtr& counters, const TActorContext* ctx)
+ : AllocCounters(counters, "kqp")
+{
+ Counters = counters;
+ KqpGroup = GetServiceCounters(counters, "kqp");
+ YdbGroup = GetServiceCounters(counters, "ydb");
QueryReplayGroup = KqpGroup->GetSubgroup("subsystem", "unified_agent_query_replay");
-
- Init();
-
- if (ctx) {
- ActorSystem = ctx->ActorSystem();
- if (AppData(ActorSystem)->FeatureFlags.GetEnableDbCounters()) {
- auto callback = MakeIntrusive<TKqpDbWatcherCallback>(this);
- DbWatcherActorId = ActorSystem->Register(NSysView::CreateDbWatcherActor(callback));
- }
- }
-
- /* Transactions */
- CreateTxKindCounters(TKqpTransactionInfo::EKind::Pure, "Pure");
- CreateTxKindCounters(TKqpTransactionInfo::EKind::ReadOnly, "ReadOnly");
- CreateTxKindCounters(TKqpTransactionInfo::EKind::WriteOnly, "WriteOnly");
- CreateTxKindCounters(TKqpTransactionInfo::EKind::ReadWrite, "ReadWrite");
-
- /* Compile service */
- CompileQueryCacheSize = YdbGroup->GetNamedCounter("name", "table.query.compilation.cached_query_count", false);
- CompileQueryCacheBytes = YdbGroup->GetNamedCounter("name", "table.query.compilation.cache_size_bytes", false);
- CompileQueryCacheEvicted = YdbGroup->GetNamedCounter("name", "table.query.compilation.cache_evictions", true);
-
- CompileQueueSize = KqpGroup->GetCounter("Compilation/QueueSize", false);
- ForceNewEngineCompileErrors = KqpGroup->GetCounter("Compilation/ForceNEErrors", true);
-
- /* Resource Manager */
- RmComputeActors = KqpGroup->GetCounter("RM/ComputeActors", false);
- RmMemory = KqpGroup->GetCounter("RM/Memory", false);
+
+ Init();
+
+ if (ctx) {
+ ActorSystem = ctx->ActorSystem();
+ if (AppData(ActorSystem)->FeatureFlags.GetEnableDbCounters()) {
+ auto callback = MakeIntrusive<TKqpDbWatcherCallback>(this);
+ DbWatcherActorId = ActorSystem->Register(NSysView::CreateDbWatcherActor(callback));
+ }
+ }
+
+ /* Transactions */
+ CreateTxKindCounters(TKqpTransactionInfo::EKind::Pure, "Pure");
+ CreateTxKindCounters(TKqpTransactionInfo::EKind::ReadOnly, "ReadOnly");
+ CreateTxKindCounters(TKqpTransactionInfo::EKind::WriteOnly, "WriteOnly");
+ CreateTxKindCounters(TKqpTransactionInfo::EKind::ReadWrite, "ReadWrite");
+
+ /* Compile service */
+ CompileQueryCacheSize = YdbGroup->GetNamedCounter("name", "table.query.compilation.cached_query_count", false);
+ CompileQueryCacheBytes = YdbGroup->GetNamedCounter("name", "table.query.compilation.cache_size_bytes", false);
+ CompileQueryCacheEvicted = YdbGroup->GetNamedCounter("name", "table.query.compilation.cache_evictions", true);
+
+ CompileQueueSize = KqpGroup->GetCounter("Compilation/QueueSize", false);
+ ForceNewEngineCompileErrors = KqpGroup->GetCounter("Compilation/ForceNEErrors", true);
+
+ /* Resource Manager */
+ RmComputeActors = KqpGroup->GetCounter("RM/ComputeActors", false);
+ RmMemory = KqpGroup->GetCounter("RM/Memory", false);
RmExternalMemory = KqpGroup->GetCounter("RM/ExternalMemory", false);
- RmNotEnoughMemory = KqpGroup->GetCounter("RM/NotEnoughMemory", true);
- RmNotEnoughComputeActors = KqpGroup->GetCounter("RM/NotEnoughComputeActors", true);
- RmExtraMemAllocs = KqpGroup->GetCounter("RM/ExtraMemAllocs", true);
- RmInternalError = KqpGroup->GetCounter("RM/InternalError", true);
-
- /* Spilling */
- SpillingWriteBlobs = KqpGroup->GetCounter("Spilling/WriteBlobs", true);
- SpillingReadBlobs = KqpGroup->GetCounter("Spilling/ReadBlobs", true);
- SpillingStoredBlobs = KqpGroup->GetCounter("Spilling/StoredBlobs", false);
- SpillingTotalSpaceUsed = KqpGroup->GetCounter("Spilling/TotalSpaceUsed", false);
- SpillingTooBigFileErrors = KqpGroup->GetCounter("Spilling/TooBigFileErrors", true);
- SpillingNoSpaceErrors = KqpGroup->GetCounter("Spilling/NoSpaceErrors", true);
- SpillingIoErrors = KqpGroup->GetCounter("Spilling/IoErrors", true);
+ RmNotEnoughMemory = KqpGroup->GetCounter("RM/NotEnoughMemory", true);
+ RmNotEnoughComputeActors = KqpGroup->GetCounter("RM/NotEnoughComputeActors", true);
+ RmExtraMemAllocs = KqpGroup->GetCounter("RM/ExtraMemAllocs", true);
+ RmInternalError = KqpGroup->GetCounter("RM/InternalError", true);
+
+ /* Spilling */
+ SpillingWriteBlobs = KqpGroup->GetCounter("Spilling/WriteBlobs", true);
+ SpillingReadBlobs = KqpGroup->GetCounter("Spilling/ReadBlobs", true);
+ SpillingStoredBlobs = KqpGroup->GetCounter("Spilling/StoredBlobs", false);
+ SpillingTotalSpaceUsed = KqpGroup->GetCounter("Spilling/TotalSpaceUsed", false);
+ SpillingTooBigFileErrors = KqpGroup->GetCounter("Spilling/TooBigFileErrors", true);
+ SpillingNoSpaceErrors = KqpGroup->GetCounter("Spilling/NoSpaceErrors", true);
+ SpillingIoErrors = KqpGroup->GetCounter("Spilling/IoErrors", true);
/* Scan queries */
ScanQueryShardDisconnect = KqpGroup->GetCounter("ScanQuery/ShardDisconnect", true);
ScanQueryShardResolve = KqpGroup->GetCounter("ScanQuery/ShardResolve", true);
ScanQueryRateLimitLatency = KqpGroup->GetHistogram(
"ScanQuery/RateLimitLatency", NMonitoring::ExponentialHistogram(20, 2, 1));
-
-
- // NewEngine
- NewEngineForcedQueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE] = KqpGroup->GetHistogram(
- "Query/ExecuteLatency_NEForced", NMonitoring::ExponentialHistogram(20, 2, 1));
- NewEngineForcedQueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED] = KqpGroup->GetHistogram(
- "Query/ExecPreparedLatency_NEForced", NMonitoring::ExponentialHistogram(20, 2, 1));
- NewEngineCompatibleQueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE] = KqpGroup->GetHistogram(
- "Query/ExecuteLatency_NECompatible", NMonitoring::ExponentialHistogram(20, 2, 1));
- NewEngineCompatibleQueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED] = KqpGroup->GetHistogram(
- "Query/ExecPreparedLatency_NECompatible", NMonitoring::ExponentialHistogram(20, 2, 1));
- NewEngineForcedComputeCpuTime = KqpGroup->GetCounter("Query/ComputeCpuTime_NEForced", true);
- NewEngineCompatibleComputeCpuTime = KqpGroup->GetCounter("Query/ComputeCpuTime_NECompatible", true);
- NewEngineForcedQueryCount = KqpGroup->GetCounter("Query/Count_NEForced", true);
- NewEngineCompatibleQueryCount = KqpGroup->GetCounter("Query/Count_NECompatible", true);
+
+
+ // NewEngine
+ NewEngineForcedQueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE] = KqpGroup->GetHistogram(
+ "Query/ExecuteLatency_NEForced", NMonitoring::ExponentialHistogram(20, 2, 1));
+ NewEngineForcedQueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED] = KqpGroup->GetHistogram(
+ "Query/ExecPreparedLatency_NEForced", NMonitoring::ExponentialHistogram(20, 2, 1));
+ NewEngineCompatibleQueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE] = KqpGroup->GetHistogram(
+ "Query/ExecuteLatency_NECompatible", NMonitoring::ExponentialHistogram(20, 2, 1));
+ NewEngineCompatibleQueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED] = KqpGroup->GetHistogram(
+ "Query/ExecPreparedLatency_NECompatible", NMonitoring::ExponentialHistogram(20, 2, 1));
+ NewEngineForcedComputeCpuTime = KqpGroup->GetCounter("Query/ComputeCpuTime_NEForced", true);
+ NewEngineCompatibleComputeCpuTime = KqpGroup->GetCounter("Query/ComputeCpuTime_NECompatible", true);
+ NewEngineForcedQueryCount = KqpGroup->GetCounter("Query/Count_NEForced", true);
+ NewEngineCompatibleQueryCount = KqpGroup->GetCounter("Query/Count_NECompatible", true);
LiteralTxTotalTimeHistogram = KqpGroup->GetHistogram(
"NE/LiteralTxTotalTimeMs", NMonitoring::ExponentialHistogram(10, 2, 1));
@@ -762,8 +762,8 @@ TKqpCounters::TKqpCounters(const NMonitoring::TDynamicCounterPtr& counters, cons
"NE/DataTxTotalTimeMs", NMonitoring::ExponentialHistogram(20, 2, 1));
ScanTxTotalTimeHistogram = KqpGroup->GetHistogram(
"NE/ScanTxTotalTimeMs", NMonitoring::ExponentialHistogram(20, 2, 1));
-}
-
+}
+
NMonitoring::TDynamicCounterPtr TKqpCounters::GetQueryReplayCounters() const {
return QueryReplayGroup;
}
@@ -784,355 +784,355 @@ void TKqpCounters::ReportProxyForwardedRequest(TKqpDbCountersPtr dbCounters) {
}
}
-void TKqpCounters::ReportSessionShutdownRequest(TKqpDbCountersPtr dbCounters) {
+void TKqpCounters::ReportSessionShutdownRequest(TKqpDbCountersPtr dbCounters) {
TKqpCountersBase::ReportSessionShutdownRequest();
- if (dbCounters) {
- dbCounters->ReportSessionShutdownRequest();
- }
-}
-
-void TKqpCounters::ReportCreateSession(TKqpDbCountersPtr dbCounters, ui64 requestSize) {
- TKqpCountersBase::ReportCreateSession(requestSize);
- if (dbCounters) {
- dbCounters->ReportCreateSession(requestSize);
- }
-}
-
-void TKqpCounters::ReportPingSession(TKqpDbCountersPtr dbCounters, ui64 requestSize) {
- TKqpCountersBase::ReportPingSession(requestSize);
- if (dbCounters) {
- dbCounters->ReportPingSession(requestSize);
- }
-}
-
-void TKqpCounters::ReportCloseSession(TKqpDbCountersPtr dbCounters, ui64 requestSize) {
- TKqpCountersBase::ReportCloseSession(requestSize);
- if (dbCounters) {
- dbCounters->ReportCloseSession(requestSize);
- }
-}
-
-void TKqpCounters::ReportQueryRequest(TKqpDbCountersPtr dbCounters, const NKikimrKqp::TQueryRequest& request) {
- TKqpCountersBase::ReportQueryRequest(request);
- if (dbCounters) {
- dbCounters->ReportQueryRequest(request);
- }
-}
-
-void TKqpCounters::ReportQueryWithRangeScan(TKqpDbCountersPtr dbCounters) {
+ if (dbCounters) {
+ dbCounters->ReportSessionShutdownRequest();
+ }
+}
+
+void TKqpCounters::ReportCreateSession(TKqpDbCountersPtr dbCounters, ui64 requestSize) {
+ TKqpCountersBase::ReportCreateSession(requestSize);
+ if (dbCounters) {
+ dbCounters->ReportCreateSession(requestSize);
+ }
+}
+
+void TKqpCounters::ReportPingSession(TKqpDbCountersPtr dbCounters, ui64 requestSize) {
+ TKqpCountersBase::ReportPingSession(requestSize);
+ if (dbCounters) {
+ dbCounters->ReportPingSession(requestSize);
+ }
+}
+
+void TKqpCounters::ReportCloseSession(TKqpDbCountersPtr dbCounters, ui64 requestSize) {
+ TKqpCountersBase::ReportCloseSession(requestSize);
+ if (dbCounters) {
+ dbCounters->ReportCloseSession(requestSize);
+ }
+}
+
+void TKqpCounters::ReportQueryRequest(TKqpDbCountersPtr dbCounters, const NKikimrKqp::TQueryRequest& request) {
+ TKqpCountersBase::ReportQueryRequest(request);
+ if (dbCounters) {
+ dbCounters->ReportQueryRequest(request);
+ }
+}
+
+void TKqpCounters::ReportQueryWithRangeScan(TKqpDbCountersPtr dbCounters) {
TKqpCountersBase::ReportQueryWithRangeScan();
- if (dbCounters) {
- dbCounters->ReportQueryWithRangeScan();
+ if (dbCounters) {
+ dbCounters->ReportQueryWithRangeScan();
}
}
-void TKqpCounters::ReportQueryWithFullScan(TKqpDbCountersPtr dbCounters) {
+void TKqpCounters::ReportQueryWithFullScan(TKqpDbCountersPtr dbCounters) {
TKqpCountersBase::ReportQueryWithFullScan();
- if (dbCounters) {
- dbCounters->ReportQueryWithFullScan();
+ if (dbCounters) {
+ dbCounters->ReportQueryWithFullScan();
}
}
-void TKqpCounters::ReportQueryAffectedShards(TKqpDbCountersPtr dbCounters, ui64 shardsCount) {
+void TKqpCounters::ReportQueryAffectedShards(TKqpDbCountersPtr dbCounters, ui64 shardsCount) {
TKqpCountersBase::ReportQueryAffectedShards(shardsCount);
- if (dbCounters) {
- dbCounters->ReportQueryAffectedShards(shardsCount);
+ if (dbCounters) {
+ dbCounters->ReportQueryAffectedShards(shardsCount);
}
}
-void TKqpCounters::ReportQueryReadSets(TKqpDbCountersPtr dbCounters, ui64 readSetsCount) {
+void TKqpCounters::ReportQueryReadSets(TKqpDbCountersPtr dbCounters, ui64 readSetsCount) {
TKqpCountersBase::ReportQueryReadSets(readSetsCount);
- if (dbCounters) {
- dbCounters->ReportQueryReadSets(readSetsCount);
+ if (dbCounters) {
+ dbCounters->ReportQueryReadSets(readSetsCount);
}
}
-void TKqpCounters::ReportQueryReadBytes(TKqpDbCountersPtr dbCounters, ui64 bytesCount) {
+void TKqpCounters::ReportQueryReadBytes(TKqpDbCountersPtr dbCounters, ui64 bytesCount) {
TKqpCountersBase::ReportQueryReadBytes(bytesCount);
- if (dbCounters) {
- dbCounters->ReportQueryReadBytes(bytesCount);
+ if (dbCounters) {
+ dbCounters->ReportQueryReadBytes(bytesCount);
}
}
-void TKqpCounters::ReportQueryReadRows(TKqpDbCountersPtr dbCounters, ui64 rowsCount) {
+void TKqpCounters::ReportQueryReadRows(TKqpDbCountersPtr dbCounters, ui64 rowsCount) {
TKqpCountersBase::ReportQueryReadRows(rowsCount);
- if (dbCounters) {
- dbCounters->ReportQueryReadRows(rowsCount);
+ if (dbCounters) {
+ dbCounters->ReportQueryReadRows(rowsCount);
}
}
-void TKqpCounters::ReportQueryMaxShardReplySize(TKqpDbCountersPtr dbCounters, ui64 replySize) {
+void TKqpCounters::ReportQueryMaxShardReplySize(TKqpDbCountersPtr dbCounters, ui64 replySize) {
TKqpCountersBase::ReportQueryMaxShardReplySize(replySize);
- if (dbCounters) {
- dbCounters->ReportQueryMaxShardReplySize(replySize);
+ if (dbCounters) {
+ dbCounters->ReportQueryMaxShardReplySize(replySize);
}
}
-void TKqpCounters::ReportQueryMaxShardProgramSize(TKqpDbCountersPtr dbCounters, ui64 programSize) {
+void TKqpCounters::ReportQueryMaxShardProgramSize(TKqpDbCountersPtr dbCounters, ui64 programSize) {
TKqpCountersBase::ReportQueryMaxShardProgramSize(programSize);
- if (dbCounters) {
- dbCounters->ReportQueryMaxShardProgramSize(programSize);
+ if (dbCounters) {
+ dbCounters->ReportQueryMaxShardProgramSize(programSize);
}
}
-void TKqpCounters::ReportResponseStatus(TKqpDbCountersPtr dbCounters, ui64 responseSize,
+void TKqpCounters::ReportResponseStatus(TKqpDbCountersPtr dbCounters, ui64 responseSize,
Ydb::StatusIds::StatusCode ydbStatus)
-{
+{
TKqpCountersBase::ReportResponseStatus(responseSize, ydbStatus);
- if (dbCounters) {
+ if (dbCounters) {
dbCounters->ReportResponseStatus(responseSize, ydbStatus);
- }
-}
-
-void TKqpCounters::ReportResultsBytes(TKqpDbCountersPtr dbCounters, ui64 resultsSize) {
- TKqpCountersBase::ReportResultsBytes(resultsSize);
- if (dbCounters) {
- dbCounters->ReportResultsBytes(resultsSize);
- }
-}
-
-void TKqpCounters::ReportIssues(TKqpDbCountersPtr dbCounters, const Ydb::Issue::IssueMessage& issue) {
- TKqpCountersBase::ReportIssues(issue);
- if (dbCounters) {
- dbCounters->ReportIssues(issue);
- }
-}
-
-void TKqpCounters::ReportQueryLatency(TKqpDbCountersPtr dbCounters,
- NKikimrKqp::EQueryAction action, const TDuration& duration)
-{
- TKqpCountersBase::ReportQueryLatency(action, duration);
- if (dbCounters) {
- dbCounters->ReportQueryLatency(action, duration);
- }
-}
-
-void TKqpCounters::ReportTransaction(TKqpDbCountersPtr dbCounters, const TKqpTransactionInfo& txInfo) {
- TKqpCountersBase::ReportTransaction(txInfo);
- if (dbCounters) {
- dbCounters->ReportTransaction(txInfo);
- }
- if (txInfo.Status == TKqpTransactionInfo::EStatus::Committed) {
- UpdateTxCounters(txInfo, TxByKind);
- }
-}
-
-void TKqpCounters::ReportSqlVersion(TKqpDbCountersPtr dbCounters, ui16 sqlVersion) {
- TKqpCountersBase::ReportSqlVersion(sqlVersion);
- if (dbCounters) {
- dbCounters->ReportSqlVersion(sqlVersion);
- }
-}
-
-void TKqpCounters::ReportWorkerCreated(TKqpDbCountersPtr dbCounters) {
- TKqpCountersBase::ReportWorkerCreated();
- if (dbCounters) {
- dbCounters->ReportWorkerCreated();
- }
-}
-
-void TKqpCounters::ReportWorkerFinished(TKqpDbCountersPtr dbCounters, TDuration lifeSpan) {
- TKqpCountersBase::ReportWorkerFinished(lifeSpan);
- if (dbCounters) {
- dbCounters->ReportWorkerFinished(lifeSpan);
- }
-}
-
-void TKqpCounters::ReportWorkerCleanupLatency(TKqpDbCountersPtr dbCounters, TDuration cleanupTime) {
- TKqpCountersBase::ReportWorkerCleanupLatency(cleanupTime);
- if (dbCounters) {
- dbCounters->ReportWorkerCleanupLatency(cleanupTime);
- }
-}
-
-void TKqpCounters::ReportWorkerClosedIdle(TKqpDbCountersPtr dbCounters) {
- TKqpCountersBase::ReportWorkerClosedIdle();
- if (dbCounters) {
- dbCounters->ReportWorkerClosedIdle();
- }
-}
-
-void TKqpCounters::ReportWorkerClosedError(TKqpDbCountersPtr dbCounters) {
- TKqpCountersBase::ReportWorkerClosedError();
- if (dbCounters) {
- dbCounters->ReportWorkerClosedError();
- }
-}
-
-void TKqpCounters::ReportWorkerClosedRequest(TKqpDbCountersPtr dbCounters) {
- TKqpCountersBase::ReportWorkerClosedRequest();
- if (dbCounters) {
- dbCounters->ReportWorkerClosedRequest();
- }
-}
-
-void TKqpCounters::ReportQueriesPerWorker(TKqpDbCountersPtr dbCounters, ui32 queryId) {
- TKqpCountersBase::ReportQueriesPerWorker(queryId);
- if (dbCounters) {
- dbCounters->ReportQueriesPerWorker(queryId);
- }
-}
-
-void TKqpCounters::ReportBeginTransaction(TKqpDbCountersPtr dbCounters,
- ui32 evictedTx, ui32 currentActiveTx, ui32 currentAbortedTx)
-{
- TKqpCountersBase::ReportBeginTransaction(evictedTx, currentActiveTx, currentAbortedTx);
- if (dbCounters) {
- dbCounters->ReportBeginTransaction(evictedTx, currentActiveTx, currentAbortedTx);
- }
-}
-
-void TKqpCounters::ReportTxCreated(TKqpDbCountersPtr dbCounters) {
- TKqpCountersBase::ReportTxCreated();
- if (dbCounters) {
- dbCounters->ReportTxCreated();
- }
-}
-
-void TKqpCounters::ReportTxAborted(TKqpDbCountersPtr dbCounters, ui32 abortedCount) {
- TKqpCountersBase::ReportTxAborted(abortedCount);
- if (dbCounters) {
- dbCounters->ReportTxAborted(abortedCount);
- }
-}
-
-void TKqpCounters::ReportQueryCacheHit(TKqpDbCountersPtr dbCounters, bool hit) {
- TKqpCountersBase::ReportQueryCacheHit(hit);
- if (dbCounters) {
- dbCounters->ReportQueryCacheHit(hit);
- }
-}
-
-void TKqpCounters::ReportCompileStart(TKqpDbCountersPtr dbCounters) {
- TKqpCountersBase::ReportCompileStart();
- if (dbCounters) {
- dbCounters->ReportCompileStart();
- }
-}
-
-void TKqpCounters::ReportCompileFinish(TKqpDbCountersPtr dbCounters) {
- TKqpCountersBase::ReportCompileFinish();
- if (dbCounters) {
- dbCounters->ReportCompileFinish();
- }
-}
-
-void TKqpCounters::ReportCompileError(TKqpDbCountersPtr dbCounters) {
- TKqpCountersBase::ReportCompileError();
- if (dbCounters) {
- dbCounters->ReportCompileError();
- }
-}
-
-void TKqpCounters::ReportCompileRequestCompile(TKqpDbCountersPtr dbCounters) {
- TKqpCountersBase::ReportCompileRequestCompile();
- if (dbCounters) {
- dbCounters->ReportCompileRequestCompile();
- }
-}
-
-void TKqpCounters::ReportCompileRequestGet(TKqpDbCountersPtr dbCounters) {
- TKqpCountersBase::ReportCompileRequestGet();
- if (dbCounters) {
- dbCounters->ReportCompileRequestGet();
- }
-}
-
-void TKqpCounters::ReportCompileRequestInvalidate(TKqpDbCountersPtr dbCounters) {
- TKqpCountersBase::ReportCompileRequestInvalidate();
- if (dbCounters) {
- dbCounters->ReportCompileRequestInvalidate();
- }
-}
-
-void TKqpCounters::ReportCompileRequestRejected(TKqpDbCountersPtr dbCounters) {
- TKqpCountersBase::ReportCompileRequestRejected();
- if (dbCounters) {
- dbCounters->ReportCompileRequestRejected();
- }
-}
-
-void TKqpCounters::ReportCompileRequestTimeout(TKqpDbCountersPtr dbCounters) {
- TKqpCountersBase::ReportCompileRequestTimeout();
- if (dbCounters) {
- dbCounters->ReportCompileRequestTimeout();
- }
-}
-
-void TKqpCounters::ReportCompileDurations(TKqpDbCountersPtr dbCounters, TDuration duration, TDuration cpuTime) {
- TKqpCountersBase::ReportCompileDurations(duration, cpuTime);
- if (dbCounters) {
- dbCounters->ReportCompileDurations(duration, cpuTime);
- }
-}
-
-void TKqpCounters::ReportRecompileRequestGet(TKqpDbCountersPtr dbCounters) {
+ }
+}
+
+void TKqpCounters::ReportResultsBytes(TKqpDbCountersPtr dbCounters, ui64 resultsSize) {
+ TKqpCountersBase::ReportResultsBytes(resultsSize);
+ if (dbCounters) {
+ dbCounters->ReportResultsBytes(resultsSize);
+ }
+}
+
+void TKqpCounters::ReportIssues(TKqpDbCountersPtr dbCounters, const Ydb::Issue::IssueMessage& issue) {
+ TKqpCountersBase::ReportIssues(issue);
+ if (dbCounters) {
+ dbCounters->ReportIssues(issue);
+ }
+}
+
+void TKqpCounters::ReportQueryLatency(TKqpDbCountersPtr dbCounters,
+ NKikimrKqp::EQueryAction action, const TDuration& duration)
+{
+ TKqpCountersBase::ReportQueryLatency(action, duration);
+ if (dbCounters) {
+ dbCounters->ReportQueryLatency(action, duration);
+ }
+}
+
+void TKqpCounters::ReportTransaction(TKqpDbCountersPtr dbCounters, const TKqpTransactionInfo& txInfo) {
+ TKqpCountersBase::ReportTransaction(txInfo);
+ if (dbCounters) {
+ dbCounters->ReportTransaction(txInfo);
+ }
+ if (txInfo.Status == TKqpTransactionInfo::EStatus::Committed) {
+ UpdateTxCounters(txInfo, TxByKind);
+ }
+}
+
+void TKqpCounters::ReportSqlVersion(TKqpDbCountersPtr dbCounters, ui16 sqlVersion) {
+ TKqpCountersBase::ReportSqlVersion(sqlVersion);
+ if (dbCounters) {
+ dbCounters->ReportSqlVersion(sqlVersion);
+ }
+}
+
+void TKqpCounters::ReportWorkerCreated(TKqpDbCountersPtr dbCounters) {
+ TKqpCountersBase::ReportWorkerCreated();
+ if (dbCounters) {
+ dbCounters->ReportWorkerCreated();
+ }
+}
+
+void TKqpCounters::ReportWorkerFinished(TKqpDbCountersPtr dbCounters, TDuration lifeSpan) {
+ TKqpCountersBase::ReportWorkerFinished(lifeSpan);
+ if (dbCounters) {
+ dbCounters->ReportWorkerFinished(lifeSpan);
+ }
+}
+
+void TKqpCounters::ReportWorkerCleanupLatency(TKqpDbCountersPtr dbCounters, TDuration cleanupTime) {
+ TKqpCountersBase::ReportWorkerCleanupLatency(cleanupTime);
+ if (dbCounters) {
+ dbCounters->ReportWorkerCleanupLatency(cleanupTime);
+ }
+}
+
+void TKqpCounters::ReportWorkerClosedIdle(TKqpDbCountersPtr dbCounters) {
+ TKqpCountersBase::ReportWorkerClosedIdle();
+ if (dbCounters) {
+ dbCounters->ReportWorkerClosedIdle();
+ }
+}
+
+void TKqpCounters::ReportWorkerClosedError(TKqpDbCountersPtr dbCounters) {
+ TKqpCountersBase::ReportWorkerClosedError();
+ if (dbCounters) {
+ dbCounters->ReportWorkerClosedError();
+ }
+}
+
+void TKqpCounters::ReportWorkerClosedRequest(TKqpDbCountersPtr dbCounters) {
+ TKqpCountersBase::ReportWorkerClosedRequest();
+ if (dbCounters) {
+ dbCounters->ReportWorkerClosedRequest();
+ }
+}
+
+void TKqpCounters::ReportQueriesPerWorker(TKqpDbCountersPtr dbCounters, ui32 queryId) {
+ TKqpCountersBase::ReportQueriesPerWorker(queryId);
+ if (dbCounters) {
+ dbCounters->ReportQueriesPerWorker(queryId);
+ }
+}
+
+void TKqpCounters::ReportBeginTransaction(TKqpDbCountersPtr dbCounters,
+ ui32 evictedTx, ui32 currentActiveTx, ui32 currentAbortedTx)
+{
+ TKqpCountersBase::ReportBeginTransaction(evictedTx, currentActiveTx, currentAbortedTx);
+ if (dbCounters) {
+ dbCounters->ReportBeginTransaction(evictedTx, currentActiveTx, currentAbortedTx);
+ }
+}
+
+void TKqpCounters::ReportTxCreated(TKqpDbCountersPtr dbCounters) {
+ TKqpCountersBase::ReportTxCreated();
+ if (dbCounters) {
+ dbCounters->ReportTxCreated();
+ }
+}
+
+void TKqpCounters::ReportTxAborted(TKqpDbCountersPtr dbCounters, ui32 abortedCount) {
+ TKqpCountersBase::ReportTxAborted(abortedCount);
+ if (dbCounters) {
+ dbCounters->ReportTxAborted(abortedCount);
+ }
+}
+
+void TKqpCounters::ReportQueryCacheHit(TKqpDbCountersPtr dbCounters, bool hit) {
+ TKqpCountersBase::ReportQueryCacheHit(hit);
+ if (dbCounters) {
+ dbCounters->ReportQueryCacheHit(hit);
+ }
+}
+
+void TKqpCounters::ReportCompileStart(TKqpDbCountersPtr dbCounters) {
+ TKqpCountersBase::ReportCompileStart();
+ if (dbCounters) {
+ dbCounters->ReportCompileStart();
+ }
+}
+
+void TKqpCounters::ReportCompileFinish(TKqpDbCountersPtr dbCounters) {
+ TKqpCountersBase::ReportCompileFinish();
+ if (dbCounters) {
+ dbCounters->ReportCompileFinish();
+ }
+}
+
+void TKqpCounters::ReportCompileError(TKqpDbCountersPtr dbCounters) {
+ TKqpCountersBase::ReportCompileError();
+ if (dbCounters) {
+ dbCounters->ReportCompileError();
+ }
+}
+
+void TKqpCounters::ReportCompileRequestCompile(TKqpDbCountersPtr dbCounters) {
+ TKqpCountersBase::ReportCompileRequestCompile();
+ if (dbCounters) {
+ dbCounters->ReportCompileRequestCompile();
+ }
+}
+
+void TKqpCounters::ReportCompileRequestGet(TKqpDbCountersPtr dbCounters) {
+ TKqpCountersBase::ReportCompileRequestGet();
+ if (dbCounters) {
+ dbCounters->ReportCompileRequestGet();
+ }
+}
+
+void TKqpCounters::ReportCompileRequestInvalidate(TKqpDbCountersPtr dbCounters) {
+ TKqpCountersBase::ReportCompileRequestInvalidate();
+ if (dbCounters) {
+ dbCounters->ReportCompileRequestInvalidate();
+ }
+}
+
+void TKqpCounters::ReportCompileRequestRejected(TKqpDbCountersPtr dbCounters) {
+ TKqpCountersBase::ReportCompileRequestRejected();
+ if (dbCounters) {
+ dbCounters->ReportCompileRequestRejected();
+ }
+}
+
+void TKqpCounters::ReportCompileRequestTimeout(TKqpDbCountersPtr dbCounters) {
+ TKqpCountersBase::ReportCompileRequestTimeout();
+ if (dbCounters) {
+ dbCounters->ReportCompileRequestTimeout();
+ }
+}
+
+void TKqpCounters::ReportCompileDurations(TKqpDbCountersPtr dbCounters, TDuration duration, TDuration cpuTime) {
+ TKqpCountersBase::ReportCompileDurations(duration, cpuTime);
+ if (dbCounters) {
+ dbCounters->ReportCompileDurations(duration, cpuTime);
+ }
+}
+
+void TKqpCounters::ReportRecompileRequestGet(TKqpDbCountersPtr dbCounters) {
TKqpCountersBase::ReportRecompileRequestGet();
- if (dbCounters) {
- dbCounters->ReportRecompileRequestGet();
- }
-}
-
-void TKqpCounters::ReportNewEngineForcedQueryStats(NKikimrKqp::EQueryAction action, TDuration duration,
- ui64 computeCpuTime)
-{
- NewEngineForcedComputeCpuTime->Add(computeCpuTime);
- NewEngineForcedQueryCount->Inc();
-
- if (auto counter = NewEngineForcedQueryLatencies.FindPtr(action)) {
- (*counter)->Collect(duration.MilliSeconds());
- }
-}
-
-void TKqpCounters::ReportNewEngineCompatibleQueryStats(NKikimrKqp::EQueryAction action, TDuration duration,
- ui64 computeCpuTime)
-{
- NewEngineCompatibleComputeCpuTime->Add(computeCpuTime);
- NewEngineCompatibleQueryCount->Inc();
-
- if (auto counter = NewEngineCompatibleQueryLatencies.FindPtr(action)) {
- (*counter)->Collect(duration.MilliSeconds());
- }
-}
-
+ if (dbCounters) {
+ dbCounters->ReportRecompileRequestGet();
+ }
+}
+
+void TKqpCounters::ReportNewEngineForcedQueryStats(NKikimrKqp::EQueryAction action, TDuration duration,
+ ui64 computeCpuTime)
+{
+ NewEngineForcedComputeCpuTime->Add(computeCpuTime);
+ NewEngineForcedQueryCount->Inc();
+
+ if (auto counter = NewEngineForcedQueryLatencies.FindPtr(action)) {
+ (*counter)->Collect(duration.MilliSeconds());
+ }
+}
+
+void TKqpCounters::ReportNewEngineCompatibleQueryStats(NKikimrKqp::EQueryAction action, TDuration duration,
+ ui64 computeCpuTime)
+{
+ NewEngineCompatibleComputeCpuTime->Add(computeCpuTime);
+ NewEngineCompatibleQueryCount->Inc();
+
+ if (auto counter = NewEngineCompatibleQueryLatencies.FindPtr(action)) {
+ (*counter)->Collect(duration.MilliSeconds());
+ }
+}
+
const NMonitoring::TDynamicCounters::TCounterPtr TKqpCounters::RecompileRequestGet() const {
return TKqpCountersBase::CompileRequestsRecompile;
}
-NMonitoring::TDynamicCounters::TCounterPtr TKqpCounters::GetQueryTypeCounter(
- NKikimrKqp::EQueryType queryType)
-{
- return QueryTypes[queryType];
-}
-
-TKqpDbCountersPtr TKqpCounters::GetDbCounters(const TString& database) {
- if (!ActorSystem || !AppData(ActorSystem)->FeatureFlags.GetEnableDbCounters() || database.empty()) {
- return {};
- }
-
- TKqpDbCountersPtr dbCounters;
- if (DbCounters.Get(database, dbCounters)) {
- return dbCounters;
- }
-
- return DbCounters.InsertIfAbsentWithInit(database, [&database, this] {
- auto counters = MakeIntrusive<TKqpDbCounters>();
-
- auto evRegister = MakeHolder<NSysView::TEvSysView::TEvRegisterDbCounters>(
- NKikimrSysView::KQP, database, counters);
- ActorSystem->Send(NSysView::MakeSysViewServiceID(ActorSystem->NodeId), evRegister.Release());
-
- if (DbWatcherActorId) {
- auto evWatch = MakeHolder<NSysView::TEvSysView::TEvWatchDatabase>(database);
- ActorSystem->Send(DbWatcherActorId, evWatch.Release());
- }
- return counters;
- });
-}
-
-void TKqpCounters::RemoveDbCounters(const TString& database) {
- DbCounters.Erase(database);
-}
-
+NMonitoring::TDynamicCounters::TCounterPtr TKqpCounters::GetQueryTypeCounter(
+ NKikimrKqp::EQueryType queryType)
+{
+ return QueryTypes[queryType];
+}
+
+TKqpDbCountersPtr TKqpCounters::GetDbCounters(const TString& database) {
+ if (!ActorSystem || !AppData(ActorSystem)->FeatureFlags.GetEnableDbCounters() || database.empty()) {
+ return {};
+ }
+
+ TKqpDbCountersPtr dbCounters;
+ if (DbCounters.Get(database, dbCounters)) {
+ return dbCounters;
+ }
+
+ return DbCounters.InsertIfAbsentWithInit(database, [&database, this] {
+ auto counters = MakeIntrusive<TKqpDbCounters>();
+
+ auto evRegister = MakeHolder<NSysView::TEvSysView::TEvRegisterDbCounters>(
+ NKikimrSysView::KQP, database, counters);
+ ActorSystem->Send(NSysView::MakeSysViewServiceID(ActorSystem->NodeId), evRegister.Release());
+
+ if (DbWatcherActorId) {
+ auto evWatch = MakeHolder<NSysView::TEvSysView::TEvWatchDatabase>(database);
+ ActorSystem->Send(DbWatcherActorId, evWatch.Release());
+ }
+ return counters;
+ });
+}
+
+void TKqpCounters::RemoveDbCounters(const TString& database) {
+ DbCounters.Erase(database);
+}
+
} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/counters/kqp_counters.h b/ydb/core/kqp/counters/kqp_counters.h
index 53a8f7d2c23..8a24403618b 100644
--- a/ydb/core/kqp/counters/kqp_counters.h
+++ b/ydb/core/kqp/counters/kqp_counters.h
@@ -1,10 +1,10 @@
#pragma once
-#include "kqp_db_counters.h"
-
-#include <library/cpp/actors/helpers/mon_histogram_helper.h>
-#include <library/cpp/monlib/metrics/histogram_collector.h>
-
+#include "kqp_db_counters.h"
+
+#include <library/cpp/actors/helpers/mon_histogram_helper.h>
+#include <library/cpp/monlib/metrics/histogram_collector.h>
+
#include <ydb/core/util/concurrent_rw_hash.h>
#include <ydb/core/kqp/common/kqp_tx_info.h>
#include <ydb/core/kqp/common/kqp_tx_info.h>
@@ -14,35 +14,35 @@
#include <ydb/library/yql/minikql/aligned_page_pool.h>
-#include <util/system/spinlock.h>
+#include <util/system/spinlock.h>
namespace NKikimr {
namespace NKqp {
-class TKqpCountersBase {
-protected:
- struct TYdbTxByKindCounters {
- NMonitoring::THistogramPtr TotalDuration;
- NMonitoring::THistogramPtr ServerDuration;
- NMonitoring::THistogramPtr ClientDuration;
+class TKqpCountersBase {
+protected:
+ struct TYdbTxByKindCounters {
+ NMonitoring::THistogramPtr TotalDuration;
+ NMonitoring::THistogramPtr ServerDuration;
+ NMonitoring::THistogramPtr ClientDuration;
};
-protected:
- void CreateYdbTxKindCounters(TKqpTransactionInfo::EKind kind, const TString& name);
- static void UpdateYdbTxCounters(const TKqpTransactionInfo& txInfo,
- THashMap<TKqpTransactionInfo::EKind, TYdbTxByKindCounters>& txCounters);
-
- void Init();
-
+protected:
+ void CreateYdbTxKindCounters(TKqpTransactionInfo::EKind kind, const TString& name);
+ static void UpdateYdbTxCounters(const TKqpTransactionInfo& txInfo,
+ THashMap<TKqpTransactionInfo::EKind, TYdbTxByKindCounters>& txCounters);
+
+ void Init();
+
void ReportQueryAction(NKikimrKqp::EQueryAction action);
void ReportQueryType(NKikimrKqp::EQueryType type);
-
+
void ReportSessionShutdownRequest();
- void ReportCreateSession(ui64 requestSize);
- void ReportPingSession(ui64 requestSize);
- void ReportCloseSession(ui64 requestSize);
- void ReportQueryRequest(const NKikimrKqp::TQueryRequest& request);
-
+ void ReportCreateSession(ui64 requestSize);
+ void ReportPingSession(ui64 requestSize);
+ void ReportCloseSession(ui64 requestSize);
+ void ReportQueryRequest(const NKikimrKqp::TQueryRequest& request);
+
void ReportQueryWithRangeScan();
void ReportQueryWithFullScan();
void ReportQueryAffectedShards(ui64 shardsCount);
@@ -53,50 +53,50 @@ protected:
void ReportQueryMaxShardProgramSize(ui64 programSize);
void ReportResponseStatus(ui64 responseSize, Ydb::StatusIds::StatusCode ydbStatus);
- void ReportResultsBytes(ui64 resultsSize);
-
- static TString GetIssueName(ui32 issueCode);
- void ReportIssues(const Ydb::Issue::IssueMessage& issue);
-
+ void ReportResultsBytes(ui64 resultsSize);
+
+ static TString GetIssueName(ui32 issueCode);
+ void ReportIssues(const Ydb::Issue::IssueMessage& issue);
+
void ReportQueryLatency(NKikimrKqp::EQueryAction action, const TDuration& duration);
-
- void ReportTransaction(const TKqpTransactionInfo& txInfo);
-
+
+ void ReportTransaction(const TKqpTransactionInfo& txInfo);
+
void ReportSqlVersion(ui16 sqlVersion);
- void ReportWorkerCreated();
- void ReportWorkerFinished(TDuration lifeSpan);
- void ReportWorkerCleanupLatency(TDuration cleanupTime);
- void ReportWorkerClosedIdle();
- void ReportWorkerClosedError();
- void ReportWorkerClosedRequest();
- void ReportQueriesPerWorker(ui32 queryId);
+ void ReportWorkerCreated();
+ void ReportWorkerFinished(TDuration lifeSpan);
+ void ReportWorkerCleanupLatency(TDuration cleanupTime);
+ void ReportWorkerClosedIdle();
+ void ReportWorkerClosedError();
+ void ReportWorkerClosedRequest();
+ void ReportQueriesPerWorker(ui32 queryId);
void ReportProxyForwardedRequest();
void ReportSessionBalancerCV(ui32 value);
- void ReportBeginTransaction(ui32 evictedTx, ui32 currentActiveTx, ui32 currentAbortedTx);
-
- void ReportTxCreated();
- void ReportTxAborted(ui32 abortedCount);
-
- void ReportQueryCacheHit(bool hit);
- void ReportCompileStart();
- void ReportCompileFinish();
- void ReportCompileError();
- void ReportCompileRequestCompile();
- void ReportCompileRequestGet();
- void ReportCompileRequestInvalidate();
- void ReportCompileRequestRejected();
- void ReportCompileRequestTimeout();
- void ReportCompileDurations(TDuration duration, TDuration cpuTime);
+ void ReportBeginTransaction(ui32 evictedTx, ui32 currentActiveTx, ui32 currentAbortedTx);
+
+ void ReportTxCreated();
+ void ReportTxAborted(ui32 abortedCount);
+
+ void ReportQueryCacheHit(bool hit);
+ void ReportCompileStart();
+ void ReportCompileFinish();
+ void ReportCompileError();
+ void ReportCompileRequestCompile();
+ void ReportCompileRequestGet();
+ void ReportCompileRequestInvalidate();
+ void ReportCompileRequestRejected();
+ void ReportCompileRequestTimeout();
+ void ReportCompileDurations(TDuration duration, TDuration cpuTime);
void ReportRecompileRequestGet();
NMonitoring::TDynamicCounterPtr GetQueryReplayCounters() const;
-
-protected:
- NMonitoring::TDynamicCounterPtr Counters;
- NMonitoring::TDynamicCounterPtr KqpGroup;
- NMonitoring::TDynamicCounterPtr YdbGroup;
+
+protected:
+ NMonitoring::TDynamicCounterPtr Counters;
+ NMonitoring::TDynamicCounterPtr KqpGroup;
+ NMonitoring::TDynamicCounterPtr YdbGroup;
NMonitoring::TDynamicCounterPtr QueryReplayGroup;
// Requests
@@ -107,10 +107,10 @@ protected:
NMonitoring::TDynamicCounters::TCounterPtr PingSessionRequests;
NMonitoring::TDynamicCounters::TCounterPtr RequestBytes;
- NMonitoring::TDynamicCounters::TCounterPtr YdbRequestBytes;
+ NMonitoring::TDynamicCounters::TCounterPtr YdbRequestBytes;
NMonitoring::TDynamicCounters::TCounterPtr QueryBytes;
NMonitoring::TDynamicCounters::TCounterPtr ParametersBytes;
- NMonitoring::TDynamicCounters::TCounterPtr YdbParametersBytes;
+ NMonitoring::TDynamicCounters::TCounterPtr YdbParametersBytes;
NMonitoring::TDynamicCounters::TCounterPtr SqlV0Translations;
NMonitoring::TDynamicCounters::TCounterPtr SqlV1Translations;
@@ -129,189 +129,189 @@ protected:
NMonitoring::THistogramPtr QueryMaxShardProgramSize;
// Request latency
- THashMap<NKikimrKqp::EQueryAction, NMonitoring::THistogramPtr> QueryLatencies;
- NMonitoring::THistogramPtr YdbQueryExecuteLatency;
+ THashMap<NKikimrKqp::EQueryAction, NMonitoring::THistogramPtr> QueryLatencies;
+ NMonitoring::THistogramPtr YdbQueryExecuteLatency;
// Responses
NMonitoring::TDynamicCounters::TCounterPtr OtherResponses;
- NMonitoring::TDynamicCounters::TCounterPtr YdbResponsesLocksInvalidated;
-
+ NMonitoring::TDynamicCounters::TCounterPtr YdbResponsesLocksInvalidated;
+
THashMap<Ydb::StatusIds::StatusCode, NMonitoring::TDynamicCounters::TCounterPtr> YdbResponses;
NMonitoring::TDynamicCounters::TCounterPtr OtherYdbResponses;
NMonitoring::TDynamicCounters::TCounterPtr ResponseBytes;
- NMonitoring::TDynamicCounters::TCounterPtr YdbResponseBytes;
+ NMonitoring::TDynamicCounters::TCounterPtr YdbResponseBytes;
NMonitoring::TDynamicCounters::TCounterPtr QueryResultsBytes;
THashMap<ui32, NMonitoring::TDynamicCounters::TCounterPtr> IssueCounters;
// Workers
- NMonitoring::THistogramPtr WorkerLifeSpan;
- NMonitoring::THistogramPtr QueriesPerWorker;
+ NMonitoring::THistogramPtr WorkerLifeSpan;
+ NMonitoring::THistogramPtr QueriesPerWorker;
NMonitoring::TDynamicCounters::TCounterPtr WorkersCreated;
NMonitoring::TDynamicCounters::TCounterPtr WorkersClosedIdle;
- NMonitoring::TDynamicCounters::TCounterPtr YdbWorkersClosedIdle;
+ NMonitoring::TDynamicCounters::TCounterPtr YdbWorkersClosedIdle;
NMonitoring::TDynamicCounters::TCounterPtr WorkersClosedError;
NMonitoring::TDynamicCounters::TCounterPtr WorkersClosedRequest;
NMonitoring::TDynamicCounters::TCounterPtr ActiveWorkers;
- NMonitoring::TDynamicCounters::TCounterPtr YdbActiveWorkers;
+ NMonitoring::TDynamicCounters::TCounterPtr YdbActiveWorkers;
NMonitoring::TDynamicCounters::TCounterPtr SessionBalancerCV;
NMonitoring::TDynamicCounters::TCounterPtr SessionBalancerShutdowns;
NMonitoring::TDynamicCounters::TCounterPtr ProxyForwardedRequests;
- NMonitoring::THistogramPtr WorkerCleanupLatency;
-
- // Transactions
- NMonitoring::TDynamicCounters::TCounterPtr TxCreated;
- NMonitoring::TDynamicCounters::TCounterPtr TxAborted;
- NMonitoring::TDynamicCounters::TCounterPtr TxCommited;
- NMonitoring::TDynamicCounters::TCounterPtr TxEvicted;
- NMonitoring::THistogramPtr TxActivePerSession;
- NMonitoring::THistogramPtr TxAbortedPerSession;
- THashMap<TKqpTransactionInfo::EKind, TYdbTxByKindCounters> YdbTxByKind;
-
- // Compile service
- NMonitoring::TDynamicCounters::TCounterPtr CompileQueryCacheHits;
- NMonitoring::TDynamicCounters::TCounterPtr CompileQueryCacheMisses;
- NMonitoring::TDynamicCounters::TCounterPtr CompileRequestsCompile;
- NMonitoring::TDynamicCounters::TCounterPtr CompileRequestsGet;
- NMonitoring::TDynamicCounters::TCounterPtr CompileRequestsInvalidate;
- NMonitoring::TDynamicCounters::TCounterPtr CompileRequestsRejected;
- NMonitoring::TDynamicCounters::TCounterPtr CompileRequestsTimeout;
+ NMonitoring::THistogramPtr WorkerCleanupLatency;
+
+ // Transactions
+ NMonitoring::TDynamicCounters::TCounterPtr TxCreated;
+ NMonitoring::TDynamicCounters::TCounterPtr TxAborted;
+ NMonitoring::TDynamicCounters::TCounterPtr TxCommited;
+ NMonitoring::TDynamicCounters::TCounterPtr TxEvicted;
+ NMonitoring::THistogramPtr TxActivePerSession;
+ NMonitoring::THistogramPtr TxAbortedPerSession;
+ THashMap<TKqpTransactionInfo::EKind, TYdbTxByKindCounters> YdbTxByKind;
+
+ // Compile service
+ NMonitoring::TDynamicCounters::TCounterPtr CompileQueryCacheHits;
+ NMonitoring::TDynamicCounters::TCounterPtr CompileQueryCacheMisses;
+ NMonitoring::TDynamicCounters::TCounterPtr CompileRequestsCompile;
+ NMonitoring::TDynamicCounters::TCounterPtr CompileRequestsGet;
+ NMonitoring::TDynamicCounters::TCounterPtr CompileRequestsInvalidate;
+ NMonitoring::TDynamicCounters::TCounterPtr CompileRequestsRejected;
+ NMonitoring::TDynamicCounters::TCounterPtr CompileRequestsTimeout;
NMonitoring::TDynamicCounters::TCounterPtr CompileRequestsRecompile;
- NMonitoring::TDynamicCounters::TCounterPtr CompileTotal;
- NMonitoring::TDynamicCounters::TCounterPtr CompileErrors;
- NMonitoring::TDynamicCounters::TCounterPtr CompileActive;
- NMonitoring::THistogramPtr CompileCpuTime;
- NMonitoring::THistogramPtr YdbCompileDuration;
-};
-
-
-class TKqpDbCounters : public NSysView::IDbCounters, public TKqpCountersBase {
- friend class TKqpCounters;
-
-public:
- // per database internal counters, not exposed
- TKqpDbCounters();
-
- // created in SVP, exposed
- explicit TKqpDbCounters(const NMonitoring::TDynamicCounterPtr& externalGroup,
- const NMonitoring::TDynamicCounterPtr& internalGroup);
-
- void ToProto(NKikimr::NSysView::TDbServiceCounters& counters) override;
- void FromProto(NKikimr::NSysView::TDbServiceCounters& counters) override;
-
-private:
- enum ESimpleCounter {
- DB_KQP_SIMPLE_COUNTERS_MAP(ENUM_VALUE_GEN_NO_VALUE)
- DB_KQP_SIMPLE_COUNTER_SIZE
- };
- enum ECumulativeCounter {
- DB_KQP_CUMULATIVE_COUNTERS_MAP(ENUM_VALUE_GEN_NO_VALUE)
- DB_KQP_CUMULATIVE_COUNTER_SIZE
- };
- enum EHistogramCounter {
- DB_KQP_HISTOGRAM_COUNTERS_MAP(ENUM_VALUE_GEN_NO_VALUE)
- DB_KQP_HISTOGRAM_COUNTER_SIZE
- };
-
- struct TDeprecatedCounter {
- void Set(ui64) {}
- ui64 Val() { return 0; }
- };
- TDeprecatedCounter DeprecatedCounter;
-};
-
-using TKqpDbCountersPtr = TIntrusivePtr<TKqpDbCounters>;
-
-
-class TKqpCounters : public TThrRefBase, public TKqpCountersBase {
-private:
- struct TTxByKindCounters {
- NMonitoring::THistogramPtr TotalDuration;
- NMonitoring::THistogramPtr ServerDuration;
- NMonitoring::THistogramPtr ClientDuration;
- NMonitoring::THistogramPtr Queries;
- };
-
-private:
- void CreateTxKindCounters(TKqpTransactionInfo::EKind kind, const TString& name);
- static void UpdateTxCounters(const TKqpTransactionInfo& txInfo,
- THashMap<TKqpTransactionInfo::EKind, TTxByKindCounters>& txCounters);
-
-public:
- explicit TKqpCounters(const NMonitoring::TDynamicCounterPtr& counters, const TActorContext* ctx = nullptr);
-
+ NMonitoring::TDynamicCounters::TCounterPtr CompileTotal;
+ NMonitoring::TDynamicCounters::TCounterPtr CompileErrors;
+ NMonitoring::TDynamicCounters::TCounterPtr CompileActive;
+ NMonitoring::THistogramPtr CompileCpuTime;
+ NMonitoring::THistogramPtr YdbCompileDuration;
+};
+
+
+class TKqpDbCounters : public NSysView::IDbCounters, public TKqpCountersBase {
+ friend class TKqpCounters;
+
+public:
+ // per database internal counters, not exposed
+ TKqpDbCounters();
+
+ // created in SVP, exposed
+ explicit TKqpDbCounters(const NMonitoring::TDynamicCounterPtr& externalGroup,
+ const NMonitoring::TDynamicCounterPtr& internalGroup);
+
+ void ToProto(NKikimr::NSysView::TDbServiceCounters& counters) override;
+ void FromProto(NKikimr::NSysView::TDbServiceCounters& counters) override;
+
+private:
+ enum ESimpleCounter {
+ DB_KQP_SIMPLE_COUNTERS_MAP(ENUM_VALUE_GEN_NO_VALUE)
+ DB_KQP_SIMPLE_COUNTER_SIZE
+ };
+ enum ECumulativeCounter {
+ DB_KQP_CUMULATIVE_COUNTERS_MAP(ENUM_VALUE_GEN_NO_VALUE)
+ DB_KQP_CUMULATIVE_COUNTER_SIZE
+ };
+ enum EHistogramCounter {
+ DB_KQP_HISTOGRAM_COUNTERS_MAP(ENUM_VALUE_GEN_NO_VALUE)
+ DB_KQP_HISTOGRAM_COUNTER_SIZE
+ };
+
+ struct TDeprecatedCounter {
+ void Set(ui64) {}
+ ui64 Val() { return 0; }
+ };
+ TDeprecatedCounter DeprecatedCounter;
+};
+
+using TKqpDbCountersPtr = TIntrusivePtr<TKqpDbCounters>;
+
+
+class TKqpCounters : public TThrRefBase, public TKqpCountersBase {
+private:
+ struct TTxByKindCounters {
+ NMonitoring::THistogramPtr TotalDuration;
+ NMonitoring::THistogramPtr ServerDuration;
+ NMonitoring::THistogramPtr ClientDuration;
+ NMonitoring::THistogramPtr Queries;
+ };
+
+private:
+ void CreateTxKindCounters(TKqpTransactionInfo::EKind kind, const TString& name);
+ static void UpdateTxCounters(const TKqpTransactionInfo& txInfo,
+ THashMap<TKqpTransactionInfo::EKind, TTxByKindCounters>& txCounters);
+
+public:
+ explicit TKqpCounters(const NMonitoring::TDynamicCounterPtr& counters, const TActorContext* ctx = nullptr);
+
void ReportProxyForwardedRequest(TKqpDbCountersPtr dbCounters);
void ReportSessionBalancerCV(TKqpDbCountersPtr dbCounters, ui32 val);
- void ReportSessionShutdownRequest(TKqpDbCountersPtr dbCounters);
- void ReportCreateSession(TKqpDbCountersPtr dbCounters, ui64 requestSize);
- void ReportPingSession(TKqpDbCountersPtr dbCounters, ui64 requestSize);
- void ReportCloseSession(TKqpDbCountersPtr dbCounters, ui64 requestSize);
- void ReportQueryRequest(TKqpDbCountersPtr dbCounters, const NKikimrKqp::TQueryRequest& request);
-
+ void ReportSessionShutdownRequest(TKqpDbCountersPtr dbCounters);
+ void ReportCreateSession(TKqpDbCountersPtr dbCounters, ui64 requestSize);
+ void ReportPingSession(TKqpDbCountersPtr dbCounters, ui64 requestSize);
+ void ReportCloseSession(TKqpDbCountersPtr dbCounters, ui64 requestSize);
+ void ReportQueryRequest(TKqpDbCountersPtr dbCounters, const NKikimrKqp::TQueryRequest& request);
+
void ReportResponseStatus(TKqpDbCountersPtr dbCounters, ui64 responseSize, Ydb::StatusIds::StatusCode ydbStatus);
- void ReportResultsBytes(TKqpDbCountersPtr dbCounters, ui64 resultsSize);
- void ReportIssues(TKqpDbCountersPtr dbCounters, const Ydb::Issue::IssueMessage& issue);
-
- void ReportQueryWithRangeScan(TKqpDbCountersPtr dbCounters);
- void ReportQueryWithFullScan(TKqpDbCountersPtr dbCounters);
- void ReportQueryAffectedShards(TKqpDbCountersPtr dbCounters, ui64 shardsCount);
- void ReportQueryReadSets(TKqpDbCountersPtr dbCounters, ui64 readSetsCount);
- void ReportQueryReadBytes(TKqpDbCountersPtr dbCounters, ui64 bytesCount);
- void ReportQueryReadRows(TKqpDbCountersPtr dbCounters, ui64 rowsCount);
- void ReportQueryMaxShardReplySize(TKqpDbCountersPtr dbCounters, ui64 replySize);
- void ReportQueryMaxShardProgramSize(TKqpDbCountersPtr dbCounters, ui64 programSize);
-
- void ReportQueryLatency(TKqpDbCountersPtr dbCounters,
- NKikimrKqp::EQueryAction action, const TDuration& duration);
- void ReportSqlVersion(TKqpDbCountersPtr dbCounters, ui16 sqlVersion);
- void ReportTransaction(TKqpDbCountersPtr dbCounters, const TKqpTransactionInfo& txInfo);
-
+ void ReportResultsBytes(TKqpDbCountersPtr dbCounters, ui64 resultsSize);
+ void ReportIssues(TKqpDbCountersPtr dbCounters, const Ydb::Issue::IssueMessage& issue);
+
+ void ReportQueryWithRangeScan(TKqpDbCountersPtr dbCounters);
+ void ReportQueryWithFullScan(TKqpDbCountersPtr dbCounters);
+ void ReportQueryAffectedShards(TKqpDbCountersPtr dbCounters, ui64 shardsCount);
+ void ReportQueryReadSets(TKqpDbCountersPtr dbCounters, ui64 readSetsCount);
+ void ReportQueryReadBytes(TKqpDbCountersPtr dbCounters, ui64 bytesCount);
+ void ReportQueryReadRows(TKqpDbCountersPtr dbCounters, ui64 rowsCount);
+ void ReportQueryMaxShardReplySize(TKqpDbCountersPtr dbCounters, ui64 replySize);
+ void ReportQueryMaxShardProgramSize(TKqpDbCountersPtr dbCounters, ui64 programSize);
+
+ void ReportQueryLatency(TKqpDbCountersPtr dbCounters,
+ NKikimrKqp::EQueryAction action, const TDuration& duration);
+ void ReportSqlVersion(TKqpDbCountersPtr dbCounters, ui16 sqlVersion);
+ void ReportTransaction(TKqpDbCountersPtr dbCounters, const TKqpTransactionInfo& txInfo);
+
// NewEngine
- void ReportNewEngineForcedQueryStats(NKikimrKqp::EQueryAction action,
- TDuration duration, ui64 computeCpuTime);
- void ReportNewEngineCompatibleQueryStats(NKikimrKqp::EQueryAction action,
- TDuration duration, ui64 computeCpuTime);
-
- void ReportWorkerCreated(TKqpDbCountersPtr dbCounters);
- void ReportWorkerFinished(TKqpDbCountersPtr dbCounters, TDuration lifeSpan);
- void ReportWorkerCleanupLatency(TKqpDbCountersPtr dbCounters, TDuration cleanupTime);
- void ReportWorkerClosedIdle(TKqpDbCountersPtr dbCounters);
- void ReportWorkerClosedError(TKqpDbCountersPtr dbCounters);
- void ReportWorkerClosedRequest(TKqpDbCountersPtr dbCounters);
- void ReportQueriesPerWorker(TKqpDbCountersPtr dbCounters, ui32 queryId);
-
- void ReportBeginTransaction(TKqpDbCountersPtr dbCounters,
- ui32 evictedTx, ui32 currentActiveTx, ui32 currentAbortedTx);
-
- void ReportTxCreated(TKqpDbCountersPtr dbCounters);
- void ReportTxAborted(TKqpDbCountersPtr dbCounters, ui32 abortedCount);
-
- void ReportQueryCacheHit(TKqpDbCountersPtr dbCounters, bool hit);
- void ReportCompileStart(TKqpDbCountersPtr dbCounters);
- void ReportCompileFinish(TKqpDbCountersPtr dbCounters);
- void ReportCompileError(TKqpDbCountersPtr dbCounters);
- void ReportCompileRequestCompile(TKqpDbCountersPtr dbCounters);
- void ReportCompileRequestGet(TKqpDbCountersPtr dbCounters);
- void ReportCompileRequestInvalidate(TKqpDbCountersPtr dbCounters);
- void ReportCompileRequestRejected(TKqpDbCountersPtr dbCounters);
- void ReportCompileRequestTimeout(TKqpDbCountersPtr dbCounters);
- void ReportCompileDurations(TKqpDbCountersPtr dbCounters, TDuration duration, TDuration cpuTime);
- void ReportRecompileRequestGet(TKqpDbCountersPtr dbCounters);
-
+ void ReportNewEngineForcedQueryStats(NKikimrKqp::EQueryAction action,
+ TDuration duration, ui64 computeCpuTime);
+ void ReportNewEngineCompatibleQueryStats(NKikimrKqp::EQueryAction action,
+ TDuration duration, ui64 computeCpuTime);
+
+ void ReportWorkerCreated(TKqpDbCountersPtr dbCounters);
+ void ReportWorkerFinished(TKqpDbCountersPtr dbCounters, TDuration lifeSpan);
+ void ReportWorkerCleanupLatency(TKqpDbCountersPtr dbCounters, TDuration cleanupTime);
+ void ReportWorkerClosedIdle(TKqpDbCountersPtr dbCounters);
+ void ReportWorkerClosedError(TKqpDbCountersPtr dbCounters);
+ void ReportWorkerClosedRequest(TKqpDbCountersPtr dbCounters);
+ void ReportQueriesPerWorker(TKqpDbCountersPtr dbCounters, ui32 queryId);
+
+ void ReportBeginTransaction(TKqpDbCountersPtr dbCounters,
+ ui32 evictedTx, ui32 currentActiveTx, ui32 currentAbortedTx);
+
+ void ReportTxCreated(TKqpDbCountersPtr dbCounters);
+ void ReportTxAborted(TKqpDbCountersPtr dbCounters, ui32 abortedCount);
+
+ void ReportQueryCacheHit(TKqpDbCountersPtr dbCounters, bool hit);
+ void ReportCompileStart(TKqpDbCountersPtr dbCounters);
+ void ReportCompileFinish(TKqpDbCountersPtr dbCounters);
+ void ReportCompileError(TKqpDbCountersPtr dbCounters);
+ void ReportCompileRequestCompile(TKqpDbCountersPtr dbCounters);
+ void ReportCompileRequestGet(TKqpDbCountersPtr dbCounters);
+ void ReportCompileRequestInvalidate(TKqpDbCountersPtr dbCounters);
+ void ReportCompileRequestRejected(TKqpDbCountersPtr dbCounters);
+ void ReportCompileRequestTimeout(TKqpDbCountersPtr dbCounters);
+ void ReportCompileDurations(TKqpDbCountersPtr dbCounters, TDuration duration, TDuration cpuTime);
+ void ReportRecompileRequestGet(TKqpDbCountersPtr dbCounters);
+
const NMonitoring::TDynamicCounters::TCounterPtr RecompileRequestGet() const;
NMonitoring::TDynamicCounterPtr GetQueryReplayCounters() const;
- NMonitoring::TDynamicCounters::TCounterPtr GetQueryTypeCounter(NKikimrKqp::EQueryType queryType);
-
- TKqpDbCountersPtr GetDbCounters(const TString& database);
- void RemoveDbCounters(const TString& database);
-
-public:
- // Transactions
- THashMap<TKqpTransactionInfo::EKind, TTxByKindCounters> TxByKind;
-
+ NMonitoring::TDynamicCounters::TCounterPtr GetQueryTypeCounter(NKikimrKqp::EQueryType queryType);
+
+ TKqpDbCountersPtr GetDbCounters(const TString& database);
+ void RemoveDbCounters(const TString& database);
+
+public:
+ // Transactions
+ THashMap<TKqpTransactionInfo::EKind, TTxByKindCounters> TxByKind;
+
// Compile service
NMonitoring::TDynamicCounters::TCounterPtr CompileQueryCacheSize;
NMonitoring::TDynamicCounters::TCounterPtr CompileQueryCacheBytes;
@@ -343,24 +343,24 @@ public:
NMonitoring::THistogramPtr ScanQueryRateLimitLatency;
// NewEngine vs OldEngine
- THashMap<NKikimrKqp::EQueryAction, NMonitoring::THistogramPtr> NewEngineForcedQueryLatencies;
- THashMap<NKikimrKqp::EQueryAction, NMonitoring::THistogramPtr> NewEngineCompatibleQueryLatencies;
- NMonitoring::TDynamicCounters::TCounterPtr NewEngineForcedComputeCpuTime;
- NMonitoring::TDynamicCounters::TCounterPtr NewEngineForcedQueryCount;
- NMonitoring::TDynamicCounters::TCounterPtr NewEngineCompatibleComputeCpuTime;
- NMonitoring::TDynamicCounters::TCounterPtr NewEngineCompatibleQueryCount;
-
+ THashMap<NKikimrKqp::EQueryAction, NMonitoring::THistogramPtr> NewEngineForcedQueryLatencies;
+ THashMap<NKikimrKqp::EQueryAction, NMonitoring::THistogramPtr> NewEngineCompatibleQueryLatencies;
+ NMonitoring::TDynamicCounters::TCounterPtr NewEngineForcedComputeCpuTime;
+ NMonitoring::TDynamicCounters::TCounterPtr NewEngineForcedQueryCount;
+ NMonitoring::TDynamicCounters::TCounterPtr NewEngineCompatibleComputeCpuTime;
+ NMonitoring::TDynamicCounters::TCounterPtr NewEngineCompatibleQueryCount;
+
// NewEngine tx duration
NMonitoring::THistogramPtr LiteralTxTotalTimeHistogram;
NMonitoring::THistogramPtr DataTxTotalTimeHistogram;
NMonitoring::THistogramPtr ScanTxTotalTimeHistogram;
TAlignedPagePoolCounters AllocCounters;
-
- // db counters
- TConcurrentRWHashMap<TString, TKqpDbCountersPtr, 256> DbCounters;
- TActorSystem* ActorSystem = nullptr;
- TActorId DbWatcherActorId;
+
+ // db counters
+ TConcurrentRWHashMap<TString, TKqpDbCountersPtr, 256> DbCounters;
+ TActorSystem* ActorSystem = nullptr;
+ TActorId DbWatcherActorId;
};
struct TKqpRequestCounters : public TThrRefBase {
diff --git a/ydb/core/kqp/counters/kqp_db_counters.h b/ydb/core/kqp/counters/kqp_db_counters.h
index 814c382d4c2..6aba15df70d 100644
--- a/ydb/core/kqp/counters/kqp_db_counters.h
+++ b/ydb/core/kqp/counters/kqp_db_counters.h
@@ -1,120 +1,120 @@
-#pragma once
-
-#include <library/cpp/deprecated/enum_codegen/enum_codegen.h>
-
-namespace NKikimr {
-namespace NKqp {
-
-#define DB_KQP_SIMPLE_COUNTERS_MAP(XX) \
- XX(DB_KQP_WORKERS_ACTIVE, ActiveWorkers) \
- XX(DB_KQP_YDB_WORKERS_ACTIVE, YdbActiveWorkers) \
- XX(DB_KQP_COMPILE_ACTIVE, CompileActive)
-
-#define DB_KQP_CUMULATIVE_COUNTERS_MAP(XX) \
- XX(DB_KQP_REQ_QUERY_EXECUTE, QueryActionRequests[NKikimrKqp::QUERY_ACTION_EXECUTE]) \
- XX(DB_KQP_REQ_QUERY_EXPLAIN, QueryActionRequests[NKikimrKqp::QUERY_ACTION_EXPLAIN]) \
- XX(DB_KQP_REQ_QUERY_VALIDATE, QueryActionRequests[NKikimrKqp::QUERY_ACTION_VALIDATE]) \
- XX(DB_KQP_REQ_QUERY_PREPARE, QueryActionRequests[NKikimrKqp::QUERY_ACTION_PREPARE]) \
- XX(DB_KQP_REQ_QUERY_EXECUTE_PREPARED, QueryActionRequests[NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED]) \
- XX(DB_KQP_REQ_QUERY_BEGIN_TX, QueryActionRequests[NKikimrKqp::QUERY_ACTION_BEGIN_TX]) \
- XX(DB_KQP_REQ_QUERY_COMMIT_TX, QueryActionRequests[NKikimrKqp::QUERY_ACTION_COMMIT_TX]) \
- XX(DB_KQP_REQ_QUERY_ROLLBACK_TX, QueryActionRequests[NKikimrKqp::QUERY_ACTION_ROLLBACK_TX]) \
- XX(DB_KQP_REQ_QUERY_PARSE, QueryActionRequests[NKikimrKqp::QUERY_ACTION_PARSE]) \
- XX(DB_KQP_REQ_QUERY_OTHER, OtherQueryRequests) \
- XX(DB_KQP_CLOSE_SESSION_REQ, CloseSessionRequests) \
- XX(DB_KQP_CREATE_SESSION_REQ, PingSessionRequests) \
- XX(DB_KQP_REQUEST_BYTES, RequestBytes) \
- XX(DB_KQP_QUERY_BYTES, QueryBytes) \
- XX(DB_KQP_PARAMS_BYTES, ParametersBytes) \
- XX(DB_KQP_SQL_V0_TRANSLATIONS, SqlV0Translations) \
- XX(DB_KQP_SQL_V1_TRANSLATIONS, SqlV1Translations) \
- XX(DB_KQP_SQL_UNKNOWN_TRANSLATIONS, SqlUnknownTranslations) \
- XX(DB_KQP_QUERY_TYPE_UNDEFINED, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_UNDEFINED]) \
- XX(DB_KQP_QUERY_TYPE_SQL_DML, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_DML]) \
- XX(DB_KQP_QUERY_TYPE_SQL_DDL, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_DDL]) \
- XX(DB_KQP_QUERY_TYPE_PREPARED_DML, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_PREPARED_DML]) \
- XX(DB_KQP_QUERY_TYPE_AST_DML, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_AST_DML]) \
- XX(DB_KQP_QUERY_TYPE_SQL_SCRIPT, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCRIPT]) \
+#pragma once
+
+#include <library/cpp/deprecated/enum_codegen/enum_codegen.h>
+
+namespace NKikimr {
+namespace NKqp {
+
+#define DB_KQP_SIMPLE_COUNTERS_MAP(XX) \
+ XX(DB_KQP_WORKERS_ACTIVE, ActiveWorkers) \
+ XX(DB_KQP_YDB_WORKERS_ACTIVE, YdbActiveWorkers) \
+ XX(DB_KQP_COMPILE_ACTIVE, CompileActive)
+
+#define DB_KQP_CUMULATIVE_COUNTERS_MAP(XX) \
+ XX(DB_KQP_REQ_QUERY_EXECUTE, QueryActionRequests[NKikimrKqp::QUERY_ACTION_EXECUTE]) \
+ XX(DB_KQP_REQ_QUERY_EXPLAIN, QueryActionRequests[NKikimrKqp::QUERY_ACTION_EXPLAIN]) \
+ XX(DB_KQP_REQ_QUERY_VALIDATE, QueryActionRequests[NKikimrKqp::QUERY_ACTION_VALIDATE]) \
+ XX(DB_KQP_REQ_QUERY_PREPARE, QueryActionRequests[NKikimrKqp::QUERY_ACTION_PREPARE]) \
+ XX(DB_KQP_REQ_QUERY_EXECUTE_PREPARED, QueryActionRequests[NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED]) \
+ XX(DB_KQP_REQ_QUERY_BEGIN_TX, QueryActionRequests[NKikimrKqp::QUERY_ACTION_BEGIN_TX]) \
+ XX(DB_KQP_REQ_QUERY_COMMIT_TX, QueryActionRequests[NKikimrKqp::QUERY_ACTION_COMMIT_TX]) \
+ XX(DB_KQP_REQ_QUERY_ROLLBACK_TX, QueryActionRequests[NKikimrKqp::QUERY_ACTION_ROLLBACK_TX]) \
+ XX(DB_KQP_REQ_QUERY_PARSE, QueryActionRequests[NKikimrKqp::QUERY_ACTION_PARSE]) \
+ XX(DB_KQP_REQ_QUERY_OTHER, OtherQueryRequests) \
+ XX(DB_KQP_CLOSE_SESSION_REQ, CloseSessionRequests) \
+ XX(DB_KQP_CREATE_SESSION_REQ, PingSessionRequests) \
+ XX(DB_KQP_REQUEST_BYTES, RequestBytes) \
+ XX(DB_KQP_QUERY_BYTES, QueryBytes) \
+ XX(DB_KQP_PARAMS_BYTES, ParametersBytes) \
+ XX(DB_KQP_SQL_V0_TRANSLATIONS, SqlV0Translations) \
+ XX(DB_KQP_SQL_V1_TRANSLATIONS, SqlV1Translations) \
+ XX(DB_KQP_SQL_UNKNOWN_TRANSLATIONS, SqlUnknownTranslations) \
+ XX(DB_KQP_QUERY_TYPE_UNDEFINED, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_UNDEFINED]) \
+ XX(DB_KQP_QUERY_TYPE_SQL_DML, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_DML]) \
+ XX(DB_KQP_QUERY_TYPE_SQL_DDL, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_DDL]) \
+ XX(DB_KQP_QUERY_TYPE_PREPARED_DML, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_PREPARED_DML]) \
+ XX(DB_KQP_QUERY_TYPE_AST_DML, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_AST_DML]) \
+ XX(DB_KQP_QUERY_TYPE_SQL_SCRIPT, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCRIPT]) \
XX(DB_KQP_QUERY_TYPE_SQL_SCRIPT_STREAMING, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCRIPT_STREAMING]) \
- XX(DB_KQP_QUERY_TYPE_SQL_SCAN, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCAN]) \
- XX(DB_KQP_QUERY_TYPE_AST_SCAN, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_AST_SCAN]) \
- XX(DB_KQP_RSP_STATUS_CODE_UNSPECIFIED, YdbResponses[Ydb::StatusIds::STATUS_CODE_UNSPECIFIED]) \
- XX(DB_KQP_RSP_SUCCESS, YdbResponses[Ydb::StatusIds::SUCCESS]) \
- XX(DB_KQP_RSP_BAD_REQUEST, YdbResponses[Ydb::StatusIds::BAD_REQUEST]) \
- XX(DB_KQP_RSP_UNAUTHORIZED, YdbResponses[Ydb::StatusIds::UNAUTHORIZED]) \
- XX(DB_KQP_RSP_INTERNAL_ERROR, YdbResponses[Ydb::StatusIds::INTERNAL_ERROR]) \
- XX(DB_KQP_RSP_ABORTED, YdbResponses[Ydb::StatusIds::ABORTED]) \
- XX(DB_KQP_RSP_UNAVAILABLE, YdbResponses[Ydb::StatusIds::UNAVAILABLE]) \
- XX(DB_KQP_RSP_OVERLOADED, YdbResponses[Ydb::StatusIds::OVERLOADED]) \
- XX(DB_KQP_RSP_SCHEME_ERROR, YdbResponses[Ydb::StatusIds::SCHEME_ERROR]) \
- XX(DB_KQP_RSP_GENERIC_ERROR, YdbResponses[Ydb::StatusIds::GENERIC_ERROR]) \
- XX(DB_KQP_RSP_TIMEOUT, YdbResponses[Ydb::StatusIds::TIMEOUT]) \
- XX(DB_KQP_RSP_BAD_SESSION, YdbResponses[Ydb::StatusIds::BAD_SESSION]) \
- XX(DB_KQP_RSP_PRECONDITION_FAILED, YdbResponses[Ydb::StatusIds::PRECONDITION_FAILED]) \
- XX(DB_KQP_RSP_ALREADY_EXISTS, YdbResponses[Ydb::StatusIds::ALREADY_EXISTS]) \
- XX(DB_KQP_RSP_NOT_FOUND, YdbResponses[Ydb::StatusIds::NOT_FOUND]) \
- XX(DB_KQP_RSP_SESSION_EXPIRED, YdbResponses[Ydb::StatusIds::SESSION_EXPIRED]) \
- XX(DB_KQP_RSP_CANCELLED, YdbResponses[Ydb::StatusIds::CANCELLED]) \
- XX(DB_KQP_RSP_UNDETERMINED, YdbResponses[Ydb::StatusIds::UNDETERMINED]) \
- XX(DB_KQP_RSP_UNSUPPORTED, YdbResponses[Ydb::StatusIds::UNSUPPORTED]) \
- XX(DB_KQP_RSP_SESSION_BUSY, YdbResponses[Ydb::StatusIds::SESSION_BUSY]) \
- XX(DB_KQP_RSP_OTHER, OtherYdbResponses) \
- XX(DB_KQP_RESPONSE_BYTES, ResponseBytes) \
- XX(DB_KQP_QUERY_RESULTS_BYTES, QueryResultsBytes) \
- XX(DB_KQP_WORKERS_CREATED, WorkersCreated) \
- XX(DB_KQP_WORKERS_CLOSED_IDLE, WorkersClosedIdle) \
- XX(DB_KQP_WORKERS_CLOSED_ERROR, WorkersClosedError) \
- XX(DB_KQP_WORKERS_CLOSED_REQUEST, WorkersClosedRequest) \
- XX(DB_KQP_TX_CREATED, TxCreated) \
- XX(DB_KQP_TX_ABORTED, TxAborted) \
- XX(DB_KQP_TX_COMMITED, TxCommited) \
- XX(DB_KQP_TX_EVICTED, TxEvicted) \
- XX(DB_KQP_COMPILE_QUERY_CACHE_HITS, CompileQueryCacheHits) \
- XX(DB_KQP_COMPILE_QUERY_CACHE_MISSES, CompileQueryCacheMisses) \
- XX(DB_KQP_COMPILE_REQ_COMPILE, CompileRequestsCompile) \
- XX(DB_KQP_COMPILE_REQ_GET, CompileRequestsGet) \
- XX(DB_KQP_COMPILE_REQ_INVALIDATE, CompileRequestsInvalidate) \
- XX(DB_KQP_COMPILE_REQ_REJECTED, CompileRequestsRejected) \
- XX(DB_KQP_COMPILE_REQ_TIMEOUT, CompileRequestsTimeout) \
- XX(DB_KQP_COMPILE_TOTAL, CompileTotal) \
- XX(DB_KQP_COMPILE_ERRORS, CompileErrors) \
- XX(DB_KQP_DEPRECATED1, &DeprecatedCounter) \
- XX(DB_KQP_YDB_WORKERS_CLOSED_IDLE, YdbWorkersClosedIdle) \
- XX(DB_KQP_YDB_REQUEST_BYTES, YdbRequestBytes) \
- XX(DB_KQP_YDB_PARAMS_BYTES, YdbParametersBytes) \
+ XX(DB_KQP_QUERY_TYPE_SQL_SCAN, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCAN]) \
+ XX(DB_KQP_QUERY_TYPE_AST_SCAN, QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_AST_SCAN]) \
+ XX(DB_KQP_RSP_STATUS_CODE_UNSPECIFIED, YdbResponses[Ydb::StatusIds::STATUS_CODE_UNSPECIFIED]) \
+ XX(DB_KQP_RSP_SUCCESS, YdbResponses[Ydb::StatusIds::SUCCESS]) \
+ XX(DB_KQP_RSP_BAD_REQUEST, YdbResponses[Ydb::StatusIds::BAD_REQUEST]) \
+ XX(DB_KQP_RSP_UNAUTHORIZED, YdbResponses[Ydb::StatusIds::UNAUTHORIZED]) \
+ XX(DB_KQP_RSP_INTERNAL_ERROR, YdbResponses[Ydb::StatusIds::INTERNAL_ERROR]) \
+ XX(DB_KQP_RSP_ABORTED, YdbResponses[Ydb::StatusIds::ABORTED]) \
+ XX(DB_KQP_RSP_UNAVAILABLE, YdbResponses[Ydb::StatusIds::UNAVAILABLE]) \
+ XX(DB_KQP_RSP_OVERLOADED, YdbResponses[Ydb::StatusIds::OVERLOADED]) \
+ XX(DB_KQP_RSP_SCHEME_ERROR, YdbResponses[Ydb::StatusIds::SCHEME_ERROR]) \
+ XX(DB_KQP_RSP_GENERIC_ERROR, YdbResponses[Ydb::StatusIds::GENERIC_ERROR]) \
+ XX(DB_KQP_RSP_TIMEOUT, YdbResponses[Ydb::StatusIds::TIMEOUT]) \
+ XX(DB_KQP_RSP_BAD_SESSION, YdbResponses[Ydb::StatusIds::BAD_SESSION]) \
+ XX(DB_KQP_RSP_PRECONDITION_FAILED, YdbResponses[Ydb::StatusIds::PRECONDITION_FAILED]) \
+ XX(DB_KQP_RSP_ALREADY_EXISTS, YdbResponses[Ydb::StatusIds::ALREADY_EXISTS]) \
+ XX(DB_KQP_RSP_NOT_FOUND, YdbResponses[Ydb::StatusIds::NOT_FOUND]) \
+ XX(DB_KQP_RSP_SESSION_EXPIRED, YdbResponses[Ydb::StatusIds::SESSION_EXPIRED]) \
+ XX(DB_KQP_RSP_CANCELLED, YdbResponses[Ydb::StatusIds::CANCELLED]) \
+ XX(DB_KQP_RSP_UNDETERMINED, YdbResponses[Ydb::StatusIds::UNDETERMINED]) \
+ XX(DB_KQP_RSP_UNSUPPORTED, YdbResponses[Ydb::StatusIds::UNSUPPORTED]) \
+ XX(DB_KQP_RSP_SESSION_BUSY, YdbResponses[Ydb::StatusIds::SESSION_BUSY]) \
+ XX(DB_KQP_RSP_OTHER, OtherYdbResponses) \
+ XX(DB_KQP_RESPONSE_BYTES, ResponseBytes) \
+ XX(DB_KQP_QUERY_RESULTS_BYTES, QueryResultsBytes) \
+ XX(DB_KQP_WORKERS_CREATED, WorkersCreated) \
+ XX(DB_KQP_WORKERS_CLOSED_IDLE, WorkersClosedIdle) \
+ XX(DB_KQP_WORKERS_CLOSED_ERROR, WorkersClosedError) \
+ XX(DB_KQP_WORKERS_CLOSED_REQUEST, WorkersClosedRequest) \
+ XX(DB_KQP_TX_CREATED, TxCreated) \
+ XX(DB_KQP_TX_ABORTED, TxAborted) \
+ XX(DB_KQP_TX_COMMITED, TxCommited) \
+ XX(DB_KQP_TX_EVICTED, TxEvicted) \
+ XX(DB_KQP_COMPILE_QUERY_CACHE_HITS, CompileQueryCacheHits) \
+ XX(DB_KQP_COMPILE_QUERY_CACHE_MISSES, CompileQueryCacheMisses) \
+ XX(DB_KQP_COMPILE_REQ_COMPILE, CompileRequestsCompile) \
+ XX(DB_KQP_COMPILE_REQ_GET, CompileRequestsGet) \
+ XX(DB_KQP_COMPILE_REQ_INVALIDATE, CompileRequestsInvalidate) \
+ XX(DB_KQP_COMPILE_REQ_REJECTED, CompileRequestsRejected) \
+ XX(DB_KQP_COMPILE_REQ_TIMEOUT, CompileRequestsTimeout) \
+ XX(DB_KQP_COMPILE_TOTAL, CompileTotal) \
+ XX(DB_KQP_COMPILE_ERRORS, CompileErrors) \
+ XX(DB_KQP_DEPRECATED1, &DeprecatedCounter) \
+ XX(DB_KQP_YDB_WORKERS_CLOSED_IDLE, YdbWorkersClosedIdle) \
+ XX(DB_KQP_YDB_REQUEST_BYTES, YdbRequestBytes) \
+ XX(DB_KQP_YDB_PARAMS_BYTES, YdbParametersBytes) \
XX(DB_KQP_YDB_RSP_LOCKS_INVALIDATED, YdbResponsesLocksInvalidated) \
XX(DB_KQP_QUERIES_WITH_RANGE_SCAN, QueriesWithRangeScan) \
- XX(DB_KQP_QUERIES_WITH_FULL_SCAN, QueriesWithFullScan) \
- XX(DB_KQP_YDB_RESPONSE_BYTES, YdbResponseBytes)
-
-#define DB_KQP_HISTOGRAM_COUNTERS_MAP(XX) \
- XX(DB_KQP_QUERY_LATENCY_EXECUTE, QueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE]) \
- XX(DB_KQP_QUERY_LATENCY_EXPLAIN, QueryLatencies[NKikimrKqp::QUERY_ACTION_EXPLAIN]) \
- XX(DB_KQP_QUERY_LATENCY_VALIDATE, QueryLatencies[NKikimrKqp::QUERY_ACTION_VALIDATE]) \
- XX(DB_KQP_QUERY_LATENCY_PREPARE, QueryLatencies[NKikimrKqp::QUERY_ACTION_PREPARE]) \
- XX(DB_KQP_QUERY_LATENCY_EXECUTE_PREPARED, QueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED]) \
- XX(DB_KQP_QUERY_LATENCY_BEGIN_TX, QueryLatencies[NKikimrKqp::QUERY_ACTION_BEGIN_TX]) \
- XX(DB_KQP_QUERY_LATENCY_COMMIT_TX, QueryLatencies[NKikimrKqp::QUERY_ACTION_COMMIT_TX]) \
- XX(DB_KQP_QUERY_LATENCY_ROLLBACK_TX, QueryLatencies[NKikimrKqp::QUERY_ACTION_ROLLBACK_TX]) \
- XX(DB_KQP_WORKER_LIFE_SPAN, WorkerLifeSpan) \
- XX(DB_KQP_QUERIES_PER_WORKER, QueriesPerWorker) \
- XX(DB_KQP_WORKER_CLEANUP_LATENCY, WorkerCleanupLatency) \
- XX(DB_KQP_TX_ACTIVE_PER_SESSION, TxActivePerSession) \
- XX(DB_KQP_TX_ABORTED_PER_SESSION, TxAbortedPerSession) \
- XX(DB_KQP_PURE_TOTAL_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::Pure].TotalDuration) \
- XX(DB_KQP_PURE_SERVER_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::Pure].ServerDuration) \
- XX(DB_KQP_PURE_CLIENT_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::Pure].ClientDuration) \
- XX(DB_KQP_RO_TOTAL_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::ReadOnly].TotalDuration) \
- XX(DB_KQP_RO_SERVER_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::ReadOnly].ServerDuration) \
- XX(DB_KQP_RO_CLIENT_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::ReadOnly].ClientDuration) \
- XX(DB_KQP_WO_TOTAL_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::WriteOnly].TotalDuration) \
- XX(DB_KQP_WO_SERVER_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::WriteOnly].ServerDuration) \
- XX(DB_KQP_WO_CLIENT_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::WriteOnly].ClientDuration) \
- XX(DB_KQP_RW_TOTAL_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::ReadWrite].TotalDuration) \
- XX(DB_KQP_RW_SERVER_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::ReadWrite].ServerDuration) \
- XX(DB_KQP_RW_CLIENT_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::ReadWrite].ClientDuration) \
- XX(DB_KQP_YDB_COMPILE_DURATION, YdbCompileDuration) \
- XX(DB_KQP_COMPILE_CPU_TIME, CompileCpuTime) \
+ XX(DB_KQP_QUERIES_WITH_FULL_SCAN, QueriesWithFullScan) \
+ XX(DB_KQP_YDB_RESPONSE_BYTES, YdbResponseBytes)
+
+#define DB_KQP_HISTOGRAM_COUNTERS_MAP(XX) \
+ XX(DB_KQP_QUERY_LATENCY_EXECUTE, QueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE]) \
+ XX(DB_KQP_QUERY_LATENCY_EXPLAIN, QueryLatencies[NKikimrKqp::QUERY_ACTION_EXPLAIN]) \
+ XX(DB_KQP_QUERY_LATENCY_VALIDATE, QueryLatencies[NKikimrKqp::QUERY_ACTION_VALIDATE]) \
+ XX(DB_KQP_QUERY_LATENCY_PREPARE, QueryLatencies[NKikimrKqp::QUERY_ACTION_PREPARE]) \
+ XX(DB_KQP_QUERY_LATENCY_EXECUTE_PREPARED, QueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED]) \
+ XX(DB_KQP_QUERY_LATENCY_BEGIN_TX, QueryLatencies[NKikimrKqp::QUERY_ACTION_BEGIN_TX]) \
+ XX(DB_KQP_QUERY_LATENCY_COMMIT_TX, QueryLatencies[NKikimrKqp::QUERY_ACTION_COMMIT_TX]) \
+ XX(DB_KQP_QUERY_LATENCY_ROLLBACK_TX, QueryLatencies[NKikimrKqp::QUERY_ACTION_ROLLBACK_TX]) \
+ XX(DB_KQP_WORKER_LIFE_SPAN, WorkerLifeSpan) \
+ XX(DB_KQP_QUERIES_PER_WORKER, QueriesPerWorker) \
+ XX(DB_KQP_WORKER_CLEANUP_LATENCY, WorkerCleanupLatency) \
+ XX(DB_KQP_TX_ACTIVE_PER_SESSION, TxActivePerSession) \
+ XX(DB_KQP_TX_ABORTED_PER_SESSION, TxAbortedPerSession) \
+ XX(DB_KQP_PURE_TOTAL_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::Pure].TotalDuration) \
+ XX(DB_KQP_PURE_SERVER_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::Pure].ServerDuration) \
+ XX(DB_KQP_PURE_CLIENT_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::Pure].ClientDuration) \
+ XX(DB_KQP_RO_TOTAL_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::ReadOnly].TotalDuration) \
+ XX(DB_KQP_RO_SERVER_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::ReadOnly].ServerDuration) \
+ XX(DB_KQP_RO_CLIENT_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::ReadOnly].ClientDuration) \
+ XX(DB_KQP_WO_TOTAL_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::WriteOnly].TotalDuration) \
+ XX(DB_KQP_WO_SERVER_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::WriteOnly].ServerDuration) \
+ XX(DB_KQP_WO_CLIENT_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::WriteOnly].ClientDuration) \
+ XX(DB_KQP_RW_TOTAL_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::ReadWrite].TotalDuration) \
+ XX(DB_KQP_RW_SERVER_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::ReadWrite].ServerDuration) \
+ XX(DB_KQP_RW_CLIENT_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::ReadWrite].ClientDuration) \
+ XX(DB_KQP_YDB_COMPILE_DURATION, YdbCompileDuration) \
+ XX(DB_KQP_COMPILE_CPU_TIME, CompileCpuTime) \
XX(DB_KQP_YDB_QUERY_LATENCY_EXECUTE, YdbQueryExecuteLatency) \
XX(DB_KQP_QUERY_AFFECTED_SHARDS, QueryAffectedShardsCount) \
XX(DB_KQP_QUERY_READ_SETS, QueryReadSetsCount) \
@@ -122,5 +122,5 @@ namespace NKqp {
XX(DB_KQP_QUERY_READ_ROWS, QueryReadRows) \
XX(DB_KQP_QUERY_MAX_SHARD_REPLY, QueryMaxShardReplySize) \
XX(DB_KQP_QUERY_MAX_SHARD_PROGRAM, QueryMaxShardProgramSize)
-}
-}
+}
+}
diff --git a/ydb/core/kqp/counters/ya.make b/ydb/core/kqp/counters/ya.make
index 45f9d3f3c90..c4d9f4fe1e2 100644
--- a/ydb/core/kqp/counters/ya.make
+++ b/ydb/core/kqp/counters/ya.make
@@ -1,20 +1,20 @@
-LIBRARY()
-
-OWNER(
- spuchin
- g:kikimr
-)
-
-SRCS(
- kqp_counters.cpp
- kqp_counters.h
- kqp_db_counters.h
-)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(
+ spuchin
+ g:kikimr
+)
+
+SRCS(
+ kqp_counters.cpp
+ kqp_counters.h
+ kqp_db_counters.h
+)
+
+PEERDIR(
ydb/core/base
ydb/core/protos
ydb/core/sys_view/service
-)
-
-END()
+)
+
+END()
diff --git a/ydb/core/kqp/executer/kqp_scan_executer.cpp b/ydb/core/kqp/executer/kqp_scan_executer.cpp
index 07c29f64e21..8bbdbaea241 100644
--- a/ydb/core/kqp/executer/kqp_scan_executer.cpp
+++ b/ydb/core/kqp/executer/kqp_scan_executer.cpp
@@ -173,7 +173,7 @@ private:
void HandleExecute(TEvDqCompute::TEvState::TPtr& ev) {
TActorId computeActor = ev->Sender;
- auto& state = ev->Get()->Record;
+ auto& state = ev->Get()->Record;
ui64 taskId = state.GetTaskId();
LOG_D("Got execution state from compute actor: " << computeActor
@@ -978,8 +978,8 @@ private:
}
TBase::ReplyErrorAndDie(status, issues);
- }
-
+ }
+
void PassAway() override {
for (auto channelPair: ResultChannelProxies) {
LOG_D("terminate result channel " << channelPair.first << " proxy at " << channelPair.second->SelfId());
diff --git a/ydb/core/kqp/host/kqp_host.cpp b/ydb/core/kqp/host/kqp_host.cpp
index 44af6d11682..6cf3495bb89 100644
--- a/ydb/core/kqp/host/kqp_host.cpp
+++ b/ydb/core/kqp/host/kqp_host.cpp
@@ -217,13 +217,13 @@ public:
TVector<const TString> queryPlans;
for (auto id : SessionCtx->Query().ExecutionOrder) {
- auto result = SessionCtx->Query().Results.FindPtr(id);
+ auto result = SessionCtx->Query().Results.FindPtr(id);
if (result) {
queryPlans.push_back(SerializeAnalyzePlan(result->QueryStats));
AddQueryStats(queryResult.QueryStats, std::move(result->QueryStats));
- }
- }
-
+ }
+ }
+
FillAstAndPlan(queryResult, GetExprRoot().Get(), GetExprContext(), PlanBuilder);
queryResult.SqlVersion = SqlVersion;
queryResult.QueryPlan = SerializeScriptPlan(queryPlans);
@@ -291,8 +291,8 @@ public:
auto& execResult = ExecuteCtx->QueryResults[0];
queryResult.QueryStats.Swap(&execResult.QueryStats);
queryResult.QueryPlan = SerializeAnalyzePlan(queryResult.QueryStats);
- }
-
+ }
+
queryResult.SqlVersion = SqlVersion;
}
diff --git a/ydb/core/kqp/host/kqp_runner.cpp b/ydb/core/kqp/host/kqp_runner.cpp
index 42b4eca7c66..cb1347ed6fd 100644
--- a/ydb/core/kqp/host/kqp_runner.cpp
+++ b/ydb/core/kqp/host/kqp_runner.cpp
@@ -409,16 +409,16 @@ private:
// OldEngine only
YQL_ENSURE(!engine.has_value() || *engine == TKqpTransactionInfo::EEngine::OldEngine);
- for (const auto& [name, table] : TransformCtx->Tables->GetTables()) {
- if (!table.Metadata->SysView.empty()) {
+ for (const auto& [name, table] : TransformCtx->Tables->GetTables()) {
+ if (!table.Metadata->SysView.empty()) {
ctx.AddError(TIssue(ctx.GetPosition(dataQuery.Pos()), TStringBuilder()
- << "Table " << table.Metadata->Name << " is a system view. "
- << "System views are not supported by data queries."
- ));
+ << "Table " << table.Metadata->Name << " is a system view. "
+ << "System views are not supported by data queries."
+ ));
return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues()));
- }
- }
-
+ }
+ }
+
auto program = BuildKiProgram(dataQuery, *TransformCtx->Tables, ctx, sysColumnsEnabled);
KqlOptimizeTransformer->Rewind();
diff --git a/ydb/core/kqp/kqp.h b/ydb/core/kqp/kqp.h
index 09f794591d3..4f183d2f33a 100644
--- a/ydb/core/kqp/kqp.h
+++ b/ydb/core/kqp/kqp.h
@@ -368,9 +368,9 @@ struct TEvKqp {
: UserToken(userToken)
, Uid(uid)
, Query(std::move(query))
- , KeepInCache(keepInCache)
+ , KeepInCache(keepInCache)
, Deadline(deadline)
- , DbCounters(dbCounters)
+ , DbCounters(dbCounters)
{
Y_ENSURE(Uid.Defined() != Query.Defined());
}
@@ -381,7 +381,7 @@ struct TEvKqp {
bool KeepInCache = false;
// it is allowed for local event to use absolute time (TInstant) instead of time interval (TDuration)
TInstant Deadline;
- TKqpDbCountersPtr DbCounters;
+ TKqpDbCountersPtr DbCounters;
TMaybe<bool> DocumentApiRestricted;
};
@@ -392,14 +392,14 @@ struct TEvKqp {
, Uid(uid)
, Query(query)
, Deadline(deadline)
- , DbCounters(dbCounters) {}
+ , DbCounters(dbCounters) {}
TString UserToken;
TString Uid;
TMaybe<TKqpQueryId> Query;
TInstant Deadline;
- TKqpDbCountersPtr DbCounters;
+ TKqpDbCountersPtr DbCounters;
};
struct TEvCompileResponse : public TEventLocal<TEvCompileResponse, TKqpEvents::EvCompileResponse> {
@@ -417,12 +417,12 @@ struct TEvKqp {
struct TEvCompileInvalidateRequest : public TEventLocal<TEvCompileInvalidateRequest,
TKqpEvents::EvCompileInvalidateRequest>
{
- TEvCompileInvalidateRequest(const TString& uid, TKqpDbCountersPtr dbCounters)
- : Uid(uid)
- , DbCounters(dbCounters) {}
+ TEvCompileInvalidateRequest(const TString& uid, TKqpDbCountersPtr dbCounters)
+ : Uid(uid)
+ , DbCounters(dbCounters) {}
TString Uid;
- TKqpDbCountersPtr DbCounters;
+ TKqpDbCountersPtr DbCounters;
};
struct TEvInitiateShutdownRequest : public TEventLocal<TEvInitiateShutdownRequest, TKqpEvents::EvInitiateShutdownRequest> {
diff --git a/ydb/core/kqp/kqp_compile_actor.cpp b/ydb/core/kqp/kqp_compile_actor.cpp
index 27cf296a03d..ecb2ad49e56 100644
--- a/ydb/core/kqp/kqp_compile_actor.cpp
+++ b/ydb/core/kqp/kqp_compile_actor.cpp
@@ -45,7 +45,7 @@ public:
TKqpCompileActor(const TActorId& owner, const TKqpSettings::TConstPtr& kqpSettings,
const TTableServiceConfig& serviceConfig, TIntrusivePtr<TModuleResolverState> moduleResolverState,
- TIntrusivePtr<TKqpCounters> counters, const TString& uid, const TKqpQueryId& query, const TString& userToken,
+ TIntrusivePtr<TKqpCounters> counters, const TString& uid, const TKqpQueryId& query, const TString& userToken,
TKqpDbCountersPtr dbCounters, bool recompileWithNewEngine)
: Owner(owner)
, ModuleResolverState(moduleResolverState)
@@ -53,7 +53,7 @@ public:
, Uid(uid)
, Query(query)
, UserToken(userToken)
- , DbCounters(dbCounters)
+ , DbCounters(dbCounters)
, Config(MakeIntrusive<TKikimrConfiguration>())
, CompilationTimeout(TDuration::MilliSeconds(serviceConfig.GetCompileTimeoutMs()))
, RecompileWithNewEngine(recompileWithNewEngine)
@@ -72,8 +72,8 @@ public:
void Bootstrap(const TActorContext& ctx) {
StartTime = TInstant::Now();
- Counters->ReportCompileStart(DbCounters);
-
+ Counters->ReportCompileStart(DbCounters);
+
LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_ACTOR, "Start compilation"
<< ", self: " << ctx.SelfID
<< ", cluster: " << Query.Cluster
@@ -211,7 +211,7 @@ private:
stats.SetCpuTimeUs(CompileCpuTime.MicroSeconds());
ctx.Send(Owner, responseEv.Release());
- Counters->ReportCompileFinish(DbCounters);
+ Counters->ReportCompileFinish(DbCounters);
Die(ctx);
}
@@ -251,9 +251,9 @@ private:
auto kqpResult = std::move(AsyncCompileResult->GetResult());
auto status = GetYdbStatus(kqpResult);
- auto database = Query.Database;
+ auto database = Query.Database;
if (kqpResult.SqlVersion) {
- Counters->ReportSqlVersion(DbCounters, *kqpResult.SqlVersion);
+ Counters->ReportSqlVersion(DbCounters, *kqpResult.SqlVersion);
}
if (status == Ydb::StatusIds::SUCCESS) {
@@ -269,7 +269,7 @@ private:
auto now = TInstant::Now();
auto duration = now - StartTime;
- Counters->ReportCompileDurations(DbCounters, duration, CompileCpuTime);
+ Counters->ReportCompileDurations(DbCounters, duration, CompileCpuTime);
LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_ACTOR, "Compilation successful"
<< ", self: " << ctx.SelfID
@@ -301,7 +301,7 @@ private:
<< ", self: " << ctx.SelfID
<< ", status: " << Ydb::StatusIds_StatusCode_Name(status)
<< ", issues: " << kqpResult.Issues().ToString());
- Counters->ReportCompileError(DbCounters);
+ Counters->ReportCompileError(DbCounters);
}
Reply(KqpCompileResult, ctx);
@@ -338,13 +338,13 @@ private:
KqpCompileResult->PreparedQueryNewEngine.reset(kqpResult.PreparingQuery.release());
auto duration = TInstant::Now() - RecompileStartTime;
- Counters->ReportCompileDurations(DbCounters, duration, CompileCpuTime);
+ Counters->ReportCompileDurations(DbCounters, duration, CompileCpuTime);
LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_ACTOR, "ReCompilation successful"
<< ", self: " << ctx.SelfID
<< ", duration: " << duration);
} else {
- Counters->ReportCompileError(DbCounters);
+ Counters->ReportCompileError(DbCounters);
Counters->ForceNewEngineCompileErrors->Inc();
LOG_ERROR_S(ctx, NKikimrServices::KQP_COMPILE_ACTOR, "ReCompilation failed"
@@ -375,7 +375,7 @@ private:
TString Uid;
TKqpQueryId Query;
TString UserToken;
- TKqpDbCountersPtr DbCounters;
+ TKqpDbCountersPtr DbCounters;
TKikimrConfiguration::TPtr Config;
TDuration CompilationTimeout;
bool RecompileWithNewEngine;
@@ -402,7 +402,7 @@ void ApplyServiceConfig(TKikimrConfiguration& kqpConfig, const TTableServiceConf
IActor* CreateKqpCompileActor(const TActorId& owner, const TKqpSettings::TConstPtr& kqpSettings,
const TTableServiceConfig& serviceConfig, TIntrusivePtr<TModuleResolverState> moduleResolverState,
- TIntrusivePtr<TKqpCounters> counters, const TString& uid, const TKqpQueryId& query, const TString& userToken,
+ TIntrusivePtr<TKqpCounters> counters, const TString& uid, const TKqpQueryId& query, const TString& userToken,
TKqpDbCountersPtr dbCounters, bool recompileWithNewEngine)
{
return new TKqpCompileActor(owner, kqpSettings, serviceConfig, moduleResolverState, counters, uid,
diff --git a/ydb/core/kqp/kqp_compile_request.cpp b/ydb/core/kqp/kqp_compile_request.cpp
index 2c18af159bc..e82d5b890c0 100644
--- a/ydb/core/kqp/kqp_compile_request.cpp
+++ b/ydb/core/kqp/kqp_compile_request.cpp
@@ -24,14 +24,14 @@ public:
}
TKqpCompileRequestActor(const TActorId& owner, const TString& userToken, const TMaybe<TString>& uid,
- TMaybe<TKqpQueryId>&& query, bool keepInCache, const TInstant& deadline, TKqpDbCountersPtr dbCounters)
+ TMaybe<TKqpQueryId>&& query, bool keepInCache, const TInstant& deadline, TKqpDbCountersPtr dbCounters)
: Owner(owner)
, UserToken(userToken)
, Uid(uid)
, Query(std::move(query))
, KeepInCache(keepInCache)
- , Deadline(deadline)
- , DbCounters(dbCounters) {}
+ , Deadline(deadline)
+ , DbCounters(dbCounters) {}
void Bootstrap(const TActorContext& ctx) {
TimeoutTimerId = CreateLongTimer(ctx, Deadline - TInstant::Now(),
@@ -41,7 +41,7 @@ public:
std::swap(Query, query);
auto compileEv = MakeHolder<TEvKqp::TEvCompileRequest>(UserToken, Uid, std::move(query),
- KeepInCache, Deadline, DbCounters);
+ KeepInCache, Deadline, DbCounters);
ctx.Send(MakeKqpCompileServiceID(ctx.SelfID.NodeId()), compileEv.Release());
Become(&TKqpCompileRequestActor::MainState);
@@ -80,7 +80,7 @@ public:
<< ", queryUid: " << compileResult.Uid);
auto recompileEv = MakeHolder<TEvKqp::TEvRecompileRequest>(UserToken, compileResult.Uid, compileResult.Query,
- Deadline, DbCounters);
+ Deadline, DbCounters);
ctx.Send(MakeKqpCompileServiceID(ctx.SelfID.NodeId()), recompileEv.Release());
DeferredResponse.Reset();
@@ -287,7 +287,7 @@ private:
TMaybe<TKqpQueryId> Query;
bool KeepInCache = false;
TInstant Deadline;
- TKqpDbCountersPtr DbCounters;
+ TKqpDbCountersPtr DbCounters;
TActorId TimeoutTimerId;
THashMap<TTableId, ui64> TableVersions;
THolder<TEvKqp::TEvCompileResponse> DeferredResponse;
@@ -295,9 +295,9 @@ private:
IActor* CreateKqpCompileRequestActor(const TActorId& owner, const TString& userToken, const TMaybe<TString>& uid,
- TMaybe<TKqpQueryId>&& query, bool keepInCache, const TInstant& deadline, TKqpDbCountersPtr dbCounters)
+ TMaybe<TKqpQueryId>&& query, bool keepInCache, const TInstant& deadline, TKqpDbCountersPtr dbCounters)
{
- return new TKqpCompileRequestActor(owner, userToken, uid, std::move(query), keepInCache, deadline, dbCounters);
+ return new TKqpCompileRequestActor(owner, userToken, uid, std::move(query), keepInCache, deadline, dbCounters);
}
} // namespace NKqp
diff --git a/ydb/core/kqp/kqp_compile_service.cpp b/ydb/core/kqp/kqp_compile_service.cpp
index da886224a2a..49114bf8f85 100644
--- a/ydb/core/kqp/kqp_compile_service.cpp
+++ b/ydb/core/kqp/kqp_compile_service.cpp
@@ -10,7 +10,7 @@
#include <library/cpp/actors/core/actor_bootstrapped.h>
#include <library/cpp/actors/core/hfunc.h>
-#include <library/cpp/cache/cache.h>
+#include <library/cpp/cache/cache.h>
#include <util/string/escape.h>
@@ -180,14 +180,14 @@ private:
struct TKqpCompileRequest {
TKqpCompileRequest(const TActorId& sender, const TString& uid, TKqpQueryId query, bool keepInCache,
- const TString& userToken, const TInstant& deadline, TKqpDbCountersPtr dbCounters)
+ const TString& userToken, const TInstant& deadline, TKqpDbCountersPtr dbCounters)
: Sender(sender)
, Query(std::move(query))
, Uid(uid)
, KeepInCache(keepInCache)
, UserToken(userToken)
- , Deadline(deadline)
- , DbCounters(dbCounters) {}
+ , Deadline(deadline)
+ , DbCounters(dbCounters) {}
TActorId Sender;
TKqpQueryId Query;
@@ -195,7 +195,7 @@ struct TKqpCompileRequest {
bool KeepInCache = false;
TString UserToken;
TInstant Deadline;
- TKqpDbCountersPtr DbCounters;
+ TKqpDbCountersPtr DbCounters;
TActorId CompileActor;
};
@@ -417,16 +417,16 @@ private:
*Counters->CompileQueryCacheBytes = QueryCache.Bytes();
auto userSid = NACLib::TUserToken(request.UserToken).GetUserSID();
- auto dbCounters = request.DbCounters;
+ auto dbCounters = request.DbCounters;
if (request.Uid) {
- Counters->ReportCompileRequestGet(dbCounters);
+ Counters->ReportCompileRequestGet(dbCounters);
auto compileResult = QueryCache.FindByUid(*request.Uid, request.KeepInCache);
if (compileResult) {
Y_ENSURE(compileResult->Query);
if (compileResult->Query->UserSid == userSid) {
- Counters->ReportQueryCacheHit(dbCounters, true);
+ Counters->ReportQueryCacheHit(dbCounters, true);
LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Served query from cache by uid"
<< ", sender: " << ev->Sender
@@ -444,7 +444,7 @@ private:
}
}
- Counters->ReportQueryCacheHit(dbCounters, false);
+ Counters->ReportQueryCacheHit(dbCounters, false);
LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Query not found"
<< ", sender: " << ev->Sender
@@ -455,7 +455,7 @@ private:
return;
}
- Counters->ReportCompileRequestCompile(dbCounters);
+ Counters->ReportCompileRequestCompile(dbCounters);
Y_ENSURE(request.Query);
auto& query = *request.Query;
@@ -468,7 +468,7 @@ private:
auto compileResult = QueryCache.FindByQuery(query, request.KeepInCache);
if (compileResult) {
- Counters->ReportQueryCacheHit(dbCounters, true);
+ Counters->ReportQueryCacheHit(dbCounters, true);
LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Served query from cache"
<< ", sender: " << ev->Sender
@@ -478,13 +478,13 @@ private:
return;
}
- Counters->ReportQueryCacheHit(dbCounters, false);
+ Counters->ReportQueryCacheHit(dbCounters, false);
TKqpCompileRequest compileRequest(ev->Sender, CreateGuidAsString(), std::move(*request.Query),
- request.KeepInCache, request.UserToken, request.Deadline, dbCounters);
+ request.KeepInCache, request.UserToken, request.Deadline, dbCounters);
if (!RequestsQueue.Enqueue(std::move(compileRequest))) {
- Counters->ReportCompileRequestRejected(dbCounters);
+ Counters->ReportCompileRequestRejected(dbCounters);
LOG_WARN_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Requests queue size limit exceeded"
<< ", sender: " << ev->Sender
@@ -519,18 +519,18 @@ private:
LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Received recompile request"
<< ", sender: " << ev->Sender);
- auto dbCounters = request.DbCounters;
- Counters->ReportRecompileRequestGet(dbCounters);
+ auto dbCounters = request.DbCounters;
+ Counters->ReportRecompileRequestGet(dbCounters);
auto compileResult = QueryCache.FindByUid(request.Uid, false);
if (compileResult || request.Query) {
- Counters->ReportCompileRequestCompile(dbCounters);
+ Counters->ReportCompileRequestCompile(dbCounters);
TKqpCompileRequest compileRequest(ev->Sender, request.Uid, compileResult ? *compileResult->Query : *request.Query,
- true, request.UserToken, request.Deadline, dbCounters);
+ true, request.UserToken, request.Deadline, dbCounters);
if (!RequestsQueue.Enqueue(std::move(compileRequest))) {
- Counters->ReportCompileRequestRejected(dbCounters);
+ Counters->ReportCompileRequestRejected(dbCounters);
LOG_WARN_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Requests queue size limit exceeded"
<< ", sender: " << ev->Sender
@@ -623,10 +623,10 @@ private:
LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Received invalidate request"
<< ", sender: " << ev->Sender
- << ", queryUid: " << request.Uid);
+ << ", queryUid: " << request.Uid);
- auto dbCounters = request.DbCounters;
- Counters->ReportCompileRequestInvalidate(dbCounters);
+ auto dbCounters = request.DbCounters;
+ Counters->ReportCompileRequestInvalidate(dbCounters);
QueryCache.EraseByUid(request.Uid);
}
@@ -657,7 +657,7 @@ private:
<< ", sender: " << request->Sender
<< ", deadline: " << request->Deadline);
- Counters->ReportCompileRequestTimeout(request->DbCounters);
+ Counters->ReportCompileRequestTimeout(request->DbCounters);
NYql::TIssue issue(NYql::TPosition(), "Compilation timed out.");
ReplyError(request->Sender, "", Ydb::StatusIds::TIMEOUT, {issue}, ctx);
diff --git a/ydb/core/kqp/kqp_ic_gateway.cpp b/ydb/core/kqp/kqp_ic_gateway.cpp
index 47489a9bd54..8013638d338 100644
--- a/ydb/core/kqp/kqp_ic_gateway.cpp
+++ b/ydb/core/kqp/kqp_ic_gateway.cpp
@@ -1007,10 +1007,10 @@ public:
}
TMaybe<NYql::TKikimrClusterConfig> GetClusterConfig(const TString& cluster) override {
- Y_UNUSED(cluster);
- return {};
- }
-
+ Y_UNUSED(cluster);
+ return {};
+ }
+
TMaybe<TString> GetSetting(const TString& cluster, const TString& name) override {
Y_UNUSED(cluster);
Y_UNUSED(name);
diff --git a/ydb/core/kqp/kqp_impl.h b/ydb/core/kqp/kqp_impl.h
index 7d6f172899b..d9f81fe65e2 100644
--- a/ydb/core/kqp/kqp_impl.h
+++ b/ydb/core/kqp/kqp_impl.h
@@ -51,11 +51,11 @@ IActor* CreateKqpCompileService(const NKikimrConfig::TTableServiceConfig& servic
IActor* CreateKqpCompileActor(const TActorId& owner, const TKqpSettings::TConstPtr& kqpSettings,
const NKikimrConfig::TTableServiceConfig& serviceConfig, TIntrusivePtr<TModuleResolverState> moduleResolverState,
- TIntrusivePtr<TKqpCounters> counters, const TString& uid, const TKqpQueryId& query, const TString& userToken,
+ TIntrusivePtr<TKqpCounters> counters, const TString& uid, const TKqpQueryId& query, const TString& userToken,
TKqpDbCountersPtr dbCounters, bool recompileWithNewEngine);
IActor* CreateKqpCompileRequestActor(const TActorId& owner, const TString& userToken, const TMaybe<TString>& uid,
- TMaybe<TKqpQueryId>&& query, bool keepInCache, const TInstant& deadline, TKqpDbCountersPtr dbCounters);
+ TMaybe<TKqpQueryId>&& query, bool keepInCache, const TInstant& deadline, TKqpDbCountersPtr dbCounters);
struct TKqpWorkerSettings {
TString Cluster;
@@ -64,14 +64,14 @@ struct TKqpWorkerSettings {
NKikimrConfig::TTableServiceConfig Service;
- TKqpDbCountersPtr DbCounters;
-
+ TKqpDbCountersPtr DbCounters;
+
TKqpWorkerSettings(const TString& cluster, const TString& database,
- const NKikimrConfig::TTableServiceConfig& serviceConfig, TKqpDbCountersPtr dbCounters)
+ const NKikimrConfig::TTableServiceConfig& serviceConfig, TKqpDbCountersPtr dbCounters)
: Cluster(cluster)
, Database(database)
- , Service(serviceConfig)
- , DbCounters(dbCounters) {}
+ , Service(serviceConfig)
+ , DbCounters(dbCounters) {}
};
IActor* CreateKqpWorkerActor(const TActorId& owner, const TString& sessionId,
diff --git a/ydb/core/kqp/kqp_worker_actor.cpp b/ydb/core/kqp/kqp_worker_actor.cpp
index cccb8a51e8e..efdc2ebb1b2 100644
--- a/ydb/core/kqp/kqp_worker_actor.cpp
+++ b/ydb/core/kqp/kqp_worker_actor.cpp
@@ -179,7 +179,7 @@ public:
void Bootstrap(const TActorContext& ctx) {
LOG_DEBUG_S(ctx, NKikimrServices::KQP_WORKER, "Worker bootstrapped, workerId: " << ctx.SelfID);
- Counters->ReportWorkerCreated(Settings.DbCounters);
+ Counters->ReportWorkerCreated(Settings.DbCounters);
std::shared_ptr<NYql::IKikimrGateway::IKqpTableMetadataLoader> loader = std::make_shared<TKqpTableMetadataLoader>(TlsActivationContext->ActorSystem(), false);
Gateway = CreateKikimrIcGateway(Settings.Cluster, Settings.Database, std::move(loader),
@@ -200,7 +200,7 @@ public:
auto requestInfo = TKqpRequestInfo(event.GetTraceId(), event.GetRequest().GetSessionId());
if (CheckRequest(requestInfo, ev->Sender, proxyRequestId, ctx)) {
LOG_INFO_S(ctx, NKikimrServices::KQP_WORKER, requestInfo << "Session closed due to explicit close event");
- Counters->ReportWorkerClosedRequest(Settings.DbCounters);
+ Counters->ReportWorkerClosedRequest(Settings.DbCounters);
FinalCleanup(ctx);
}
}
@@ -448,7 +448,7 @@ public:
if (timerId == IdleTimerId) {
LOG_NOTICE_S(ctx, NKikimrServices::KQP_WORKER, TKqpRequestInfo("", SessionId)
<< "Worker idle timeout, worker destroyed");
- Counters->ReportWorkerClosedIdle(Settings.DbCounters);
+ Counters->ReportWorkerClosedIdle(Settings.DbCounters);
FinalCleanup(ctx);
}
}
@@ -527,7 +527,7 @@ public:
TKqpRequestInfo(QueryState->TraceId, SessionId), Ydb::StatusIds::BAD_SESSION,
"Session is being closed", ctx);
- Counters->ReportWorkerClosedError(Settings.DbCounters);
+ Counters->ReportWorkerClosedError(Settings.DbCounters);
FinalCleanup(ctx);
}
@@ -876,7 +876,7 @@ private:
}
auto compileRequestActor = CreateKqpCompileRequestActor(ctx.SelfID, QueryState->UserToken, uid,
- std::move(query), keepInCache, compileDeadline, Settings.DbCounters);
+ std::move(query), keepInCache, compileDeadline, Settings.DbCounters);
ctx.ExecutorThread.RegisterActor(compileRequestActor);
Become(&TKqpWorkerActor::CompileQueryState);
@@ -907,7 +907,7 @@ private:
if (Settings.LongSession) {
QueryState.Reset();
} else {
- Counters->ReportWorkerClosedError(Settings.DbCounters);
+ Counters->ReportWorkerClosedError(Settings.DbCounters);
FinalCleanup(ctx);
}
};
@@ -931,11 +931,11 @@ private:
case Ydb::Table::TransactionControl::kBeginTx: {
beginTxResult = BeginTransaction(txControl.begin_tx());
- Counters->ReportBeginTransaction(Settings.DbCounters,
- beginTxResult.EvictedTx,
- beginTxResult.CurrentActiveTx,
- beginTxResult.CurrentAbortedTx);
-
+ Counters->ReportBeginTransaction(Settings.DbCounters,
+ beginTxResult.EvictedTx,
+ beginTxResult.CurrentActiveTx,
+ beginTxResult.CurrentAbortedTx);
+
if (!beginTxResult.Success()) {
QueryState->AsyncQueryResult = MakeKikimrResultHolder(
NCommon::ResultFromErrors<TQueryResult>(beginTxResult.Issues()));
@@ -944,7 +944,7 @@ private:
return;
}
- Counters->ReportTxCreated(Settings.DbCounters);
+ Counters->ReportTxCreated(Settings.DbCounters);
QueryState->TxId = beginTxResult.TxId;
if (commit) {
@@ -1206,7 +1206,7 @@ private:
if (isFinal) {
StopIdleTimer(ctx);
- Counters->ReportQueriesPerWorker(Settings.DbCounters, QueryId);
+ Counters->ReportQueriesPerWorker(Settings.DbCounters, QueryId);
MakeNewQueryState();
}
@@ -1214,7 +1214,7 @@ private:
if (Settings.LongSession) {
if (isFinal) {
auto abortedCount = KqpHost->AbortAll();
- Counters->ReportTxAborted(Settings.DbCounters, abortedCount);
+ Counters->ReportTxAborted(Settings.DbCounters, abortedCount);
}
CleanupState->AsyncResult = KqpHost->RollbackAborted();
} else {
@@ -1234,16 +1234,16 @@ private:
Y_VERIFY(CleanupState);
if (CleanupState->AsyncResult) {
- auto cleanupTime = TInstant::Now() - CleanupState->Start;
- Counters->ReportWorkerCleanupLatency(Settings.DbCounters, cleanupTime);
+ auto cleanupTime = TInstant::Now() - CleanupState->Start;
+ Counters->ReportWorkerCleanupLatency(Settings.DbCounters, cleanupTime);
}
bool isFinal = CleanupState->Final;
CleanupState.Reset();
if (isFinal) {
- auto lifeSpan = TInstant::Now() - CreationTime;
- Counters->ReportWorkerFinished(Settings.DbCounters, lifeSpan);
+ auto lifeSpan = TInstant::Now() - CreationTime;
+ Counters->ReportWorkerFinished(Settings.DbCounters, lifeSpan);
auto closeEv = MakeHolder<TEvKqp::TEvCloseSessionResponse>();
closeEv->Record.SetStatus(Ydb::StatusIds::SUCCESS);
@@ -1523,20 +1523,20 @@ private:
if (status == Ydb::StatusIds::INTERNAL_ERROR) {
LOG_DEBUG_S(ctx, NKikimrServices::KQP_WORKER, requestInfo
<< "Worker destroyed due to internal error");
- Counters->ReportWorkerClosedError(Settings.DbCounters);
+ Counters->ReportWorkerClosedError(Settings.DbCounters);
return false;
}
if (status == Ydb::StatusIds::BAD_SESSION) {
LOG_DEBUG_S(ctx, NKikimrServices::KQP_WORKER, requestInfo
<< "Worker destroyed due to session error");
- Counters->ReportWorkerClosedError(Settings.DbCounters);
+ Counters->ReportWorkerClosedError(Settings.DbCounters);
return false;
}
} else {
if (status != Ydb::StatusIds::SUCCESS) {
LOG_DEBUG_S(ctx, NKikimrServices::KQP_WORKER, requestInfo
<< "Worker destroyed due to query error");
- Counters->ReportWorkerClosedError(Settings.DbCounters);
+ Counters->ReportWorkerClosedError(Settings.DbCounters);
return false;
}
}
@@ -1544,7 +1544,7 @@ private:
if (!keepSession) {
LOG_DEBUG_S(ctx, NKikimrServices::KQP_WORKER, requestInfo
<< "Worker destroyed due to negative keep session flag");
- Counters->ReportWorkerClosedRequest(Settings.DbCounters);
+ Counters->ReportWorkerClosedRequest(Settings.DbCounters);
return false;
}
@@ -1616,17 +1616,17 @@ private:
auto& record = responseEv->Record.GetRef();
auto status = record.GetYdbStatus();
- auto now = TInstant::Now();
- auto queryDuration = now - QueryState->StartTime;
+ auto now = TInstant::Now();
+ auto queryDuration = now - QueryState->StartTime;
if (status == Ydb::StatusIds::SUCCESS) {
- Counters->ReportQueryLatency(Settings.DbCounters, queryRequest.GetAction(), queryDuration);
+ Counters->ReportQueryLatency(Settings.DbCounters, queryRequest.GetAction(), queryDuration);
auto maxReadType = ExtractMostHeavyReadType(queryResult.QueryPlan);
if (maxReadType == ETableReadType::FullScan) {
- Counters->ReportQueryWithFullScan(Settings.DbCounters);
+ Counters->ReportQueryWithFullScan(Settings.DbCounters);
} else if (maxReadType == ETableReadType::Scan) {
- Counters->ReportQueryWithRangeScan(Settings.DbCounters);
+ Counters->ReportQueryWithRangeScan(Settings.DbCounters);
}
ui32 affectedShardsCount = 0;
@@ -1640,12 +1640,12 @@ private:
}
}
- Counters->ReportQueryAffectedShards(Settings.DbCounters, affectedShardsCount);
- Counters->ReportQueryReadRows(Settings.DbCounters, readRowsCount);
- Counters->ReportQueryReadBytes(Settings.DbCounters, readBytesCount);
- Counters->ReportQueryReadSets(Settings.DbCounters, queryResult.QueryStats.GetReadSetsCount());
- Counters->ReportQueryMaxShardReplySize(Settings.DbCounters, queryResult.QueryStats.GetMaxShardReplySize());
- Counters->ReportQueryMaxShardProgramSize(Settings.DbCounters, queryResult.QueryStats.GetMaxShardProgramSize());
+ Counters->ReportQueryAffectedShards(Settings.DbCounters, affectedShardsCount);
+ Counters->ReportQueryReadRows(Settings.DbCounters, readRowsCount);
+ Counters->ReportQueryReadBytes(Settings.DbCounters, readBytesCount);
+ Counters->ReportQueryReadSets(Settings.DbCounters, queryResult.QueryStats.GetReadSetsCount());
+ Counters->ReportQueryMaxShardReplySize(Settings.DbCounters, queryResult.QueryStats.GetMaxShardReplySize());
+ Counters->ReportQueryMaxShardProgramSize(Settings.DbCounters, queryResult.QueryStats.GetMaxShardProgramSize());
if (QueryState->QueryCompileResult && QueryState->QueryCompileResult->PreparedQueryNewEngine
&& QueryState->NewEngineCompatibleQuery)
@@ -1665,7 +1665,7 @@ private:
}
if (queryResult.SqlVersion) {
- Counters->ReportSqlVersion(Settings.DbCounters, *queryResult.SqlVersion);
+ Counters->ReportSqlVersion(Settings.DbCounters, *queryResult.SqlVersion);
}
if (Settings.LongSession && status != Ydb::StatusIds::SUCCESS) {
@@ -1712,7 +1712,7 @@ private:
if (IsExecuteAction(queryRequest.GetAction())) {
auto ru = CalcRequestUnit(stats);
record.SetConsumedRu(ru);
- CollectSystemViewQueryStats(ctx, &stats, queryDuration, queryRequest.GetDatabase(), ru);
+ CollectSystemViewQueryStats(ctx, &stats, queryDuration, queryRequest.GetDatabase(), ru);
SlowLogQuery(ctx, requestInfo, queryDuration, status, [&record](){
ui64 resultsSize = 0;
for (auto& result : record.GetResponse().GetResults()) {
@@ -1796,38 +1796,38 @@ private:
"Pending previous query completion", ctx);
}
- void CollectSystemViewQueryStats(const TActorContext& ctx,
- const NKqpProto::TKqpStatsQuery* stats, TDuration queryDuration,
- const TString& database, ui64 requestUnits)
- {
- auto type = QueryState->Request.GetType();
- switch (type) {
- case NKikimrKqp::QUERY_TYPE_SQL_DML:
- case NKikimrKqp::QUERY_TYPE_PREPARED_DML:
- case NKikimrKqp::QUERY_TYPE_SQL_SCAN:
+ void CollectSystemViewQueryStats(const TActorContext& ctx,
+ const NKqpProto::TKqpStatsQuery* stats, TDuration queryDuration,
+ const TString& database, ui64 requestUnits)
+ {
+ auto type = QueryState->Request.GetType();
+ switch (type) {
+ case NKikimrKqp::QUERY_TYPE_SQL_DML:
+ case NKikimrKqp::QUERY_TYPE_PREPARED_DML:
+ case NKikimrKqp::QUERY_TYPE_SQL_SCAN:
case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT:
case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT_STREAMING: {
- auto userSID = NACLib::TUserToken(QueryState->UserToken).GetUserSID();
- NSysView::CollectQueryStats(ctx, stats, queryDuration, ExtractQueryText(),
- userSID, QueryState->ParametersSize, database, type, requestUnits);
+ auto userSID = NACLib::TUserToken(QueryState->UserToken).GetUserSID();
+ NSysView::CollectQueryStats(ctx, stats, queryDuration, ExtractQueryText(),
+ userSID, QueryState->ParametersSize, database, type, requestUnits);
break;
- }
- default:
- break;
- }
- }
-
- TString ExtractQueryText() const {
- auto compileResult = QueryState->QueryCompileResult;
- if (compileResult) {
- if (compileResult->Query) {
- return compileResult->Query->Text;
- }
- return {};
- }
- return QueryState->Request.GetQuery();
- }
-
+ }
+ default:
+ break;
+ }
+ }
+
+ TString ExtractQueryText() const {
+ auto compileResult = QueryState->QueryCompileResult;
+ if (compileResult) {
+ if (compileResult->Query) {
+ return compileResult->Query->Text;
+ }
+ return {};
+ }
+ return QueryState->Request.GetQuery();
+ }
+
void SlowLogQuery(const TActorContext &ctx, const TKqpRequestInfo& requestInfo, const TDuration& duration,
Ydb::StatusIds::StatusCode status, const std::function<ui64()>& resultsSizeFunc)
{
@@ -1858,7 +1858,7 @@ private:
username = "UNAUTHENTICATED";
}
- auto queryText = ExtractQueryText();
+ auto queryText = ExtractQueryText();
auto paramsText = TStringBuilder()
<< ToString(QueryState->ParametersSize)
@@ -2037,7 +2037,7 @@ private:
TString replyTxId;
if (txInfo) {
- Counters->ReportTransaction(Settings.DbCounters, *txInfo);
+ Counters->ReportTransaction(Settings.DbCounters, *txInfo);
switch (txInfo->Status) {
case TKqpTransactionInfo::EStatus::Active:
@@ -2057,7 +2057,7 @@ private:
}
TString InvalidateQuery(const TKqpCompileResult& compileResult, const TActorContext& ctx) {
- auto invalidateEv = MakeHolder<TEvKqp::TEvCompileInvalidateRequest>(compileResult.Uid, Settings.DbCounters);
+ auto invalidateEv = MakeHolder<TEvKqp::TEvCompileInvalidateRequest>(compileResult.Uid, Settings.DbCounters);
ctx.Send(MakeKqpCompileServiceID(ctx.SelfID.NodeId()), invalidateEv.Release());
return compileResult.Uid;
diff --git a/ydb/core/kqp/node/kqp_node.cpp b/ydb/core/kqp/node/kqp_node.cpp
index 24f8f2ac79a..7acf87f65f3 100644
--- a/ydb/core/kqp/node/kqp_node.cpp
+++ b/ydb/core/kqp/node/kqp_node.cpp
@@ -16,7 +16,7 @@
#include <library/cpp/actors/core/actor_bootstrapped.h>
#include <library/cpp/monlib/service/pages/templates.h>
-#include <util/string/join.h>
+#include <util/string/join.h>
namespace NKikimr {
namespace NKqp {
diff --git a/ydb/core/kqp/prepare/kqp_prepare.h b/ydb/core/kqp/prepare/kqp_prepare.h
index d114a449458..295f55ecbbd 100644
--- a/ydb/core/kqp/prepare/kqp_prepare.h
+++ b/ydb/core/kqp/prepare/kqp_prepare.h
@@ -127,7 +127,7 @@ struct TKqlTransformContext : TThrRefBase {
ReplyTarget = {};
AnalyzeResults = {};
MkqlResults.clear();
- QueryStats = {};
+ QueryStats = {};
PreparingKql = nullptr;
PreparedKql = nullptr;
PhysicalQuery = nullptr;
diff --git a/ydb/core/kqp/prepare/kqp_query_exec.cpp b/ydb/core/kqp/prepare/kqp_query_exec.cpp
index bc2d4e94ce9..f07f557b360 100644
--- a/ydb/core/kqp/prepare/kqp_query_exec.cpp
+++ b/ydb/core/kqp/prepare/kqp_query_exec.cpp
@@ -387,7 +387,7 @@ private:
void ExtractQueryStats(NKqpProto::TKqpStatsQuery& dst, const NKikimrQueryStats::TTxStats& txStats) {
auto& dstExec = *dst.AddExecutions();
NKqpProto::TKqpExecutionExtraStats executionExtraStats;
-
+
dstExec.SetDurationUs(txStats.GetDurationUs());
dstExec.SetCpuTimeUs(txStats.GetComputeCpuTimeUsec());
@@ -444,18 +444,18 @@ void ExtractQueryStats(NKqpProto::TKqpStatsQuery& dst, const NKikimrQueryStats::
// NOTE: This might be incorrect in case when single shard has several
// tables, i.e. collocated tables.
affectedShards += table.GetShardCount();
- }
-
+ }
+
executionExtraStats.SetAffectedShards(affectedShards);
dstExec.MutableExtra()->PackFrom(executionExtraStats);
-}
-
+}
+
} // namespace
void TKqlTransformContext::AddMkqlStats(const TString& program, NKikimrQueryStats::TTxStats&& txStats) {
Y_UNUSED(program);
-
+
ExtractQueryStats(QueryStats, txStats);
}
diff --git a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp
index 035d6b77012..f0ac0ccd54b 100644
--- a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp
@@ -17,13 +17,13 @@ public:
: SessionCtx(sessionCtx) {}
private:
- TStatus HandleClusterConfig(TKiClusterConfig node, TExprContext& ctx) override {
- Y_UNUSED(node);
- Y_UNUSED(ctx);
-
- return TStatus::Ok;
- }
-
+ TStatus HandleClusterConfig(TKiClusterConfig node, TExprContext& ctx) override {
+ Y_UNUSED(node);
+ Y_UNUSED(ctx);
+
+ return TStatus::Ok;
+ }
+
TStatus HandleWriteTable(TKiWriteTable node, TExprContext& ctx) override {
Y_UNUSED(ctx);
@@ -332,35 +332,35 @@ public:
return KikimrProviderName;
}
- TExprNode::TPtr GetClusterInfo(const TString& cluster, TExprContext& ctx) override {
- auto config = Gateway->GetClusterConfig(cluster);
- if (!config) {
- return {};
- }
-
+ TExprNode::TPtr GetClusterInfo(const TString& cluster, TExprContext& ctx) override {
+ auto config = Gateway->GetClusterConfig(cluster);
+ if (!config) {
+ return {};
+ }
+
TPositionHandle pos;
-
- TVector<TExprBase> locators;
- auto& grpcData = config->GetGrpc();
- for (size_t index = 0; index < grpcData.LocatorsSize(); ++index) {
- locators.push_back(Build<TCoAtom>(ctx, pos)
- .Value(grpcData.GetLocators(index))
- .Done());
- }
-
- return Build<TKiClusterConfig>(ctx, pos)
- .GrpcData<TKiGrpcData>()
- .Locators<TCoAtomList>()
- .Add(locators)
- .Build()
- .TimeoutMs<TCoAtom>().Build(ToString(grpcData.GetTimeoutMs()))
- .MaxMessageSizeBytes<TCoAtom>().Build(ToString(grpcData.GetMaxMessageSizeBytes()))
- .MaxInFlight<TCoAtom>().Build(ToString(grpcData.GetMaxInFlight()))
- .Build()
- .TvmId<TCoAtom>().Build(ToString(config->GetTvmId()))
- .Done().Ptr();
- }
-
+
+ TVector<TExprBase> locators;
+ auto& grpcData = config->GetGrpc();
+ for (size_t index = 0; index < grpcData.LocatorsSize(); ++index) {
+ locators.push_back(Build<TCoAtom>(ctx, pos)
+ .Value(grpcData.GetLocators(index))
+ .Done());
+ }
+
+ return Build<TKiClusterConfig>(ctx, pos)
+ .GrpcData<TKiGrpcData>()
+ .Locators<TCoAtomList>()
+ .Add(locators)
+ .Build()
+ .TimeoutMs<TCoAtom>().Build(ToString(grpcData.GetTimeoutMs()))
+ .MaxMessageSizeBytes<TCoAtom>().Build(ToString(grpcData.GetMaxMessageSizeBytes()))
+ .MaxInFlight<TCoAtom>().Build(ToString(grpcData.GetMaxInFlight()))
+ .Build()
+ .TvmId<TCoAtom>().Build(ToString(config->GetTvmId()))
+ .Done().Ptr();
+ }
+
IGraphTransformer& GetIntentDeterminationTransformer() override {
return *IntentDeterminationTransformer;
}
@@ -698,10 +698,10 @@ IGraphTransformer::TStatus TKiSinkVisitorTransformer::DoTransform(TExprNode::TPt
auto callable = TCallable(input);
- if (auto node = callable.Maybe<TKiClusterConfig>()) {
- return HandleClusterConfig(node.Cast(), ctx);
- }
-
+ if (auto node = callable.Maybe<TKiClusterConfig>()) {
+ return HandleClusterConfig(node.Cast(), ctx);
+ }
+
if (auto node = callable.Maybe<TKiWriteTable>()) {
return HandleWriteTable(node.Cast(), ctx);
}
diff --git a/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json b/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json
index c54d82a2cac..2b4225c182a 100644
--- a/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json
+++ b/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json
@@ -426,24 +426,24 @@
]
},
{
- "Name": "TKiGrpcData",
- "Base": "TExprBase",
- "Match": {"Type": "Tuple"},
- "Children": [
- {"Index": 0, "Name": "Locators", "Type": "TCoAtomList"},
- {"Index": 1, "Name": "TimeoutMs", "Type": "TCoAtom"},
- {"Index": 2, "Name": "MaxMessageSizeBytes", "Type": "TCoAtom"},
- {"Index": 3, "Name": "MaxInFlight", "Type": "TCoAtom"}
- ]
- },
- {
- "Name": "TKiClusterConfig",
- "Base": "TCallable",
- "Match": {"Type": "Callable", "Name": "KiClusterConfig"},
- "Children": [
- {"Index": 0, "Name": "GrpcData", "Type": "TKiGrpcData"},
- {"Index": 1, "Name": "TvmId", "Type": "TCoAtom"}
- ]
+ "Name": "TKiGrpcData",
+ "Base": "TExprBase",
+ "Match": {"Type": "Tuple"},
+ "Children": [
+ {"Index": 0, "Name": "Locators", "Type": "TCoAtomList"},
+ {"Index": 1, "Name": "TimeoutMs", "Type": "TCoAtom"},
+ {"Index": 2, "Name": "MaxMessageSizeBytes", "Type": "TCoAtom"},
+ {"Index": 3, "Name": "MaxInFlight", "Type": "TCoAtom"}
+ ]
+ },
+ {
+ "Name": "TKiClusterConfig",
+ "Base": "TCallable",
+ "Match": {"Type": "Callable", "Name": "KiClusterConfig"},
+ "Children": [
+ {"Index": 0, "Name": "GrpcData", "Type": "TKiGrpcData"},
+ {"Index": 1, "Name": "TvmId", "Type": "TCoAtom"}
+ ]
}
]
}
diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.h b/ydb/core/kqp/provider/yql_kikimr_gateway.h
index 7598494a2b9..d944b1aebdd 100644
--- a/ydb/core/kqp/provider/yql_kikimr_gateway.h
+++ b/ydb/core/kqp/provider/yql_kikimr_gateway.h
@@ -585,7 +585,7 @@ public:
virtual bool HasCluster(const TString& cluster) = 0;
virtual TVector<TString> GetClusters() = 0;
virtual TString GetDefaultCluster() = 0;
- virtual TMaybe<TKikimrClusterConfig> GetClusterConfig(const TString& cluster) = 0;
+ virtual TMaybe<TKikimrClusterConfig> GetClusterConfig(const TString& cluster) = 0;
virtual TMaybe<TString> GetSetting(const TString& cluster, const TString& name) = 0;
virtual void SetToken(const TString& cluster, const TString& token) = 0;
diff --git a/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp b/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp
index 0ba0b349524..3ffa3a61140 100644
--- a/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp
@@ -475,7 +475,7 @@ TExprNode::TPtr KiBuildQuery(TExprBase node, const TMaybe<bool>& useNewEngine, T
TKiExploreTxResults txExplore;
if (!ExploreTx(commit.World(), ctx, kiDataSink, txExplore)) {
- return node.Ptr();
+ return node.Ptr();
}
if (txExplore.HasExecute) {
@@ -484,7 +484,7 @@ TExprNode::TPtr KiBuildQuery(TExprBase node, const TMaybe<bool>& useNewEngine, T
auto txSyncSet = txExplore.GetSyncSet();
if (!CheckTx(commit.World(), kiDataSink, txExplore.Ops, txSyncSet)) {
- return node.Ptr();
+ return node.Ptr();
}
bool hasScheme;
@@ -578,7 +578,7 @@ TExprNode::TPtr KiBuildResult(TExprBase node, const TString& cluster, TExprCont
auto resFill = node.Cast<TResFill>();
if (resFill.DelegatedSource().Value() != KikimrProviderName) {
- return node.Ptr();
+ return node.Ptr();
}
if (resFill.Data().Maybe<TCoNth>().Tuple().Maybe<TCoRight>().Input().Maybe<TKiExecDataQuery>()) {
diff --git a/ydb/core/kqp/provider/yql_kikimr_provider.cpp b/ydb/core/kqp/provider/yql_kikimr_provider.cpp
index 635d1648279..8ee0475fe9f 100644
--- a/ydb/core/kqp/provider/yql_kikimr_provider.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_provider.cpp
@@ -38,7 +38,7 @@ struct TKikimrData {
DataSourceNames.insert(TKiReadTableScheme::CallableName());
DataSourceNames.insert(TKiReadTableList::CallableName());
- DataSinkNames.insert(TKiClusterConfig::CallableName());
+ DataSinkNames.insert(TKiClusterConfig::CallableName());
DataSinkNames.insert(TKiWriteTable::CallableName());
DataSinkNames.insert(TKiUpdateTable::CallableName());
DataSinkNames.insert(TKiDeleteTable::CallableName());
diff --git a/ydb/core/kqp/provider/yql_kikimr_provider_impl.h b/ydb/core/kqp/provider/yql_kikimr_provider_impl.h
index e1aa3cda2b4..31e0cdf1d8c 100644
--- a/ydb/core/kqp/provider/yql_kikimr_provider_impl.h
+++ b/ydb/core/kqp/provider/yql_kikimr_provider_impl.h
@@ -27,7 +27,7 @@ public:
TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final;
private:
- virtual TStatus HandleClusterConfig(NNodes::TKiClusterConfig, TExprContext& ctx) = 0;
+ virtual TStatus HandleClusterConfig(NNodes::TKiClusterConfig, TExprContext& ctx) = 0;
virtual TStatus HandleWriteTable(NNodes::TKiWriteTable node, TExprContext& ctx) = 0;
virtual TStatus HandleUpdateTable(NNodes::TKiUpdateTable node, TExprContext& ctx) = 0;
virtual TStatus HandleDeleteTable(NNodes::TKiDeleteTable node, TExprContext& ctx) = 0;
diff --git a/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp b/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp
index 692165204ae..1dccb61e093 100644
--- a/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp
@@ -269,19 +269,19 @@ public:
, SessionCtx(sessionCtx) {}
private:
- virtual TStatus HandleClusterConfig(TKiClusterConfig node, TExprContext& ctx) override {
- if (!EnsureTuple(node.GrpcData().Ref(), ctx)) {
- return TStatus::Error;
- }
-
- if (!EnsureAtom(node.TvmId().Ref(), ctx)) {
- return TStatus::Error;
- }
-
+ virtual TStatus HandleClusterConfig(TKiClusterConfig node, TExprContext& ctx) override {
+ if (!EnsureTuple(node.GrpcData().Ref(), ctx)) {
+ return TStatus::Error;
+ }
+
+ if (!EnsureAtom(node.TvmId().Ref(), ctx)) {
+ return TStatus::Error;
+ }
+
node.Ptr()->SetTypeAnn(ctx.MakeType<TUnitExprType>());
- return TStatus::Ok;
- }
-
+ return TStatus::Ok;
+ }
+
virtual TStatus HandleWriteTable(TKiWriteTable node, TExprContext& ctx) override {
if (!EnsureWorldType(node.World().Ref(), ctx)) {
return TStatus::Error;
diff --git a/ydb/core/kqp/proxy/kqp_proxy_service.cpp b/ydb/core/kqp/proxy/kqp_proxy_service.cpp
index decd2cb3011..24a764ed414 100644
--- a/ydb/core/kqp/proxy/kqp_proxy_service.cpp
+++ b/ydb/core/kqp/proxy/kqp_proxy_service.cpp
@@ -122,11 +122,11 @@ class TLocalSessionsRegistry {
public:
TLocalSessionsRegistry() = default;
- TKqpSessionInfo* Create(const TString& sessionId, const TActorId& workerId,
- const TString& database, TKqpDbCountersPtr dbCounters)
- {
+ TKqpSessionInfo* Create(const TString& sessionId, const TActorId& workerId,
+ const TString& database, TKqpDbCountersPtr dbCounters)
+ {
RecentUsedSessions.push_front(sessionId);
- auto result = LocalSessions.emplace(sessionId, TKqpSessionInfo(sessionId, workerId, database, dbCounters));
+ auto result = LocalSessions.emplace(sessionId, TKqpSessionInfo(sessionId, workerId, database, dbCounters));
Links.emplace(sessionId, RecentUsedSessions.begin());
Y_VERIFY(result.second, "Duplicate session id!");
TargetIdIndex.emplace(workerId, sessionId);
@@ -434,7 +434,7 @@ public:
KQP_PROXY_LOG_I("Received tenant pool status, serving tenants: " << JoinRange(", ", Tenants.begin(), Tenants.end()));
for (auto& [_, sessionInfo] : LocalSessions) {
- if (!sessionInfo.Database.empty() && !Tenants.contains(sessionInfo.Database)) {
+ if (!sessionInfo.Database.empty() && !Tenants.contains(sessionInfo.Database)) {
auto closeSessionEv = MakeHolder<TEvKqp::TEvCloseSessionRequest>();
closeSessionEv->Record.MutableRequest()->SetSessionId(sessionInfo.SessionId);
Send(sessionInfo.WorkerId, closeSessionEv.Release());
@@ -514,7 +514,7 @@ public:
auto responseEv = MakeHolder<TEvKqp::TEvCreateSessionResponse>();
TProcessResult<TKqpSessionInfo*> result;
- TKqpDbCountersPtr dbCounters;
+ TKqpDbCountersPtr dbCounters;
const auto deadline = TInstant::MicroSeconds(event.GetDeadlineUs());
@@ -523,18 +523,18 @@ public:
{
auto& response = *responseEv->Record.MutableResponse();
response.SetSessionId(result.Value->SessionId);
- dbCounters = result.Value->DbCounters;
- } else {
- dbCounters = Counters->GetDbCounters(request.GetDatabase());
+ dbCounters = result.Value->DbCounters;
+ } else {
+ dbCounters = Counters->GetDbCounters(request.GetDatabase());
}
- LogRequest(request, requestInfo, ev->Sender, dbCounters);
-
+ LogRequest(request, requestInfo, ev->Sender, dbCounters);
+
responseEv->Record.SetResourceExhausted(result.ResourceExhausted);
responseEv->Record.SetYdbStatus(result.YdbStatus);
responseEv->Record.SetError(result.Error);
- LogResponse(event.GetTraceId(), responseEv->Record, dbCounters);
+ LogResponse(event.GetTraceId(), responseEv->Record, dbCounters);
Send(ev->Sender, responseEv.Release(), 0, ev->Cookie);
}
@@ -543,20 +543,20 @@ public:
auto& request = *event.MutableRequest();
TString traceId = event.GetTraceId();
TKqpRequestInfo requestInfo(traceId);
- ui64 requestId = PendingRequests.RegisterRequest(ev->Sender, ev->Cookie, traceId, TKqpEvents::EvQueryRequest);
-
- auto dbCounters = GetDbCountersForSession(request.GetSessionId());
+ ui64 requestId = PendingRequests.RegisterRequest(ev->Sender, ev->Cookie, traceId, TKqpEvents::EvQueryRequest);
+ auto dbCounters = GetDbCountersForSession(request.GetSessionId());
+
auto queryLimitBytes = TableServiceConfig.GetQueryLimitBytes();
if (queryLimitBytes && IsSqlQuery(request.GetType())) {
auto querySizeBytes = request.GetQuery().size();
if (querySizeBytes > queryLimitBytes) {
TString error = TStringBuilder() << "Query text size exceeds limit (" << querySizeBytes << "b > " << queryLimitBytes << "b)";
ReplyProcessError(Ydb::StatusIds::BAD_REQUEST, error, requestId);
- if (!dbCounters) {
- dbCounters = Counters->GetDbCounters(request.GetDatabase());
- }
- LogRequest(request, requestInfo, ev->Sender, requestId, dbCounters);
+ if (!dbCounters) {
+ dbCounters = Counters->GetDbCounters(request.GetDatabase());
+ }
+ LogRequest(request, requestInfo, ev->Sender, requestId, dbCounters);
return;
}
}
@@ -565,11 +565,11 @@ public:
if (paramsLimitBytes) {
auto paramsBytes = request.GetParameters().ByteSizeLong();
if (paramsBytes > paramsLimitBytes) {
- if (!dbCounters) {
- dbCounters = Counters->GetDbCounters(request.GetDatabase());
- }
- LogRequest(request, requestInfo, ev->Sender, requestId, dbCounters);
-
+ if (!dbCounters) {
+ dbCounters = Counters->GetDbCounters(request.GetDatabase());
+ }
+ LogRequest(request, requestInfo, ev->Sender, requestId, dbCounters);
+
TString error = TStringBuilder() << "Parameters size exceeds limit (" << paramsBytes << "b > " << paramsLimitBytes << "b)";
ReplyProcessError(Ydb::StatusIds::BAD_REQUEST, error, requestId);
return;
@@ -580,17 +580,17 @@ public:
if (!request.GetSessionId().empty()) {
TProcessResult<TActorId> result;
if (!TryGetSessionTargetActor(request.GetSessionId(), requestInfo, result)) {
- if (!dbCounters) {
- dbCounters = Counters->GetDbCounters(request.GetDatabase());
- }
- LogRequest(request, requestInfo, ev->Sender, requestId, dbCounters);
+ if (!dbCounters) {
+ dbCounters = Counters->GetDbCounters(request.GetDatabase());
+ }
+ LogRequest(request, requestInfo, ev->Sender, requestId, dbCounters);
ReplyProcessError(result.YdbStatus, result.Error, requestId);
return;
}
targetId = result.Value;
-
- LogRequest(request, requestInfo, ev->Sender, requestId, dbCounters);
+
+ LogRequest(request, requestInfo, ev->Sender, requestId, dbCounters);
} else {
TString cluster = request.GetCluster();
if (cluster.empty()) {
@@ -599,23 +599,23 @@ public:
TProcessResult<TKqpSessionInfo*> result;
if (!CreateNewSessionWorker(requestInfo, cluster, false, request.GetDatabase(), result)) {
- if (!dbCounters) {
- dbCounters = Counters->GetDbCounters(request.GetDatabase());
- }
- LogRequest(request, requestInfo, ev->Sender, requestId, dbCounters);
+ if (!dbCounters) {
+ dbCounters = Counters->GetDbCounters(request.GetDatabase());
+ }
+ LogRequest(request, requestInfo, ev->Sender, requestId, dbCounters);
ReplyProcessError(result.YdbStatus, result.Error, requestId);
return;
}
targetId = result.Value->WorkerId;
request.SetSessionId(result.Value->SessionId);
- dbCounters = result.Value->DbCounters;
-
- LogRequest(request, requestInfo, ev->Sender, requestId, dbCounters);
+ dbCounters = result.Value->DbCounters;
+
+ LogRequest(request, requestInfo, ev->Sender, requestId, dbCounters);
}
TString sessionId = request.GetSessionId();
- PendingRequests.SetSessionId(requestId, sessionId, dbCounters);
+ PendingRequests.SetSessionId(requestId, sessionId, dbCounters);
// We add extra milliseconds to the user-specified timeout, so it means we give additional priority for worker replies,
// because it is much better to give detailed error message rather than generic timeout.
// For example, it helps to avoid race in event order when worker and proxy recieve timeout at the same moment.
@@ -633,10 +633,10 @@ public:
TKqpRequestInfo requestInfo(event.GetTraceId());
TString sessionId = request.GetSessionId();
- auto dbCounters = GetDbCountersForSession(sessionId);
-
- LogRequest(request, requestInfo, ev->Sender, dbCounters);
-
+ auto dbCounters = GetDbCountersForSession(sessionId);
+
+ LogRequest(request, requestInfo, ev->Sender, dbCounters);
+
if (!sessionId.empty()) {
TProcessResult<TActorId> result;
if (TryGetSessionTargetActor(sessionId, requestInfo, result)) {
@@ -651,15 +651,15 @@ public:
auto traceId = event.GetTraceId();
TKqpRequestInfo requestInfo(traceId);
- auto sessionId = request.GetSessionId();
- ui64 requestId = PendingRequests.RegisterRequest(ev->Sender, ev->Cookie, traceId, TKqpEvents::EvPingSessionRequest);
-
- auto dbCounters = GetDbCountersForSession(sessionId);
-
- LogRequest(request, requestInfo, ev->Sender, requestId, dbCounters);
+ auto sessionId = request.GetSessionId();
+ ui64 requestId = PendingRequests.RegisterRequest(ev->Sender, ev->Cookie, traceId, TKqpEvents::EvPingSessionRequest);
+ auto dbCounters = GetDbCountersForSession(sessionId);
+
+ LogRequest(request, requestInfo, ev->Sender, requestId, dbCounters);
+
TProcessResult<TActorId> result;
- if (!TryGetSessionTargetActor(sessionId, requestInfo, result)) {
+ if (!TryGetSessionTargetActor(sessionId, requestInfo, result)) {
ReplyProcessError(result.YdbStatus, result.Error, requestId);
return;
}
@@ -669,7 +669,7 @@ public:
timeout = TDuration::MilliSeconds(Min(timeout.MilliSeconds(), (ui64)request.GetTimeoutMs()));
}
- PendingRequests.SetSessionId(requestId, sessionId, dbCounters);
+ PendingRequests.SetSessionId(requestId, sessionId, dbCounters);
StartQueryTimeout(requestId, timeout);
Send(result.Value, ev->Release().Release(), IEventHandle::FlagTrackDelivery, requestId);
}
@@ -685,7 +685,7 @@ public:
return;
}
- LogResponse(proxyRequest->TraceId, ev->Get()->Record, proxyRequest->DbCounters);
+ LogResponse(proxyRequest->TraceId, ev->Get()->Record, proxyRequest->DbCounters);
Send(proxyRequest->Sender, ev->Release().Release(), 0, proxyRequest->SenderCookie);
TKqpRequestInfo requestInfo(proxyRequest->TraceId);
@@ -993,7 +993,7 @@ public:
private:
void LogResponse(const TKqpRequestInfo& requestInfo,
- const NKikimrKqp::TEvProcessResponse& event, TKqpDbCountersPtr dbCounters)
+ const NKikimrKqp::TEvProcessResponse& event, TKqpDbCountersPtr dbCounters)
{
auto status = event.GetYdbStatus();
if (status != Ydb::StatusIds::SUCCESS) {
@@ -1004,74 +1004,74 @@ private:
}
void LogResponse(const TKqpRequestInfo& requestInfo,
- const TEvKqp::TProtoArenaHolder<NKikimrKqp::TEvQueryResponse>& holder,
- TKqpDbCountersPtr dbCounters)
+ const TEvKqp::TProtoArenaHolder<NKikimrKqp::TEvQueryResponse>& holder,
+ TKqpDbCountersPtr dbCounters)
{
Y_UNUSED(requestInfo);
const auto& event = holder.GetRef();
- Counters->ReportResponseStatus(dbCounters, event.ByteSize(),
+ Counters->ReportResponseStatus(dbCounters, event.ByteSize(),
event.GetYdbStatus());
-
+
for (auto& issue : event.GetResponse().GetQueryIssues()) {
- Counters->ReportIssues(dbCounters, issue);
+ Counters->ReportIssues(dbCounters, issue);
}
- ui64 resultsBytes = 0;
+ ui64 resultsBytes = 0;
for (auto& result : event.GetResponse().GetResults()) {
- resultsBytes += result.ByteSize();
+ resultsBytes += result.ByteSize();
}
- Counters->ReportResultsBytes(dbCounters, resultsBytes);
+ Counters->ReportResultsBytes(dbCounters, resultsBytes);
}
void LogResponse(const TKqpRequestInfo& requestInfo,
- const NKikimrKqp::TEvCreateSessionResponse& event, TKqpDbCountersPtr dbCounters)
+ const NKikimrKqp::TEvCreateSessionResponse& event, TKqpDbCountersPtr dbCounters)
{
Y_UNUSED(requestInfo);
-
- Counters->ReportResponseStatus(dbCounters, event.ByteSize(),
+
+ Counters->ReportResponseStatus(dbCounters, event.ByteSize(),
event.GetYdbStatus());
}
void LogResponse(const TKqpRequestInfo& requestInfo,
- const NKikimrKqp::TEvPingSessionResponse& event, TKqpDbCountersPtr dbCounters)
+ const NKikimrKqp::TEvPingSessionResponse& event, TKqpDbCountersPtr dbCounters)
{
Y_UNUSED(requestInfo);
-
- Counters->ReportResponseStatus(dbCounters, event.ByteSize(), event.GetStatus());
+
+ Counters->ReportResponseStatus(dbCounters, event.ByteSize(), event.GetStatus());
}
void LogRequest(const NKikimrKqp::TCloseSessionRequest& request,
- const TKqpRequestInfo& requestInfo, const TActorId& sender,
- TKqpDbCountersPtr dbCounters)
+ const TKqpRequestInfo& requestInfo, const TActorId& sender,
+ TKqpDbCountersPtr dbCounters)
{
KQP_PROXY_LOG_D(requestInfo << "Received close session request, sender: " << sender << ", SessionId: " << request.GetSessionId());
- Counters->ReportCloseSession(dbCounters, request.ByteSize());
+ Counters->ReportCloseSession(dbCounters, request.ByteSize());
}
void LogRequest(const NKikimrKqp::TQueryRequest& request,
- const TKqpRequestInfo& requestInfo, const TActorId& sender, ui64 requestId,
- TKqpDbCountersPtr dbCounters)
+ const TKqpRequestInfo& requestInfo, const TActorId& sender, ui64 requestId,
+ TKqpDbCountersPtr dbCounters)
{
KQP_PROXY_LOG_D(requestInfo << "Received new query request, sender: " << sender << ", RequestId: " << requestId
<< ", Query: \"" << request.GetQuery().substr(0, 10000) << "\"");
- Counters->ReportQueryRequest(dbCounters, request);
+ Counters->ReportQueryRequest(dbCounters, request);
}
void LogRequest(const NKikimrKqp::TCreateSessionRequest& request,
- const TKqpRequestInfo& requestInfo, const TActorId& sender,
- TKqpDbCountersPtr dbCounters)
+ const TKqpRequestInfo& requestInfo, const TActorId& sender,
+ TKqpDbCountersPtr dbCounters)
{
KQP_PROXY_LOG_D(requestInfo << "Received create session request, sender: " << sender);
- Counters->ReportCreateSession(dbCounters, request.ByteSize());
+ Counters->ReportCreateSession(dbCounters, request.ByteSize());
}
void LogRequest(const NKikimrKqp::TPingSessionRequest& request,
- const TKqpRequestInfo& requestInfo, const TActorId& sender, ui64 requestId,
- TKqpDbCountersPtr dbCounters)
+ const TKqpRequestInfo& requestInfo, const TActorId& sender, ui64 requestId,
+ TKqpDbCountersPtr dbCounters)
{
KQP_PROXY_LOG_D(requestInfo << "Received ping session request, sender: " << sender << " selfID: " << SelfId() << ", RequestId: " << requestId);
- Counters->ReportPingSession(dbCounters, request.ByteSize());
+ Counters->ReportPingSession(dbCounters, request.ByteSize());
}
bool ReplyProcessError(Ydb::StatusIds::StatusCode ydbStatus, const TString& message, ui64 requestId)
@@ -1155,16 +1155,16 @@ private:
auto sessionId = EncodeSessionId(SelfId().NodeId(), CreateGuidAsString());
- auto dbCounters = Counters->GetDbCounters(database);
-
- TKqpWorkerSettings workerSettings(cluster, database, TableServiceConfig, dbCounters);
+ auto dbCounters = Counters->GetDbCounters(database);
+
+ TKqpWorkerSettings workerSettings(cluster, database, TableServiceConfig, dbCounters);
workerSettings.LongSession = longSession;
IActor* workerActor = AppData()->FeatureFlags.GetEnableKqpSessionActor()
? CreateKqpSessionActor(SelfId(), sessionId, KqpSettings, workerSettings, ModuleResolverState, Counters)
: CreateKqpWorkerActor(SelfId(), sessionId, KqpSettings, workerSettings, ModuleResolverState, Counters);
auto workerId = TlsActivationContext->ExecutorThread.RegisterActor(workerActor, TMailboxType::HTSwap, AppData()->UserPoolId);
- TKqpSessionInfo* sessionInfo = LocalSessions.Create(sessionId, workerId, database, dbCounters);
+ TKqpSessionInfo* sessionInfo = LocalSessions.Create(sessionId, workerId, database, dbCounters);
KQP_PROXY_LOG_D(requestInfo << "Created new session"
<< ", sessionId: " << sessionInfo->SessionId
@@ -1256,11 +1256,11 @@ private:
NYql::NDq::SetYqlLogLevels(yqlPriority);
}
- TKqpDbCountersPtr GetDbCountersForSession(const TString& sessionId) const {
- auto localSession = LocalSessions.FindPtr(sessionId);
- return localSession ? localSession->DbCounters : nullptr;
- }
-
+ TKqpDbCountersPtr GetDbCountersForSession(const TString& sessionId) const {
+ auto localSession = LocalSessions.FindPtr(sessionId);
+ return localSession ? localSession->DbCounters : nullptr;
+ }
+
private:
NYql::NLog::YqlLoggerScope YqlLoggerScope;
NKikimrConfig::TLogConfig LogConfig;
@@ -1281,7 +1281,7 @@ private:
TIntrusivePtr<TKqpCounters> Counters;
TLocalSessionsRegistry LocalSessions;
-
+
bool ServerWorkerBalancerComplete = false;
std::optional<TString> SelfDataCenterId;
TVector<NKikimrKqp::TKqpProxyNodeResources> PeerProxyNodeResources;
diff --git a/ydb/core/kqp/proxy/kqp_proxy_service.h b/ydb/core/kqp/proxy/kqp_proxy_service.h
index f5059d3f36b..4c1c92bdbe7 100644
--- a/ydb/core/kqp/proxy/kqp_proxy_service.h
+++ b/ydb/core/kqp/proxy/kqp_proxy_service.h
@@ -16,10 +16,10 @@ struct TKqpProxyRequest {
TString TraceId;
ui32 EventType;
TString SessionId;
- TKqpDbCountersPtr DbCounters;
+ TKqpDbCountersPtr DbCounters;
TKqpProxyRequest(const TActorId& sender, ui64 senderCookie, const TString& traceId,
- ui32 eventType)
+ ui32 eventType)
: Sender(sender)
, SenderCookie(senderCookie)
, TraceId(traceId)
@@ -27,9 +27,9 @@ struct TKqpProxyRequest {
, SessionId()
{}
- void SetSessionId(const TString& sessionId, TKqpDbCountersPtr dbCounters) {
+ void SetSessionId(const TString& sessionId, TKqpDbCountersPtr dbCounters) {
SessionId = sessionId;
- DbCounters = dbCounters;
+ DbCounters = dbCounters;
}
};
@@ -43,9 +43,9 @@ public:
: RequestId(0)
{}
- ui64 RegisterRequest(const TActorId& sender, ui64 senderCookie, const TString& traceId, ui32 eventType) {
+ ui64 RegisterRequest(const TActorId& sender, ui64 senderCookie, const TString& traceId, ui32 eventType) {
ui64 NewRequestId = ++RequestId;
- PendingRequests.emplace(NewRequestId, TKqpProxyRequest(sender, senderCookie, traceId, eventType));
+ PendingRequests.emplace(NewRequestId, TKqpProxyRequest(sender, senderCookie, traceId, eventType));
return NewRequestId;
}
@@ -53,9 +53,9 @@ public:
return PendingRequests.FindPtr(requestId);
}
- void SetSessionId(ui64 requestId, const TString& sessionId, TKqpDbCountersPtr dbCounters) {
+ void SetSessionId(ui64 requestId, const TString& sessionId, TKqpDbCountersPtr dbCounters) {
TKqpProxyRequest* ptr = PendingRequests.FindPtr(requestId);
- ptr->SetSessionId(sessionId, dbCounters);
+ ptr->SetSessionId(sessionId, dbCounters);
}
void Erase(ui64 requestId) {
@@ -77,18 +77,18 @@ struct TKqpSessionInfo {
TString SessionId;
TActorId WorkerId;
TString Database;
- TKqpDbCountersPtr DbCounters;
+ TKqpDbCountersPtr DbCounters;
ui32 UseFrequency;
TInstant LastRequestAt;
TInstant CreatedAt;
TInstant ShutdownStartedAt;
- TKqpSessionInfo(const TString& sessionId, const TActorId& workerId,
- const TString& database, TKqpDbCountersPtr dbCounters)
+ TKqpSessionInfo(const TString& sessionId, const TActorId& workerId,
+ const TString& database, TKqpDbCountersPtr dbCounters)
: SessionId(sessionId)
, WorkerId(workerId)
, Database(database)
- , DbCounters(dbCounters)
+ , DbCounters(dbCounters)
, UseFrequency(0)
, ShutdownStartedAt()
{
diff --git a/ydb/core/kqp/rm/kqp_rm.cpp b/ydb/core/kqp/rm/kqp_rm.cpp
index 30fc08ff5f8..d187e4c6415 100644
--- a/ydb/core/kqp/rm/kqp_rm.cpp
+++ b/ydb/core/kqp/rm/kqp_rm.cpp
@@ -231,7 +231,7 @@ public:
void Bootstrap() {
ActorSystem = TlsActivationContext->ActorSystem();
if (!Counters) {
- Counters = MakeIntrusive<TKqpCounters>(AppData()->Counters);
+ Counters = MakeIntrusive<TKqpCounters>(AppData()->Counters);
}
LOG_D("Start KqpResourceManagerActor at " << SelfId() << " with ResourceBroker at " << ResourceBrokerId);
diff --git a/ydb/core/kqp/runtime/kqp_compute.h b/ydb/core/kqp/runtime/kqp_compute.h
index c55a4e2a21b..e6f8492b41a 100644
--- a/ydb/core/kqp/runtime/kqp_compute.h
+++ b/ydb/core/kqp/runtime/kqp_compute.h
@@ -4,18 +4,18 @@
#include <ydb/core/scheme/scheme_tabledefs.h>
#include <ydb/core/tablet_flat/flat_row_eggs.h>
-
+
// TODO rename file to runtime_compute_context.h
namespace NKikimr {
namespace NMiniKQL {
class TKqpComputeContextBase : public NYql::NDq::TDqComputeContextBase {
-public:
- struct TColumn {
- NTable::TTag Tag;
- NScheme::TTypeId Type;
- };
+public:
+ struct TColumn {
+ NTable::TTag Tag;
+ NScheme::TTypeId Type;
+ };
// used only at then building of a computation graph, to inject taskId in runtime nodes
void SetCurrentTaskId(ui64 taskId) { CurrentTaskId = taskId; }
@@ -23,7 +23,7 @@ public:
private:
ui64 CurrentTaskId = 0;
-};
+};
TComputationNodeFactory GetKqpBaseComputeFactory(const TKqpComputeContextBase* computeCtx);
diff --git a/ydb/core/kqp/runtime/kqp_program_builder.cpp b/ydb/core/kqp/runtime/kqp_program_builder.cpp
index b5169c19d82..ff5de8411fb 100644
--- a/ydb/core/kqp/runtime/kqp_program_builder.cpp
+++ b/ydb/core/kqp/runtime/kqp_program_builder.cpp
@@ -73,8 +73,8 @@ TRuntimeNode BuildKeyPrefixIndicesList(const TProgramBuilder& builder, const TSt
TRuntimeNode BuildTableIdLiteral(const TTableId& tableId, TProgramBuilder& builder) {
TVector<TRuntimeNode> tupleItems {
- builder.NewDataLiteral<ui64>(tableId.PathId.OwnerId),
- builder.NewDataLiteral<ui64>(tableId.PathId.LocalPathId),
+ builder.NewDataLiteral<ui64>(tableId.PathId.OwnerId),
+ builder.NewDataLiteral<ui64>(tableId.PathId.LocalPathId),
builder.NewDataLiteral<NUdf::EDataSlot::String>(tableId.SysViewInfo),
builder.NewDataLiteral<ui64>(tableId.SchemaVersion),
};
diff --git a/ydb/core/kqp/runtime/kqp_read_table.cpp b/ydb/core/kqp/runtime/kqp_read_table.cpp
index 358ebd0a3eb..e566dc0bea2 100644
--- a/ydb/core/kqp/runtime/kqp_read_table.cpp
+++ b/ydb/core/kqp/runtime/kqp_read_table.cpp
@@ -1,11 +1,11 @@
-#include "kqp_read_table.h"
-#include "kqp_scan_data.h"
-
+#include "kqp_read_table.h"
+#include "kqp_scan_data.h"
+
#include <ydb/core/engine/mkql_keys.h>
-
+
#include <ydb/library/yql/minikql/mkql_node_cast.h>
-
-namespace NKikimr {
+
+namespace NKikimr {
namespace NKqp {
@@ -32,43 +32,43 @@ NUdf::TDataTypeId UnwrapDataTypeFromStruct(const TStructType& structType, ui32 i
} // namespace NKqp
-namespace NMiniKQL {
-
-using namespace NTable;
-using namespace NUdf;
-
-static TCell BuildKeyTupleCell(const TType* type, const TUnboxedValue& value, const TTypeEnvironment& env) {
- MKQL_ENSURE_S(type);
-
- auto keyType = type;
- auto keyValue = value;
-
- if (type->IsOptional()) {
- if (!value) {
- return TCell();
- }
-
- keyType = AS_TYPE(TOptionalType, type)->GetItemType();
- keyValue = value.GetOptionalValue();
- }
-
- return MakeCell(AS_TYPE(TDataType, keyType)->GetSchemeType(), keyValue, env, true);
-}
-
-void BuildKeyTupleCells(const TTupleType* tupleType, const TUnboxedValue& tupleValue, TVector<TCell>& cells,
- const TTypeEnvironment& env)
-{
- MKQL_ENSURE_S(tupleType);
-
- cells.resize(tupleType->GetElementsCount());
-
- for (ui32 i = 0; i < tupleType->GetElementsCount(); ++i) {
- auto keyType = tupleType->GetElementType(i);
- auto keyValue = tupleValue.GetElement(i);
- cells[i] = BuildKeyTupleCell(keyType, keyValue, env);
- }
-}
-
+namespace NMiniKQL {
+
+using namespace NTable;
+using namespace NUdf;
+
+static TCell BuildKeyTupleCell(const TType* type, const TUnboxedValue& value, const TTypeEnvironment& env) {
+ MKQL_ENSURE_S(type);
+
+ auto keyType = type;
+ auto keyValue = value;
+
+ if (type->IsOptional()) {
+ if (!value) {
+ return TCell();
+ }
+
+ keyType = AS_TYPE(TOptionalType, type)->GetItemType();
+ keyValue = value.GetOptionalValue();
+ }
+
+ return MakeCell(AS_TYPE(TDataType, keyType)->GetSchemeType(), keyValue, env, true);
+}
+
+void BuildKeyTupleCells(const TTupleType* tupleType, const TUnboxedValue& tupleValue, TVector<TCell>& cells,
+ const TTypeEnvironment& env)
+{
+ MKQL_ENSURE_S(tupleType);
+
+ cells.resize(tupleType->GetElementsCount());
+
+ for (ui32 i = 0; i < tupleType->GetElementsCount(); ++i) {
+ auto keyType = tupleType->GetElementType(i);
+ auto keyValue = tupleValue.GetElement(i);
+ cells[i] = BuildKeyTupleCell(keyType, keyValue, env);
+ }
+}
+
void ParseReadColumns(const TType* readType, const TRuntimeNode& tagsNode,
TSmallVec<TKqpComputeContextBase::TColumn>& columns, TSmallVec<TKqpComputeContextBase::TColumn>& systemColumns)
{
@@ -106,37 +106,37 @@ void ParseWideReadColumns(const TCallable& callable, const TRuntimeNode& tagsNod
{
auto tags = AS_VALUE(TStructLiteral, tagsNode);
MKQL_ENSURE_S(tags);
-
+
TType* returnType = callable.GetType()->GetReturnType();
MKQL_ENSURE_S(returnType->GetKind() == TType::EKind::Flow);
-
+
auto itemType = AS_TYPE(TFlowType, returnType)->GetItemType();
MKQL_ENSURE_S(itemType->GetKind() == TType::EKind::Tuple);
-
+
auto tupleType = AS_TYPE(TTupleType, itemType);
MKQL_ENSURE_S(tags->GetValuesCount() == tupleType->GetElementsCount());
-
+
columns.reserve(tupleType->GetElementsCount());
-
+
for (ui32 i = 0; i < tupleType->GetElementsCount(); ++i) {
auto memberType = tupleType->GetElementType(i);
-
+
if (memberType->GetKind() == TType::EKind::Optional) {
memberType = AS_TYPE(TOptionalType, memberType)->GetItemType();
}
-
+
MKQL_ENSURE_S(memberType->GetKind() == TType::EKind::Data);
-
+
NTable::TTag columnId = AS_VALUE(TDataLiteral, tags->GetValue(i))->AsValue().Get<ui32>();
-
+
if (IsSystemColumn(columnId)) {
systemColumns.push_back({columnId, AS_TYPE(TDataType, memberType)->GetSchemeType()});
} else {
columns.push_back({columnId, AS_TYPE(TDataType, memberType)->GetSchemeType()});
}
}
-}
-
+}
+
TParseReadTableResult ParseWideReadTable(TCallable& callable) {
MKQL_ENSURE_S(callable.GetInputsCount() >= 4);
@@ -230,7 +230,7 @@ TParseReadTableRangesResult ParseWideReadTableRanges(TCallable& callable) {
}
namespace {
-
+
class TKqpScanWideReadTableWrapperBase : public TStatelessWideFlowCodegeneratorNode<TKqpScanWideReadTableWrapperBase> {
using TBase = TStatelessWideFlowCodegeneratorNode<TKqpScanWideReadTableWrapperBase>;
public:
diff --git a/ydb/core/kqp/runtime/kqp_read_table.h b/ydb/core/kqp/runtime/kqp_read_table.h
index 99c9cc3f050..67133361b93 100644
--- a/ydb/core/kqp/runtime/kqp_read_table.h
+++ b/ydb/core/kqp/runtime/kqp_read_table.h
@@ -1,29 +1,29 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/kqp/runtime/kqp_scan_data.h>
#include <ydb/core/tablet_flat/flat_database.h>
#include <ydb/core/scheme/scheme_tabledefs.h>
-
+
#include <ydb/library/yql/minikql/computation/mkql_computation_node_codegen.h>
#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
-
-namespace NKikimr {
-namespace NMiniKQL {
-
-void BuildKeyTupleCells(const TTupleType* tupleType, const NUdf::TUnboxedValue& tupleValue,
- TVector<TCell>& cells, const TTypeEnvironment& env);
-
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+void BuildKeyTupleCells(const TTupleType* tupleType, const NUdf::TUnboxedValue& tupleValue,
+ TVector<TCell>& cells, const TTypeEnvironment& env);
+
struct TParseReadTableResultBase {
- ui32 CallableId = 0;
- TTableId TableId;
-
- TSmallVec<TKqpComputeContextBase::TColumn> Columns;
+ ui32 CallableId = 0;
+ TTableId TableId;
+
+ TSmallVec<TKqpComputeContextBase::TColumn> Columns;
TSmallVec<TKqpComputeContextBase::TColumn> SystemColumns;
- TSmallVec<bool> SkipNullKeys;
+ TSmallVec<bool> SkipNullKeys;
TNode* ItemsLimit = nullptr;
bool Reverse = false;
-};
-
+};
+
struct TParseReadTableResult : TParseReadTableResultBase {
TTupleLiteral* FromTuple = nullptr;
bool FromInclusive = false;
@@ -40,12 +40,12 @@ void ParseReadColumns(const TType* readType, const TRuntimeNode& tagsNode,
TParseReadTableResult ParseWideReadTable(TCallable& callable);
TParseReadTableRangesResult ParseWideReadTableRanges(TCallable& callable);
-
+
IComputationNode* WrapKqpScanWideReadTableRanges(TCallable& callable, const TComputationNodeFactoryContext& ctx,
TKqpScanComputeContext& computeCtx);
IComputationNode* WrapKqpScanWideReadTable(TCallable& callable, const TComputationNodeFactoryContext& ctx,
TKqpScanComputeContext& computeCtx);
-
+
} // namespace NMiniKQL
} // namespace NKikimr
diff --git a/ydb/core/kqp/runtime/kqp_scan_data.cpp b/ydb/core/kqp/runtime/kqp_scan_data.cpp
index c4d8e241966..fb09ed4be27 100644
--- a/ydb/core/kqp/runtime/kqp_scan_data.cpp
+++ b/ydb/core/kqp/runtime/kqp_scan_data.cpp
@@ -1,14 +1,14 @@
-#include "kqp_scan_data.h"
-
+#include "kqp_scan_data.h"
+
#include <ydb/core/engine/minikql/minikql_engine_host.h>
#include <ydb/core/protos/tx_datashard.pb.h>
-
+
#include <ydb/library/yql/minikql/mkql_string_util.h>
#include <ydb/library/yql/utils/yql_panic.h>
-namespace NKikimr {
-namespace NMiniKQL {
-
+namespace NKikimr {
+namespace NMiniKQL {
+
namespace {
struct TBytesStatistics {
@@ -248,15 +248,15 @@ std::pair<ui64, ui64> GetUnboxedValueSizeForTests(const NUdf::TUnboxedValue& val
return {sizes.AllocatedBytes, sizes.DataBytes};
}
-TKqpScanComputeContext::TScanData::TScanData(const TTableId& tableId, const TTableRange& range,
+TKqpScanComputeContext::TScanData::TScanData(const TTableId& tableId, const TTableRange& range,
const TSmallVec<TColumn>& columns, const TSmallVec<TColumn>& systemColumns, const TSmallVec<bool>& skipNullKeys)
- : TableId(tableId)
- , Range(range)
+ : TableId(tableId)
+ , Range(range)
, SkipNullKeys(skipNullKeys)
- , Columns(columns)
+ , Columns(columns)
, SystemColumns(systemColumns)
-{}
-
+{}
+
TKqpScanComputeContext::TScanData::TScanData(const NKikimrTxDataShard::TKqpTransaction_TScanTaskMeta& meta,
NYql::NDqProto::EDqStatsMode statsMode)
{
@@ -378,8 +378,8 @@ ui64 TKqpScanComputeContext::TScanData::AddRows(const arrow::RecordBatch& batch,
}
return stats.AllocatedBytes;
-}
-
+}
+
NUdf::TUnboxedValue TKqpScanComputeContext::TScanData::TakeRow() {
YQL_ENSURE(!RowBatches.empty());
auto& batch = RowBatches.front();
@@ -396,7 +396,7 @@ NUdf::TUnboxedValue TKqpScanComputeContext::TScanData::TakeRow() {
void TKqpScanComputeContext::AddTableScan(ui32, const TTableId& tableId, const TTableRange& range,
const TSmallVec<TColumn>& columns, const TSmallVec<TColumn>& systemColumns, const TSmallVec<bool>& skipNullKeys)
-{
+{
auto scanData = TKqpScanComputeContext::TScanData(tableId, range, columns, systemColumns, skipNullKeys);
if (Y_UNLIKELY(StatsMode >= NYql::NDqProto::DQ_STATS_MODE_BASIC)) {
@@ -408,9 +408,9 @@ void TKqpScanComputeContext::AddTableScan(ui32, const TTableId& tableId, const T
}
auto result = Scans.emplace(0, std::move(scanData));
- Y_ENSURE(result.second);
-}
-
+ Y_ENSURE(result.second);
+}
+
void TKqpScanComputeContext::AddTableScan(ui32, const NKikimrTxDataShard::TKqpTransaction_TScanTaskMeta& meta,
NYql::NDqProto::EDqStatsMode statsMode)
{
@@ -422,15 +422,15 @@ void TKqpScanComputeContext::AddTableScan(ui32, const NKikimrTxDataShard::TKqpTr
TKqpScanComputeContext::TScanData& TKqpScanComputeContext::GetTableScan(ui32) {
auto scanData = Scans.FindPtr(0);
- Y_ENSURE(scanData);
-
- return *scanData;
-}
-
-TMap<ui32, TKqpScanComputeContext::TScanData>& TKqpScanComputeContext::GetTableScans() {
- return Scans;
-}
-
+ Y_ENSURE(scanData);
+
+ return *scanData;
+}
+
+TMap<ui32, TKqpScanComputeContext::TScanData>& TKqpScanComputeContext::GetTableScans() {
+ return Scans;
+}
+
const TMap<ui32, TKqpScanComputeContext::TScanData>& TKqpScanComputeContext::GetTableScans() const {
return Scans;
}
@@ -438,29 +438,29 @@ const TMap<ui32, TKqpScanComputeContext::TScanData>& TKqpScanComputeContext::Get
TIntrusivePtr<IKqpTableReader> TKqpScanComputeContext::ReadTable(ui32) const {
auto scanData = Scans.FindPtr(0);
Y_ENSURE(scanData);
- Y_ENSURE(scanData->TableReader);
-
- return scanData->TableReader;
-}
-
-class TKqpTableReader : public IKqpTableReader {
-public:
+ Y_ENSURE(scanData->TableReader);
+
+ return scanData->TableReader;
+}
+
+class TKqpTableReader : public IKqpTableReader {
+public:
TKqpTableReader(TKqpScanComputeContext::TScanData& scanData)
: ScanData(scanData)
{}
-
+
NUdf::EFetchStatus Next(NUdf::TUnboxedValue& result) override {
if (ScanData.IsEmpty()) {
if (ScanData.IsFinished()) {
- return NUdf::EFetchStatus::Finish;
- }
- return NUdf::EFetchStatus::Yield;
- }
-
+ return NUdf::EFetchStatus::Finish;
+ }
+ return NUdf::EFetchStatus::Yield;
+ }
+
result = std::move(ScanData.TakeRow());
- return NUdf::EFetchStatus::Ok;
- }
-
+ return NUdf::EFetchStatus::Ok;
+ }
+
EFetchResult Next(NUdf::TUnboxedValue* const* result) override {
if (ScanData.IsEmpty()) {
if (ScanData.IsFinished()) {
@@ -479,13 +479,13 @@ public:
return EFetchResult::One;
}
-private:
- TKqpScanComputeContext::TScanData& ScanData;
-};
-
+private:
+ TKqpScanComputeContext::TScanData& ScanData;
+};
+
TIntrusivePtr<IKqpTableReader> CreateKqpTableReader(TKqpScanComputeContext::TScanData& scanData) {
return MakeIntrusive<TKqpTableReader>(scanData);
-}
-
+}
+
} // namespace NMiniKQL
-} // namespace NKikimr
+} // namespace NKikimr
diff --git a/ydb/core/kqp/runtime/kqp_scan_data.h b/ydb/core/kqp/runtime/kqp_scan_data.h
index c8e7d82b328..9dfe941e499 100644
--- a/ydb/core/kqp/runtime/kqp_scan_data.h
+++ b/ydb/core/kqp/runtime/kqp_scan_data.h
@@ -1,13 +1,13 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/protos/services.pb.h>
-#include "kqp_compute.h"
-
+#include "kqp_compute.h"
+
#include <ydb/core/engine/minikql/minikql_engine_host.h>
#include <ydb/core/formats/arrow_helpers.h>
#include <ydb/core/scheme/scheme_tabledefs.h>
#include <ydb/core/tablet_flat/flat_database.h>
-
+
#include <ydb/library/yql/dq/actors/protos/dq_stats.pb.h>
#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
@@ -19,9 +19,9 @@ namespace NKikimrTxDataShard {
class TKqpTransaction_TScanTaskMeta;
}
-namespace NKikimr {
-namespace NMiniKQL {
-
+namespace NKikimr {
+namespace NMiniKQL {
+
std::pair<ui64, ui64> GetUnboxedValueSizeForTests(const NUdf::TUnboxedValue& value, NScheme::TTypeId type);
class IKqpTableReader : public TSimpleRefCount<IKqpTableReader> {
@@ -87,13 +87,13 @@ public:
public:
ui64 TaskId = 0;
- TTableId TableId;
+ TTableId TableId;
TString TablePath;
- TSerializedTableRange Range;
+ TSerializedTableRange Range;
TSmallVec<bool> SkipNullKeys;
// shared with actor via TableReader
- TIntrusivePtr<IKqpTableReader> TableReader;
+ TIntrusivePtr<IKqpTableReader> TableReader;
struct TBasicStats {
size_t Rows = 0;
@@ -126,37 +126,37 @@ public:
TQueue<RowBatch> RowBatches;
ui64 StoredBytes = 0;
bool Finished = false;
- };
-
-public:
+ };
+
+public:
explicit TKqpScanComputeContext(NYql::NDqProto::EDqStatsMode statsMode)
: StatsMode(statsMode) {}
-
- TIntrusivePtr<IKqpTableReader> ReadTable(ui32 callableId) const;
-
- void AddTableScan(ui32 callableId, const TTableId& tableId, const TTableRange& range,
+
+ TIntrusivePtr<IKqpTableReader> ReadTable(ui32 callableId) const;
+
+ void AddTableScan(ui32 callableId, const TTableId& tableId, const TTableRange& range,
const TSmallVec<TColumn>& columns, const TSmallVec<TColumn>& systemColumns, const TSmallVec<bool>& skipNullKeys);
-
+
void AddTableScan(ui32 callableId, const NKikimrTxDataShard::TKqpTransaction_TScanTaskMeta& meta,
NYql::NDqProto::EDqStatsMode statsMode);
- TScanData& GetTableScan(ui32 callableId);
- TMap<ui32, TScanData>& GetTableScans();
+ TScanData& GetTableScan(ui32 callableId);
+ TMap<ui32, TScanData>& GetTableScans();
const TMap<ui32, TScanData>& GetTableScans() const;
-
+
void Clear() {
for (auto& scan: Scans) {
scan.second.Clear();
}
Scans.clear();
}
-
-private:
+
+private:
const NYql::NDqProto::EDqStatsMode StatsMode;
- TMap<ui32, TScanData> Scans;
-};
-
+ TMap<ui32, TScanData> Scans;
+};
+
TIntrusivePtr<IKqpTableReader> CreateKqpTableReader(TKqpScanComputeContext::TScanData& scanData);
-
+
} // namespace NMiniKQL
-} // namespace NKikimr
+} // namespace NKikimr
diff --git a/ydb/core/kqp/runtime/ya.make b/ydb/core/kqp/runtime/ya.make
index 1143b6bc892..9543fea453c 100644
--- a/ydb/core/kqp/runtime/ya.make
+++ b/ydb/core/kqp/runtime/ya.make
@@ -13,11 +13,11 @@ SRCS(
kqp_output_stream.cpp
kqp_program_builder.cpp
kqp_program_builder.h
- kqp_read_table.cpp
- kqp_read_table.h
+ kqp_read_table.cpp
+ kqp_read_table.h
kqp_runtime_impl.h
- kqp_scan_data.cpp
- kqp_scan_data.h
+ kqp_scan_data.cpp
+ kqp_scan_data.h
kqp_spilling_file.cpp
kqp_tasks_runner.cpp
kqp_tasks_runner.h
diff --git a/ydb/core/kqp/ut/common/kqp_ut_common.cpp b/ydb/core/kqp/ut/common/kqp_ut_common.cpp
index 9bc7889f92f..403ff67a850 100644
--- a/ydb/core/kqp/ut/common/kqp_ut_common.cpp
+++ b/ydb/core/kqp/ut/common/kqp_ut_common.cpp
@@ -131,15 +131,15 @@ TKikimrRunner::TKikimrRunner(const TKikimrSettings& settings) {
}
TKikimrRunner::TKikimrRunner(const TVector<NKikimrKqp::TKqpSetting>& kqpSettings, const TString& authToken,
- const TString& domainRoot, ui32 nodeCount)
+ const TString& domainRoot, ui32 nodeCount)
: TKikimrRunner(TKikimrSettings()
.SetKqpSettings(kqpSettings)
.SetAuthToken(authToken)
.SetDomainRoot(domainRoot)
.SetNodeCount(nodeCount)) {}
-TKikimrRunner::TKikimrRunner(const NKikimrConfig::TAppConfig& appConfig, const TString& authToken,
- const TString& domainRoot, ui32 nodeCount)
+TKikimrRunner::TKikimrRunner(const NKikimrConfig::TAppConfig& appConfig, const TString& authToken,
+ const TString& domainRoot, ui32 nodeCount)
: TKikimrRunner(TKikimrSettings()
.SetAppConfig(appConfig)
.SetAuthToken(authToken)
diff --git a/ydb/core/kqp/ut/common/kqp_ut_common.h b/ydb/core/kqp/ut/common/kqp_ut_common.h
index c1d64b65c3e..b8033b495ff 100644
--- a/ydb/core/kqp/ut/common/kqp_ut_common.h
+++ b/ydb/core/kqp/ut/common/kqp_ut_common.h
@@ -99,10 +99,10 @@ public:
TKikimrRunner(const TKikimrSettings& settings);
TKikimrRunner(const TVector<NKikimrKqp::TKqpSetting>& kqpSettings, const TString& authToken = "",
- const TString& domainRoot = KikimrDefaultUtDomainRoot, ui32 nodeCount = 1);
+ const TString& domainRoot = KikimrDefaultUtDomainRoot, ui32 nodeCount = 1);
TKikimrRunner(const NKikimrConfig::TAppConfig& appConfig, const TString& authToken = "",
- const TString& domainRoot = KikimrDefaultUtDomainRoot, ui32 nodeCount = 1);
+ const TString& domainRoot = KikimrDefaultUtDomainRoot, ui32 nodeCount = 1);
TKikimrRunner(const NKikimrConfig::TAppConfig& appConfig, const TVector<NKikimrKqp::TKqpSetting>& kqpSettings,
const TString& authToken = "", const TString& domainRoot = KikimrDefaultUtDomainRoot, ui32 nodeCount = 1);
@@ -110,7 +110,7 @@ public:
TKikimrRunner(const NKikimrConfig::TFeatureFlags& featureFlags, const TString& authToken = "",
const TString& domainRoot = KikimrDefaultUtDomainRoot, ui32 nodeCount = 1);
- TKikimrRunner(const TString& authToken = "", const TString& domainRoot = KikimrDefaultUtDomainRoot,
+ TKikimrRunner(const TString& authToken = "", const TString& domainRoot = KikimrDefaultUtDomainRoot,
ui32 nodeCount = 1);
~TKikimrRunner() {
@@ -119,7 +119,7 @@ public:
Client.Reset();
}
- const TString& GetEndpoint() const { return Endpoint; }
+ const TString& GetEndpoint() const { return Endpoint; }
const NYdb::TDriver& GetDriver() const { return *Driver; }
NYdb::NScheme::TSchemeClient GetSchemeClient() const { return NYdb::NScheme::TSchemeClient(*Driver); }
Tests::TClient& GetTestClient() const { return *Client; }
diff --git a/ydb/core/kqp/ut/kqp_acl_ut.cpp b/ydb/core/kqp/ut/kqp_acl_ut.cpp
index 203c8ba6188..0764e66f4d5 100644
--- a/ydb/core/kqp/ut/kqp_acl_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_acl_ut.cpp
@@ -1,126 +1,126 @@
#include <ydb/core/kqp/ut/common/kqp_ut_common.h>
-
+
#include <ydb/public/sdk/cpp/client/ydb_scheme/scheme.h>
#include <ydb/public/sdk/cpp/client/ydb_table/table.h>
-
-namespace NKikimr {
-namespace NKqp {
-
-using namespace NYdb;
-using namespace NYdb::NTable;
-
-Y_UNIT_TEST_SUITE(KqpNewEngineAcl) {
- Y_UNIT_TEST(FailNavigate) {
+
+namespace NKikimr {
+namespace NKqp {
+
+using namespace NYdb;
+using namespace NYdb::NTable;
+
+Y_UNIT_TEST_SUITE(KqpNewEngineAcl) {
+ Y_UNIT_TEST(FailNavigate) {
TKikimrRunner kikimr("user0@builtin");
-
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
+
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
SELECT * FROM `/Root/TwoShard`;
- )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR);
- }
-
- Y_UNIT_TEST(FailResolve) {
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR);
+ }
+
+ Y_UNIT_TEST(FailResolve) {
TKikimrRunner kikimr;
- {
- NYdb::NScheme::TPermissions permissions("user0@builtin",
- {"ydb.deprecated.describe_schema"}
- );
- auto schemeClient = kikimr.GetSchemeClient();
- auto result = schemeClient.ModifyPermissions("/Root/TwoShard",
- NYdb::NScheme::TModifyPermissionsSettings().AddGrantPermissions(permissions)
- ).ExtractValueSync();
- AssertSuccessResult(result);
- }
-
- auto driverConfig = TDriverConfig()
- .SetEndpoint(kikimr.GetEndpoint())
- .SetAuthToken("user0@builtin");
- auto driver = TDriver(driverConfig);
+ {
+ NYdb::NScheme::TPermissions permissions("user0@builtin",
+ {"ydb.deprecated.describe_schema"}
+ );
+ auto schemeClient = kikimr.GetSchemeClient();
+ auto result = schemeClient.ModifyPermissions("/Root/TwoShard",
+ NYdb::NScheme::TModifyPermissionsSettings().AddGrantPermissions(permissions)
+ ).ExtractValueSync();
+ AssertSuccessResult(result);
+ }
+
+ auto driverConfig = TDriverConfig()
+ .SetEndpoint(kikimr.GetEndpoint())
+ .SetAuthToken("user0@builtin");
+ auto driver = TDriver(driverConfig);
auto db = NYdb::NTable::TTableClient(driver);
-
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- {
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
+
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ {
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
SELECT * FROM `/Root/TwoShard`;
- )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR);
- }
- {
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
+ }
+ {
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
UPSERT INTO `/Root/TwoShard` (Key, Value1, Value2) VALUES
- (10u, "One", -10);
- )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ (10u, "One", -10);
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR);
- }
+ }
driver.Stop(true);
- }
-
- Y_UNIT_TEST(ReadSuccess) {
+ }
+
+ Y_UNIT_TEST(ReadSuccess) {
TKikimrRunner kikimr;
- {
- NYdb::NScheme::TPermissions permissions("user0@builtin",
- {"ydb.deprecated.describe_schema", "ydb.deprecated.select_row"}
- );
- auto schemeClient = kikimr.GetSchemeClient();
- auto result = schemeClient.ModifyPermissions("/Root/TwoShard",
- NYdb::NScheme::TModifyPermissionsSettings().AddGrantPermissions(permissions)
- ).ExtractValueSync();
- AssertSuccessResult(result);
- }
-
- auto driverConfig = TDriverConfig()
- .SetEndpoint(kikimr.GetEndpoint())
- .SetAuthToken("user0@builtin");
- auto driver = TDriver(driverConfig);
+ {
+ NYdb::NScheme::TPermissions permissions("user0@builtin",
+ {"ydb.deprecated.describe_schema", "ydb.deprecated.select_row"}
+ );
+ auto schemeClient = kikimr.GetSchemeClient();
+ auto result = schemeClient.ModifyPermissions("/Root/TwoShard",
+ NYdb::NScheme::TModifyPermissionsSettings().AddGrantPermissions(permissions)
+ ).ExtractValueSync();
+ AssertSuccessResult(result);
+ }
+
+ auto driverConfig = TDriverConfig()
+ .SetEndpoint(kikimr.GetEndpoint())
+ .SetAuthToken("user0@builtin");
+ auto driver = TDriver(driverConfig);
auto db = NYdb::NTable::TTableClient(driver);
-
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
+
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
SELECT * FROM `/Root/TwoShard`;
- )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- AssertSuccessResult(result);
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ AssertSuccessResult(result);
driver.Stop(true);
- }
-
- Y_UNIT_TEST(WriteSuccess) {
+ }
+
+ Y_UNIT_TEST(WriteSuccess) {
TKikimrRunner kikimr;
- {
- NYdb::NScheme::TPermissions permissions("user0@builtin",
- {"ydb.deprecated.describe_schema", "ydb.deprecated.update_row"}
- );
- auto schemeClient = kikimr.GetSchemeClient();
- auto result = schemeClient.ModifyPermissions("/Root/TwoShard",
- NYdb::NScheme::TModifyPermissionsSettings().AddGrantPermissions(permissions)
- ).ExtractValueSync();
- AssertSuccessResult(result);
- }
-
- auto driverConfig = TDriverConfig()
- .SetEndpoint(kikimr.GetEndpoint())
- .SetAuthToken("user0@builtin");
- auto driver = TDriver(driverConfig);
+ {
+ NYdb::NScheme::TPermissions permissions("user0@builtin",
+ {"ydb.deprecated.describe_schema", "ydb.deprecated.update_row"}
+ );
+ auto schemeClient = kikimr.GetSchemeClient();
+ auto result = schemeClient.ModifyPermissions("/Root/TwoShard",
+ NYdb::NScheme::TModifyPermissionsSettings().AddGrantPermissions(permissions)
+ ).ExtractValueSync();
+ AssertSuccessResult(result);
+ }
+
+ auto driverConfig = TDriverConfig()
+ .SetEndpoint(kikimr.GetEndpoint())
+ .SetAuthToken("user0@builtin");
+ auto driver = TDriver(driverConfig);
auto db = NYdb::NTable::TTableClient(driver);
-
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
+
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
UPSERT INTO `/Root/TwoShard` (Key, Value1, Value2) VALUES
- (10u, "One", -10);
- )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
- AssertSuccessResult(result);
+ (10u, "One", -10);
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ AssertSuccessResult(result);
driver.Stop(true);
- }
+ }
Y_UNIT_TEST(RecursiveCreateTableShouldSuccess) {
TKikimrRunner kikimr;
@@ -158,7 +158,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngineAcl) {
driver.Stop(true);
}
-}
-
+}
+
} // namespace NKqp
-} // namespace NKikimr
+} // namespace NKikimr
diff --git a/ydb/core/kqp/ut/kqp_scan_ut.cpp b/ydb/core/kqp/ut/kqp_scan_ut.cpp
index 347ca354412..f5dcb825004 100644
--- a/ydb/core/kqp/ut/kqp_scan_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_scan_ut.cpp
@@ -1724,7 +1724,7 @@ Y_UNIT_TEST_SUITE(KqpScan) {
UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
CompareYson(R"([[[1u];["One"]];[[2u];["Two"]]])", StreamResultToYson(it));
TKqpCounters counters(kikimr.GetTestServer().GetRuntime()->GetAppData().Counters);
- UNIT_ASSERT_VALUES_EQUAL(1, counters.GetQueryTypeCounter(NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCAN)->Val());
+ UNIT_ASSERT_VALUES_EQUAL(1, counters.GetQueryTypeCounter(NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCAN)->Val());
}
Y_UNIT_TEST(DropRedundantSortByPk) {
diff --git a/ydb/core/kqp/ut/kqp_sys_view_ut.cpp b/ydb/core/kqp/ut/kqp_sys_view_ut.cpp
index 1f8988517c4..10437bf79e6 100644
--- a/ydb/core/kqp/ut/kqp_sys_view_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_sys_view_ut.cpp
@@ -1,24 +1,24 @@
#include <ydb/core/kqp/ut/common/kqp_ut_common.h>
-
+
#include <ydb/core/sys_view/service/query_history.h>
-
-namespace NKikimr {
-namespace NKqp {
-
-using namespace NYdb;
-using namespace NYdb::NTable;
-using namespace NYdb::NScheme;
-
-Y_UNIT_TEST_SUITE(KqpSystemView) {
-
+
+namespace NKikimr {
+namespace NKqp {
+
+using namespace NYdb;
+using namespace NYdb::NTable;
+using namespace NYdb::NScheme;
+
+Y_UNIT_TEST_SUITE(KqpSystemView) {
+
Y_UNIT_TEST(Join) {
return; // nodes table is currently switched off
-
+
TKikimrRunner kikimr;
- auto client = kikimr.GetTableClient();
+ auto client = kikimr.GetTableClient();
while (true) {
- auto it = client.StreamExecuteScanQuery(
+ auto it = client.StreamExecuteScanQuery(
"select NodeId from `/Root/.sys/partition_stats` where Path = '/Root/KeyValue' limit 1"
).GetValueSync();
UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
@@ -31,7 +31,7 @@ Y_UNIT_TEST_SUITE(KqpSystemView) {
}
}
- auto it = client.StreamExecuteScanQuery(R"(
+ auto it = client.StreamExecuteScanQuery(R"(
--!syntax_v1
select n.Host, ps.Path, ps.RowCount
from `/Root/.sys/partition_stats` as ps
@@ -44,42 +44,42 @@ Y_UNIT_TEST_SUITE(KqpSystemView) {
CompareYson(R"([[["::1"];["/Root/KeyValue"];[2u]]])", StreamResultToYson(it));
}
- Y_UNIT_TEST(PartitionStatsSimple) {
- TKikimrRunner kikimr;
- auto client = kikimr.GetTableClient();
-
- auto it = client.StreamExecuteScanQuery(R"(
+ Y_UNIT_TEST(PartitionStatsSimple) {
+ TKikimrRunner kikimr;
+ auto client = kikimr.GetTableClient();
+
+ auto it = client.StreamExecuteScanQuery(R"(
SELECT OwnerId, PartIdx, Path, PathId
FROM `/Root/.sys/partition_stats`
- ORDER BY PathId, PartIdx;
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- CompareYson(R"([
- [[72057594046644480u];[0u];["/Root/TwoShard"];[2u]];
- [[72057594046644480u];[1u];["/Root/TwoShard"];[2u]];
- [[72057594046644480u];[0u];["/Root/EightShard"];[3u]];
- [[72057594046644480u];[1u];["/Root/EightShard"];[3u]];
- [[72057594046644480u];[2u];["/Root/EightShard"];[3u]];
- [[72057594046644480u];[3u];["/Root/EightShard"];[3u]];
- [[72057594046644480u];[4u];["/Root/EightShard"];[3u]];
- [[72057594046644480u];[5u];["/Root/EightShard"];[3u]];
- [[72057594046644480u];[6u];["/Root/EightShard"];[3u]];
- [[72057594046644480u];[7u];["/Root/EightShard"];[3u]];
- [[72057594046644480u];[0u];["/Root/Logs"];[4u]];
- [[72057594046644480u];[1u];["/Root/Logs"];[4u]];
- [[72057594046644480u];[2u];["/Root/Logs"];[4u]];
- [[72057594046644480u];[0u];["/Root/BatchUpload"];[5u]];
- [[72057594046644480u];[1u];["/Root/BatchUpload"];[5u]];
- [[72057594046644480u];[2u];["/Root/BatchUpload"];[5u]];
- [[72057594046644480u];[3u];["/Root/BatchUpload"];[5u]];
- [[72057594046644480u];[4u];["/Root/BatchUpload"];[5u]];
- [[72057594046644480u];[5u];["/Root/BatchUpload"];[5u]];
- [[72057594046644480u];[6u];["/Root/BatchUpload"];[5u]];
- [[72057594046644480u];[7u];["/Root/BatchUpload"];[5u]];
- [[72057594046644480u];[8u];["/Root/BatchUpload"];[5u]];
- [[72057594046644480u];[9u];["/Root/BatchUpload"];[5u]];
+ ORDER BY PathId, PartIdx;
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ CompareYson(R"([
+ [[72057594046644480u];[0u];["/Root/TwoShard"];[2u]];
+ [[72057594046644480u];[1u];["/Root/TwoShard"];[2u]];
+ [[72057594046644480u];[0u];["/Root/EightShard"];[3u]];
+ [[72057594046644480u];[1u];["/Root/EightShard"];[3u]];
+ [[72057594046644480u];[2u];["/Root/EightShard"];[3u]];
+ [[72057594046644480u];[3u];["/Root/EightShard"];[3u]];
+ [[72057594046644480u];[4u];["/Root/EightShard"];[3u]];
+ [[72057594046644480u];[5u];["/Root/EightShard"];[3u]];
+ [[72057594046644480u];[6u];["/Root/EightShard"];[3u]];
+ [[72057594046644480u];[7u];["/Root/EightShard"];[3u]];
+ [[72057594046644480u];[0u];["/Root/Logs"];[4u]];
+ [[72057594046644480u];[1u];["/Root/Logs"];[4u]];
+ [[72057594046644480u];[2u];["/Root/Logs"];[4u]];
+ [[72057594046644480u];[0u];["/Root/BatchUpload"];[5u]];
+ [[72057594046644480u];[1u];["/Root/BatchUpload"];[5u]];
+ [[72057594046644480u];[2u];["/Root/BatchUpload"];[5u]];
+ [[72057594046644480u];[3u];["/Root/BatchUpload"];[5u]];
+ [[72057594046644480u];[4u];["/Root/BatchUpload"];[5u]];
+ [[72057594046644480u];[5u];["/Root/BatchUpload"];[5u]];
+ [[72057594046644480u];[6u];["/Root/BatchUpload"];[5u]];
+ [[72057594046644480u];[7u];["/Root/BatchUpload"];[5u]];
+ [[72057594046644480u];[8u];["/Root/BatchUpload"];[5u]];
+ [[72057594046644480u];[9u];["/Root/BatchUpload"];[5u]];
[[72057594046644480u];[0u];["/Root/KeyValue"];[6u]];
[[72057594046644480u];[0u];["/Root/KeyValue2"];[7u]];
[[72057594046644480u];[0u];["/Root/Test"];[8u]];
@@ -87,24 +87,24 @@ Y_UNIT_TEST_SUITE(KqpSystemView) {
[[72057594046644480u];[1u];["/Root/Join1"];[9u]];
[[72057594046644480u];[0u];["/Root/Join2"];[10u]];
[[72057594046644480u];[1u];["/Root/Join2"];[10u]]
- ])", StreamResultToYson(it));
- }
-
- Y_UNIT_TEST(PartitionStatsRange1) {
- TKikimrRunner kikimr;
- auto client = kikimr.GetTableClient();
-
+ ])", StreamResultToYson(it));
+ }
+
+ Y_UNIT_TEST(PartitionStatsRange1) {
+ TKikimrRunner kikimr;
+ auto client = kikimr.GetTableClient();
+
TString enablePredicateExtractor = R"(
PRAGMA Kikimr.OptEnablePredicateExtract = "true";
)";
TString query = R"(
- SELECT OwnerId, PathId, PartIdx, Path
+ SELECT OwnerId, PathId, PartIdx, Path
FROM `/Root/.sys/partition_stats`
WHERE OwnerId = 72057594046644480ul AND PathId > 5u AND PathId <= 9u
- ORDER BY PathId, PartIdx;
+ ORDER BY PathId, PartIdx;
)";
-
+
TString expectedYson = R"([
[[72057594046644480u];[6u];[0u];["/Root/KeyValue"]];
[[72057594046644480u];[7u];[0u];["/Root/KeyValue2"]];
@@ -120,22 +120,22 @@ Y_UNIT_TEST_SUITE(KqpSystemView) {
it = client.StreamExecuteScanQuery(enablePredicateExtractor + query).GetValueSync();
UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
CompareYson(expectedYson, StreamResultToYson(it));
- }
-
- Y_UNIT_TEST(PartitionStatsRange2) {
- TKikimrRunner kikimr;
- auto client = kikimr.GetTableClient();
+ }
+
+ Y_UNIT_TEST(PartitionStatsRange2) {
+ TKikimrRunner kikimr;
+ auto client = kikimr.GetTableClient();
TString enablePredicateExtractor = R"(
PRAGMA Kikimr.OptEnablePredicateExtract = "true";
)";
-
+
TString query = R"(
- SELECT OwnerId, PathId, PartIdx, Path
+ SELECT OwnerId, PathId, PartIdx, Path
FROM `/Root/.sys/partition_stats`
WHERE OwnerId = 72057594046644480ul AND PathId >= 6u AND PathId < 9u
- ORDER BY PathId, PartIdx;
+ ORDER BY PathId, PartIdx;
)";
-
+
TString expectedYson = R"([
[[72057594046644480u];[6u];[0u];["/Root/KeyValue"]];
[[72057594046644480u];[7u];[0u];["/Root/KeyValue2"]];
@@ -149,111 +149,111 @@ Y_UNIT_TEST_SUITE(KqpSystemView) {
it = client.StreamExecuteScanQuery(enablePredicateExtractor + query).GetValueSync();
UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
CompareYson(expectedYson, StreamResultToYson(it));
- }
-
- Y_UNIT_TEST(PartitionStatsRange3) {
- TKikimrRunner kikimr;
- auto client = kikimr.GetTableClient();
-
- auto it = client.StreamExecuteScanQuery(R"(
- SELECT OwnerId, PathId, PartIdx, Path
+ }
+
+ Y_UNIT_TEST(PartitionStatsRange3) {
+ TKikimrRunner kikimr;
+ auto client = kikimr.GetTableClient();
+
+ auto it = client.StreamExecuteScanQuery(R"(
+ SELECT OwnerId, PathId, PartIdx, Path
FROM `/Root/.sys/partition_stats`
- WHERE OwnerId = 72057594046644480ul AND PathId = 5u AND PartIdx > 1u AND PartIdx < 7u
- ORDER BY PathId, PartIdx;
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- CompareYson(R"([
+ WHERE OwnerId = 72057594046644480ul AND PathId = 5u AND PartIdx > 1u AND PartIdx < 7u
+ ORDER BY PathId, PartIdx;
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ CompareYson(R"([
[[72057594046644480u];[5u];[2u];["/Root/BatchUpload"]];
[[72057594046644480u];[5u];[3u];["/Root/BatchUpload"]];
[[72057594046644480u];[5u];[4u];["/Root/BatchUpload"]];
[[72057594046644480u];[5u];[5u];["/Root/BatchUpload"]];
[[72057594046644480u];[5u];[6u];["/Root/BatchUpload"]];
- ])", StreamResultToYson(it));
- }
-
- Y_UNIT_TEST(NodesSimple) {
- return; // nodes table is currenty switched off
-
- TKikimrRunner kikimr("", KikimrDefaultUtDomainRoot, 3);
- auto client = kikimr.GetTableClient();
-
- ui32 offset = kikimr.GetTestServer().GetRuntime()->GetNodeId(0);
-
- auto it = client.StreamExecuteScanQuery(R"(
- SELECT NodeId, Host
+ ])", StreamResultToYson(it));
+ }
+
+ Y_UNIT_TEST(NodesSimple) {
+ return; // nodes table is currenty switched off
+
+ TKikimrRunner kikimr("", KikimrDefaultUtDomainRoot, 3);
+ auto client = kikimr.GetTableClient();
+
+ ui32 offset = kikimr.GetTestServer().GetRuntime()->GetNodeId(0);
+
+ auto it = client.StreamExecuteScanQuery(R"(
+ SELECT NodeId, Host
FROM `/Root/.sys/nodes`;
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- auto expected = Sprintf(R"([
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ auto expected = Sprintf(R"([
[[%du];["::1"]];
[[%du];["::1"]];
[[%du];["::1"]]
- ])", offset, offset + 1, offset + 2);
-
- CompareYson(expected, StreamResultToYson(it));
- }
-
- Y_UNIT_TEST(NodesRange1) {
- return; // nodes table is currenty switched off
-
- TKikimrRunner kikimr("", KikimrDefaultUtDomainRoot, 5);
- auto client = kikimr.GetTableClient();
-
- ui32 offset = kikimr.GetTestServer().GetRuntime()->GetNodeId(0);
-
- auto query = Sprintf(R"(
- SELECT NodeId, Host
+ ])", offset, offset + 1, offset + 2);
+
+ CompareYson(expected, StreamResultToYson(it));
+ }
+
+ Y_UNIT_TEST(NodesRange1) {
+ return; // nodes table is currenty switched off
+
+ TKikimrRunner kikimr("", KikimrDefaultUtDomainRoot, 5);
+ auto client = kikimr.GetTableClient();
+
+ ui32 offset = kikimr.GetTestServer().GetRuntime()->GetNodeId(0);
+
+ auto query = Sprintf(R"(
+ SELECT NodeId, Host
FROM `/Root/.sys/nodes`
- WHERE NodeId >= %du AND NodeId <= %du
- )", offset + 1, offset + 3);
-
- auto it = client.StreamExecuteScanQuery(query).GetValueSync();
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- auto expected = Sprintf(R"([
+ WHERE NodeId >= %du AND NodeId <= %du
+ )", offset + 1, offset + 3);
+
+ auto it = client.StreamExecuteScanQuery(query).GetValueSync();
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ auto expected = Sprintf(R"([
[[%du];["::1"]];
[[%du];["::1"]];
[[%du];["::1"]]
- ])", offset + 1, offset + 2, offset + 3);
-
- CompareYson(expected, StreamResultToYson(it));
- }
-
- Y_UNIT_TEST(NodesRange2) {
- return; // nodes table is currenty switched off
-
- TKikimrRunner kikimr("", KikimrDefaultUtDomainRoot, 5);
- auto client = kikimr.GetTableClient();
-
- ui32 offset = kikimr.GetTestServer().GetRuntime()->GetNodeId(0);
-
- auto query = Sprintf(R"(
- SELECT NodeId, Host
+ ])", offset + 1, offset + 2, offset + 3);
+
+ CompareYson(expected, StreamResultToYson(it));
+ }
+
+ Y_UNIT_TEST(NodesRange2) {
+ return; // nodes table is currenty switched off
+
+ TKikimrRunner kikimr("", KikimrDefaultUtDomainRoot, 5);
+ auto client = kikimr.GetTableClient();
+
+ ui32 offset = kikimr.GetTestServer().GetRuntime()->GetNodeId(0);
+
+ auto query = Sprintf(R"(
+ SELECT NodeId, Host
FROM `/Root/.sys/nodes`
- WHERE NodeId > %du AND NodeId < %du
- )", offset + 1, offset + 3);
-
- auto it = client.StreamExecuteScanQuery(query).GetValueSync();
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- auto expected = Sprintf(R"([
+ WHERE NodeId > %du AND NodeId < %du
+ )", offset + 1, offset + 3);
+
+ auto it = client.StreamExecuteScanQuery(query).GetValueSync();
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ auto expected = Sprintf(R"([
[[%du];["::1"]];
- ])", offset + 2);
-
- CompareYson(expected, StreamResultToYson(it));
- }
-
+ ])", offset + 2);
+
+ CompareYson(expected, StreamResultToYson(it));
+ }
+
Y_UNIT_TEST_NEW_ENGINE(QueryStatsSimple) {
- auto checkTable = [&] (const TStringBuf tableName) {
- TKikimrRunner kikimr("", KikimrDefaultUtDomainRoot, 3);
-
- auto client = kikimr.GetTableClient();
- auto session = client.CreateSession().GetValueSync().GetSession();
- {
+ auto checkTable = [&] (const TStringBuf tableName) {
+ TKikimrRunner kikimr("", KikimrDefaultUtDomainRoot, 3);
+
+ auto client = kikimr.GetTableClient();
+ auto session = client.CreateSession().GetValueSync().GetSession();
+ {
auto result = session.ExecuteDataQuery("SELECT 1;",
TTxControl::BeginTx().CommitTx()).GetValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
@@ -262,21 +262,21 @@ Y_UNIT_TEST_SUITE(KqpSystemView) {
auto result = session.ExecuteDataQuery(Q_(R"(
SELECT * FROM `/Root/TwoShard`
)"), TTxControl::BeginTx().CommitTx()).GetValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- }
- {
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ }
+ {
auto result = session.ExecuteDataQuery(Q_(R"(
SELECT * FROM `/Root/EightShard`
)"), TTxControl::BeginTx().CommitTx()).GetValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- }
-
- TStringStream request;
- request << "SELECT ReadBytes FROM " << tableName << " ORDER BY ReadBytes";
-
- auto it = client.StreamExecuteScanQuery(request.Str()).GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ }
+
+ TStringStream request;
+ request << "SELECT ReadBytes FROM " << tableName << " ORDER BY ReadBytes";
+
+ auto it = client.StreamExecuteScanQuery(request.Str()).GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
TSet<ui64> readBytesSet;
for (;;) {
@@ -301,87 +301,87 @@ Y_UNIT_TEST_SUITE(KqpSystemView) {
UNIT_ASSERT(readBytesSet.contains(0)); // Pure
UNIT_ASSERT(readBytesSet.contains(79)); // TwoShard
UNIT_ASSERT(readBytesSet.contains(432)); // EightShard
- };
-
+ };
+
checkTable("`/Root/.sys/top_queries_by_read_bytes_one_minute`");
checkTable("`/Root/.sys/top_queries_by_read_bytes_one_hour`");
checkTable("`/Root/.sys/top_queries_by_duration_one_minute`");
checkTable("`/Root/.sys/top_queries_by_duration_one_hour`");
checkTable("`/Root/.sys/top_queries_by_cpu_time_one_minute`");
checkTable("`/Root/.sys/top_queries_by_cpu_time_one_hour`");
- }
-
- Y_UNIT_TEST(FailNavigate) {
- TKikimrRunner kikimr("user0@builtin");
- auto client = kikimr.GetTableClient();
-
- auto it = client.StreamExecuteScanQuery(R"(
+ }
+
+ Y_UNIT_TEST(FailNavigate) {
+ TKikimrRunner kikimr("user0@builtin");
+ auto client = kikimr.GetTableClient();
+
+ auto it = client.StreamExecuteScanQuery(R"(
SELECT PathId FROM `/Root/.sys/partition_stats`;
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- auto streamPart = it.ReadNext().GetValueSync();
- UNIT_ASSERT_VALUES_EQUAL(streamPart.GetStatus(), EStatus::SCHEME_ERROR);
- }
-
- Y_UNIT_TEST(FailResolve) {
- TKikimrRunner kikimr;
- {
- TPermissions permissions("user0@builtin",
- {"ydb.deprecated.describe_schema"}
- );
- auto schemeClient = kikimr.GetSchemeClient();
- auto result = schemeClient.ModifyPermissions("/Root",
- TModifyPermissionsSettings().AddGrantPermissions(permissions)
- ).ExtractValueSync();
- AssertSuccessResult(result);
- }
-
- auto driverConfig = TDriverConfig()
- .SetEndpoint(kikimr.GetEndpoint())
- .SetAuthToken("user0@builtin");
- auto driver = TDriver(driverConfig);
-
- TTableClient client(driver);
- auto it = client.StreamExecuteScanQuery(R"(
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ auto streamPart = it.ReadNext().GetValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(streamPart.GetStatus(), EStatus::SCHEME_ERROR);
+ }
+
+ Y_UNIT_TEST(FailResolve) {
+ TKikimrRunner kikimr;
+ {
+ TPermissions permissions("user0@builtin",
+ {"ydb.deprecated.describe_schema"}
+ );
+ auto schemeClient = kikimr.GetSchemeClient();
+ auto result = schemeClient.ModifyPermissions("/Root",
+ TModifyPermissionsSettings().AddGrantPermissions(permissions)
+ ).ExtractValueSync();
+ AssertSuccessResult(result);
+ }
+
+ auto driverConfig = TDriverConfig()
+ .SetEndpoint(kikimr.GetEndpoint())
+ .SetAuthToken("user0@builtin");
+ auto driver = TDriver(driverConfig);
+
+ TTableClient client(driver);
+ auto it = client.StreamExecuteScanQuery(R"(
SELECT PathId FROM `/Root/.sys/partition_stats`;
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- auto streamPart = it.ReadNext().GetValueSync();
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ auto streamPart = it.ReadNext().GetValueSync();
UNIT_ASSERT_VALUES_EQUAL(streamPart.GetStatus(), EStatus::SCHEME_ERROR);
driver.Stop(true);
- }
-
- Y_UNIT_TEST(ReadSuccess) {
- TKikimrRunner kikimr;
- {
- TPermissions permissions("user0@builtin",
- {"ydb.deprecated.describe_schema", "ydb.deprecated.select_row"}
- );
- auto schemeClient = kikimr.GetSchemeClient();
- auto result = schemeClient.ModifyPermissions("/Root",
- TModifyPermissionsSettings().AddGrantPermissions(permissions)
- ).ExtractValueSync();
- AssertSuccessResult(result);
- }
-
- auto driverConfig = TDriverConfig()
- .SetEndpoint(kikimr.GetEndpoint())
- .SetAuthToken("user0@builtin");
- auto driver = TDriver(driverConfig);
-
- TTableClient client(driver);
- auto it = client.StreamExecuteScanQuery(R"(
+ }
+
+ Y_UNIT_TEST(ReadSuccess) {
+ TKikimrRunner kikimr;
+ {
+ TPermissions permissions("user0@builtin",
+ {"ydb.deprecated.describe_schema", "ydb.deprecated.select_row"}
+ );
+ auto schemeClient = kikimr.GetSchemeClient();
+ auto result = schemeClient.ModifyPermissions("/Root",
+ TModifyPermissionsSettings().AddGrantPermissions(permissions)
+ ).ExtractValueSync();
+ AssertSuccessResult(result);
+ }
+
+ auto driverConfig = TDriverConfig()
+ .SetEndpoint(kikimr.GetEndpoint())
+ .SetAuthToken("user0@builtin");
+ auto driver = TDriver(driverConfig);
+
+ TTableClient client(driver);
+ auto it = client.StreamExecuteScanQuery(R"(
SELECT PathId FROM `/Root/.sys/partition_stats`;
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- auto streamPart = it.ReadNext().GetValueSync();
- UNIT_ASSERT_VALUES_EQUAL(streamPart.GetStatus(), EStatus::SUCCESS);
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ auto streamPart = it.ReadNext().GetValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(streamPart.GetStatus(), EStatus::SUCCESS);
driver.Stop(true);
- }
-}
-
-} // namspace NKqp
-} // namespace NKikimr
+ }
+}
+
+} // namspace NKqp
+} // namespace NKikimr
diff --git a/ydb/core/kqp/ut/ya.make b/ydb/core/kqp/ut/ya.make
index 609ec3237f8..97135d645ef 100644
--- a/ydb/core/kqp/ut/ya.make
+++ b/ydb/core/kqp/ut/ya.make
@@ -19,7 +19,7 @@ ELSE()
ENDIF()
SRCS(
- kqp_acl_ut.cpp
+ kqp_acl_ut.cpp
kqp_arrow_in_channels_ut.cpp
kqp_document_api_ut.cpp
kqp_effects_perf_ut.cpp
@@ -49,7 +49,7 @@ SRCS(
kqp_sort_ut.cpp
kqp_stats_ut.cpp
kqp_sqlin_ut.cpp
- kqp_sys_view_ut.cpp
+ kqp_sys_view_ut.cpp
kqp_sys_col_ut.cpp
kqp_table_predicate_ut.cpp
kqp_tx_ut.cpp
diff --git a/ydb/core/mind/bscontroller/bsc.cpp b/ydb/core/mind/bscontroller/bsc.cpp
index 779381fd513..6be79630f3d 100644
--- a/ydb/core/mind/bscontroller/bsc.cpp
+++ b/ydb/core/mind/bscontroller/bsc.cpp
@@ -1,7 +1,7 @@
#include "impl.h"
#include "config.h"
#include "self_heal.h"
-#include "sys_view.h"
+#include "sys_view.h"
namespace NKikimr {
@@ -121,9 +121,9 @@ void TBlobStorageController::OnActivateExecutor(const TActorContext&) {
// create stat processor
StatProcessorActorId = Register(CreateStatProcessorActor());
- // create system views collector
+ // create system views collector
SystemViewsCollectorId = Register(CreateSystemViewsCollector());
-
+
Executor()->RegisterExternalTabletCounters(TabletCountersPtr);
if (!ResponsivenessPinger) {
ResponsivenessPinger = new TTabletResponsivenessPinger(TabletCounters->Simple()[NBlobStorageController::COUNTER_RESPONSE_TIME_USEC], TDuration::Seconds(1));
diff --git a/ydb/core/mind/bscontroller/impl.h b/ydb/core/mind/bscontroller/impl.h
index ba311a254c3..3ceeac69601 100644
--- a/ydb/core/mind/bscontroller/impl.h
+++ b/ydb/core/mind/bscontroller/impl.h
@@ -1335,10 +1335,10 @@ private:
friend class TConfigState;
class TNodeWardenUpdateNotifier;
-
- struct TEvPrivate {
- enum EEv {
- EvUpdateSystemViews = EventSpaceBegin(TEvents::ES_PRIVATE),
+
+ struct TEvPrivate {
+ enum EEv {
+ EvUpdateSystemViews = EventSpaceBegin(TEvents::ES_PRIVATE),
EvUpdateSelfHealCounters,
EvHostRecordsTimeToLiveExceeded,
EvDropDonor,
@@ -1346,9 +1346,9 @@ private:
EvVSlotReadyUpdate,
EvVSlotNotReadyHistogramUpdate,
EvProcessIncomingEvent,
- };
-
- struct TEvUpdateSystemViews : public TEventLocal<TEvUpdateSystemViews, EvUpdateSystemViews> {};
+ };
+
+ struct TEvUpdateSystemViews : public TEventLocal<TEvUpdateSystemViews, EvUpdateSystemViews> {};
struct TEvUpdateSelfHealCounters : TEventLocal<TEvUpdateSelfHealCounters, EvUpdateSelfHealCounters> {};
struct TEvHostRecordsTimeToLiveExceeded : TEventLocal<TEvHostRecordsTimeToLiveExceeded, EvHostRecordsTimeToLiveExceeded> {};
@@ -1359,10 +1359,10 @@ private:
struct TEvScrub : TEventLocal<TEvScrub, EvScrub> {};
struct TEvVSlotReadyUpdate : TEventLocal<TEvVSlotReadyUpdate, EvVSlotReadyUpdate> {};
struct TEvVSlotNotReadyHistogramUpdate : TEventLocal<TEvVSlotNotReadyHistogramUpdate, EvVSlotNotReadyHistogramUpdate> {};
- };
-
- static constexpr TDuration UpdateSystemViewsPeriod = TDuration::Seconds(5);
-
+ };
+
+ static constexpr TDuration UpdateSystemViewsPeriod = TDuration::Seconds(5);
+
std::unordered_set<TPDiskId, THash<TPDiskId>> SysViewChangedPDisks;
std::unordered_set<TVSlotId, THash<TVSlotId>> SysViewChangedVSlots;
std::unordered_set<TGroupId, THash<TGroupId>> SysViewChangedGroups;
@@ -1371,7 +1371,7 @@ private:
IActor* CreateSystemViewsCollector();
void UpdateSystemViews();
-
+
bool CommitConfigUpdates(TConfigState& state, bool suppressFailModelChecking, bool suppressDegradedGroupsChecking,
TTransactionContext& txc, TString *errorDescription);
diff --git a/ydb/core/mind/bscontroller/sys_view.cpp b/ydb/core/mind/bscontroller/sys_view.cpp
index ba9b50cde00..c1153e42d9f 100644
--- a/ydb/core/mind/bscontroller/sys_view.cpp
+++ b/ydb/core/mind/bscontroller/sys_view.cpp
@@ -1,52 +1,52 @@
-#include "sys_view.h"
+#include "sys_view.h"
#include "group_geometry_info.h"
-
-namespace NKikimr::NBsController {
-
-using namespace NSysView;
-
-TPDiskId TransformKey(const NKikimrSysView::TPDiskKey& key) {
- return TPDiskId(key.GetNodeId(), key.GetPDiskId());
-}
-
-void FillKey(NKikimrSysView::TPDiskKey* key, const TPDiskId& id) {
- key->SetNodeId(id.NodeId);
- key->SetPDiskId(id.PDiskId);
-}
-
-TVSlotId TransformKey(const NKikimrSysView::TVSlotKey& key) {
- return TVSlotId(key.GetNodeId(), key.GetPDiskId(), key.GetVSlotId());
-}
-
-void FillKey(NKikimrSysView::TVSlotKey* key, const TVSlotId& id) {
- key->SetNodeId(id.NodeId);
- key->SetPDiskId(id.PDiskId);
- key->SetVSlotId(id.VSlotId);
-}
-
-TGroupId TransformKey(const NKikimrSysView::TGroupKey& key) {
- return key.GetGroupId();
-}
-
-void FillKey(NKikimrSysView::TGroupKey* key, const TGroupId& id) {
- key->SetGroupId(id);
-}
-
-TBlobStorageController::TBoxStoragePoolId TransformKey(const NKikimrSysView::TStoragePoolKey& key) {
- return std::make_tuple(key.GetBoxId(), key.GetStoragePoolId());
-}
-
-void FillKey(NKikimrSysView::TStoragePoolKey* key, const TBlobStorageController::TBoxStoragePoolId& id) {
- key->SetBoxId(std::get<0>(id));
- key->SetStoragePoolId(std::get<1>(id));
-}
-
+
+namespace NKikimr::NBsController {
+
+using namespace NSysView;
+
+TPDiskId TransformKey(const NKikimrSysView::TPDiskKey& key) {
+ return TPDiskId(key.GetNodeId(), key.GetPDiskId());
+}
+
+void FillKey(NKikimrSysView::TPDiskKey* key, const TPDiskId& id) {
+ key->SetNodeId(id.NodeId);
+ key->SetPDiskId(id.PDiskId);
+}
+
+TVSlotId TransformKey(const NKikimrSysView::TVSlotKey& key) {
+ return TVSlotId(key.GetNodeId(), key.GetPDiskId(), key.GetVSlotId());
+}
+
+void FillKey(NKikimrSysView::TVSlotKey* key, const TVSlotId& id) {
+ key->SetNodeId(id.NodeId);
+ key->SetPDiskId(id.PDiskId);
+ key->SetVSlotId(id.VSlotId);
+}
+
+TGroupId TransformKey(const NKikimrSysView::TGroupKey& key) {
+ return key.GetGroupId();
+}
+
+void FillKey(NKikimrSysView::TGroupKey* key, const TGroupId& id) {
+ key->SetGroupId(id);
+}
+
+TBlobStorageController::TBoxStoragePoolId TransformKey(const NKikimrSysView::TStoragePoolKey& key) {
+ return std::make_tuple(key.GetBoxId(), key.GetStoragePoolId());
+}
+
+void FillKey(NKikimrSysView::TStoragePoolKey* key, const TBlobStorageController::TBoxStoragePoolId& id) {
+ key->SetBoxId(std::get<0>(id));
+ key->SetStoragePoolId(std::get<1>(id));
+}
+
struct TGroupDiskInfo {
const NKikimrBlobStorage::TPDiskMetrics *PDiskMetrics;
const NKikimrBlobStorage::TVDiskMetrics *VDiskMetrics;
ui32 ExpectedSlotCount;
};
-
+
void CalculateGroupUsageStats(NKikimrSysView::TGroupInfo *info, const std::vector<TGroupDiskInfo>& disks,
TBlobStorageGroupType type) {
ui64 allocatedSize = 0;
@@ -75,8 +75,8 @@ void CalculateGroupUsageStats(NKikimrSysView::TGroupInfo *info, const std::vecto
info->SetAvailableSize(b < a ? a - b : 0);
}
-class TSystemViewsCollector : public TActor<TSystemViewsCollector> {
- TControllerSystemViewsState State;
+class TSystemViewsCollector : public TActor<TSystemViewsCollector> {
+ TControllerSystemViewsState State;
std::optional<std::vector<NKikimrSysView::TStorageStatsEntry>> StorageStats;
std::vector<std::pair<TPDiskId, const NKikimrSysView::TPDiskInfo*>> PDiskIndex;
std::vector<std::pair<TVSlotId, const NKikimrSysView::TVSlotInfo*>> VSlotIndex;
@@ -88,32 +88,32 @@ class TSystemViewsCollector : public TActor<TSystemViewsCollector> {
NMonitoring::TDynamicCounterPtr Counters;
std::unordered_set<std::tuple<TString>> PDiskFilterCounters;
std::unordered_set<std::tuple<TString, TString>> ErasureCounters;
-
-public:
- static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
- return NKikimrServices::TActivity::BSC_SYSTEM_VIEWS_COLLECTOR;
- }
-
+
+public:
+ static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
+ return NKikimrServices::TActivity::BSC_SYSTEM_VIEWS_COLLECTOR;
+ }
+
TSystemViewsCollector(NMonitoring::TDynamicCounterPtr counters)
- : TActor(&TSystemViewsCollector::StateWork)
+ : TActor(&TSystemViewsCollector::StateWork)
, Counters(std::move(counters))
- {}
-
+ {}
+
~TSystemViewsCollector() {
Counters->RemoveSubgroup("subsystem", "storage_stats");
}
- STRICT_STFUNC(StateWork,
- hFunc(TEvControllerUpdateSystemViews, Handle);
- hFunc(TEvSysView::TEvGetPDisksRequest, Handle);
- hFunc(TEvSysView::TEvGetVSlotsRequest, Handle);
- hFunc(TEvSysView::TEvGetGroupsRequest, Handle);
- hFunc(TEvSysView::TEvGetStoragePoolsRequest, Handle);
+ STRICT_STFUNC(StateWork,
+ hFunc(TEvControllerUpdateSystemViews, Handle);
+ hFunc(TEvSysView::TEvGetPDisksRequest, Handle);
+ hFunc(TEvSysView::TEvGetVSlotsRequest, Handle);
+ hFunc(TEvSysView::TEvGetGroupsRequest, Handle);
+ hFunc(TEvSysView::TEvGetStoragePoolsRequest, Handle);
hFunc(TEvSysView::TEvGetStorageStatsRequest, Handle);
- cFunc(TEvents::TSystem::Poison, PassAway);
- )
-
- void Handle(TEvControllerUpdateSystemViews::TPtr& ev) {
+ cFunc(TEvents::TSystem::Poison, PassAway);
+ )
+
+ void Handle(TEvControllerUpdateSystemViews::TPtr& ev) {
auto *msg = ev->Get();
auto& newState = msg->State;
Merge(State.PDisks, newState.PDisks, msg->DeletedPDisks, PDiskIndex);
@@ -124,8 +124,8 @@ public:
GroupReserveMin = msg->GroupReserveMin;
GroupReservePart = msg->GroupReservePart;
GenerateStorageStats();
- }
-
+ }
+
template<typename TDest, typename TSrc, typename TDeleted, typename TIndex>
void Merge(TDest& dest, TSrc& src, const TDeleted& deleted, TIndex& index) {
if (!src.empty() || !deleted.empty()) {
@@ -142,9 +142,9 @@ public:
template <typename TResponse, typename TRequest, typename TMap, typename TIndex>
void Reply(TRequest& request, const TMap& entries, TIndex& index) {
- const auto& record = request->Get()->Record;
+ const auto& record = request->Get()->Record;
auto response = MakeHolder<TResponse>();
-
+
if (index.empty() && !entries.empty()) {
index.reserve(entries.size());
for (const auto& [key, value] : entries) {
@@ -152,51 +152,51 @@ public:
}
std::sort(index.begin(), index.end());
}
-
+
auto begin = index.begin();
auto end = index.end();
auto comp = [](const auto& kv, const auto& key) { return kv.first < key; };
- if (record.HasFrom()) {
+ if (record.HasFrom()) {
auto from = TransformKey(record.GetFrom());
begin = std::lower_bound(index.begin(), index.end(), from, comp);
if (begin != index.end() && begin->first == from && record.HasInclusiveFrom() && !record.GetInclusiveFrom()) {
++begin;
- }
- }
-
- if (record.HasTo()) {
+ }
+ }
+
+ if (record.HasTo()) {
auto to = TransformKey(record.GetTo());
end = std::lower_bound(index.begin(), index.end(), to, comp);
if (end != index.end() && end->first == to && record.GetInclusiveTo()) {
++end;
- }
- }
-
+ }
+ }
+
for (; begin < end; ++begin) {
- auto* entry = response->Record.AddEntries();
+ auto* entry = response->Record.AddEntries();
FillKey(entry->MutableKey(), begin->first);
entry->MutableInfo()->CopyFrom(*begin->second);
- }
-
- Send(request->Sender, response.Release());
- }
-
- void Handle(TEvSysView::TEvGetPDisksRequest::TPtr& ev) {
+ }
+
+ Send(request->Sender, response.Release());
+ }
+
+ void Handle(TEvSysView::TEvGetPDisksRequest::TPtr& ev) {
Reply<TEvSysView::TEvGetPDisksResponse>(ev, State.PDisks, PDiskIndex);
- }
-
- void Handle(TEvSysView::TEvGetVSlotsRequest::TPtr& ev) {
+ }
+
+ void Handle(TEvSysView::TEvGetVSlotsRequest::TPtr& ev) {
Reply<TEvSysView::TEvGetVSlotsResponse>(ev, State.VSlots, VSlotIndex);
- }
-
- void Handle(TEvSysView::TEvGetGroupsRequest::TPtr& ev) {
+ }
+
+ void Handle(TEvSysView::TEvGetGroupsRequest::TPtr& ev) {
Reply<TEvSysView::TEvGetGroupsResponse>(ev, State.Groups, GroupIndex);
- }
-
- void Handle(TEvSysView::TEvGetStoragePoolsRequest::TPtr& ev) {
+ }
+
+ void Handle(TEvSysView::TEvGetStoragePoolsRequest::TPtr& ev) {
Reply<TEvSysView::TEvGetStoragePoolsResponse>(ev, State.StoragePools, StoragePoolIndex);
- }
+ }
void Handle(TEvSysView::TEvGetStorageStatsRequest::TPtr& ev) {
auto response = std::make_unique<TEvSysView::TEvGetStorageStatsResponse>();
@@ -386,37 +386,37 @@ public:
->RemoveSubgroup("pdiskFilter", std::get<0>(item));
}
}
-};
-
+};
+
IActor* TBlobStorageController::CreateSystemViewsCollector() {
return new TSystemViewsCollector(GetServiceCounters(AppData()->Counters, "storage_pool_stat"));
-}
-
+}
+
void TBlobStorageController::ForwardToSystemViewsCollector(STATEFN_SIG) {
TActivationContext::Send(ev->Forward(SystemViewsCollectorId));
-}
-
+}
+
void TBlobStorageController::Handle(TEvPrivate::TEvUpdateSystemViews::TPtr&) {
UpdateSystemViews();
-}
-
-void CopyInfo(NKikimrSysView::TPDiskInfo* info, const THolder<TBlobStorageController::TPDiskInfo>& pDiskInfo) {
- TPDiskCategory category(pDiskInfo->Kind);
- info->SetType(category.TypeStrShort());
- info->SetKind(category.Kind());
+}
+
+void CopyInfo(NKikimrSysView::TPDiskInfo* info, const THolder<TBlobStorageController::TPDiskInfo>& pDiskInfo) {
+ TPDiskCategory category(pDiskInfo->Kind);
+ info->SetType(category.TypeStrShort());
+ info->SetKind(category.Kind());
info->SetCategory(category);
- info->SetPath(pDiskInfo->Path);
- info->SetGuid(pDiskInfo->Guid);
+ info->SetPath(pDiskInfo->Path);
+ info->SetGuid(pDiskInfo->Guid);
info->SetBoxId(pDiskInfo->BoxId);
- if (pDiskInfo->SharedWithOs) {
- info->SetSharedWithOs(*pDiskInfo->SharedWithOs);
- }
- if (pDiskInfo->ReadCentric) {
- info->SetReadCentric(*pDiskInfo->ReadCentric);
- }
- info->SetAvailableSize(pDiskInfo->Metrics.GetAvailableSize());
- info->SetTotalSize(pDiskInfo->Metrics.GetTotalSize());
- info->SetStatusV2(NKikimrBlobStorage::EDriveStatus_Name(pDiskInfo->Status));
+ if (pDiskInfo->SharedWithOs) {
+ info->SetSharedWithOs(*pDiskInfo->SharedWithOs);
+ }
+ if (pDiskInfo->ReadCentric) {
+ info->SetReadCentric(*pDiskInfo->ReadCentric);
+ }
+ info->SetAvailableSize(pDiskInfo->Metrics.GetAvailableSize());
+ info->SetTotalSize(pDiskInfo->Metrics.GetTotalSize());
+ info->SetStatusV2(NKikimrBlobStorage::EDriveStatus_Name(pDiskInfo->Status));
if (pDiskInfo->StatusTimestamp != TInstant::Zero()) {
info->SetStatusChangeTimestamp(pDiskInfo->StatusTimestamp.GetValue());
}
@@ -425,8 +425,8 @@ void CopyInfo(NKikimrSysView::TPDiskInfo* info, const THolder<TBlobStorageContro
}
info->SetExpectedSlotCount(pDiskInfo->ExpectedSlotCount);
info->SetNumActiveSlots(pDiskInfo->NumActiveSlots + pDiskInfo->StaticSlotUsage);
-}
-
+}
+
void SerializeVSlotInfo(NKikimrSysView::TVSlotInfo *pb, const TVDiskID& vdiskId, const NKikimrBlobStorage::TVDiskMetrics& m,
NKikimrBlobStorage::EVDiskStatus status, NKikimrBlobStorage::TVDiskKind::EVDiskKind kind, bool isBeingDeleted) {
pb->SetGroupId(vdiskId.GroupID);
@@ -447,68 +447,68 @@ void SerializeVSlotInfo(NKikimrSysView::TVSlotInfo *pb, const TVDiskID& vdiskId,
}
}
-void CopyInfo(NKikimrSysView::TVSlotInfo* info, const THolder<TBlobStorageController::TVSlotInfo>& vSlotInfo) {
+void CopyInfo(NKikimrSysView::TVSlotInfo* info, const THolder<TBlobStorageController::TVSlotInfo>& vSlotInfo) {
SerializeVSlotInfo(info, vSlotInfo->GetVDiskId(), vSlotInfo->Metrics, vSlotInfo->GetStatus(), vSlotInfo->Kind,
vSlotInfo->IsBeingDeleted());
-}
-
-void CopyInfo(NKikimrSysView::TGroupInfo* info, const THolder<TBlobStorageController::TGroupInfo>& groupInfo) {
- info->SetGeneration(groupInfo->Generation);
- info->SetErasureSpeciesV2(TErasureType::ErasureSpeciesName(groupInfo->ErasureSpecies));
+}
+
+void CopyInfo(NKikimrSysView::TGroupInfo* info, const THolder<TBlobStorageController::TGroupInfo>& groupInfo) {
+ info->SetGeneration(groupInfo->Generation);
+ info->SetErasureSpeciesV2(TErasureType::ErasureSpeciesName(groupInfo->ErasureSpecies));
info->SetBoxId(std::get<0>(groupInfo->StoragePoolId));
info->SetStoragePoolId(std::get<1>(groupInfo->StoragePoolId));
- if (groupInfo->EncryptionMode) {
- info->SetEncryptionMode(*groupInfo->EncryptionMode);
- }
- if (groupInfo->LifeCyclePhase) {
- info->SetLifeCyclePhase(*groupInfo->LifeCyclePhase);
- }
+ if (groupInfo->EncryptionMode) {
+ info->SetEncryptionMode(*groupInfo->EncryptionMode);
+ }
+ if (groupInfo->LifeCyclePhase) {
+ info->SetLifeCyclePhase(*groupInfo->LifeCyclePhase);
+ }
std::vector<TGroupDiskInfo> disks;
for (const auto& vslot : groupInfo->VDisksInGroup) {
disks.push_back({&vslot->PDisk->Metrics, &vslot->Metrics, vslot->PDisk->ExpectedSlotCount});
- }
+ }
CalculateGroupUsageStats(info, disks, TBlobStorageGroupType(groupInfo->ErasureSpecies));
- info->SetSeenOperational(groupInfo->SeenOperational);
- const auto& latencyStats = groupInfo->LatencyStats;
- if (latencyStats.PutTabletLog) {
- info->SetPutTabletLogLatency(latencyStats.PutTabletLog->MicroSeconds());
- }
- if (latencyStats.PutUserData) {
- info->SetPutUserDataLatency(latencyStats.PutUserData->MicroSeconds());
- }
- if (latencyStats.GetFast) {
- info->SetGetFastLatency(latencyStats.GetFast->MicroSeconds());
- }
-}
-
-void CopyInfo(NKikimrSysView::TStoragePoolInfo* info, const TBlobStorageController::TStoragePoolInfo& poolInfo) {
- info->SetName(poolInfo.Name);
- if (poolInfo.Generation) {
- info->SetGeneration(*poolInfo.Generation);
- }
- info->SetErasureSpeciesV2(TErasureType::ErasureSpeciesName(poolInfo.ErasureSpecies));
- info->SetVDiskKindV2(NKikimrBlobStorage::TVDiskKind::EVDiskKind_Name(poolInfo.VDiskKind));
- info->SetKind(poolInfo.Kind);
- info->SetNumGroups(poolInfo.NumGroups);
- if (poolInfo.EncryptionMode) {
- info->SetEncryptionMode(*poolInfo.EncryptionMode);
- }
- if (poolInfo.SchemeshardId) {
- info->SetSchemeshardId(*poolInfo.SchemeshardId);
- }
- if (poolInfo.PathItemId) {
- info->SetPathId(*poolInfo.PathItemId);
- }
+ info->SetSeenOperational(groupInfo->SeenOperational);
+ const auto& latencyStats = groupInfo->LatencyStats;
+ if (latencyStats.PutTabletLog) {
+ info->SetPutTabletLogLatency(latencyStats.PutTabletLog->MicroSeconds());
+ }
+ if (latencyStats.PutUserData) {
+ info->SetPutUserDataLatency(latencyStats.PutUserData->MicroSeconds());
+ }
+ if (latencyStats.GetFast) {
+ info->SetGetFastLatency(latencyStats.GetFast->MicroSeconds());
+ }
+}
+
+void CopyInfo(NKikimrSysView::TStoragePoolInfo* info, const TBlobStorageController::TStoragePoolInfo& poolInfo) {
+ info->SetName(poolInfo.Name);
+ if (poolInfo.Generation) {
+ info->SetGeneration(*poolInfo.Generation);
+ }
+ info->SetErasureSpeciesV2(TErasureType::ErasureSpeciesName(poolInfo.ErasureSpecies));
+ info->SetVDiskKindV2(NKikimrBlobStorage::TVDiskKind::EVDiskKind_Name(poolInfo.VDiskKind));
+ info->SetKind(poolInfo.Kind);
+ info->SetNumGroups(poolInfo.NumGroups);
+ if (poolInfo.EncryptionMode) {
+ info->SetEncryptionMode(*poolInfo.EncryptionMode);
+ }
+ if (poolInfo.SchemeshardId) {
+ info->SetSchemeshardId(*poolInfo.SchemeshardId);
+ }
+ if (poolInfo.PathItemId) {
+ info->SetPathId(*poolInfo.PathItemId);
+ }
info->SetPDiskFilter(TBlobStorageController::TStoragePoolInfo::TPDiskFilter::ToString(poolInfo.PDiskFilters));
TStringStream pdiskFilterData;
Save(&pdiskFilterData, poolInfo.PDiskFilters);
info->SetPDiskFilterData(pdiskFilterData.Str());
-}
-
+}
+
template<typename TDstMap, typename TDeletedSet, typename TSrcMap, typename TChangedSet>
void CopyInfo(TDstMap& dst, TDeletedSet& deleted, const TSrcMap& src, TChangedSet& changed) {
for (const auto& key : changed) {
@@ -517,29 +517,29 @@ void CopyInfo(TDstMap& dst, TDeletedSet& deleted, const TSrcMap& src, TChangedSe
} else {
deleted.insert(key);
}
- }
+ }
changed.clear();
-}
-
+}
+
void TBlobStorageController::UpdateSystemViews() {
if (!AppData()->FeatureFlags.GetEnableSystemViews()) {
- return;
- }
-
+ return;
+ }
+
if (!SysViewChangedPDisks.empty() || !SysViewChangedVSlots.empty() || !SysViewChangedGroups.empty() ||
!SysViewChangedStoragePools.empty()) {
auto update = MakeHolder<TEvControllerUpdateSystemViews>();
update->HostRecords = HostRecords;
update->GroupReserveMin = GroupReserveMin;
update->GroupReservePart = GroupReservePart;
-
+
auto& state = update->State;
CopyInfo(state.PDisks, update->DeletedPDisks, PDisks, SysViewChangedPDisks);
CopyInfo(state.VSlots, update->DeletedVSlots, VSlots, SysViewChangedVSlots);
CopyInfo(state.Groups, update->DeletedGroups, GroupMap, SysViewChangedGroups);
CopyInfo(state.StoragePools, update->DeletedStoragePools, StoragePools, SysViewChangedStoragePools);
SysViewChangedSettings = false;
-
+
// process static slots and static groups
for (const auto& [pdiskId, pdisk] : StaticPDisks) {
if (SysViewChangedPDisks.count(pdiskId) && !FindPDisk(pdiskId)) {
@@ -579,7 +579,7 @@ void TBlobStorageController::UpdateSystemViews() {
pb->SetLifeCyclePhase(group.GetLifeCyclePhase());
pb->SetSeenOperational(true);
pb->SetErasureSpeciesV2(TBlobStorageGroupType::ErasureSpeciesName(group.GetErasureSpecies()));
-
+
const NKikimrBlobStorage::TVDiskMetrics zero;
std::vector<TGroupDiskInfo> disks;
for (const auto& realm : group.GetRings()) {
@@ -607,6 +607,6 @@ void TBlobStorageController::UpdateSystemViews() {
}
Schedule(UpdateSystemViewsPeriod, new TEvPrivate::TEvUpdateSystemViews);
-}
-
-} // NKikimr::NBsController
+}
+
+} // NKikimr::NBsController
diff --git a/ydb/core/mind/bscontroller/sys_view.h b/ydb/core/mind/bscontroller/sys_view.h
index 9effe8c5bb8..a354aef9f4c 100644
--- a/ydb/core/mind/bscontroller/sys_view.h
+++ b/ydb/core/mind/bscontroller/sys_view.h
@@ -1,21 +1,21 @@
-#pragma once
-
-#include "impl.h"
-
-namespace NKikimr::NBsController {
-
-struct TControllerSystemViewsState {
+#pragma once
+
+#include "impl.h"
+
+namespace NKikimr::NBsController {
+
+struct TControllerSystemViewsState {
std::unordered_map<TPDiskId, NKikimrSysView::TPDiskInfo, THash<TPDiskId>> PDisks;
std::unordered_map<TVSlotId, NKikimrSysView::TVSlotInfo, THash<TVSlotId>> VSlots;
std::unordered_map<TGroupId, NKikimrSysView::TGroupInfo, THash<TGroupId>> Groups;
std::unordered_map<TBlobStorageController::TBoxStoragePoolId, NKikimrSysView::TStoragePoolInfo,
THash<TBlobStorageController::TBoxStoragePoolId>> StoragePools;
-};
-
-struct TEvControllerUpdateSystemViews :
- TEventLocal<TEvControllerUpdateSystemViews, TEvBlobStorage::EvControllerUpdateSystemViews>
-{
- TControllerSystemViewsState State;
+};
+
+struct TEvControllerUpdateSystemViews :
+ TEventLocal<TEvControllerUpdateSystemViews, TEvBlobStorage::EvControllerUpdateSystemViews>
+{
+ TControllerSystemViewsState State;
std::unordered_set<TPDiskId, THash<TPDiskId>> DeletedPDisks;
std::unordered_set<TVSlotId, THash<TVSlotId>> DeletedVSlots;
std::unordered_set<TGroupId, THash<TGroupId>> DeletedGroups;
@@ -23,7 +23,7 @@ struct TEvControllerUpdateSystemViews :
TBlobStorageController::THostRecordMap HostRecords;
ui32 GroupReserveMin;
ui32 GroupReservePart;
-};
-
-} // NKikimr::NBsController
-
+};
+
+} // NKikimr::NBsController
+
diff --git a/ydb/core/mind/bscontroller/ya.make b/ydb/core/mind/bscontroller/ya.make
index 275a4dc2385..f11b7ff6865 100644
--- a/ydb/core/mind/bscontroller/ya.make
+++ b/ydb/core/mind/bscontroller/ya.make
@@ -50,8 +50,8 @@ SRCS(
stat_processor.cpp
stat_processor.h
storage_pool_stat.h
- sys_view.cpp
- sys_view.h
+ sys_view.cpp
+ sys_view.h
table_merger.h
types.h
update_group_latencies.cpp
diff --git a/ydb/core/mind/configured_tablet_bootstrapper.cpp b/ydb/core/mind/configured_tablet_bootstrapper.cpp
index 008acd2b400..e72a6d57590 100644
--- a/ydb/core/mind/configured_tablet_bootstrapper.cpp
+++ b/ydb/core/mind/configured_tablet_bootstrapper.cpp
@@ -211,9 +211,9 @@ TIntrusivePtr<TTabletSetupInfo> MakeTabletSetupInfo(
case TTabletTypes::Kesus:
createFunc = &NKesus::CreateKesusTablet;
break;
- case TTabletTypes::SysViewProcessor:
- createFunc = &NSysView::CreateSysViewProcessor;
- break;
+ case TTabletTypes::SysViewProcessor:
+ createFunc = &NSysView::CreateSysViewProcessor;
+ break;
case TTabletTypes::TestShard:
createFunc = &NTestShard::CreateTestShard;
break;
diff --git a/ydb/core/mind/hive/hive_impl.cpp b/ydb/core/mind/hive/hive_impl.cpp
index 58ca2ac3e43..f895b84d01b 100644
--- a/ydb/core/mind/hive/hive_impl.cpp
+++ b/ydb/core/mind/hive/hive_impl.cpp
@@ -2322,8 +2322,8 @@ STFUNC(THive::StateWork) {
hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle);
hFunc(NConsole::TEvConsole::TEvConfigNotificationRequest, Handle);
hFunc(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse, Handle);
- hFunc(NSysView::TEvSysView::TEvGetTabletIdsRequest, Handle);
- hFunc(NSysView::TEvSysView::TEvGetTabletsRequest, Handle);
+ hFunc(NSysView::TEvSysView::TEvGetTabletIdsRequest, Handle);
+ hFunc(NSysView::TEvSysView::TEvGetTabletsRequest, Handle);
default:
if (!HandleDefaultEvents(ev, ctx)) {
BLOG_W("THive::StateWork unhandled event type: " << ev->GetTypeRewrite()
@@ -2387,89 +2387,89 @@ void THive::Handle(TEvHive::TEvConfigureHive::TPtr& ev) {
}
-void THive::Handle(NSysView::TEvSysView::TEvGetTabletIdsRequest::TPtr& ev) {
- const auto& request = ev->Get()->Record;
- auto fromId = request.GetFrom();
- auto toId = request.GetTo();
-
- auto response = MakeHolder<NSysView::TEvSysView::TEvGetTabletIdsResponse>();
- auto& record = response->Record;
-
- for (const auto& [tabletId, _] : Tablets) {
- if (tabletId >= fromId && tabletId <= toId) {
- record.AddTabletIds(tabletId);
- }
- }
-
- Send(ev->Sender, response.Release());
-}
-
-void THive::Handle(NSysView::TEvSysView::TEvGetTabletsRequest::TPtr& ev) {
- const auto& request = ev->Get()->Record;
-
- auto response = MakeHolder<NSysView::TEvSysView::TEvGetTabletsResponse>();
- auto& record = response->Record;
-
- auto limit = request.GetBatchSizeLimit();
- size_t count = 0;
-
- for (size_t i = 0; i < request.TabletIdsSize(); ++i) {
- auto tabletId = request.GetTabletIds(i);
- auto lookup = Tablets.find(tabletId);
- if (lookup == Tablets.end()) {
- continue;
- }
-
- auto* entry = record.AddEntries();
- ++count;
-
- const auto& tablet = lookup->second;
- auto tabletTypeName = TTabletTypes::TypeToStr(tablet.Type);
-
- entry->SetTabletId(tabletId);
+void THive::Handle(NSysView::TEvSysView::TEvGetTabletIdsRequest::TPtr& ev) {
+ const auto& request = ev->Get()->Record;
+ auto fromId = request.GetFrom();
+ auto toId = request.GetTo();
+
+ auto response = MakeHolder<NSysView::TEvSysView::TEvGetTabletIdsResponse>();
+ auto& record = response->Record;
+
+ for (const auto& [tabletId, _] : Tablets) {
+ if (tabletId >= fromId && tabletId <= toId) {
+ record.AddTabletIds(tabletId);
+ }
+ }
+
+ Send(ev->Sender, response.Release());
+}
+
+void THive::Handle(NSysView::TEvSysView::TEvGetTabletsRequest::TPtr& ev) {
+ const auto& request = ev->Get()->Record;
+
+ auto response = MakeHolder<NSysView::TEvSysView::TEvGetTabletsResponse>();
+ auto& record = response->Record;
+
+ auto limit = request.GetBatchSizeLimit();
+ size_t count = 0;
+
+ for (size_t i = 0; i < request.TabletIdsSize(); ++i) {
+ auto tabletId = request.GetTabletIds(i);
+ auto lookup = Tablets.find(tabletId);
+ if (lookup == Tablets.end()) {
+ continue;
+ }
+
+ auto* entry = record.AddEntries();
+ ++count;
+
+ const auto& tablet = lookup->second;
+ auto tabletTypeName = TTabletTypes::TypeToStr(tablet.Type);
+
+ entry->SetTabletId(tabletId);
entry->SetFollowerId(0);
-
- entry->SetType(tabletTypeName);
- entry->SetState(ETabletStateName(tablet.State));
- entry->SetVolatileState(TTabletInfo::EVolatileStateName(tablet.GetVolatileState()));
- entry->SetBootState(tablet.BootState);
- entry->SetGeneration(tablet.KnownGeneration);
- entry->SetNodeId(tablet.NodeId);
-
- const auto& resourceValues = tablet.GetResourceValues();
- entry->SetCPU(resourceValues.GetCPU());
- entry->SetMemory(resourceValues.GetMemory());
- entry->SetNetwork(resourceValues.GetNetwork());
-
+
+ entry->SetType(tabletTypeName);
+ entry->SetState(ETabletStateName(tablet.State));
+ entry->SetVolatileState(TTabletInfo::EVolatileStateName(tablet.GetVolatileState()));
+ entry->SetBootState(tablet.BootState);
+ entry->SetGeneration(tablet.KnownGeneration);
+ entry->SetNodeId(tablet.NodeId);
+
+ const auto& resourceValues = tablet.GetResourceValues();
+ entry->SetCPU(resourceValues.GetCPU());
+ entry->SetMemory(resourceValues.GetMemory());
+ entry->SetNetwork(resourceValues.GetNetwork());
+
for (const auto& follower : tablet.Followers) {
- auto* entry = record.AddEntries();
- ++count;
-
- entry->SetTabletId(tabletId);
+ auto* entry = record.AddEntries();
+ ++count;
+
+ entry->SetTabletId(tabletId);
entry->SetFollowerId(follower.Id);
-
- entry->SetType(tabletTypeName);
- // state is null
+
+ entry->SetType(tabletTypeName);
+ // state is null
entry->SetVolatileState(TTabletInfo::EVolatileStateName(follower.GetVolatileState()));
entry->SetBootState(follower.BootState);
- // generation is null
+ // generation is null
entry->SetNodeId(follower.NodeId);
-
+
const auto& resourceValues = follower.GetResourceValues();
- entry->SetCPU(resourceValues.GetCPU());
- entry->SetMemory(resourceValues.GetMemory());
- entry->SetNetwork(resourceValues.GetNetwork());
- }
-
- if (count >= limit && i < request.TabletIdsSize() - 1) {
- record.SetNextTabletId(request.GetTabletIds(i + 1));
- break;
- }
- }
-
- Send(ev->Sender, response.Release());
-}
-
+ entry->SetCPU(resourceValues.GetCPU());
+ entry->SetMemory(resourceValues.GetMemory());
+ entry->SetNetwork(resourceValues.GetNetwork());
+ }
+
+ if (count >= limit && i < request.TabletIdsSize() - 1) {
+ record.SetNextTabletId(request.GetTabletIds(i + 1));
+ break;
+ }
+ }
+
+ Send(ev->Sender, response.Release());
+}
+
const TTabletMetricsAggregates& THive::GetDefaultResourceMetricsAggregates() const {
return DefaultResourceMetricsAggregates;
}
diff --git a/ydb/core/mind/hive/hive_impl.h b/ydb/core/mind/hive/hive_impl.h
index 70c4530fee0..e2bf6cbad03 100644
--- a/ydb/core/mind/hive/hive_impl.h
+++ b/ydb/core/mind/hive/hive_impl.h
@@ -496,8 +496,8 @@ protected:
void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev);
void Handle(NConsole::TEvConsole::TEvConfigNotificationRequest::TPtr& ev);
void Handle(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse::TPtr& ev);
- void Handle(NSysView::TEvSysView::TEvGetTabletIdsRequest::TPtr& ev);
- void Handle(NSysView::TEvSysView::TEvGetTabletsRequest::TPtr& ev);
+ void Handle(NSysView::TEvSysView::TEvGetTabletIdsRequest::TPtr& ev);
+ void Handle(NSysView::TEvSysView::TEvGetTabletsRequest::TPtr& ev);
protected:
void RestartPipeTx(ui64 tabletId);
diff --git a/ydb/core/mind/hive/hive_schema.h b/ydb/core/mind/hive/hive_schema.h
index 6289499cc31..364b6bfb2a6 100644
--- a/ydb/core/mind/hive/hive_schema.h
+++ b/ydb/core/mind/hive/hive_schema.h
@@ -214,19 +214,19 @@ struct Schema : NIceDb::Schema {
using TColumns = TableColumns<TabletID, CPU, Memory, Network, Metrics>;
};
- struct Metrics : Table<16> {
- struct TabletID : Column<1, Tablet::ID::ColumnType> {};
+ struct Metrics : Table<16> {
+ struct TabletID : Column<1, Tablet::ID::ColumnType> {};
struct FollowerID : Column<2, TabletFollowerTablet::FollowerID::ColumnType> {};
- struct ProtoMetrics : Column<3, NScheme::NTypeIds::String> { using Type = NKikimrTabletBase::TMetrics; };
-
+ struct ProtoMetrics : Column<3, NScheme::NTypeIds::String> { using Type = NKikimrTabletBase::TMetrics; };
+
struct MaximumCPU : Column<100 + (int)NMetrics::EResource::CPU, NScheme::NTypeIds::String> { using Type = NKikimrMetricsProto::TMaximumValueUI64; };
struct MaximumMemory : Column<100 + (int)NMetrics::EResource::Memory, NScheme::NTypeIds::String> { using Type = NKikimrMetricsProto::TMaximumValueUI64; };
struct MaximumNetwork : Column<100 + (int)NMetrics::EResource::Network, NScheme::NTypeIds::String> { using Type = NKikimrMetricsProto::TMaximumValueUI64; };
using TKey = TableKey<TabletID, FollowerID>;
using TColumns = TableColumns<TabletID, FollowerID, ProtoMetrics, MaximumCPU, MaximumMemory, MaximumNetwork>;
- };
-
+ };
+
struct TabletTypeMetrics : Table<13> {
struct TabletType : Column<1, Tablet::TabletType::ColumnType> { using Type = TTabletTypes::EType; };
struct AllowedMetricIDs : Column<2, NScheme::NTypeIds::String> { using Type = TVector<i64>; };
@@ -294,7 +294,7 @@ struct Schema : NIceDb::Schema {
TabletFollowerGroup,
TabletFollowerTablet,
TabletTypeMetrics,
- Sequences,
+ Sequences,
Metrics,
SubDomain,
BlockedOwner,
diff --git a/ydb/core/mind/hive/monitoring.cpp b/ydb/core/mind/hive/monitoring.cpp
index 7e9a83b354e..5f2fc40f66e 100644
--- a/ydb/core/mind/hive/monitoring.cpp
+++ b/ydb/core/mind/hive/monitoring.cpp
@@ -1084,8 +1084,8 @@ public:
return "BP";
case TTabletTypes::Kesus:
return "K";
- case TTabletTypes::SysViewProcessor:
- return "SV";
+ case TTabletTypes::SysViewProcessor:
+ return "SV";
case TTabletTypes::FileStore:
return "FS";
case TTabletTypes::TestShard:
diff --git a/ydb/core/mind/hive/tx__delete_tablet_result.cpp b/ydb/core/mind/hive/tx__delete_tablet_result.cpp
index bbe89acae48..5543d1a3d27 100644
--- a/ydb/core/mind/hive/tx__delete_tablet_result.cpp
+++ b/ydb/core/mind/hive/tx__delete_tablet_result.cpp
@@ -27,7 +27,7 @@ public:
if (Tablet != nullptr) {
if (msg->Status == NKikimrProto::OK) {
NIceDb::TNiceDb db(txc.DB);
- db.Table<Schema::Metrics>().Key(Tablet->Id, 0).Delete();
+ db.Table<Schema::Metrics>().Key(Tablet->Id, 0).Delete();
for (const TTabletChannelInfo& channelInfo : Tablet->TabletStorageInfo->Channels) {
for (const TTabletChannelInfo::THistoryEntry& historyInfo : channelInfo.History) {
db.Table<Schema::TabletChannelGen>().Key(Tablet->Id, channelInfo.Channel, historyInfo.FromGeneration).Delete();
@@ -37,7 +37,7 @@ public:
for (TFollowerTabletInfo& follower : Tablet->Followers) {
auto fullTabletId = follower.GetFullTabletId();
db.Table<Schema::TabletFollowerTablet>().Key(fullTabletId).Delete();
- db.Table<Schema::Metrics>().Key(fullTabletId).Delete();
+ db.Table<Schema::Metrics>().Key(fullTabletId).Delete();
}
for (TFollowerGroup& group : Tablet->FollowerGroups) {
db.Table<Schema::TabletFollowerGroup>().Key(Tablet->Id, group.Id).Delete();
diff --git a/ydb/core/mind/hive/tx__load_everything.cpp b/ydb/core/mind/hive/tx__load_everything.cpp
index e4c1cf1f51e..3f98a0b3549 100644
--- a/ydb/core/mind/hive/tx__load_everything.cpp
+++ b/ydb/core/mind/hive/tx__load_everything.cpp
@@ -45,7 +45,7 @@ public:
auto tabletRowset = db.Table<Schema::Tablet>().Range().Select();
auto tabletChannelRowset = db.Table<Schema::TabletChannel>().Range().Select();
auto tabletChannelGenRowset = db.Table<Schema::TabletChannelGen>().Range().Select();
- auto metrics = db.Table<Schema::Metrics>().Range().Select();
+ auto metrics = db.Table<Schema::Metrics>().Range().Select();
auto tabletFollowerGroupRowset = db.Table<Schema::TabletFollowerGroup>().Range().Select();
auto tabletFollowerRowset = db.Table<Schema::TabletFollowerTablet>().Range().Select();
auto tabletTypeAllowedMetrics = db.Table<Schema::TabletTypeMetrics>().Range().Select();
@@ -57,7 +57,7 @@ public:
if (!tabletRowset.IsReady()
|| !tabletChannelRowset.IsReady()
|| !tabletChannelGenRowset.IsReady()
- || !metrics.IsReady()
+ || !metrics.IsReady()
|| !tabletFollowerGroupRowset.IsReady()
|| !tabletFollowerRowset.IsReady()
|| !tabletTypeAllowedMetrics.IsReady()
@@ -470,10 +470,10 @@ public:
return false;
}
- auto metricsRowset = db.Table<Schema::Metrics>().Range(tabletId).Select();
- if (!metricsRowset.IsReady())
- return false;
- while (!metricsRowset.EndOfSet()) {
+ auto metricsRowset = db.Table<Schema::Metrics>().Range(tabletId).Select();
+ if (!metricsRowset.IsReady())
+ return false;
+ while (!metricsRowset.EndOfSet()) {
TFollowerId followerId = metricsRowset.GetValue<Schema::Metrics::FollowerID>();
auto* leaderOrFollower = tablet.FindTablet(followerId);
if (leaderOrFollower) {
@@ -482,11 +482,11 @@ public:
leaderOrFollower->MutableResourceMetricsAggregates().MaximumNetwork.InitiaizeFrom(metricsRowset.GetValueOrDefault<Schema::Metrics::MaximumNetwork>());
// do not reorder
leaderOrFollower->UpdateResourceUsage(metricsRowset.GetValueOrDefault<Schema::Metrics::ProtoMetrics>());
- }
- if (!metricsRowset.Next())
- return false;
- }
-
+ }
+ if (!metricsRowset.Next())
+ return false;
+ }
+
for (auto& pr : followersPerGroup) {
TFollowerGroup& followerGroup(*pr.first);
while (followerGroup.GetComputedFollowerCount(Self->GetDataCenters()) > pr.second) {
diff --git a/ydb/core/mind/labels_maintainer.cpp b/ydb/core/mind/labels_maintainer.cpp
index 19138c7d23b..f3e1d04dbb1 100644
--- a/ydb/core/mind/labels_maintainer.cpp
+++ b/ydb/core/mind/labels_maintainer.cpp
@@ -246,26 +246,26 @@ private:
oldGroup = oldGroup->GetSubgroup("subsystem", subSvc);
TIntrusivePtr<NMonitoring::TDynamicCounters> serviceGroup = new NMonitoring::TDynamicCounters;
TIntrusivePtr<NMonitoring::TDynamicCounters> curGroup = serviceGroup;
-
- const auto* actualLabels = &labels;
-
- TSmallVec<std::pair<TString, TString>> ydbLabels;
- if (DatabaseAttributeSensorServices.contains(service)) {
- // explicitly remove "slot" label for external services ("ydb")
- ydbLabels = labels;
- if (auto it = std::find_if(ydbLabels.begin(), ydbLabels.end(), [](auto& el){ return el.first == SLOT_LABEL; });
- it != ydbLabels.end())
- {
- ydbLabels.erase(it);
- actualLabels = &ydbLabels;
- }
- }
-
- for (size_t i = 0; i < actualLabels->size() - 1; ++i) {
- curGroup = curGroup->GetSubgroup((*actualLabels)[i].first, (*actualLabels)[i].second);
- }
- curGroup->RegisterSubgroup(actualLabels->back().first, actualLabels->back().second, oldGroup);
-
+
+ const auto* actualLabels = &labels;
+
+ TSmallVec<std::pair<TString, TString>> ydbLabels;
+ if (DatabaseAttributeSensorServices.contains(service)) {
+ // explicitly remove "slot" label for external services ("ydb")
+ ydbLabels = labels;
+ if (auto it = std::find_if(ydbLabels.begin(), ydbLabels.end(), [](auto& el){ return el.first == SLOT_LABEL; });
+ it != ydbLabels.end())
+ {
+ ydbLabels.erase(it);
+ actualLabels = &ydbLabels;
+ }
+ }
+
+ for (size_t i = 0; i < actualLabels->size() - 1; ++i) {
+ curGroup = curGroup->GetSubgroup((*actualLabels)[i].first, (*actualLabels)[i].second);
+ }
+ curGroup->RegisterSubgroup(actualLabels->back().first, actualLabels->back().second, oldGroup);
+
auto rt = GetServiceCountersRoot(root, service);
rt->ReplaceSubgroup(subSvc.empty() ? "counters" : "subsystem", subSvc.empty() ? svc : subSvc, serviceGroup);
}
diff --git a/ydb/core/mind/local.cpp b/ydb/core/mind/local.cpp
index 09fc15f76dd..34a697ff7c8 100644
--- a/ydb/core/mind/local.cpp
+++ b/ydb/core/mind/local.cpp
@@ -631,15 +631,15 @@ class TLocalNodeRegistrar : public TActorBootstrapped<TLocalNodeRegistrar> {
const auto& poolStats(info.GetPoolStats(AppData()->UserPoolId));
UserPoolUsage = poolStats.usage() * poolStats.threads() * 1000000; // uS
}
- if (info.HasMemoryUsedInAlloc()) {
- MemUsage = info.GetMemoryUsedInAlloc();
+ if (info.HasMemoryUsedInAlloc()) {
+ MemUsage = info.GetMemoryUsedInAlloc();
}
/*if (info.HasMemoryUsed()) {
MemUsage = info.GetMemoryUsed();
}*/
-
+
double usage = 0;
-
+
if (info.HasMemoryLimit()) {
if (MemLimit != info.GetMemoryLimit()) {
MemLimit = info.GetMemoryLimit();
diff --git a/ydb/core/mind/tenant_ut_pool.cpp b/ydb/core/mind/tenant_ut_pool.cpp
index 9ac6b6843c3..d11bcfcf82b 100644
--- a/ydb/core/mind/tenant_ut_pool.cpp
+++ b/ydb/core/mind/tenant_ut_pool.cpp
@@ -186,18 +186,18 @@ void CheckLabels(TIntrusivePtr<NMonitoring::TDynamicCounters> counters,
allServices.insert(attrServices.begin(), attrServices.end());
for (auto &service : allServices) {
- THashSet<TString> allLabels = GetDatabaseAttributeLabels();
- allLabels.insert(DATABASE_LABEL);
- allLabels.insert(SLOT_LABEL);
-
+ THashSet<TString> allLabels = GetDatabaseAttributeLabels();
+ allLabels.insert(DATABASE_LABEL);
+ allLabels.insert(SLOT_LABEL);
+
THashMap<TString, TString> labels;
- if (dbServices.contains(service)) {
+ if (dbServices.contains(service)) {
labels.insert(dbLabels.begin(), dbLabels.end());
- if (attrServices.contains(service)) {
- labels.erase(SLOT_LABEL); // no slot now
- allLabels.erase(SLOT_LABEL);
- }
- }
+ if (attrServices.contains(service)) {
+ labels.erase(SLOT_LABEL); // no slot now
+ allLabels.erase(SLOT_LABEL);
+ }
+ }
if (attrServices.contains(service))
labels.insert(attrLabels.begin(), attrLabels.end());
auto serviceGroup = GetServiceCounters(counters, service, false);
@@ -496,17 +496,17 @@ Y_UNIT_TEST_SUITE(TTenantPoolTests) {
runtime.WaitForHiveState({{{TENANT1_1_NAME, 1, 1, 1}}});
- auto &attrServices = GetDatabaseAttributeSensorServices();
+ auto &attrServices = GetDatabaseAttributeSensorServices();
for (auto &service : services) {
auto serviceGroup = GetServiceCounters(counters, service, false);
auto tenantGroup = serviceGroup->FindSubgroup(DATABASE_LABEL, TENANT1_1_NAME);
UNIT_ASSERT(tenantGroup);
- TIntrusivePtr<NMonitoring::TDynamicCounters> slotGroup;
- if (attrServices.contains(service)) {
- slotGroup = tenantGroup->FindSubgroup(HOST_LABEL, "slot-1");
- } else {
- slotGroup = tenantGroup->FindSubgroup(SLOT_LABEL, "static")->FindSubgroup(HOST_LABEL, "slot-1");
- }
+ TIntrusivePtr<NMonitoring::TDynamicCounters> slotGroup;
+ if (attrServices.contains(service)) {
+ slotGroup = tenantGroup->FindSubgroup(HOST_LABEL, "slot-1");
+ } else {
+ slotGroup = tenantGroup->FindSubgroup(SLOT_LABEL, "static")->FindSubgroup(HOST_LABEL, "slot-1");
+ }
UNIT_ASSERT(slotGroup);
auto counter = slotGroup->GetCounter("counter", true);
UNIT_ASSERT(*counter == 0);
@@ -585,18 +585,18 @@ Y_UNIT_TEST_SUITE(TTenantPoolTests) {
runtime.WaitForHiveState({{{TENANT1_1_NAME, 1, 1, 1}}});
- auto &attrServices = GetDatabaseAttributeSensorServices();
-
+ auto &attrServices = GetDatabaseAttributeSensorServices();
+
for (auto &service : services) {
auto serviceGroup = GetServiceCounters(counters, service, false);
auto tenantGroup = serviceGroup->FindSubgroup(DATABASE_LABEL, TENANT1_1_NAME);
UNIT_ASSERT(tenantGroup);
- NMonitoring::TDynamicCounterPtr slotGroup;
- if (attrServices.contains(service)) {
- slotGroup = tenantGroup->FindSubgroup(HOST_LABEL, "slot-1");
- } else {
- slotGroup = tenantGroup->FindSubgroup(SLOT_LABEL, "static")->FindSubgroup(HOST_LABEL, "slot-1");
- }
+ NMonitoring::TDynamicCounterPtr slotGroup;
+ if (attrServices.contains(service)) {
+ slotGroup = tenantGroup->FindSubgroup(HOST_LABEL, "slot-1");
+ } else {
+ slotGroup = tenantGroup->FindSubgroup(SLOT_LABEL, "static")->FindSubgroup(HOST_LABEL, "slot-1");
+ }
UNIT_ASSERT(slotGroup);
auto counter = slotGroup->GetCounter("counter", true);
UNIT_ASSERT(*counter == 0);
@@ -612,12 +612,12 @@ Y_UNIT_TEST_SUITE(TTenantPoolTests) {
auto serviceGroup = GetServiceCounters(counters, service, false);
auto tenantGroup = serviceGroup->FindSubgroup(DATABASE_LABEL, CanonizePath(DOMAIN1_NAME));
UNIT_ASSERT(tenantGroup);
- NMonitoring::TDynamicCounterPtr slotGroup;
- if (attrServices.contains(service)) {
- slotGroup = tenantGroup;
- } else {
- slotGroup = tenantGroup->FindSubgroup(SLOT_LABEL, "static");
- }
+ NMonitoring::TDynamicCounterPtr slotGroup;
+ if (attrServices.contains(service)) {
+ slotGroup = tenantGroup;
+ } else {
+ slotGroup = tenantGroup->FindSubgroup(SLOT_LABEL, "static");
+ }
UNIT_ASSERT(slotGroup);
auto counter = slotGroup->GetCounter("counter", true);
UNIT_ASSERT(*counter == 0);
@@ -633,12 +633,12 @@ Y_UNIT_TEST_SUITE(TTenantPoolTests) {
auto serviceGroup = GetServiceCounters(counters, service, false);
auto tenantGroup = serviceGroup->FindSubgroup(DATABASE_LABEL, "<none>");
UNIT_ASSERT(tenantGroup);
- NMonitoring::TDynamicCounterPtr slotGroup;
- if (attrServices.contains(service)) {
- slotGroup = tenantGroup->FindSubgroup(HOST_LABEL, "unassigned");
- } else {
- slotGroup = tenantGroup->FindSubgroup(SLOT_LABEL, "static")->FindSubgroup(HOST_LABEL, "unassigned");
- }
+ NMonitoring::TDynamicCounterPtr slotGroup;
+ if (attrServices.contains(service)) {
+ slotGroup = tenantGroup->FindSubgroup(HOST_LABEL, "unassigned");
+ } else {
+ slotGroup = tenantGroup->FindSubgroup(SLOT_LABEL, "static")->FindSubgroup(HOST_LABEL, "unassigned");
+ }
UNIT_ASSERT(slotGroup);
auto counter = slotGroup->GetCounter("counter", true);
UNIT_ASSERT(*counter == 0);
diff --git a/ydb/core/mon_alloc/monitor.cpp b/ydb/core/mon_alloc/monitor.cpp
index 8afc7852c97..30bd2218d97 100644
--- a/ydb/core/mon_alloc/monitor.cpp
+++ b/ydb/core/mon_alloc/monitor.cpp
@@ -1,5 +1,5 @@
#include "monitor.h"
-#include "tcmalloc.h"
+#include "tcmalloc.h"
#include <ydb/core/base/counters.h>
#include <ydb/core/base/appdata.h>
@@ -8,7 +8,7 @@
#include <library/cpp/actors/core/actor_bootstrapped.h>
#include <library/cpp/actors/core/hfunc.h>
#include <library/cpp/actors/prof/tag.h>
-#include <library/cpp/html/pcdata/pcdata.h>
+#include <library/cpp/html/pcdata/pcdata.h>
#include <library/cpp/lfalloc/dbg_info/dbg_info.h>
#include <library/cpp/malloc/api/malloc.h>
#include <library/cpp/monlib/dynamic_counters/counters.h>
@@ -72,7 +72,7 @@ namespace NKikimr {
}
void Update() override {
-#ifdef PROFILE_MEMORY_ALLOCATIONS
+#ifdef PROFILE_MEMORY_ALLOCATIONS
int maxTag = 0;
int numSizes = 0;
auto info = NAllocDbg::GetPerTagAllocInfo(true, maxTag, numSizes);
@@ -142,21 +142,21 @@ namespace NKikimr {
bySize.Set(bucket.Count, bucket.Size);
}
}
-#endif
+#endif
}
- void Dump(IOutputStream& out, const TString& relPath) override {
- Y_UNUSED(relPath);
-#ifdef PROFILE_MEMORY_ALLOCATIONS
+ void Dump(IOutputStream& out, const TString& relPath) override {
+ Y_UNUSED(relPath);
+#ifdef PROFILE_MEMORY_ALLOCATIONS
int maxTag = 0;
int numSizes = 0;
auto info = NAllocDbg::GetPerTagAllocInfo(true, maxTag, numSizes);
HTML(out) {
- H3() {
- out << "LFAlloc" << Endl;
- }
- out << "<hr>" << Endl;
+ H3() {
+ out << "LFAlloc" << Endl;
+ }
+ out << "<hr>" << Endl;
TABLE_SORTABLE_CLASS("table") {
TABLEHEAD() {
TABLER() {
@@ -244,15 +244,15 @@ namespace NKikimr {
"{$('[data-toggle=\"tooltip\"]').tooltip();}</script>";
}
}
-#else
- HTML(out) {
- H3() {
- out << "LFAlloc" << Endl;
- }
- out << "<hr>" << Endl;
- out << "PROFILE_MEMORY_ALLOCATIONS is off" << Endl;
- }
-#endif
+#else
+ HTML(out) {
+ H3() {
+ out << "LFAlloc" << Endl;
+ }
+ out << "<hr>" << Endl;
+ out << "PROFILE_MEMORY_ALLOCATIONS is off" << Endl;
+ }
+#endif
}
};
@@ -267,7 +267,7 @@ namespace NKikimr {
}
void Update() override {
-#ifdef PROFILE_MEMORY_ALLOCATIONS
+#ifdef PROFILE_MEMORY_ALLOCATIONS
using namespace NYT::NYTAlloc;
size_t maxTag = NProfiling::GetTagsCount();
@@ -295,12 +295,12 @@ namespace NKikimr {
*perTag = usages[tag];
}
-#endif
+#endif
}
- void Dump(IOutputStream& out, const TString& relPath) override {
- Y_UNUSED(relPath);
-#ifdef PROFILE_MEMORY_ALLOCATIONS
+ void Dump(IOutputStream& out, const TString& relPath) override {
+ Y_UNUSED(relPath);
+#ifdef PROFILE_MEMORY_ALLOCATIONS
using namespace NYT::NYTAlloc;
size_t maxTag = NProfiling::GetTagsCount();
@@ -312,10 +312,10 @@ namespace NKikimr {
GetMemoryUsageForTags(tags.data(), tags.size(), usages.data());
HTML(out) {
- H3() {
- out << "YTAlloc" << Endl;
- }
- out << "<hr>" << Endl;
+ H3() {
+ out << "YTAlloc" << Endl;
+ }
+ out << "<hr>" << Endl;
TABLE_SORTABLE_CLASS("table") {
TABLEHEAD() {
TABLER() {
@@ -354,15 +354,15 @@ namespace NKikimr {
}
}
}
-#else
- HTML(out) {
- H3() {
- out << "YTAlloc" << Endl;
- }
- out << "<hr>" << Endl;
- out << "PROFILE_MEMORY_ALLOCATIONS is off" << Endl;
- }
-#endif
+#else
+ HTML(out) {
+ H3() {
+ out << "YTAlloc" << Endl;
+ }
+ out << "<hr>" << Endl;
+ out << "PROFILE_MEMORY_ALLOCATIONS is off" << Endl;
+ }
+#endif
}
};
@@ -370,50 +370,50 @@ namespace NKikimr {
void Update() override {
}
- void Dump(IOutputStream& out, const TString& relPath) override {
+ void Dump(IOutputStream& out, const TString& relPath) override {
Y_UNUSED(out);
- Y_UNUSED(relPath);
+ Y_UNUSED(relPath);
}
};
std::unique_ptr<IAllocMonitor> CreateAllocMonitor(TDynamicCountersPtr group) {
const auto& info = NMalloc::MallocInfo();
- TStringBuf name(info.Name);
+ TStringBuf name(info.Name);
- std::unique_ptr<IAllocMonitor> monitor;
+ std::unique_ptr<IAllocMonitor> monitor;
if (name.StartsWith("lf")) {
- monitor = std::make_unique<TLfAllocMonitor>(std::move(group));
+ monitor = std::make_unique<TLfAllocMonitor>(std::move(group));
} else if (name.StartsWith("yt")) {
- monitor = std::make_unique<TYtAllocMonitor>(std::move(group));
- } else if (name.StartsWith("tc")) {
- monitor = std::move(CreateTcMallocMonitor(std::move(group)));
+ monitor = std::make_unique<TYtAllocMonitor>(std::move(group));
+ } else if (name.StartsWith("tc")) {
+ monitor = std::move(CreateTcMallocMonitor(std::move(group)));
}
-
- return monitor ? std::move(monitor) : std::make_unique<TFakeAllocMonitor>();
+
+ return monitor ? std::move(monitor) : std::make_unique<TFakeAllocMonitor>();
}
class TMemProfMonitor: public TActorBootstrapped<TMemProfMonitor> {
- struct TDumpLogConfig {
- static constexpr double RssUsageHard = 0.9;
- static constexpr double RssUsageSoft = 0.85;
- static constexpr TDuration RepeatInterval = TDuration::Seconds(10);
- static constexpr TDuration DumpInterval = TDuration::Minutes(10);
- };
-
- enum {
- EvDumpLogStats = EventSpaceBegin(TEvents::ES_PRIVATE),
- EvEnd
- };
-
- struct TEvDumpLogStats : public TEventLocal<TEvDumpLogStats, EvDumpLogStats> {};
-
+ struct TDumpLogConfig {
+ static constexpr double RssUsageHard = 0.9;
+ static constexpr double RssUsageSoft = 0.85;
+ static constexpr TDuration RepeatInterval = TDuration::Seconds(10);
+ static constexpr TDuration DumpInterval = TDuration::Minutes(10);
+ };
+
+ enum {
+ EvDumpLogStats = EventSpaceBegin(TEvents::ES_PRIVATE),
+ EvEnd
+ };
+
+ struct TEvDumpLogStats : public TEventLocal<TEvDumpLogStats, EvDumpLogStats> {};
+
const TDuration Interval;
const std::unique_ptr<IAllocMonitor> AllocMonitor;
- TInstant LogMemoryStatsTime = TInstant::Now() - TDumpLogConfig::DumpInterval;
-
- bool IsDangerous = false;
-
+ TInstant LogMemoryStatsTime = TInstant::Now() - TDumpLogConfig::DumpInterval;
+
+ bool IsDangerous = false;
+
public:
static constexpr EActivityType ActorActivityType() {
return ACTORLIB_STATS;
@@ -422,7 +422,7 @@ namespace NKikimr {
TMemProfMonitor(TDuration interval, std::unique_ptr<IAllocMonitor> allocMonitor)
: Interval(interval)
, AllocMonitor(std::move(allocMonitor))
- {}
+ {}
void Bootstrap(const TActorContext& ctx) {
NActors::TMon* mon = AppData(ctx)->Mon;
@@ -433,14 +433,14 @@ namespace NKikimr {
return;
}
- auto* indexPage = mon->RegisterIndexPage("memory", "Memory");
+ auto* indexPage = mon->RegisterIndexPage("memory", "Memory");
mon->RegisterActorPage(
- indexPage, "statistics", "Statistics",
+ indexPage, "statistics", "Statistics",
false, ctx.ExecutorThread.ActorSystem, ctx.SelfID);
- AllocMonitor->RegisterPages(mon, ctx.ExecutorThread.ActorSystem, ctx.SelfID);
- AllocMonitor->RegisterControls(AppData(ctx)->Icb);
-
+ AllocMonitor->RegisterPages(mon, ctx.ExecutorThread.ActorSystem, ctx.SelfID);
+ AllocMonitor->RegisterControls(AppData(ctx)->Icb);
+
Become(&TThis::StateWork);
ctx.Schedule(Interval, new TEvents::TEvWakeup());
}
@@ -449,65 +449,65 @@ namespace NKikimr {
STFUNC(StateWork) {
switch (ev->GetTypeRewrite()) {
CFunc(TEvents::TEvWakeup::EventType, HandleWakeup);
- HFunc(TEvDumpLogStats, HandleDump);
+ HFunc(TEvDumpLogStats, HandleDump);
HFunc(NMon::TEvHttpInfo, HandleHttpInfo);
}
}
- void LogMemoryStats(const TActorContext& ctx, size_t limit) noexcept {
- LogMemoryStatsTime = TInstant::Now();
- TStringStream out;
- AllocMonitor->DumpForLog(out, limit);
-
- TVector<TString> split;
- Split(out.Str(), "\n", split);
- for (const auto& line : split) {
- LOG_WARN_S(ctx, NKikimrServices::MEMORY_PROFILER, line);
- }
- }
-
- void LogMemoryStatsIfNeeded(const TActorContext& ctx) noexcept {
- auto memoryUsage = TAllocState::GetMemoryUsage();
-
- if (IsDangerous && memoryUsage < TDumpLogConfig::RssUsageSoft) {
- IsDangerous = false;
- } else if (!IsDangerous && memoryUsage > TDumpLogConfig::RssUsageHard) {
- if (TInstant::Now() - LogMemoryStatsTime > TDumpLogConfig::DumpInterval) {
- IsDangerous = true;
- LOG_WARN_S(ctx, NKikimrServices::MEMORY_PROFILER,
- "RSS usage " << memoryUsage * 100. << "%");
- LogMemoryStats(ctx, 256);
- ctx.Schedule(TDumpLogConfig::RepeatInterval, new TEvDumpLogStats);
- }
- }
- }
-
+ void LogMemoryStats(const TActorContext& ctx, size_t limit) noexcept {
+ LogMemoryStatsTime = TInstant::Now();
+ TStringStream out;
+ AllocMonitor->DumpForLog(out, limit);
+
+ TVector<TString> split;
+ Split(out.Str(), "\n", split);
+ for (const auto& line : split) {
+ LOG_WARN_S(ctx, NKikimrServices::MEMORY_PROFILER, line);
+ }
+ }
+
+ void LogMemoryStatsIfNeeded(const TActorContext& ctx) noexcept {
+ auto memoryUsage = TAllocState::GetMemoryUsage();
+
+ if (IsDangerous && memoryUsage < TDumpLogConfig::RssUsageSoft) {
+ IsDangerous = false;
+ } else if (!IsDangerous && memoryUsage > TDumpLogConfig::RssUsageHard) {
+ if (TInstant::Now() - LogMemoryStatsTime > TDumpLogConfig::DumpInterval) {
+ IsDangerous = true;
+ LOG_WARN_S(ctx, NKikimrServices::MEMORY_PROFILER,
+ "RSS usage " << memoryUsage * 100. << "%");
+ LogMemoryStats(ctx, 256);
+ ctx.Schedule(TDumpLogConfig::RepeatInterval, new TEvDumpLogStats);
+ }
+ }
+ }
+
void HandleWakeup(const TActorContext& ctx) noexcept {
AllocMonitor->Update();
- LogMemoryStatsIfNeeded(ctx);
+ LogMemoryStatsIfNeeded(ctx);
ctx.Schedule(Interval, new TEvents::TEvWakeup());
}
- void HandleDump(TEvDumpLogStats::TPtr&, const TActorContext& ctx) noexcept {
- if (IsDangerous) {
- auto memoryUsage = TAllocState::GetMemoryUsage();
- LOG_WARN_S(ctx, NKikimrServices::MEMORY_PROFILER,
- "RSS usage " << memoryUsage * 100. << "%");
- LogMemoryStats(ctx, 256);
- }
- }
-
+ void HandleDump(TEvDumpLogStats::TPtr&, const TActorContext& ctx) noexcept {
+ if (IsDangerous) {
+ auto memoryUsage = TAllocState::GetMemoryUsage();
+ LOG_WARN_S(ctx, NKikimrServices::MEMORY_PROFILER,
+ "RSS usage " << memoryUsage * 100. << "%");
+ LogMemoryStats(ctx, 256);
+ }
+ }
+
void HandleHttpInfo(NMon::TEvHttpInfo::TPtr& ev, const TActorContext& ctx) noexcept {
- TString action = ev->Get()->Request.GetParams().Get("action");
- TString relPath(ev->Get()->Request.GetPath());
+ TString action = ev->Get()->Request.GetParams().Get("action");
+ TString relPath(ev->Get()->Request.GetPath());
TStringStream out;
- if (action == "log" && relPath == "/memory/heap") {
- LogMemoryStats(ctx, 2048);
- out << "<p>Output dumped to log</p>"
- << "<p><a href=\"/memory/heap\">Return</a></p>\n";
- } else {
- AllocMonitor->Dump(out, relPath);
- }
+ if (action == "log" && relPath == "/memory/heap") {
+ LogMemoryStats(ctx, 2048);
+ out << "<p>Output dumped to log</p>"
+ << "<p><a href=\"/memory/heap\">Return</a></p>\n";
+ } else {
+ AllocMonitor->Dump(out, relPath);
+ }
ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes(out.Str()));
}
};
diff --git a/ydb/core/mon_alloc/monitor.h b/ydb/core/mon_alloc/monitor.h
index f6a8404b65d..c1e44be27ae 100644
--- a/ydb/core/mon_alloc/monitor.h
+++ b/ydb/core/mon_alloc/monitor.h
@@ -1,49 +1,49 @@
#pragma once
#include <ydb/core/control/immediate_control_board_impl.h>
-
+
#include <library/cpp/actors/core/defs.h>
#include <library/cpp/actors/core/actor.h>
#include <library/cpp/monlib/dynamic_counters/counters.h>
-namespace NActors {
- class TMon;
-}
-
+namespace NActors {
+ class TMon;
+}
+
namespace NKikimr {
inline NActors::TActorId MakeMemProfMonitorID(ui32 node = 0) {
char x[12] = {'m', 'e', 'm', 'p', 'r', 'o', 'f', 'm', 'o', 'n', 'i', 't'};
return NActors::TActorId(node, TStringBuf(x, 12));
}
- struct IAllocMonitor {
- virtual ~IAllocMonitor() = default;
-
- virtual void RegisterPages(
- NActors::TMon* mon,
- NActors::TActorSystem* actorSystem,
- NActors::TActorId actorId)
- {
- Y_UNUSED(mon);
- Y_UNUSED(actorSystem);
- Y_UNUSED(actorId);
- }
-
- virtual void RegisterControls(TIntrusivePtr<TControlBoard> icb) {
- Y_UNUSED(icb);
- }
-
- virtual void Update() = 0;
-
- virtual void Dump(IOutputStream& out, const TString& relPath) = 0;
-
- virtual void DumpForLog(IOutputStream& out, size_t limit) {
- Y_UNUSED(out);
- Y_UNUSED(limit);
- }
- };
-
+ struct IAllocMonitor {
+ virtual ~IAllocMonitor() = default;
+
+ virtual void RegisterPages(
+ NActors::TMon* mon,
+ NActors::TActorSystem* actorSystem,
+ NActors::TActorId actorId)
+ {
+ Y_UNUSED(mon);
+ Y_UNUSED(actorSystem);
+ Y_UNUSED(actorId);
+ }
+
+ virtual void RegisterControls(TIntrusivePtr<TControlBoard> icb) {
+ Y_UNUSED(icb);
+ }
+
+ virtual void Update() = 0;
+
+ virtual void Dump(IOutputStream& out, const TString& relPath) = 0;
+
+ virtual void DumpForLog(IOutputStream& out, size_t limit) {
+ Y_UNUSED(out);
+ Y_UNUSED(limit);
+ }
+ };
+
NActors::IActor* CreateMemProfMonitor(
ui32 intervalSec,
TIntrusivePtr<NMonitoring::TDynamicCounters> counters);
diff --git a/ydb/core/mon_alloc/profiler.cpp b/ydb/core/mon_alloc/profiler.cpp
index bfff45c1804..b4464e51293 100644
--- a/ydb/core/mon_alloc/profiler.cpp
+++ b/ydb/core/mon_alloc/profiler.cpp
@@ -1,13 +1,13 @@
#include "profiler.h"
-#include "tcmalloc.h"
+#include "tcmalloc.h"
#include <library/cpp/actors/core/actorsystem.h>
#include <library/cpp/actors/core/hfunc.h>
#include <library/cpp/actors/core/mon.h>
-#include <library/cpp/actors/core/log.h>
+#include <library/cpp/actors/core/log.h>
#include <library/cpp/actors/prof/tag.h>
#include <library/cpp/html/pcdata/pcdata.h>
-#include <library/cpp/malloc/api/malloc.h>
+#include <library/cpp/malloc/api/malloc.h>
#include <library/cpp/monlib/service/pages/templates.h>
#if defined(PROFILE_MEMORY_ALLOCATIONS)
@@ -49,8 +49,8 @@ namespace NActors {
NAllocProfiler::StartAllocationSampling(true);
}
- void Stop(IOutputStream& out, size_t limit, bool forLog) override {
- Y_UNUSED(forLog);
+ void Stop(IOutputStream& out, size_t limit, bool forLog) override {
+ Y_UNUSED(forLog);
TAllocDumper dumper(out);
NAllocProfiler::StopAllocationSampling(dumper, limit);
}
@@ -66,8 +66,8 @@ namespace NActors {
SetEnabled(true);
}
- void Stop(IOutputStream& out, size_t limit, bool forLog) override {
- Y_UNUSED(forLog);
+ void Stop(IOutputStream& out, size_t limit, bool forLog) override {
+ Y_UNUSED(forLog);
DumpStats(out, limit);
SetEnabled(false);
}
@@ -165,9 +165,9 @@ namespace NActors {
BeginProfiling();
}
- void Stop(IOutputStream& out, size_t limit, bool forLog) override {
+ void Stop(IOutputStream& out, size_t limit, bool forLog) override {
Y_UNUSED(limit);
- Y_UNUSED(forLog);
+ Y_UNUSED(forLog);
char* buf = nullptr;
size_t len = 0;
@@ -188,35 +188,35 @@ namespace NActors {
void Start() override {
}
- void Stop(IOutputStream& out, size_t limit, bool forLog) override {
+ void Stop(IOutputStream& out, size_t limit, bool forLog) override {
Y_UNUSED(out);
Y_UNUSED(limit);
- Y_UNUSED(forLog);
+ Y_UNUSED(forLog);
}
};
std::unique_ptr<IProfilerLogic> CreateProfiler() {
const auto& info = NMalloc::MallocInfo();
- TStringBuf name(info.Name);
+ TStringBuf name(info.Name);
- std::unique_ptr<IProfilerLogic> profiler;
-
-#if defined(PROFILE_MEMORY_ALLOCATIONS)
+ std::unique_ptr<IProfilerLogic> profiler;
+
+#if defined(PROFILE_MEMORY_ALLOCATIONS)
if (name.StartsWith("lf")) {
- profiler = std::make_unique<TLfAllocProfiler>();
+ profiler = std::make_unique<TLfAllocProfiler>();
} else if (name.StartsWith("yt")) {
- profiler = std::make_unique<TYtAllocProfiler>();
+ profiler = std::make_unique<TYtAllocProfiler>();
}
#endif // PROFILE_MEMORY_ALLOCATIONS
- if (name.StartsWith("tc")) {
- profiler = std::move(NKikimr::CreateTcMallocProfiler());
- }
-
- if (profiler) {
- return std::move(profiler);
- }
-
+ if (name.StartsWith("tc")) {
+ profiler = std::move(NKikimr::CreateTcMallocProfiler());
+ }
+
+ if (profiler) {
+ return std::move(profiler);
+ }
+
#if defined(EXEC_PROFILER_ENABLED)
return std::make_unique<TExecProfiler>();
#endif // EXEC_PROFILER_ENABLED
@@ -265,8 +265,8 @@ namespace NActors {
}
}
- void DumpProfilingTimes(IOutputStream& out);
-
+ void DumpProfilingTimes(IOutputStream& out);
+
void HandleStart(TEvProfiler::TEvStart::TPtr& ev, const TActorContext& ctx);
void HandleStop(TEvProfiler::TEvStop::TPtr& ev, const TActorContext& ctx);
void HandleMonInfo(NMon::TEvHttpInfo::TPtr& ev, const TActorContext& ctx);
@@ -274,18 +274,18 @@ namespace NActors {
bool StartProfiler();
bool StopProfiler(TString& profile);
bool StopProfilerFile(TString& outFileName, TString& err);
- bool StopProfilerDumpToLog(const TActorContext& ctx);
+ bool StopProfilerDumpToLog(const TActorContext& ctx);
void OutputControlHtml(IOutputStream& os, const TString& action, bool isOk);
void OutputResultHtml(IOutputStream& os, const TString& action, const TString& profile, const TString& fileName, bool isOk, const TString& err);
};
- void TProfilerActor::DumpProfilingTimes(IOutputStream& out) {
- out << "Profiling started: " << StartTime << Endl
- << "Profiling stopped: " << StopTime << Endl
- << Endl;
- }
-
+ void TProfilerActor::DumpProfilingTimes(IOutputStream& out) {
+ out << "Profiling started: " << StartTime << Endl
+ << "Profiling stopped: " << StopTime << Endl
+ << Endl;
+ }
+
void TProfilerActor::HandleStart(TEvProfiler::TEvStart::TPtr& ev, const NActors::TActorContext& ctx) {
bool isOk = StartProfiler();
ctx.Send(ev->Sender, new TEvProfiler::TEvStartResult(ev->Get()->Cookie(), isOk));
@@ -310,8 +310,8 @@ namespace NActors {
isOk = StopProfiler(profile);
} else if (action == "stop-save") {
isOk = StopProfilerFile(fileName, err);
- } else if (action == "stop-log") {
- isOk = StopProfilerDumpToLog(ctx);
+ } else if (action == "stop-log") {
+ isOk = StopProfilerDumpToLog(ctx);
}
TStringStream out;
@@ -359,34 +359,34 @@ namespace NActors {
}
TStringOutput out(profile);
- DumpProfilingTimes(out);
- Profiler->Stop(out, 256, false);
- return true;
- }
-
- bool TProfilerActor::StopProfilerDumpToLog(const TActorContext& ctx) {
- if (!IsProfiling) {
- return false;
- }
-
- IsProfiling = false;
- StopTime = TInstant::Now();
- if (DynamicCounters) {
- *Counters.IsProfiling = 0;
- }
-
- TStringStream out;
- DumpProfilingTimes(out);
- Profiler->Stop(out, 2048, true);
-
- TVector<TString> split;
- Split(out.Str(), "\n", split);
- for (const auto& line : split) {
- LOG_WARN_S(ctx, NKikimrServices::MEMORY_PROFILER, line);
- }
+ DumpProfilingTimes(out);
+ Profiler->Stop(out, 256, false);
return true;
}
+ bool TProfilerActor::StopProfilerDumpToLog(const TActorContext& ctx) {
+ if (!IsProfiling) {
+ return false;
+ }
+
+ IsProfiling = false;
+ StopTime = TInstant::Now();
+ if (DynamicCounters) {
+ *Counters.IsProfiling = 0;
+ }
+
+ TStringStream out;
+ DumpProfilingTimes(out);
+ Profiler->Stop(out, 2048, true);
+
+ TVector<TString> split;
+ Split(out.Str(), "\n", split);
+ for (const auto& line : split) {
+ LOG_WARN_S(ctx, NKikimrServices::MEMORY_PROFILER, line);
+ }
+ return true;
+ }
+
bool TProfilerActor::StopProfilerFile(TString& outFileName, TString& err) {
if (!IsProfiling) {
outFileName.clear();
@@ -404,8 +404,8 @@ namespace NActors {
try {
TFileOutput out(outFileName);
- DumpProfilingTimes(out);
- Profiler->Stop(out, 4096, false);
+ DumpProfilingTimes(out);
+ Profiler->Stop(out, 4096, false);
} catch (const yexception& e) {
err = "Failed to dump profile: ";
err += e.what();
@@ -420,7 +420,7 @@ namespace NActors {
os << "<p>";
if (IsProfiling) {
os << "<a class=\"btn btn-primary\" href=\"?action=stop-display\">Stop & display</a> \n";
- os << "<a class=\"btn btn-primary\" href=\"?action=stop-log\">Stop & dump to log</a>\n";
+ os << "<a class=\"btn btn-primary\" href=\"?action=stop-log\">Stop & dump to log</a>\n";
os << "<a class=\"btn btn-primary\" href=\"?action=stop-save\">Stop & save to file</a>\n";
} else {
os << "<a class=\"btn btn-primary\" href=\"?action=start\">Start</a>\n";
@@ -450,12 +450,12 @@ namespace NActors {
os << "filename: " << fileName;
os << "</p><p>Error: " << err << "</p>";
}
- } else if (action == "stop-log") {
- if (isOk) {
- os << "<p>Output dumped to log</p>";
- } else {
- os << "<p>Error stopping profiling.</p>";
- }
+ } else if (action == "stop-log") {
+ if (isOk) {
+ os << "<p>Output dumped to log</p>";
+ } else {
+ os << "<p>Error stopping profiling.</p>";
+ }
} else if (action == "start") {
if (isOk) {
os << "<p>Profiling started OK.</p>";
diff --git a/ydb/core/mon_alloc/profiler.h b/ydb/core/mon_alloc/profiler.h
index 4fba8941da9..6769418a66b 100644
--- a/ydb/core/mon_alloc/profiler.h
+++ b/ydb/core/mon_alloc/profiler.h
@@ -107,7 +107,7 @@ namespace NActors {
struct IProfilerLogic {
virtual ~IProfilerLogic() = default;
virtual void Start() = 0;
- virtual void Stop(IOutputStream& out, size_t limit, bool forLog) = 0;
+ virtual void Stop(IOutputStream& out, size_t limit, bool forLog) = 0;
};
inline TActorId MakeProfilerID(ui32 nodeId) {
diff --git a/ydb/core/mon_alloc/stats.cpp b/ydb/core/mon_alloc/stats.cpp
index 050822241c6..6052ed8f6b5 100644
--- a/ydb/core/mon_alloc/stats.cpp
+++ b/ydb/core/mon_alloc/stats.cpp
@@ -1,11 +1,11 @@
#include "stats.h"
-#include "tcmalloc.h"
+#include "tcmalloc.h"
#include <ydb/core/base/counters.h>
#include <library/cpp/actors/core/actor_bootstrapped.h>
#include <library/cpp/actors/core/hfunc.h>
-#include <library/cpp/actors/core/process_stats.h>
+#include <library/cpp/actors/core/process_stats.h>
#include <library/cpp/lfalloc/dbg_info/dbg_info.h>
#include <library/cpp/malloc/api/malloc.h>
#include <library/cpp/monlib/dynamic_counters/counters.h>
@@ -203,18 +203,18 @@ namespace NKikimr {
std::unique_ptr<IAllocStats> CreateAllocStats(TDynamicCountersPtr group) {
const auto& info = NMalloc::MallocInfo();
- TStringBuf name(info.Name);
+ TStringBuf name(info.Name);
- std::unique_ptr<IAllocStats> stats;
+ std::unique_ptr<IAllocStats> stats;
if (name.StartsWith("lf")) {
- stats = std::make_unique<TLfAllocStats>(std::move(group));
+ stats = std::make_unique<TLfAllocStats>(std::move(group));
} else if (name.StartsWith("yt")) {
- stats = std::make_unique<TYtAllocStats>(std::move(group));
- } else if (name.StartsWith("tc")) {
- stats = std::move(CreateTcMallocStats(std::move(group)));
+ stats = std::make_unique<TYtAllocStats>(std::move(group));
+ } else if (name.StartsWith("tc")) {
+ stats = std::move(CreateTcMallocStats(std::move(group)));
}
-
- return stats ? std::move(stats) : std::make_unique<TFakeAllocStats>();
+
+ return stats ? std::move(stats) : std::make_unique<TFakeAllocStats>();
}
class TMemStatsCollector: public TActorBootstrapped<TMemStatsCollector> {
@@ -250,39 +250,39 @@ namespace NKikimr {
ctx.Schedule(Interval, new TEvents::TEvWakeup());
}
};
-
- struct TLfAllocState: public IAllocState {
- ui64 GetAllocatedMemoryEstimate() const override {
- ui64 result = 0;
- result += NAllocDbg::GetAllocationCounterFast(NAllocDbg::CT_SYSTEM_ALLOC);
- result -= NAllocDbg::GetAllocationCounterFast(NAllocDbg::CT_SYSTEM_FREE);
- result += NAllocDbg::GetAllocationCounterFast(NAllocDbg::CT_SMALL_ALLOC);
- result -= NAllocDbg::GetAllocationCounterFast(NAllocDbg::CT_SMALL_FREE);
- result += NAllocDbg::GetAllocationCounterFast(NAllocDbg::CT_LARGE_ALLOC);
- result -= NAllocDbg::GetAllocationCounterFast(NAllocDbg::CT_LARGE_FREE);
- return result;
- }
- };
-
- struct TFakeAllocState: public IAllocState {
- ui64 GetAllocatedMemoryEstimate() const override {
- return 0;
- }
- };
-
- std::unique_ptr<IAllocState> CreateAllocState() {
- const auto& info = NMalloc::MallocInfo();
- TStringBuf name(info.Name);
-
- std::unique_ptr<IAllocState> state;
- if (name.StartsWith("lf")) {
- state = std::make_unique<TLfAllocState>();
- } else if (name.StartsWith("tc")) {
- state = std::move(CreateTcMallocState());
- }
-
- return state ? std::move(state) : std::make_unique<TFakeAllocState>();
- }
+
+ struct TLfAllocState: public IAllocState {
+ ui64 GetAllocatedMemoryEstimate() const override {
+ ui64 result = 0;
+ result += NAllocDbg::GetAllocationCounterFast(NAllocDbg::CT_SYSTEM_ALLOC);
+ result -= NAllocDbg::GetAllocationCounterFast(NAllocDbg::CT_SYSTEM_FREE);
+ result += NAllocDbg::GetAllocationCounterFast(NAllocDbg::CT_SMALL_ALLOC);
+ result -= NAllocDbg::GetAllocationCounterFast(NAllocDbg::CT_SMALL_FREE);
+ result += NAllocDbg::GetAllocationCounterFast(NAllocDbg::CT_LARGE_ALLOC);
+ result -= NAllocDbg::GetAllocationCounterFast(NAllocDbg::CT_LARGE_FREE);
+ return result;
+ }
+ };
+
+ struct TFakeAllocState: public IAllocState {
+ ui64 GetAllocatedMemoryEstimate() const override {
+ return 0;
+ }
+ };
+
+ std::unique_ptr<IAllocState> CreateAllocState() {
+ const auto& info = NMalloc::MallocInfo();
+ TStringBuf name(info.Name);
+
+ std::unique_ptr<IAllocState> state;
+ if (name.StartsWith("lf")) {
+ state = std::make_unique<TLfAllocState>();
+ } else if (name.StartsWith("tc")) {
+ state = std::move(CreateTcMallocState());
+ }
+
+ return state ? std::move(state) : std::make_unique<TFakeAllocState>();
+ }
}
IActor* CreateMemStatsCollector(ui32 intervalSec, TDynamicCountersPtr counters) {
@@ -290,19 +290,19 @@ namespace NKikimr {
TDuration::Seconds(intervalSec),
CreateAllocStats(GetServiceCounters(counters, "utils")));
}
-
- std::unique_ptr<IAllocState> TAllocState::AllocState = CreateAllocState();
-
- ui64 TAllocState::GetAllocatedMemoryEstimate() {
- return AllocState->GetAllocatedMemoryEstimate();
- }
-
- double TAllocState::GetMemoryUsage() {
- NActors::TProcStat procStat;
- procStat.Fill(getpid());
- if (!procStat.CGroupMemLim) {
- return 0;
- }
- return (double)procStat.AnonRss / procStat.CGroupMemLim;
- }
+
+ std::unique_ptr<IAllocState> TAllocState::AllocState = CreateAllocState();
+
+ ui64 TAllocState::GetAllocatedMemoryEstimate() {
+ return AllocState->GetAllocatedMemoryEstimate();
+ }
+
+ double TAllocState::GetMemoryUsage() {
+ NActors::TProcStat procStat;
+ procStat.Fill(getpid());
+ if (!procStat.CGroupMemLim) {
+ return 0;
+ }
+ return (double)procStat.AnonRss / procStat.CGroupMemLim;
+ }
}
diff --git a/ydb/core/mon_alloc/stats.h b/ydb/core/mon_alloc/stats.h
index a5528e5afd3..a9925412450 100644
--- a/ydb/core/mon_alloc/stats.h
+++ b/ydb/core/mon_alloc/stats.h
@@ -6,25 +6,25 @@
#include <library/cpp/monlib/dynamic_counters/counters.h>
namespace NKikimr {
- struct IAllocStats {
- virtual ~IAllocStats() = default;
- virtual void Update() = 0;
- };
-
+ struct IAllocStats {
+ virtual ~IAllocStats() = default;
+ virtual void Update() = 0;
+ };
+
NActors::IActor* CreateMemStatsCollector(
ui32 intervalSec,
TIntrusivePtr<NMonitoring::TDynamicCounters> counters);
-
-
- struct IAllocState {
- virtual ~IAllocState() = default;
- virtual ui64 GetAllocatedMemoryEstimate() const = 0;
- };
-
- struct TAllocState {
- static std::unique_ptr<IAllocState> AllocState;
-
- static ui64 GetAllocatedMemoryEstimate();
- static double GetMemoryUsage();
- };
+
+
+ struct IAllocState {
+ virtual ~IAllocState() = default;
+ virtual ui64 GetAllocatedMemoryEstimate() const = 0;
+ };
+
+ struct TAllocState {
+ static std::unique_ptr<IAllocState> AllocState;
+
+ static ui64 GetAllocatedMemoryEstimate();
+ static double GetMemoryUsage();
+ };
}
diff --git a/ydb/core/mon_alloc/tcmalloc.cpp b/ydb/core/mon_alloc/tcmalloc.cpp
index 6c8689205f9..7c7890d25fc 100644
--- a/ydb/core/mon_alloc/tcmalloc.cpp
+++ b/ydb/core/mon_alloc/tcmalloc.cpp
@@ -1,791 +1,791 @@
-#include "tcmalloc.h"
-
-#include <contrib/libs/tcmalloc/tcmalloc/malloc_extension.h>
-
-#include <library/cpp/actors/prof/tag.h>
-#include <library/cpp/cache/cache.h>
-#include <library/cpp/html/pcdata/pcdata.h>
-#include <library/cpp/monlib/service/pages/templates.h>
-
+#include "tcmalloc.h"
+
+#include <contrib/libs/tcmalloc/tcmalloc/malloc_extension.h>
+
+#include <library/cpp/actors/prof/tag.h>
+#include <library/cpp/cache/cache.h>
+#include <library/cpp/html/pcdata/pcdata.h>
+#include <library/cpp/monlib/service/pages/templates.h>
+
#include <ydb/core/mon/mon.h>
-
-#include <util/stream/format.h>
-
-using namespace NActors;
-
-namespace NKikimr {
-
-using TDynamicCountersPtr = TIntrusivePtr<NMonitoring::TDynamicCounters>;
-using TDynamicCounterPtr = NMonitoring::TDynamicCounters::TCounterPtr;
-using THistogramPtr = NMonitoring::THistogramPtr;
-
-static void FormatPrettyNumber(IOutputStream& out, ssize_t val) {
- if (val == 0) {
- out << "0";
- return;
- }
- if (val < 0) {
- out << "- ";
- }
- ui64 value = val < 0 ? -val : val;
- TSmallVec<ui64> parts;
- while (value > 0) {
- auto head = value / 1000;
- parts.push_back(value - head * 1000);
- value = head;
- }
- out << parts.back();
- parts.pop_back();
- while (!parts.empty()) {
- out << Sprintf(" %03" PRIu64, parts.back());
- parts.pop_back();
- }
-}
-
-static void FormatSize(IOutputStream& out, size_t size) {
- if (size < (1 << 10)) {
- out << size;
- } else if (size < (1 << 20)) {
- out << size << " ("
- << Sprintf("%.1f", size / float(1 << 10)) << " KiB)";
- } else {
- out << size << " ("
- << Sprintf("%.1f", size / float(1 << 20)) << " MiB)";
- }
-}
-
-static ui64 GetProperty(
- const std::map<std::string, tcmalloc::MallocExtension::Property>& properties,
- const char* name)
-{
- auto found = properties.find(name);
- return found != properties.end() ? found->second.value : 0;
-}
-
-static ui64 GetProperty(const char* name) {
- const auto properties = tcmalloc::MallocExtension::GetProperties();
- return GetProperty(properties, name);
-}
-
-static ui64 GetCachesSize(
- const std::map<std::string, tcmalloc::MallocExtension::Property>& properties)
-{
- return
- GetProperty(properties, "tcmalloc.page_heap_free") +
- GetProperty(properties, "tcmalloc.central_cache_free") +
- GetProperty(properties, "tcmalloc.transfer_cache_free") +
- GetProperty(properties, "tcmalloc.sharded_transfer_cache_free") +
- GetProperty(properties, "tcmalloc.cpu_free") +
- GetProperty(properties, "tcmalloc.thread_cache_free");
-}
-
-struct TAllocationStats {
- size_t Size;
- size_t Count;
- size_t RequestedSize;
-
- static constexpr size_t MaxSizeIndex = 32;
- size_t Sizes[MaxSizeIndex];
- size_t Counts[MaxSizeIndex];
-
- static constexpr ui32 MaxTag = 1024;
- size_t TagSizes[MaxTag];
- size_t TagCounts[MaxTag];
-
- TAllocationStats() {
- Zero(*this);
- }
-
- static size_t SizeUpperBound(size_t index) {
- return 1 << std::min(MaxSizeIndex - 1, index);
- }
-
- static size_t GetIndex(size_t size) {
- size_t index = 0;
- if (size > 0) {
- index = std::min(MaxSizeIndex - 1, (size_t)GetValueBitCount(size - 1));
- }
- return index;
- }
-
- static const char* GetTagName(ui32 tag) {
- const char* name = NProfiling::GetTag(tag);
- return name ? name : "_DEFAULT";
- }
-
- void Add(const tcmalloc::Profile::Sample& sample) {
- Size += sample.sum;
- Count += sample.count;
- RequestedSize += sample.count * sample.requested_size;
-
- auto index = GetIndex(sample.allocated_size);
- Sizes[index] += sample.sum;
- Counts[index] += sample.count;
-
- ui32 tag = reinterpret_cast<uintptr_t>(sample.user_data);
- tag = (tag < MaxTag) ? tag : 0;
- TagSizes[tag] += sample.sum;
- TagCounts[tag] += sample.count;
- }
-
- void DumpSizeStats(IOutputStream& out, const char* marker, const char* sep) const {
- out << marker << sep << sep;
- for (size_t b = 0; b < MaxSizeIndex; ++b) {
- if (b == MaxSizeIndex - 1) {
- out << "inf : ";
- } else {
- out << "<= " << Sprintf("%-10" PRIu64, SizeUpperBound(b)) << ": ";
- }
- if (!Sizes[b]) {
- out << "0" << sep;
- continue;
- }
- out << "size ";
- FormatSize(out, Sizes[b]);
- out << ", count " << Counts[b] << sep;
- }
- out << Endl;
- }
-
- void DumpTagStats(IOutputStream& out, const char* marker, const char* sep) const {
- out << marker << sep << sep;
- for (ui32 t = 0; t < MaxTag; ++t) {
- if (!TagSizes[t]) {
- continue;
- }
- out << Sprintf("%-32s", TAllocationStats::GetTagName(t)) << ": size ";
- FormatSize(out, TagSizes[t]);
- out << ", count " << TagCounts[t] << sep;
- }
- out << Endl;
- }
-};
-
-
-class TAllocationAnalyzer {
- const tcmalloc::Profile Profile;
-
- struct TSymbol {
- const void* Address;
- TString Name;
- };
- TLFUCache<void*, TSymbol> SymbolCache;
-
- struct TStackKey {
- int Depth;
- void* Stack[tcmalloc::Profile::Sample::kMaxStackDepth];
-
- explicit TStackKey(const tcmalloc::Profile::Sample& sample) {
- Depth = std::min(sample.depth, tcmalloc::Profile::Sample::kMaxStackDepth);
- std::memcpy(Stack, sample.stack, Depth * sizeof(void*));
- }
-
- bool operator==(const TStackKey& key) const {
- if (Depth != key.Depth) {
- return false;
- }
- return std::memcmp(Stack, key.Stack, Depth * sizeof(void*)) == 0;
- }
-
- struct THash {
- size_t operator()(const TStackKey& key) const {
- std::string_view view((char*)key.Stack, key.Depth * sizeof(void*));
- return std::hash<std::string_view>()(view);
- }
- };
- };
-
- struct TStackStats {
- std::vector<tcmalloc::Profile::Sample> Samples;
- size_t Size = 0;
- size_t Count = 0;
- ui32 Tag = 0;
- };
-
- std::unordered_map<TStackKey, TStackStats, TStackKey::THash> Stacks;
- size_t SampleCount = 0;
- size_t Size = 0;
- size_t Count = 0;
- size_t RequestedSize = 0;
-
- bool Prepared = false;
-
-private:
- void PrintBackTrace(IOutputStream& out, void* const* stack, size_t sz,
- const char* sep)
- {
- char name[1024];
- for (size_t i = 0; i < sz; ++i) {
- TSymbol symbol;
- auto it = SymbolCache.Find(stack[i]);
- if (it != SymbolCache.End()) {
- symbol = it.Value();
- } else {
- TResolvedSymbol rs = ResolveSymbol(stack[i], name, sizeof(name));
- symbol = {rs.NearestSymbol, rs.Name};
- SymbolCache.Insert(stack[i], symbol);
- }
-
- out << Hex((intptr_t)stack[i], HF_FULL) << " " << symbol.Name;
- intptr_t offset = (intptr_t)stack[i] - (intptr_t)symbol.Address;
- if (offset)
- out << " +" << offset;
- out << sep;
- }
- }
-
- void PrintSample(IOutputStream& out, const tcmalloc::Profile::Sample* sample,
- const char* sep) const
- {
- out << sample->sum << " = "
- << sample->count << " * "
- << sample->allocated_size << " ("
- << sample->requested_size << ")"
- << sep;
- }
-
- void PrintStack(IOutputStream& out, const TStackStats* stats, size_t sampleCountLimit,
- const char* marker, const char* sep)
- {
- std::vector<const tcmalloc::Profile::Sample*> samples;
- samples.reserve(stats->Samples.size());
- size_t size = 0;
- size_t count = 0;
-
- for (const auto& sample : stats->Samples) {
- samples.push_back(&sample);
- size += sample.sum;
- count += sample.count;
- }
-
- std::sort(samples.begin(), samples.end(), [] (const auto* l, const auto* r) {
- return l->sum > r->sum;
- });
-
- out << marker << sep << sep << "size ";
- FormatSize(out, size);
- out << ", count " << count
- << ", " << TAllocationStats::GetTagName(stats->Tag) << sep;
-
- size_t i = 0;
- for (const auto* sample : samples) {
- PrintSample(out, sample, sep);
- if (++i >= sampleCountLimit) {
- break;
- }
- }
-
- if (samples.size() > sampleCountLimit) {
- out << "....TRUNCATED, first " << sampleCountLimit << " samples are shown" << sep;
- }
- out << sep;
-
- if (samples.size()) {
- const auto& sample = samples.front();
- PrintBackTrace(out, sample->stack, sample->depth, sep);
- }
- out << Endl;
- }
-
- void PrintAllocatorDetails(IOutputStream& out, const char* sep) const {
- const auto properties = tcmalloc::MallocExtension::GetProperties();
- out << sep
- << "Physical memory used: ";
- FormatSize(out, GetProperty(properties, "generic.physical_memory_used"));
- out << sep
- << " Application: ";
- FormatSize(out, GetProperty(properties, "generic.bytes_in_use_by_app"));
- out << sep
- << " Allocator caches: ";
- FormatSize(out, GetCachesSize(properties));
- out << sep
- << " Allocator metadata: ";
- FormatSize(out, GetProperty(properties, "tcmalloc.metadata_bytes"));
- out << sep;
- }
-
-public:
- explicit TAllocationAnalyzer(tcmalloc::Profile&& profile)
- : Profile(std::move(profile))
- , SymbolCache(2048)
- {}
-
- void Prepare(TAllocationStats* allocationStats) {
- Y_VERIFY(!Prepared);
-
- TAllocationStats allocations;
-
- Profile.Iterate([&] (const tcmalloc::Profile::Sample& sample) {
- if (!sample.sum) {
- return;
- }
-
- TStackKey key(sample);
- auto& stats = Stacks[key];
- stats.Samples.push_back(sample);
- stats.Size += sample.sum;
- stats.Count += sample.count;
- stats.Tag = reinterpret_cast<uintptr_t>(sample.user_data);
-
- allocations.Add(sample);
-
- ++SampleCount;
- });
-
- Size = allocations.Size;
- Count = allocations.Count;
- RequestedSize = allocations.RequestedSize;
-
- if (allocationStats) {
- *allocationStats = allocations;
- }
-
- Prepared = true;
- }
-
- void Dump(IOutputStream& out, size_t stackCountLimit, size_t sampleCountLimit,
- bool isHeap, bool forLog)
- {
- Y_VERIFY(Prepared);
-
- const char* marker = forLog ?
- (isHeap ? "HEAP" : "PROFILE") : "--------------------------------";
- const char* sep = forLog ? " | " : "\n";
-
- out << marker << sep << sep;
- out << "Stack count: " << Stacks.size() << sep
- << "Sample count: " << SampleCount << sep
- << "Allocated bytes: ";
- FormatSize(out, Size);
- out << sep
- << "Allocation count: " << Count << sep;
-
- if (isHeap) {
- out << sep
- << "Requested bytes: ";
- FormatSize(out, RequestedSize);
- out << sep
- << "Wasted bytes: ";
- FormatSize(out, Size - RequestedSize);
- out << sep;
-
- PrintAllocatorDetails(out, sep);
- }
-
- if (!forLog) {
- out << sep
- << "Details: size = count * allocated_size (requested_size)" << sep;
- }
-
- out << Endl;
-
- std::vector<const TStackStats*> stats;
- stats.reserve(Stacks.size());
- for (auto& [key, stackStats] : Stacks) {
- stats.push_back(&stackStats);
- }
-
- std::sort(stats.begin(), stats.end(), [] (const auto* l, const auto* r) {
- return l->Size > r->Size;
- });
-
- size_t i = 0;
- for (const auto* stackStats : stats) {
- PrintStack(out, stackStats, sampleCountLimit, marker, sep);
- if (++i >= stackCountLimit) {
- break;
- }
- }
-
- if (stats.size() > stackCountLimit) {
- out << "....TRUNCATED, first " << stackCountLimit << " stacks are shown" << sep;
- }
- out << Endl;
- }
-};
-
-
-class TTcMallocStats : public IAllocStats {
- TDynamicCountersPtr CounterGroup;
-
- static constexpr const char* Names[] = {
- "generic.virtual_memory_used",
- "generic.physical_memory_used",
- "generic.bytes_in_use_by_app",
- "tcmalloc.page_heap_free",
- "tcmalloc.metadata_bytes",
- "tcmalloc.thread_cache_count",
- "tcmalloc.central_cache_free",
- "tcmalloc.transfer_cache_free",
- "tcmalloc.sharded_transfer_cache_free",
- "tcmalloc.cpu_free",
- "tcmalloc.per_cpu_caches_active",
- "tcmalloc.thread_cache_free",
- "tcmalloc.page_heap_unmapped",
- };
-
- std::unordered_map<TString, TDynamicCounterPtr> Counters;
-
- TDynamicCounterPtr CachesSize;
-
-public:
- explicit TTcMallocStats(TDynamicCountersPtr group) {
- CounterGroup = group->GetSubgroup("component", "tcmalloc");
- for (auto name : Names) {
- Counters[name] = CounterGroup->GetCounter(name, false);
- }
- CachesSize = CounterGroup->GetCounter("tcmalloc.caches_bytes", false);
- }
-
- void Update() override {
- auto properties = tcmalloc::MallocExtension::GetProperties();
- for (auto name : Names) {
- if (auto it = properties.find(name); it != properties.end()) {
- Counters[name]->Set(it->second.value);
- }
- }
- CachesSize->Set(GetCachesSize(properties));
- }
-};
-
-
-class TTcMallocState : public IAllocState {
-public:
- ui64 GetAllocatedMemoryEstimate() const override {
- return GetProperty("generic.physical_memory_used");
- }
-};
-
-
-class TTcMallocMonitor : public IAllocMonitor {
- TDynamicCountersPtr CounterGroup;
-
- THistogramPtr SizeHistogram;
- THistogramPtr CountHistogram;
- std::unordered_map<ui32, std::pair<TDynamicCounterPtr, TDynamicCounterPtr>> TaggedCounters;
-
- struct TControls {
- static constexpr size_t MaxSamplingRate = 4ll << 30;
-
- TControlWrapper ProfileSamplingRate;
- TControlWrapper GuardedSamplingRate;
- TControlWrapper BackgroundReleaseRate;
- TControlWrapper MemoryLimit;
-
- TControls()
- : ProfileSamplingRate(tcmalloc::MallocExtension::GetProfileSamplingRate(),
- 64 << 10, MaxSamplingRate)
- , GuardedSamplingRate(MaxSamplingRate,
- 64 << 10, MaxSamplingRate)
- , BackgroundReleaseRate(0,
- 0, 64 << 20)
- , MemoryLimit(0,
- 0, std::numeric_limits<i64>::max())
- {}
-
- void Register(TIntrusivePtr<TControlBoard> icb) {
- icb->RegisterSharedControl(ProfileSamplingRate, "TCMallocControls.ProfileSamplingRate");
- icb->RegisterSharedControl(GuardedSamplingRate, "TCMallocControls.GuardedSamplingRate");
- icb->RegisterSharedControl(BackgroundReleaseRate, "TCMallocControls.BackgroundReleaseRate");
- icb->RegisterSharedControl(MemoryLimit, "TCMallocControls.MemoryLimit");
- }
- };
- TControls Controls;
-
-private:
- void UpdateCounters() {
- auto profile = tcmalloc::MallocExtension::SnapshotCurrent(tcmalloc::ProfileType::kHeap);
-
- TAllocationAnalyzer analyzer(std::move(profile));
- TAllocationStats allocationStats;
- analyzer.Prepare(&allocationStats);
-
- auto updateHistogram = [] (THistogramPtr histogram, size_t* numbers) {
- histogram->Reset();
- auto snapshot = histogram->Snapshot();
- auto count = std::min(snapshot->Count(), (ui32)TAllocationStats::MaxSizeIndex);
- for (ui32 b = 0; b < count; ++b) {
- histogram->Collect(snapshot->UpperBound(b), numbers[b]);
- }
- };
-
- updateHistogram(SizeHistogram, allocationStats.Sizes);
- updateHistogram(CountHistogram, allocationStats.Counts);
-
- for (ui32 t = 0; t < TAllocationStats::MaxTag; ++t) {
- auto found = TaggedCounters.find(t);
- if (!allocationStats.TagSizes[t] && found == TaggedCounters.end()) {
- continue;
- }
-
- TDynamicCounterPtr size, count;
-
- if (found == TaggedCounters.end()) {
- auto name = TAllocationStats::GetTagName(t);
- auto group = CounterGroup->GetSubgroup("activity", name);
- size = group->GetCounter("tcmalloc.sampled_size_by_activity", false);
- count = group->GetCounter("tcmalloc.sampled_count_by_activity", false);
- TaggedCounters.emplace(t, std::make_pair(size, count));
- } else {
- size = found->second.first;
- count = found->second.second;
- }
-
- size->Set(allocationStats.TagSizes[t]);
- count->Set(allocationStats.TagCounts[t]);
- }
- }
-
- void UpdateControls() {
- tcmalloc::MallocExtension::SetProfileSamplingRate(Controls.ProfileSamplingRate);
-
- if (Controls.GuardedSamplingRate != TControls::MaxSamplingRate) {
- tcmalloc::MallocExtension::ActivateGuardedSampling();
- }
- tcmalloc::MallocExtension::SetGuardedSamplingRate(Controls.GuardedSamplingRate);
-
- tcmalloc::MallocExtension::BytesPerSecond rate{(size_t)Controls.BackgroundReleaseRate};
- tcmalloc::MallocExtension::SetBackgroundReleaseRate(rate);
-
- tcmalloc::MallocExtension::MemoryLimit limit;
- limit.hard = false;
- limit.limit = Controls.MemoryLimit ?
- (size_t)Controls.MemoryLimit : std::numeric_limits<size_t>::max();
- tcmalloc::MallocExtension::SetMemoryLimit(limit);
- }
-
- void DumpCurrent(IOutputStream& out, tcmalloc::ProfileType type,
- size_t stackCountLimit = 256, size_t sampleCountLimit = 1024, bool forLog = false)
- {
- auto start = TInstant::Now();
- auto profile = tcmalloc::MallocExtension::SnapshotCurrent(type);
- auto end = TInstant::Now();
-
- TAllocationAnalyzer analyzer(std::move(profile));
- TAllocationStats allocationStats;
- analyzer.Prepare(&allocationStats);
-
- bool isHeap = (type == tcmalloc::ProfileType::kHeap);
-
- if (forLog) {
- const char* marker = "HEAP";
- const char* sep = " | ";
- out << marker << sep
- << "Snapshot calculation time: " << (end - start).MicroSeconds() << " us" << sep
- << Endl;
- allocationStats.DumpSizeStats(out, marker, sep);
- allocationStats.DumpTagStats(out, marker, sep);
- analyzer.Dump(out, stackCountLimit, sampleCountLimit, isHeap, true);
- return;
- }
-
- HTML(out) {
- if (isHeap) {
- out << "<p><a class=\"btn btn-primary\" href=\"?action=log\">Dump to log</a></p>\n";
- }
- TABLE_SORTABLE_CLASS("table") {
- TABLEHEAD() {
- TABLER() {
- TABLEH() { out << "upper bound"; }
- TABLEH() { out << "size"; }
- TABLEH() { out << "count"; }
- }
- out << Endl;
- }
- TABLEBODY() {
- for (size_t b = 0; b < TAllocationStats::MaxSizeIndex; ++b) {
- if (!allocationStats.Sizes[b]) {
- continue;
- }
- TABLER() {
- if (b == TAllocationStats::MaxSizeIndex - 1) {
- TABLED() { out << "inf"; }
- } else {
- TABLED() { out << TAllocationStats::SizeUpperBound(b); }
- }
- TABLED() { out << allocationStats.Sizes[b]; }
- TABLED() { out << allocationStats.Counts[b]; }
- }
- out << Endl;
- }
- TABLER() {
- TABLED() { out << "<b>total</b>"; }
- TABLED() { out << "<b>" << allocationStats.Size << "</b>"; }
- TABLED() { out << "<b>" << allocationStats.Count << "</b>"; }
- }
- out << Endl;
- }
- }
- TABLE_SORTABLE_CLASS("table") {
- TABLEHEAD() {
- TABLER() {
- TABLEH() { out << "activity"; }
- TABLEH() { out << "size"; }
- TABLEH() { out << "count"; }
- }
- out << Endl;
- }
- TABLEBODY() {
- for (ui32 t = 0; t < TAllocationStats::MaxTag; ++t) {
- if (!allocationStats.TagSizes[t]) {
- continue;
- }
- TABLER() {
- TABLED() { out << TAllocationStats::GetTagName(t); }
- TABLED() { out << allocationStats.TagSizes[t]; }
- TABLED() { out << allocationStats.TagCounts[t]; }
- }
- out << Endl;
- }
- }
- }
- PRE() {
- out << "Snapshot calculation time: " << (end - start).MicroSeconds() << " us" << Endl
- << Endl;
- analyzer.Dump(out, stackCountLimit, sampleCountLimit, isHeap, false);
- }
- }
- }
-
-public:
- explicit TTcMallocMonitor(TDynamicCountersPtr group) {
- CounterGroup = group->GetSubgroup("component", "tcmalloc");
-
- SizeHistogram = CounterGroup->GetHistogram("tcmalloc.sampled_size",
- NMonitoring::ExponentialHistogram(TAllocationStats::MaxSizeIndex, 2, 1), false);
-
- CountHistogram = CounterGroup->GetHistogram("tcmalloc.sampled_count",
- NMonitoring::ExponentialHistogram(TAllocationStats::MaxSizeIndex, 2, 1), false);
- }
-
- void RegisterPages(TMon* mon, TActorSystem* actorSystem, TActorId actorId) override {
- auto* indexPage = mon->FindIndexPage("memory");
- if (!indexPage) {
- return;
- }
- mon->RegisterActorPage(indexPage, "heap", "Heap", false, actorSystem, actorId);
- mon->RegisterActorPage(indexPage, "peakheap", "Peak heap", false, actorSystem, actorId);
- mon->RegisterActorPage(indexPage, "fragmentation", "Fragmentation", false, actorSystem, actorId);
- }
-
- void RegisterControls(TIntrusivePtr<TControlBoard> icb) override {
- Controls.Register(icb);
- }
-
- void Update() override {
- UpdateCounters();
- UpdateControls();
- }
-
- void Dump(IOutputStream& out, const TString& relPath) override {
- if (relPath == "/memory/heap") {
- DumpCurrent(out, tcmalloc::ProfileType::kHeap);
- return;
- } else if (relPath == "/memory/peakheap") {
- DumpCurrent(out, tcmalloc::ProfileType::kPeakHeap);
- return;
- } else if (relPath == "/memory/fragmentation") {
- DumpCurrent(out, tcmalloc::ProfileType::kFragmentation);
- return;
- }
-
- auto properties = tcmalloc::MallocExtension::GetProperties();
- auto stats = tcmalloc::MallocExtension::GetStats();
-
- HTML(out) {
- H3() {
- out << "TCMalloc" << Endl;
- }
- out << "<hr>" << Endl;
- H4() {
- out << "Allocator properties" << Endl;
- }
- TABLE_CLASS("table") {
- TABLEHEAD() {
- TABLER() {
- TABLEH() { out << "Property"; }
- TABLEH() { out << "Value"; }
- }
- out << Endl;
- }
- TABLEBODY() {
- for (const auto& [name, value] : properties) {
- TABLER() {
- TABLED() { out << name; }
- TABLED() { FormatPrettyNumber(out, value.value); }
- }
- out << Endl;
- }
- }
- }
- out << "<hr>" << Endl;
- H4() {
- out << "<a href=\"https://github.com/google/tcmalloc/blob/master/docs/stats.md\">Internal stats</a>" << Endl;
- }
- PRE() {
- out << EncodeHtmlPcdata(stats) << Endl;
- }
- }
- }
-
- void DumpForLog(IOutputStream& out, size_t limit) override {
- auto properties = tcmalloc::MallocExtension::GetProperties();
- auto stats = tcmalloc::MallocExtension::GetStats();
-
- const char* sep = " | ";
- out << "======== TCMALLOC PROPERTIES" << sep;
- for (const auto& [name, value] : properties) {
- out << name << ": " << value.value << sep;
- }
- out << Endl;
-
- out << "======== TCMALLOC STATISTICS" << Endl;
- out << stats << Endl;
-
- out << "======== TCMALLOC HEAP" << Endl;
- DumpCurrent(out, tcmalloc::ProfileType::kHeap, limit, 128, true);
- }
-};
-
-
-class TTcMallocProfiler : public IProfilerLogic {
- tcmalloc::MallocExtension::AllocationProfilingToken Token;
-
-public:
- void Start() override {
- Token = tcmalloc::MallocExtension::StartAllocationProfiling();
- }
-
- void Stop(IOutputStream& out, size_t countLimit, bool forLog) override {
- auto profile = std::move(Token).Stop();
-
- TAllocationAnalyzer analyzer(std::move(profile));
- TAllocationStats allocationStats;
- analyzer.Prepare(&allocationStats);
-
- const char* marker = forLog ? "PROFILE" : "--------------------------------";
- const char* sep = forLog ? " | " : "\n";
- allocationStats.DumpSizeStats(out, marker, sep);
- allocationStats.DumpTagStats(out, marker, sep);
- analyzer.Dump(out, countLimit, 1024, false, forLog);
- }
-};
-
-
-std::unique_ptr<IAllocStats> CreateTcMallocStats(TDynamicCountersPtr group) {
- return std::make_unique<TTcMallocStats>(std::move(group));
-}
-
-std::unique_ptr<IAllocState> CreateTcMallocState() {
- return std::make_unique<TTcMallocState>();
-}
-
-std::unique_ptr<IAllocMonitor> CreateTcMallocMonitor(TDynamicCountersPtr group) {
- return std::make_unique<TTcMallocMonitor>(std::move(group));
-}
-
-std::unique_ptr<IProfilerLogic> CreateTcMallocProfiler() {
- return std::make_unique<TTcMallocProfiler>();
-}
-
-}
+
+#include <util/stream/format.h>
+
+using namespace NActors;
+
+namespace NKikimr {
+
+using TDynamicCountersPtr = TIntrusivePtr<NMonitoring::TDynamicCounters>;
+using TDynamicCounterPtr = NMonitoring::TDynamicCounters::TCounterPtr;
+using THistogramPtr = NMonitoring::THistogramPtr;
+
+static void FormatPrettyNumber(IOutputStream& out, ssize_t val) {
+ if (val == 0) {
+ out << "0";
+ return;
+ }
+ if (val < 0) {
+ out << "- ";
+ }
+ ui64 value = val < 0 ? -val : val;
+ TSmallVec<ui64> parts;
+ while (value > 0) {
+ auto head = value / 1000;
+ parts.push_back(value - head * 1000);
+ value = head;
+ }
+ out << parts.back();
+ parts.pop_back();
+ while (!parts.empty()) {
+ out << Sprintf(" %03" PRIu64, parts.back());
+ parts.pop_back();
+ }
+}
+
+static void FormatSize(IOutputStream& out, size_t size) {
+ if (size < (1 << 10)) {
+ out << size;
+ } else if (size < (1 << 20)) {
+ out << size << " ("
+ << Sprintf("%.1f", size / float(1 << 10)) << " KiB)";
+ } else {
+ out << size << " ("
+ << Sprintf("%.1f", size / float(1 << 20)) << " MiB)";
+ }
+}
+
+static ui64 GetProperty(
+ const std::map<std::string, tcmalloc::MallocExtension::Property>& properties,
+ const char* name)
+{
+ auto found = properties.find(name);
+ return found != properties.end() ? found->second.value : 0;
+}
+
+static ui64 GetProperty(const char* name) {
+ const auto properties = tcmalloc::MallocExtension::GetProperties();
+ return GetProperty(properties, name);
+}
+
+static ui64 GetCachesSize(
+ const std::map<std::string, tcmalloc::MallocExtension::Property>& properties)
+{
+ return
+ GetProperty(properties, "tcmalloc.page_heap_free") +
+ GetProperty(properties, "tcmalloc.central_cache_free") +
+ GetProperty(properties, "tcmalloc.transfer_cache_free") +
+ GetProperty(properties, "tcmalloc.sharded_transfer_cache_free") +
+ GetProperty(properties, "tcmalloc.cpu_free") +
+ GetProperty(properties, "tcmalloc.thread_cache_free");
+}
+
+struct TAllocationStats {
+ size_t Size;
+ size_t Count;
+ size_t RequestedSize;
+
+ static constexpr size_t MaxSizeIndex = 32;
+ size_t Sizes[MaxSizeIndex];
+ size_t Counts[MaxSizeIndex];
+
+ static constexpr ui32 MaxTag = 1024;
+ size_t TagSizes[MaxTag];
+ size_t TagCounts[MaxTag];
+
+ TAllocationStats() {
+ Zero(*this);
+ }
+
+ static size_t SizeUpperBound(size_t index) {
+ return 1 << std::min(MaxSizeIndex - 1, index);
+ }
+
+ static size_t GetIndex(size_t size) {
+ size_t index = 0;
+ if (size > 0) {
+ index = std::min(MaxSizeIndex - 1, (size_t)GetValueBitCount(size - 1));
+ }
+ return index;
+ }
+
+ static const char* GetTagName(ui32 tag) {
+ const char* name = NProfiling::GetTag(tag);
+ return name ? name : "_DEFAULT";
+ }
+
+ void Add(const tcmalloc::Profile::Sample& sample) {
+ Size += sample.sum;
+ Count += sample.count;
+ RequestedSize += sample.count * sample.requested_size;
+
+ auto index = GetIndex(sample.allocated_size);
+ Sizes[index] += sample.sum;
+ Counts[index] += sample.count;
+
+ ui32 tag = reinterpret_cast<uintptr_t>(sample.user_data);
+ tag = (tag < MaxTag) ? tag : 0;
+ TagSizes[tag] += sample.sum;
+ TagCounts[tag] += sample.count;
+ }
+
+ void DumpSizeStats(IOutputStream& out, const char* marker, const char* sep) const {
+ out << marker << sep << sep;
+ for (size_t b = 0; b < MaxSizeIndex; ++b) {
+ if (b == MaxSizeIndex - 1) {
+ out << "inf : ";
+ } else {
+ out << "<= " << Sprintf("%-10" PRIu64, SizeUpperBound(b)) << ": ";
+ }
+ if (!Sizes[b]) {
+ out << "0" << sep;
+ continue;
+ }
+ out << "size ";
+ FormatSize(out, Sizes[b]);
+ out << ", count " << Counts[b] << sep;
+ }
+ out << Endl;
+ }
+
+ void DumpTagStats(IOutputStream& out, const char* marker, const char* sep) const {
+ out << marker << sep << sep;
+ for (ui32 t = 0; t < MaxTag; ++t) {
+ if (!TagSizes[t]) {
+ continue;
+ }
+ out << Sprintf("%-32s", TAllocationStats::GetTagName(t)) << ": size ";
+ FormatSize(out, TagSizes[t]);
+ out << ", count " << TagCounts[t] << sep;
+ }
+ out << Endl;
+ }
+};
+
+
+class TAllocationAnalyzer {
+ const tcmalloc::Profile Profile;
+
+ struct TSymbol {
+ const void* Address;
+ TString Name;
+ };
+ TLFUCache<void*, TSymbol> SymbolCache;
+
+ struct TStackKey {
+ int Depth;
+ void* Stack[tcmalloc::Profile::Sample::kMaxStackDepth];
+
+ explicit TStackKey(const tcmalloc::Profile::Sample& sample) {
+ Depth = std::min(sample.depth, tcmalloc::Profile::Sample::kMaxStackDepth);
+ std::memcpy(Stack, sample.stack, Depth * sizeof(void*));
+ }
+
+ bool operator==(const TStackKey& key) const {
+ if (Depth != key.Depth) {
+ return false;
+ }
+ return std::memcmp(Stack, key.Stack, Depth * sizeof(void*)) == 0;
+ }
+
+ struct THash {
+ size_t operator()(const TStackKey& key) const {
+ std::string_view view((char*)key.Stack, key.Depth * sizeof(void*));
+ return std::hash<std::string_view>()(view);
+ }
+ };
+ };
+
+ struct TStackStats {
+ std::vector<tcmalloc::Profile::Sample> Samples;
+ size_t Size = 0;
+ size_t Count = 0;
+ ui32 Tag = 0;
+ };
+
+ std::unordered_map<TStackKey, TStackStats, TStackKey::THash> Stacks;
+ size_t SampleCount = 0;
+ size_t Size = 0;
+ size_t Count = 0;
+ size_t RequestedSize = 0;
+
+ bool Prepared = false;
+
+private:
+ void PrintBackTrace(IOutputStream& out, void* const* stack, size_t sz,
+ const char* sep)
+ {
+ char name[1024];
+ for (size_t i = 0; i < sz; ++i) {
+ TSymbol symbol;
+ auto it = SymbolCache.Find(stack[i]);
+ if (it != SymbolCache.End()) {
+ symbol = it.Value();
+ } else {
+ TResolvedSymbol rs = ResolveSymbol(stack[i], name, sizeof(name));
+ symbol = {rs.NearestSymbol, rs.Name};
+ SymbolCache.Insert(stack[i], symbol);
+ }
+
+ out << Hex((intptr_t)stack[i], HF_FULL) << " " << symbol.Name;
+ intptr_t offset = (intptr_t)stack[i] - (intptr_t)symbol.Address;
+ if (offset)
+ out << " +" << offset;
+ out << sep;
+ }
+ }
+
+ void PrintSample(IOutputStream& out, const tcmalloc::Profile::Sample* sample,
+ const char* sep) const
+ {
+ out << sample->sum << " = "
+ << sample->count << " * "
+ << sample->allocated_size << " ("
+ << sample->requested_size << ")"
+ << sep;
+ }
+
+ void PrintStack(IOutputStream& out, const TStackStats* stats, size_t sampleCountLimit,
+ const char* marker, const char* sep)
+ {
+ std::vector<const tcmalloc::Profile::Sample*> samples;
+ samples.reserve(stats->Samples.size());
+ size_t size = 0;
+ size_t count = 0;
+
+ for (const auto& sample : stats->Samples) {
+ samples.push_back(&sample);
+ size += sample.sum;
+ count += sample.count;
+ }
+
+ std::sort(samples.begin(), samples.end(), [] (const auto* l, const auto* r) {
+ return l->sum > r->sum;
+ });
+
+ out << marker << sep << sep << "size ";
+ FormatSize(out, size);
+ out << ", count " << count
+ << ", " << TAllocationStats::GetTagName(stats->Tag) << sep;
+
+ size_t i = 0;
+ for (const auto* sample : samples) {
+ PrintSample(out, sample, sep);
+ if (++i >= sampleCountLimit) {
+ break;
+ }
+ }
+
+ if (samples.size() > sampleCountLimit) {
+ out << "....TRUNCATED, first " << sampleCountLimit << " samples are shown" << sep;
+ }
+ out << sep;
+
+ if (samples.size()) {
+ const auto& sample = samples.front();
+ PrintBackTrace(out, sample->stack, sample->depth, sep);
+ }
+ out << Endl;
+ }
+
+ void PrintAllocatorDetails(IOutputStream& out, const char* sep) const {
+ const auto properties = tcmalloc::MallocExtension::GetProperties();
+ out << sep
+ << "Physical memory used: ";
+ FormatSize(out, GetProperty(properties, "generic.physical_memory_used"));
+ out << sep
+ << " Application: ";
+ FormatSize(out, GetProperty(properties, "generic.bytes_in_use_by_app"));
+ out << sep
+ << " Allocator caches: ";
+ FormatSize(out, GetCachesSize(properties));
+ out << sep
+ << " Allocator metadata: ";
+ FormatSize(out, GetProperty(properties, "tcmalloc.metadata_bytes"));
+ out << sep;
+ }
+
+public:
+ explicit TAllocationAnalyzer(tcmalloc::Profile&& profile)
+ : Profile(std::move(profile))
+ , SymbolCache(2048)
+ {}
+
+ void Prepare(TAllocationStats* allocationStats) {
+ Y_VERIFY(!Prepared);
+
+ TAllocationStats allocations;
+
+ Profile.Iterate([&] (const tcmalloc::Profile::Sample& sample) {
+ if (!sample.sum) {
+ return;
+ }
+
+ TStackKey key(sample);
+ auto& stats = Stacks[key];
+ stats.Samples.push_back(sample);
+ stats.Size += sample.sum;
+ stats.Count += sample.count;
+ stats.Tag = reinterpret_cast<uintptr_t>(sample.user_data);
+
+ allocations.Add(sample);
+
+ ++SampleCount;
+ });
+
+ Size = allocations.Size;
+ Count = allocations.Count;
+ RequestedSize = allocations.RequestedSize;
+
+ if (allocationStats) {
+ *allocationStats = allocations;
+ }
+
+ Prepared = true;
+ }
+
+ void Dump(IOutputStream& out, size_t stackCountLimit, size_t sampleCountLimit,
+ bool isHeap, bool forLog)
+ {
+ Y_VERIFY(Prepared);
+
+ const char* marker = forLog ?
+ (isHeap ? "HEAP" : "PROFILE") : "--------------------------------";
+ const char* sep = forLog ? " | " : "\n";
+
+ out << marker << sep << sep;
+ out << "Stack count: " << Stacks.size() << sep
+ << "Sample count: " << SampleCount << sep
+ << "Allocated bytes: ";
+ FormatSize(out, Size);
+ out << sep
+ << "Allocation count: " << Count << sep;
+
+ if (isHeap) {
+ out << sep
+ << "Requested bytes: ";
+ FormatSize(out, RequestedSize);
+ out << sep
+ << "Wasted bytes: ";
+ FormatSize(out, Size - RequestedSize);
+ out << sep;
+
+ PrintAllocatorDetails(out, sep);
+ }
+
+ if (!forLog) {
+ out << sep
+ << "Details: size = count * allocated_size (requested_size)" << sep;
+ }
+
+ out << Endl;
+
+ std::vector<const TStackStats*> stats;
+ stats.reserve(Stacks.size());
+ for (auto& [key, stackStats] : Stacks) {
+ stats.push_back(&stackStats);
+ }
+
+ std::sort(stats.begin(), stats.end(), [] (const auto* l, const auto* r) {
+ return l->Size > r->Size;
+ });
+
+ size_t i = 0;
+ for (const auto* stackStats : stats) {
+ PrintStack(out, stackStats, sampleCountLimit, marker, sep);
+ if (++i >= stackCountLimit) {
+ break;
+ }
+ }
+
+ if (stats.size() > stackCountLimit) {
+ out << "....TRUNCATED, first " << stackCountLimit << " stacks are shown" << sep;
+ }
+ out << Endl;
+ }
+};
+
+
+class TTcMallocStats : public IAllocStats {
+ TDynamicCountersPtr CounterGroup;
+
+ static constexpr const char* Names[] = {
+ "generic.virtual_memory_used",
+ "generic.physical_memory_used",
+ "generic.bytes_in_use_by_app",
+ "tcmalloc.page_heap_free",
+ "tcmalloc.metadata_bytes",
+ "tcmalloc.thread_cache_count",
+ "tcmalloc.central_cache_free",
+ "tcmalloc.transfer_cache_free",
+ "tcmalloc.sharded_transfer_cache_free",
+ "tcmalloc.cpu_free",
+ "tcmalloc.per_cpu_caches_active",
+ "tcmalloc.thread_cache_free",
+ "tcmalloc.page_heap_unmapped",
+ };
+
+ std::unordered_map<TString, TDynamicCounterPtr> Counters;
+
+ TDynamicCounterPtr CachesSize;
+
+public:
+ explicit TTcMallocStats(TDynamicCountersPtr group) {
+ CounterGroup = group->GetSubgroup("component", "tcmalloc");
+ for (auto name : Names) {
+ Counters[name] = CounterGroup->GetCounter(name, false);
+ }
+ CachesSize = CounterGroup->GetCounter("tcmalloc.caches_bytes", false);
+ }
+
+ void Update() override {
+ auto properties = tcmalloc::MallocExtension::GetProperties();
+ for (auto name : Names) {
+ if (auto it = properties.find(name); it != properties.end()) {
+ Counters[name]->Set(it->second.value);
+ }
+ }
+ CachesSize->Set(GetCachesSize(properties));
+ }
+};
+
+
+class TTcMallocState : public IAllocState {
+public:
+ ui64 GetAllocatedMemoryEstimate() const override {
+ return GetProperty("generic.physical_memory_used");
+ }
+};
+
+
+class TTcMallocMonitor : public IAllocMonitor {
+ TDynamicCountersPtr CounterGroup;
+
+ THistogramPtr SizeHistogram;
+ THistogramPtr CountHistogram;
+ std::unordered_map<ui32, std::pair<TDynamicCounterPtr, TDynamicCounterPtr>> TaggedCounters;
+
+ struct TControls {
+ static constexpr size_t MaxSamplingRate = 4ll << 30;
+
+ TControlWrapper ProfileSamplingRate;
+ TControlWrapper GuardedSamplingRate;
+ TControlWrapper BackgroundReleaseRate;
+ TControlWrapper MemoryLimit;
+
+ TControls()
+ : ProfileSamplingRate(tcmalloc::MallocExtension::GetProfileSamplingRate(),
+ 64 << 10, MaxSamplingRate)
+ , GuardedSamplingRate(MaxSamplingRate,
+ 64 << 10, MaxSamplingRate)
+ , BackgroundReleaseRate(0,
+ 0, 64 << 20)
+ , MemoryLimit(0,
+ 0, std::numeric_limits<i64>::max())
+ {}
+
+ void Register(TIntrusivePtr<TControlBoard> icb) {
+ icb->RegisterSharedControl(ProfileSamplingRate, "TCMallocControls.ProfileSamplingRate");
+ icb->RegisterSharedControl(GuardedSamplingRate, "TCMallocControls.GuardedSamplingRate");
+ icb->RegisterSharedControl(BackgroundReleaseRate, "TCMallocControls.BackgroundReleaseRate");
+ icb->RegisterSharedControl(MemoryLimit, "TCMallocControls.MemoryLimit");
+ }
+ };
+ TControls Controls;
+
+private:
+ void UpdateCounters() {
+ auto profile = tcmalloc::MallocExtension::SnapshotCurrent(tcmalloc::ProfileType::kHeap);
+
+ TAllocationAnalyzer analyzer(std::move(profile));
+ TAllocationStats allocationStats;
+ analyzer.Prepare(&allocationStats);
+
+ auto updateHistogram = [] (THistogramPtr histogram, size_t* numbers) {
+ histogram->Reset();
+ auto snapshot = histogram->Snapshot();
+ auto count = std::min(snapshot->Count(), (ui32)TAllocationStats::MaxSizeIndex);
+ for (ui32 b = 0; b < count; ++b) {
+ histogram->Collect(snapshot->UpperBound(b), numbers[b]);
+ }
+ };
+
+ updateHistogram(SizeHistogram, allocationStats.Sizes);
+ updateHistogram(CountHistogram, allocationStats.Counts);
+
+ for (ui32 t = 0; t < TAllocationStats::MaxTag; ++t) {
+ auto found = TaggedCounters.find(t);
+ if (!allocationStats.TagSizes[t] && found == TaggedCounters.end()) {
+ continue;
+ }
+
+ TDynamicCounterPtr size, count;
+
+ if (found == TaggedCounters.end()) {
+ auto name = TAllocationStats::GetTagName(t);
+ auto group = CounterGroup->GetSubgroup("activity", name);
+ size = group->GetCounter("tcmalloc.sampled_size_by_activity", false);
+ count = group->GetCounter("tcmalloc.sampled_count_by_activity", false);
+ TaggedCounters.emplace(t, std::make_pair(size, count));
+ } else {
+ size = found->second.first;
+ count = found->second.second;
+ }
+
+ size->Set(allocationStats.TagSizes[t]);
+ count->Set(allocationStats.TagCounts[t]);
+ }
+ }
+
+ void UpdateControls() {
+ tcmalloc::MallocExtension::SetProfileSamplingRate(Controls.ProfileSamplingRate);
+
+ if (Controls.GuardedSamplingRate != TControls::MaxSamplingRate) {
+ tcmalloc::MallocExtension::ActivateGuardedSampling();
+ }
+ tcmalloc::MallocExtension::SetGuardedSamplingRate(Controls.GuardedSamplingRate);
+
+ tcmalloc::MallocExtension::BytesPerSecond rate{(size_t)Controls.BackgroundReleaseRate};
+ tcmalloc::MallocExtension::SetBackgroundReleaseRate(rate);
+
+ tcmalloc::MallocExtension::MemoryLimit limit;
+ limit.hard = false;
+ limit.limit = Controls.MemoryLimit ?
+ (size_t)Controls.MemoryLimit : std::numeric_limits<size_t>::max();
+ tcmalloc::MallocExtension::SetMemoryLimit(limit);
+ }
+
+ void DumpCurrent(IOutputStream& out, tcmalloc::ProfileType type,
+ size_t stackCountLimit = 256, size_t sampleCountLimit = 1024, bool forLog = false)
+ {
+ auto start = TInstant::Now();
+ auto profile = tcmalloc::MallocExtension::SnapshotCurrent(type);
+ auto end = TInstant::Now();
+
+ TAllocationAnalyzer analyzer(std::move(profile));
+ TAllocationStats allocationStats;
+ analyzer.Prepare(&allocationStats);
+
+ bool isHeap = (type == tcmalloc::ProfileType::kHeap);
+
+ if (forLog) {
+ const char* marker = "HEAP";
+ const char* sep = " | ";
+ out << marker << sep
+ << "Snapshot calculation time: " << (end - start).MicroSeconds() << " us" << sep
+ << Endl;
+ allocationStats.DumpSizeStats(out, marker, sep);
+ allocationStats.DumpTagStats(out, marker, sep);
+ analyzer.Dump(out, stackCountLimit, sampleCountLimit, isHeap, true);
+ return;
+ }
+
+ HTML(out) {
+ if (isHeap) {
+ out << "<p><a class=\"btn btn-primary\" href=\"?action=log\">Dump to log</a></p>\n";
+ }
+ TABLE_SORTABLE_CLASS("table") {
+ TABLEHEAD() {
+ TABLER() {
+ TABLEH() { out << "upper bound"; }
+ TABLEH() { out << "size"; }
+ TABLEH() { out << "count"; }
+ }
+ out << Endl;
+ }
+ TABLEBODY() {
+ for (size_t b = 0; b < TAllocationStats::MaxSizeIndex; ++b) {
+ if (!allocationStats.Sizes[b]) {
+ continue;
+ }
+ TABLER() {
+ if (b == TAllocationStats::MaxSizeIndex - 1) {
+ TABLED() { out << "inf"; }
+ } else {
+ TABLED() { out << TAllocationStats::SizeUpperBound(b); }
+ }
+ TABLED() { out << allocationStats.Sizes[b]; }
+ TABLED() { out << allocationStats.Counts[b]; }
+ }
+ out << Endl;
+ }
+ TABLER() {
+ TABLED() { out << "<b>total</b>"; }
+ TABLED() { out << "<b>" << allocationStats.Size << "</b>"; }
+ TABLED() { out << "<b>" << allocationStats.Count << "</b>"; }
+ }
+ out << Endl;
+ }
+ }
+ TABLE_SORTABLE_CLASS("table") {
+ TABLEHEAD() {
+ TABLER() {
+ TABLEH() { out << "activity"; }
+ TABLEH() { out << "size"; }
+ TABLEH() { out << "count"; }
+ }
+ out << Endl;
+ }
+ TABLEBODY() {
+ for (ui32 t = 0; t < TAllocationStats::MaxTag; ++t) {
+ if (!allocationStats.TagSizes[t]) {
+ continue;
+ }
+ TABLER() {
+ TABLED() { out << TAllocationStats::GetTagName(t); }
+ TABLED() { out << allocationStats.TagSizes[t]; }
+ TABLED() { out << allocationStats.TagCounts[t]; }
+ }
+ out << Endl;
+ }
+ }
+ }
+ PRE() {
+ out << "Snapshot calculation time: " << (end - start).MicroSeconds() << " us" << Endl
+ << Endl;
+ analyzer.Dump(out, stackCountLimit, sampleCountLimit, isHeap, false);
+ }
+ }
+ }
+
+public:
+ explicit TTcMallocMonitor(TDynamicCountersPtr group) {
+ CounterGroup = group->GetSubgroup("component", "tcmalloc");
+
+ SizeHistogram = CounterGroup->GetHistogram("tcmalloc.sampled_size",
+ NMonitoring::ExponentialHistogram(TAllocationStats::MaxSizeIndex, 2, 1), false);
+
+ CountHistogram = CounterGroup->GetHistogram("tcmalloc.sampled_count",
+ NMonitoring::ExponentialHistogram(TAllocationStats::MaxSizeIndex, 2, 1), false);
+ }
+
+ void RegisterPages(TMon* mon, TActorSystem* actorSystem, TActorId actorId) override {
+ auto* indexPage = mon->FindIndexPage("memory");
+ if (!indexPage) {
+ return;
+ }
+ mon->RegisterActorPage(indexPage, "heap", "Heap", false, actorSystem, actorId);
+ mon->RegisterActorPage(indexPage, "peakheap", "Peak heap", false, actorSystem, actorId);
+ mon->RegisterActorPage(indexPage, "fragmentation", "Fragmentation", false, actorSystem, actorId);
+ }
+
+ void RegisterControls(TIntrusivePtr<TControlBoard> icb) override {
+ Controls.Register(icb);
+ }
+
+ void Update() override {
+ UpdateCounters();
+ UpdateControls();
+ }
+
+ void Dump(IOutputStream& out, const TString& relPath) override {
+ if (relPath == "/memory/heap") {
+ DumpCurrent(out, tcmalloc::ProfileType::kHeap);
+ return;
+ } else if (relPath == "/memory/peakheap") {
+ DumpCurrent(out, tcmalloc::ProfileType::kPeakHeap);
+ return;
+ } else if (relPath == "/memory/fragmentation") {
+ DumpCurrent(out, tcmalloc::ProfileType::kFragmentation);
+ return;
+ }
+
+ auto properties = tcmalloc::MallocExtension::GetProperties();
+ auto stats = tcmalloc::MallocExtension::GetStats();
+
+ HTML(out) {
+ H3() {
+ out << "TCMalloc" << Endl;
+ }
+ out << "<hr>" << Endl;
+ H4() {
+ out << "Allocator properties" << Endl;
+ }
+ TABLE_CLASS("table") {
+ TABLEHEAD() {
+ TABLER() {
+ TABLEH() { out << "Property"; }
+ TABLEH() { out << "Value"; }
+ }
+ out << Endl;
+ }
+ TABLEBODY() {
+ for (const auto& [name, value] : properties) {
+ TABLER() {
+ TABLED() { out << name; }
+ TABLED() { FormatPrettyNumber(out, value.value); }
+ }
+ out << Endl;
+ }
+ }
+ }
+ out << "<hr>" << Endl;
+ H4() {
+ out << "<a href=\"https://github.com/google/tcmalloc/blob/master/docs/stats.md\">Internal stats</a>" << Endl;
+ }
+ PRE() {
+ out << EncodeHtmlPcdata(stats) << Endl;
+ }
+ }
+ }
+
+ void DumpForLog(IOutputStream& out, size_t limit) override {
+ auto properties = tcmalloc::MallocExtension::GetProperties();
+ auto stats = tcmalloc::MallocExtension::GetStats();
+
+ const char* sep = " | ";
+ out << "======== TCMALLOC PROPERTIES" << sep;
+ for (const auto& [name, value] : properties) {
+ out << name << ": " << value.value << sep;
+ }
+ out << Endl;
+
+ out << "======== TCMALLOC STATISTICS" << Endl;
+ out << stats << Endl;
+
+ out << "======== TCMALLOC HEAP" << Endl;
+ DumpCurrent(out, tcmalloc::ProfileType::kHeap, limit, 128, true);
+ }
+};
+
+
+class TTcMallocProfiler : public IProfilerLogic {
+ tcmalloc::MallocExtension::AllocationProfilingToken Token;
+
+public:
+ void Start() override {
+ Token = tcmalloc::MallocExtension::StartAllocationProfiling();
+ }
+
+ void Stop(IOutputStream& out, size_t countLimit, bool forLog) override {
+ auto profile = std::move(Token).Stop();
+
+ TAllocationAnalyzer analyzer(std::move(profile));
+ TAllocationStats allocationStats;
+ analyzer.Prepare(&allocationStats);
+
+ const char* marker = forLog ? "PROFILE" : "--------------------------------";
+ const char* sep = forLog ? " | " : "\n";
+ allocationStats.DumpSizeStats(out, marker, sep);
+ allocationStats.DumpTagStats(out, marker, sep);
+ analyzer.Dump(out, countLimit, 1024, false, forLog);
+ }
+};
+
+
+std::unique_ptr<IAllocStats> CreateTcMallocStats(TDynamicCountersPtr group) {
+ return std::make_unique<TTcMallocStats>(std::move(group));
+}
+
+std::unique_ptr<IAllocState> CreateTcMallocState() {
+ return std::make_unique<TTcMallocState>();
+}
+
+std::unique_ptr<IAllocMonitor> CreateTcMallocMonitor(TDynamicCountersPtr group) {
+ return std::make_unique<TTcMallocMonitor>(std::move(group));
+}
+
+std::unique_ptr<IProfilerLogic> CreateTcMallocProfiler() {
+ return std::make_unique<TTcMallocProfiler>();
+}
+
+}
diff --git a/ydb/core/mon_alloc/tcmalloc.h b/ydb/core/mon_alloc/tcmalloc.h
index db69ea16a87..d394b32c9b3 100644
--- a/ydb/core/mon_alloc/tcmalloc.h
+++ b/ydb/core/mon_alloc/tcmalloc.h
@@ -1,20 +1,20 @@
-#pragma once
-
-#include "monitor.h"
-#include "profiler.h"
-#include "stats.h"
-
-namespace NKikimr {
-
-std::unique_ptr<IAllocStats> CreateTcMallocStats(
- TIntrusivePtr<NMonitoring::TDynamicCounters> group);
-
-std::unique_ptr<IAllocState> CreateTcMallocState();
-
-std::unique_ptr<IAllocMonitor> CreateTcMallocMonitor(
- TIntrusivePtr<NMonitoring::TDynamicCounters> group);
-
-std::unique_ptr<NActors::IProfilerLogic> CreateTcMallocProfiler();
-
-
-}
+#pragma once
+
+#include "monitor.h"
+#include "profiler.h"
+#include "stats.h"
+
+namespace NKikimr {
+
+std::unique_ptr<IAllocStats> CreateTcMallocStats(
+ TIntrusivePtr<NMonitoring::TDynamicCounters> group);
+
+std::unique_ptr<IAllocState> CreateTcMallocState();
+
+std::unique_ptr<IAllocMonitor> CreateTcMallocMonitor(
+ TIntrusivePtr<NMonitoring::TDynamicCounters> group);
+
+std::unique_ptr<NActors::IProfilerLogic> CreateTcMallocProfiler();
+
+
+}
diff --git a/ydb/core/mon_alloc/tcmalloc_null.cpp b/ydb/core/mon_alloc/tcmalloc_null.cpp
index e5139fb8b88..a4de4d5dccd 100644
--- a/ydb/core/mon_alloc/tcmalloc_null.cpp
+++ b/ydb/core/mon_alloc/tcmalloc_null.cpp
@@ -1,27 +1,27 @@
-#include "tcmalloc.h"
-
-namespace NKikimr {
-
-std::unique_ptr<IAllocStats> CreateTcMallocStats(
- TIntrusivePtr<NMonitoring::TDynamicCounters> group)
-{
- Y_UNUSED(group);
- return {};
-}
-
-std::unique_ptr<IAllocState> CreateTcMallocState() {
- return {};
-}
-
-std::unique_ptr<IAllocMonitor> CreateTcMallocMonitor(
- TIntrusivePtr<NMonitoring::TDynamicCounters> group)
-{
- Y_UNUSED(group);
- return {};
-}
-
-std::unique_ptr<NActors::IProfilerLogic> CreateTcMallocProfiler() {
- return {};
-}
-
-}
+#include "tcmalloc.h"
+
+namespace NKikimr {
+
+std::unique_ptr<IAllocStats> CreateTcMallocStats(
+ TIntrusivePtr<NMonitoring::TDynamicCounters> group)
+{
+ Y_UNUSED(group);
+ return {};
+}
+
+std::unique_ptr<IAllocState> CreateTcMallocState() {
+ return {};
+}
+
+std::unique_ptr<IAllocMonitor> CreateTcMallocMonitor(
+ TIntrusivePtr<NMonitoring::TDynamicCounters> group)
+{
+ Y_UNUSED(group);
+ return {};
+}
+
+std::unique_ptr<NActors::IProfilerLogic> CreateTcMallocProfiler() {
+ return {};
+}
+
+}
diff --git a/ydb/core/mon_alloc/ya.make b/ydb/core/mon_alloc/ya.make
index 972ea082f34..8b60e88262f 100644
--- a/ydb/core/mon_alloc/ya.make
+++ b/ydb/core/mon_alloc/ya.make
@@ -34,10 +34,10 @@ IF (ALLOCATOR == "TCMALLOC_256K")
PEERDIR(
contrib/libs/tcmalloc
)
-ELSE()
+ELSE()
SRCS(
tcmalloc_null.cpp
)
-ENDIF()
-
+ENDIF()
+
END()
diff --git a/ydb/core/protos/blobstorage.proto b/ydb/core/protos/blobstorage.proto
index a08ccbf865c..2f997e69a3a 100644
--- a/ydb/core/protos/blobstorage.proto
+++ b/ydb/core/protos/blobstorage.proto
@@ -1469,14 +1469,14 @@ message TEvTestLoadRequest {
TStockWorkload Stock = 13;
}
}
-
- message TMemoryLoadStart {
- optional uint64 Tag = 1;
- optional uint32 DurationSeconds = 2;
- optional uint64 BlockSize = 3;
- optional uint64 IntervalUs = 4;
- }
-
+
+ message TMemoryLoadStart {
+ optional uint64 Tag = 1;
+ optional uint32 DurationSeconds = 2;
+ optional uint64 BlockSize = 3;
+ optional uint64 IntervalUs = 4;
+ }
+
optional uint64 Cookie = 1;
oneof Command {
TLoadStart LoadStart = 2;
@@ -1487,7 +1487,7 @@ message TEvTestLoadRequest {
TPDiskLogLoadStart PDiskLogLoadStart = 7;
TKeyValueLoadStart KeyValueLoadStart = 8;
TKqpLoadStart KqpLoadStart = 9;
- TMemoryLoadStart MemoryLoadStart = 10;
+ TMemoryLoadStart MemoryLoadStart = 10;
}
}
diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto
index d64169d4fc0..f3dc06a28c0 100644
--- a/ydb/core/protos/config.proto
+++ b/ydb/core/protos/config.proto
@@ -628,7 +628,7 @@ message TFeatureFlags {
optional bool AllowHugeKeyValueDeletes = 13 [default = true]; // delete when all clients limit deletes per request
optional bool SendSchemaVersionToDatashard = 14 [default = true]; // deprecated: always true
optional bool EnableSchemeBoardCache = 15 [default = true]; // deprecated: always true
- optional bool EnableSystemViews = 16 [default = true];
+ optional bool EnableSystemViews = 16 [default = true];
optional bool EnableExternalHive = 17 [default = true];
optional bool UseSchemeBoardCacheForSchemeRequests = 18 [default = true]; // deprecated: always true
optional bool CompileMinikqlWithVersion = 19 [default = true]; // deprecated: always true
@@ -637,7 +637,7 @@ message TFeatureFlags {
optional bool EnableOfflineSlaves = 22 [default = true]; // deprecated: always true
optional bool CheckDatabaseAccessPermission = 23 [default = false];
optional bool AllowOnlineIndexBuild = 24 [default = true]; // deprecated: always true
- optional bool EnablePersistentQueryStats = 25 [default = false];
+ optional bool EnablePersistentQueryStats = 25 [default = false];
optional bool DisableDataShardBarrier = 26 [default = false];
optional bool EnablePutBatchingForBlobStorage = 27 [default = true];
optional bool EnableKqpWideFlow = 28 [default = true]; // deprecated: always true
@@ -651,7 +651,7 @@ message TFeatureFlags {
optional bool EnableGracefulShutdown = 36 [default = true];
optional bool EnableDrainOnShutdown = 37 [default = true];
optional bool EnableConfigurationCache = 38 [default = false];
- optional bool EnableDbCounters = 39 [default = false];
+ optional bool EnableDbCounters = 39 [default = false];
optional bool EnableClockGettimeForUserCpuAccounting = 40 [default = false];
optional bool EnableAsyncIndexes = 41 [default = true];
optional bool AllowStreamExecuteYqlScript = 42 [default = true];
diff --git a/ydb/core/protos/counters_schemeshard.proto b/ydb/core/protos/counters_schemeshard.proto
index a839915c98f..4d6adcbbdf2 100644
--- a/ydb/core/protos/counters_schemeshard.proto
+++ b/ydb/core/protos/counters_schemeshard.proto
@@ -85,7 +85,7 @@ enum ESimpleCounters {
COUNTER_IN_FLIGHT_OPS_TxAlterSolomonVolume = 71 [(CounterOpts) = {Name: "InFlightOps/AlterSolomonVolume"}];
COUNTER_IN_FLIGHT_OPS_TxDropLock = 72 [(CounterOpts) = {Name: "InFlightOps/DropLock"}];
COUNTER_IN_FLIGHT_OPS_TxDropTableIndexAtMainTable = 73 [(CounterOpts) = {Name: "InFlightOps/DropTableIndexAtMainTable"}];
- COUNTER_SYS_VIEW_PROCESSOR_COUNT = 74 [(CounterOpts) = {Name: "SysViewProcessors"}];
+ COUNTER_SYS_VIEW_PROCESSOR_COUNT = 74 [(CounterOpts) = {Name: "SysViewProcessors"}];
COUNTER_FILESTORE_COUNT = 75 [(CounterOpts) = {Name: "FileStores"}];
COUNTER_FILESTORE_SHARD_COUNT = 76 [(CounterOpts) = {Name: "FileStoreShards"}];
diff --git a/ydb/core/protos/counters_sysview_processor.proto b/ydb/core/protos/counters_sysview_processor.proto
index 63c03cb6533..e7df0b8334d 100644
--- a/ydb/core/protos/counters_sysview_processor.proto
+++ b/ydb/core/protos/counters_sysview_processor.proto
@@ -1,17 +1,17 @@
import "ydb/core/protos/counters.proto";
-
-package NKikimr.NSysView;
-
-option java_package = "ru.yandex.kikimr.proto";
-
-option (TabletTypeName) = "SysViewProcessor";
-
-enum ETxTypes {
- TXTYPE_INIT_SCHEMA = 0 [(TxTypeOpts) = {Name: "TxInitSchema"}];
- TXTYPE_INIT = 1 [(TxTypeOpts) = {Name: "TxInit"}];
- TXTYPE_CONFIGURE = 2 [(TxTypeOpts) = {Name: "TxConfigure"}];
- TXTYPE_COLLECT = 3 [(TxTypeOpts) = {Name: "TxCollect"}];
- TXTYPE_AGGREGATE = 4 [(TxTypeOpts) = {Name: "TxAggregate"}];
- TXTYPE_INTERVAL_SUMMARY = 5 [(TxTypeOpts) = {Name: "TxIntervalSummary"}];
- TXTYPE_INTERVAL_METRICS = 6 [(TxTypeOpts) = {Name: "TxIntervalMetrics"}];
-}
+
+package NKikimr.NSysView;
+
+option java_package = "ru.yandex.kikimr.proto";
+
+option (TabletTypeName) = "SysViewProcessor";
+
+enum ETxTypes {
+ TXTYPE_INIT_SCHEMA = 0 [(TxTypeOpts) = {Name: "TxInitSchema"}];
+ TXTYPE_INIT = 1 [(TxTypeOpts) = {Name: "TxInit"}];
+ TXTYPE_CONFIGURE = 2 [(TxTypeOpts) = {Name: "TxConfigure"}];
+ TXTYPE_COLLECT = 3 [(TxTypeOpts) = {Name: "TxCollect"}];
+ TXTYPE_AGGREGATE = 4 [(TxTypeOpts) = {Name: "TxAggregate"}];
+ TXTYPE_INTERVAL_SUMMARY = 5 [(TxTypeOpts) = {Name: "TxIntervalSummary"}];
+ TXTYPE_INTERVAL_METRICS = 6 [(TxTypeOpts) = {Name: "TxIntervalMetrics"}];
+}
diff --git a/ydb/core/protos/flat_tx_scheme.proto b/ydb/core/protos/flat_tx_scheme.proto
index dc06523e5cc..141a0f49459 100644
--- a/ydb/core/protos/flat_tx_scheme.proto
+++ b/ydb/core/protos/flat_tx_scheme.proto
@@ -310,7 +310,7 @@ message TEvSyncTenantSchemeShard {
optional uint64 UserAttributesVersion = 7;
optional uint64 TenantHive = 8;
- optional uint64 TenantSysViewProcessor = 9;
+ optional uint64 TenantSysViewProcessor = 9;
optional string TenantRootACL = 10;
}
@@ -330,7 +330,7 @@ message TEvUpdateTenantSchemeShard {
optional uint64 UserAttributesVersion = 9;
optional uint64 TenantHive = 10;
- optional uint64 TenantSysViewProcessor = 11;
+ optional uint64 TenantSysViewProcessor = 11;
optional NKikimrSubDomains.TSchemeQuotas DeclaredSchemeQuotas = 12;
optional Ydb.Cms.DatabaseQuotas DatabaseQuotas = 14;
diff --git a/ydb/core/protos/node_whiteboard.proto b/ydb/core/protos/node_whiteboard.proto
index b96713ca298..e5fa3054b5a 100644
--- a/ydb/core/protos/node_whiteboard.proto
+++ b/ydb/core/protos/node_whiteboard.proto
@@ -286,7 +286,7 @@ message TSystemStateInfo {
optional uint64 MemoryUsed = 26;
optional uint64 MemoryLimit = 27;
optional EConfigState ConfigState = 28 [default = Consistent];
- optional uint64 MemoryUsedInAlloc = 29;
+ optional uint64 MemoryUsedInAlloc = 29;
optional double MaxDiskUsage = 30;
optional NActorsInterconnect.TNodeLocation Location = 31;
}
diff --git a/ydb/core/protos/query_stats.proto b/ydb/core/protos/query_stats.proto
index 453b3034013..d97cd3ae6b1 100644
--- a/ydb/core/protos/query_stats.proto
+++ b/ydb/core/protos/query_stats.proto
@@ -28,7 +28,7 @@ message TTableAccessStats {
optional TReadOpStats SelectRange = 3;
optional TWriteOpStats UpdateRow = 4;
optional TWriteOpStats EraseRow = 5;
- optional uint64 ShardCount = 6;
+ optional uint64 ShardCount = 6;
}
message TPerShardStats {
diff --git a/ydb/core/protos/services.proto b/ydb/core/protos/services.proto
index c17c8a7dc37..7276c61c869 100644
--- a/ydb/core/protos/services.proto
+++ b/ydb/core/protos/services.proto
@@ -271,9 +271,9 @@ enum EServiceKikimr {
DATASHARD_RESTORE = 801;
IMPORT = 802;
S3_WRAPPER = 803;
-
- // System views
- SYSTEM_VIEWS = 900;
+
+ // System views
+ SYSTEM_VIEWS = 900;
BUILD_INDEX = 1000;
@@ -693,7 +693,7 @@ message TActivity {
INTERCONNECT_DESTRUCT_ACTOR = 378;
BS_MIGRATION_ACTOR = 379;
SQS_GARBAGE_COLLECTOR_ACTOR = 380;
- SYSTEM_VIEW_SERVICE = 381;
+ SYSTEM_VIEW_SERVICE = 381;
CMS_SENTINEL_ACTOR = 382;
CMS_SENTINEL_CONFIG_UPDATER_ACTOR = 383;
CMS_SENTINEL_STATE_UPDATER_ACTOR = 384;
@@ -838,12 +838,12 @@ message TActivity {
BS_RESTORE_CORRUPTED_BLOB_ACTOR = 533;
YQ_CONTROL_PLANE_STORAGE_ACTOR = 534;
YQ_CONTROL_PLANE_PROXY_ACTOR = 535;
- DB_WATCHER_ACTOR = 495;
+ DB_WATCHER_ACTOR = 495;
YQL_PENDING_FETCHER_ACTOR = 536;
YQL_PINGER_ACTOR = 537;
YQL_PRIVATE_PROXY_ACTOR = 538;
YQL_PRIVATE_PING_TASK_ACTOR = 539;
- EXT_COUNTERS_UPDATER_ACTOR = 540;
+ EXT_COUNTERS_UPDATER_ACTOR = 540;
YQL_PRIVATE_GET_TASK_ACTOR = 541;
YQL_PRIVATE_WRITE_TASK_ACTOR = 542;
YQL_NODES_MANAGER_ACTOR = 543;
diff --git a/ydb/core/protos/subdomains.proto b/ydb/core/protos/subdomains.proto
index 5d665ff3813..22d7d53d85c 100644
--- a/ydb/core/protos/subdomains.proto
+++ b/ydb/core/protos/subdomains.proto
@@ -19,7 +19,7 @@ message TSubDomainSettings {
optional bool ExternalSchemeShard = 7 [default = false];
optional bool ExternalHive = 8 [default = false];
optional TDomainKey ResourcesDomainKey = 9;
- optional bool ExternalSysViewProcessor = 10 [default = false];
+ optional bool ExternalSysViewProcessor = 10 [default = false];
optional TSchemeQuotas DeclaredSchemeQuotas = 11;
optional Ydb.Cms.DatabaseQuotas DatabaseQuotas = 12;
}
@@ -33,7 +33,7 @@ message TProcessingParams {
optional fixed64 SchemeShard = 6;
optional fixed64 Hive = 7;
- optional fixed64 SysViewProcessor = 8;
+ optional fixed64 SysViewProcessor = 8;
//put there SubSchemeShard and SubHive at the future
}
diff --git a/ydb/core/protos/sys_view.proto b/ydb/core/protos/sys_view.proto
index cfc03662725..94b3095e6d8 100644
--- a/ydb/core/protos/sys_view.proto
+++ b/ydb/core/protos/sys_view.proto
@@ -1,85 +1,85 @@
-package NKikimrSysView;
-
-option java_package = "ru.yandex.kikimr.proto";
-
+package NKikimrSysView;
+
+option java_package = "ru.yandex.kikimr.proto";
+
import "ydb/core/protos/tablet.proto";
-
-message TPartitionStatsKey {
- optional uint64 OwnerId = 1;
- optional uint64 PathId = 2;
- optional uint64 PartIdx = 3;
-}
-
+
+message TPartitionStatsKey {
+ optional uint64 OwnerId = 1;
+ optional uint64 PathId = 2;
+ optional uint64 PartIdx = 3;
+}
+
message TTtlStats {
optional uint64 LastRunTime = 1; // milliseconds since epoch
optional uint64 LastRowsProcessed = 2;
optional uint64 LastRowsErased = 3;
}
-message TPartitionStats {
- optional uint64 DataSize = 1;
- optional uint64 RowCount = 2;
- optional uint64 IndexSize = 3;
- optional double CPUCores = 4;
-
- optional fixed64 TabletId = 5;
- optional uint32 NodeId = 6;
-
- optional uint64 StartTime = 7; // milliseconds since epoch
- optional uint64 AccessTime = 8;
- optional uint64 UpdateTime = 9;
-
- optional uint64 InFlightTxCount = 10;
-
- optional uint64 RowUpdates = 11;
- optional uint64 RowDeletes = 12;
- optional uint64 RowReads = 13;
- optional uint64 RangeReads = 14;
- optional uint64 RangeReadRows = 15;
-
- optional uint64 ImmediateTxCompleted = 16;
- optional uint64 PlannedTxCompleted = 17;
- optional uint64 TxRejectedByOverload = 18;
- optional uint64 TxRejectedBySpace = 19;
+message TPartitionStats {
+ optional uint64 DataSize = 1;
+ optional uint64 RowCount = 2;
+ optional uint64 IndexSize = 3;
+ optional double CPUCores = 4;
+
+ optional fixed64 TabletId = 5;
+ optional uint32 NodeId = 6;
+
+ optional uint64 StartTime = 7; // milliseconds since epoch
+ optional uint64 AccessTime = 8;
+ optional uint64 UpdateTime = 9;
+
+ optional uint64 InFlightTxCount = 10;
+
+ optional uint64 RowUpdates = 11;
+ optional uint64 RowDeletes = 12;
+ optional uint64 RowReads = 13;
+ optional uint64 RangeReads = 14;
+ optional uint64 RangeReadRows = 15;
+
+ optional uint64 ImmediateTxCompleted = 16;
+ optional uint64 PlannedTxCompleted = 17;
+ optional uint64 TxRejectedByOverload = 18;
+ optional uint64 TxRejectedBySpace = 19;
optional TTtlStats TtlStats = 20;
-}
-
-message TPartitionStatsResult {
- optional TPartitionStatsKey Key = 1;
- optional TPartitionStats Stats = 2;
- optional string Path = 3;
-}
-
-message TEvGetPartitionStats {
- optional uint64 DomainKeyOwnerId = 1;
- optional uint64 DomainKeyPathId = 2;
-
- optional TPartitionStatsKey From = 3;
- optional bool FromInclusive = 4;
-
- optional TPartitionStatsKey To = 5;
- optional bool ToInclusive = 6;
-
- optional bool IncludePathColumn = 7;
-}
-
-message TEvGetPartitionStatsResult {
- repeated TPartitionStatsResult Stats = 1;
- optional TPartitionStatsKey Next = 2;
- optional bool LastBatch = 3;
- optional bool Overloaded = 4;
-}
-
+}
+
+message TPartitionStatsResult {
+ optional TPartitionStatsKey Key = 1;
+ optional TPartitionStats Stats = 2;
+ optional string Path = 3;
+}
+
+message TEvGetPartitionStats {
+ optional uint64 DomainKeyOwnerId = 1;
+ optional uint64 DomainKeyPathId = 2;
+
+ optional TPartitionStatsKey From = 3;
+ optional bool FromInclusive = 4;
+
+ optional TPartitionStatsKey To = 5;
+ optional bool ToInclusive = 6;
+
+ optional bool IncludePathColumn = 7;
+}
+
+message TEvGetPartitionStatsResult {
+ repeated TPartitionStatsResult Stats = 1;
+ optional TPartitionStatsKey Next = 2;
+ optional bool LastBatch = 3;
+ optional bool Overloaded = 4;
+}
+
message TQueryDataStats {
optional uint64 ReadRows = 1;
optional uint64 ReadBytes = 2;
optional uint64 UpdateRows = 3;
optional uint64 UpdateBytes = 4;
optional uint64 DeleteRows = 5;
- optional uint64 DeleteBytes = 6; // deprecated
- optional uint64 PartitionCount = 7;
-}
+ optional uint64 DeleteBytes = 6; // deprecated
+ optional uint64 PartitionCount = 7;
+}
message TStatsAggr {
optional uint64 Min = 1;
@@ -88,229 +88,229 @@ message TStatsAggr {
optional uint64 Cnt = 4;
}
-message TQueryStats {
- optional string QueryText = 1;
- optional uint64 QueryTextHash = 2;
- optional uint64 DurationMs = 3;
- optional uint64 EndTimeMs = 4;
+message TQueryStats {
+ optional string QueryText = 1;
+ optional uint64 QueryTextHash = 2;
+ optional uint64 DurationMs = 3;
+ optional uint64 EndTimeMs = 4;
optional TQueryDataStats Stats = 5;
- optional string UserSID = 6;
- optional uint64 ParametersSize = 7;
- optional uint64 CompileDurationMs = 8;
- optional bool FromQueryCache = 9;
- optional uint32 NodeId = 10;
+ optional string UserSID = 6;
+ optional uint64 ParametersSize = 7;
+ optional uint64 CompileDurationMs = 8;
+ optional bool FromQueryCache = 9;
+ optional uint32 NodeId = 10;
optional TStatsAggr ShardsCpuTimeUs = 11;
optional TStatsAggr ComputeCpuTimeUs = 12;
- optional uint64 CompileCpuTimeUs = 13;
- optional uint64 ProcessCpuTimeUs = 14;
- optional uint64 TotalCpuTimeUs = 15;
- optional string Type = 16;
- optional uint64 RequestUnits = 17;
-}
-
-message TTopQueryStats {
- repeated TQueryStats Stats = 1;
-}
-
-enum EStatsType {
- TOP_DURATION_ONE_MINUTE = 1;
- TOP_DURATION_ONE_HOUR = 2;
- TOP_READ_BYTES_ONE_MINUTE = 3;
- TOP_READ_BYTES_ONE_HOUR = 4;
- TOP_CPU_TIME_ONE_MINUTE = 5;
- TOP_CPU_TIME_ONE_HOUR = 6;
- METRICS_ONE_MINUTE = 7;
- METRICS_ONE_HOUR = 8;
- TOP_REQUEST_UNITS_ONE_MINUTE = 9;
- TOP_REQUEST_UNITS_ONE_HOUR = 10;
-}
-
-message TEvGetQueryStats {
- optional EStatsType StatsType = 1;
- optional uint64 StartBucket = 2;
- optional string TenantName = 3;
-}
-
-message THashedQueryText {
- optional uint64 Hash = 1;
- optional string Text = 2;
-}
-
-message TEvGetQueryStatsResult {
- repeated TTopQueryStats Buckets = 1;
- optional uint64 LastBucket = 2;
- optional uint64 BucketSizeMs = 3;
- repeated THashedQueryText QueryTexts = 4;
-}
-
-message TPDiskKey {
- optional uint32 NodeId = 1;
- optional uint32 PDiskId = 2;
-}
-
-message TPDiskInfo {
- optional string Type = 1;
- optional uint64 Kind = 2;
- optional string Path = 3;
- optional uint64 Guid = 4;
- optional uint64 BoxId = 5;
- optional bool SharedWithOs = 6;
- optional bool ReadCentric = 7;
- optional uint64 AvailableSize = 8;
- optional uint64 TotalSize = 9;
- reserved 10;
+ optional uint64 CompileCpuTimeUs = 13;
+ optional uint64 ProcessCpuTimeUs = 14;
+ optional uint64 TotalCpuTimeUs = 15;
+ optional string Type = 16;
+ optional uint64 RequestUnits = 17;
+}
+
+message TTopQueryStats {
+ repeated TQueryStats Stats = 1;
+}
+
+enum EStatsType {
+ TOP_DURATION_ONE_MINUTE = 1;
+ TOP_DURATION_ONE_HOUR = 2;
+ TOP_READ_BYTES_ONE_MINUTE = 3;
+ TOP_READ_BYTES_ONE_HOUR = 4;
+ TOP_CPU_TIME_ONE_MINUTE = 5;
+ TOP_CPU_TIME_ONE_HOUR = 6;
+ METRICS_ONE_MINUTE = 7;
+ METRICS_ONE_HOUR = 8;
+ TOP_REQUEST_UNITS_ONE_MINUTE = 9;
+ TOP_REQUEST_UNITS_ONE_HOUR = 10;
+}
+
+message TEvGetQueryStats {
+ optional EStatsType StatsType = 1;
+ optional uint64 StartBucket = 2;
+ optional string TenantName = 3;
+}
+
+message THashedQueryText {
+ optional uint64 Hash = 1;
+ optional string Text = 2;
+}
+
+message TEvGetQueryStatsResult {
+ repeated TTopQueryStats Buckets = 1;
+ optional uint64 LastBucket = 2;
+ optional uint64 BucketSizeMs = 3;
+ repeated THashedQueryText QueryTexts = 4;
+}
+
+message TPDiskKey {
+ optional uint32 NodeId = 1;
+ optional uint32 PDiskId = 2;
+}
+
+message TPDiskInfo {
+ optional string Type = 1;
+ optional uint64 Kind = 2;
+ optional string Path = 3;
+ optional uint64 Guid = 4;
+ optional uint64 BoxId = 5;
+ optional bool SharedWithOs = 6;
+ optional bool ReadCentric = 7;
+ optional uint64 AvailableSize = 8;
+ optional uint64 TotalSize = 9;
+ reserved 10;
reserved 11;
- optional string StatusV2 = 12;
+ optional string StatusV2 = 12;
optional uint64 StatusChangeTimestamp = 13;
optional uint64 EnforcedDynamicSlotSize = 14;
optional uint32 ExpectedSlotCount = 15;
optional uint32 NumActiveSlots = 16;
optional uint64 Category = 17;
- // metrics ?
- // physical location ?
- // config ?
-}
-
-message TPDiskEntry {
- optional TPDiskKey Key = 1;
- optional TPDiskInfo Info = 2;
-}
-
-message TEvGetPDisksRequest {
- optional TPDiskKey From = 1;
- optional bool InclusiveFrom = 2;
-
- optional TPDiskKey To = 3;
- optional bool InclusiveTo = 4;
-}
-
-message TEvGetPDisksResponse {
- repeated TPDiskEntry Entries = 1;
-}
-
-message TVSlotKey {
- optional uint32 NodeId = 1;
- optional uint32 PDiskId = 2;
- optional uint32 VSlotId = 3;
-}
-
-message TVSlotInfo {
- reserved 1;
- optional uint32 GroupId = 2;
- optional uint32 GroupGeneration = 3;
+ // metrics ?
+ // physical location ?
+ // config ?
+}
+
+message TPDiskEntry {
+ optional TPDiskKey Key = 1;
+ optional TPDiskInfo Info = 2;
+}
+
+message TEvGetPDisksRequest {
+ optional TPDiskKey From = 1;
+ optional bool InclusiveFrom = 2;
+
+ optional TPDiskKey To = 3;
+ optional bool InclusiveTo = 4;
+}
+
+message TEvGetPDisksResponse {
+ repeated TPDiskEntry Entries = 1;
+}
+
+message TVSlotKey {
+ optional uint32 NodeId = 1;
+ optional uint32 PDiskId = 2;
+ optional uint32 VSlotId = 3;
+}
+
+message TVSlotInfo {
+ reserved 1;
+ optional uint32 GroupId = 2;
+ optional uint32 GroupGeneration = 3;
optional uint32 FailRealm = 4;
- optional uint32 FailDomain = 5;
- optional uint32 VDisk = 6;
- optional uint64 AllocatedSize = 7;
- optional uint64 AvailableSize = 8;
- reserved 9;
+ optional uint32 FailDomain = 5;
+ optional uint32 VDisk = 6;
+ optional uint64 AllocatedSize = 7;
+ optional uint64 AvailableSize = 8;
+ reserved 9;
reserved 10;
- optional string StatusV2 = 12;
- optional string Kind = 13;
+ optional string StatusV2 = 12;
+ optional string Kind = 13;
optional bool IsBeingDeleted = 14;
- // metrics ?
-}
-
-message TVSlotEntry {
- optional TVSlotKey Key = 1;
- optional TVSlotInfo Info = 2;
-}
-
-message TEvGetVSlotsRequest {
- optional TVSlotKey From = 1;
- optional bool InclusiveFrom = 2;
-
- optional TVSlotKey To = 3;
- optional bool InclusiveTo = 4;
-}
-
-message TEvGetVSlotsResponse {
- repeated TVSlotEntry Entries = 1;
-}
-
-message TGroupKey {
- optional uint32 GroupId = 1;
-}
-
-message TGroupInfo {
- optional uint32 Generation = 1;
- reserved 2;
- optional uint64 BoxId = 3;
- optional uint64 StoragePoolId = 4;
- optional uint32 EncryptionMode = 5;
- optional uint32 LifeCyclePhase = 6;
- optional uint64 AllocatedSize = 7;
- optional uint64 AvailableSize = 8;
+ // metrics ?
+}
+
+message TVSlotEntry {
+ optional TVSlotKey Key = 1;
+ optional TVSlotInfo Info = 2;
+}
+
+message TEvGetVSlotsRequest {
+ optional TVSlotKey From = 1;
+ optional bool InclusiveFrom = 2;
+
+ optional TVSlotKey To = 3;
+ optional bool InclusiveTo = 4;
+}
+
+message TEvGetVSlotsResponse {
+ repeated TVSlotEntry Entries = 1;
+}
+
+message TGroupKey {
+ optional uint32 GroupId = 1;
+}
+
+message TGroupInfo {
+ optional uint32 Generation = 1;
+ reserved 2;
+ optional uint64 BoxId = 3;
+ optional uint64 StoragePoolId = 4;
+ optional uint32 EncryptionMode = 5;
+ optional uint32 LifeCyclePhase = 6;
+ optional uint64 AllocatedSize = 7;
+ optional uint64 AvailableSize = 8;
reserved 9;
reserved 10;
- optional bool SeenOperational = 11;
- optional string ErasureSpeciesV2 = 12;
- optional uint64 PutTabletLogLatency = 13;
- optional uint64 PutUserDataLatency = 14;
- optional uint64 GetFastLatency = 15;
- // owner ?
- // storage pool name ?
- // desired disk categories ?
- // down/persisted down ?
- // metrics ?
-}
-
-message TGroupEntry {
- optional TGroupKey Key = 1;
- optional TGroupInfo Info = 2;
-}
-
-message TEvGetGroupsRequest {
- optional TGroupKey From = 1;
- optional bool InclusiveFrom = 2;
-
- optional TGroupKey To = 3;
- optional bool InclusiveTo = 4;
-}
-
-message TEvGetGroupsResponse {
- repeated TGroupEntry Entries = 1;
-}
-
-message TStoragePoolKey {
- optional uint64 BoxId = 1;
- optional uint64 StoragePoolId = 2;
-}
-
-message TStoragePoolInfo {
- optional string Name = 1;
- optional uint64 Generation = 2;
- reserved 3;
- reserved 4;
- optional string Kind = 5;
- optional uint32 NumGroups = 6;
- optional uint32 EncryptionMode = 7;
- optional uint64 SchemeshardId = 8;
- optional uint64 PathId = 9;
- optional string ErasureSpeciesV2 = 10;
- optional string VDiskKindV2 = 12;
+ optional bool SeenOperational = 11;
+ optional string ErasureSpeciesV2 = 12;
+ optional uint64 PutTabletLogLatency = 13;
+ optional uint64 PutUserDataLatency = 14;
+ optional uint64 GetFastLatency = 15;
+ // owner ?
+ // storage pool name ?
+ // desired disk categories ?
+ // down/persisted down ?
+ // metrics ?
+}
+
+message TGroupEntry {
+ optional TGroupKey Key = 1;
+ optional TGroupInfo Info = 2;
+}
+
+message TEvGetGroupsRequest {
+ optional TGroupKey From = 1;
+ optional bool InclusiveFrom = 2;
+
+ optional TGroupKey To = 3;
+ optional bool InclusiveTo = 4;
+}
+
+message TEvGetGroupsResponse {
+ repeated TGroupEntry Entries = 1;
+}
+
+message TStoragePoolKey {
+ optional uint64 BoxId = 1;
+ optional uint64 StoragePoolId = 2;
+}
+
+message TStoragePoolInfo {
+ optional string Name = 1;
+ optional uint64 Generation = 2;
+ reserved 3;
+ reserved 4;
+ optional string Kind = 5;
+ optional uint32 NumGroups = 6;
+ optional uint32 EncryptionMode = 7;
+ optional uint64 SchemeshardId = 8;
+ optional uint64 PathId = 9;
+ optional string ErasureSpeciesV2 = 10;
+ optional string VDiskKindV2 = 12;
optional string PDiskFilter = 13;
optional bytes PDiskFilterData = 14;
- // metrics ?
-}
-
-message TStoragePoolEntry {
- optional TStoragePoolKey Key = 1;
- optional TStoragePoolInfo Info = 2;
-}
-
-message TEvGetStoragePoolsRequest {
- optional TStoragePoolKey From = 1;
- optional bool InclusiveFrom = 2;
-
- optional TStoragePoolKey To = 3;
- optional bool InclusiveTo = 4;
-}
-
-message TEvGetStoragePoolsResponse {
- repeated TStoragePoolEntry Entries = 1;
-}
-
+ // metrics ?
+}
+
+message TStoragePoolEntry {
+ optional TStoragePoolKey Key = 1;
+ optional TStoragePoolInfo Info = 2;
+}
+
+message TEvGetStoragePoolsRequest {
+ optional TStoragePoolKey From = 1;
+ optional bool InclusiveFrom = 2;
+
+ optional TStoragePoolKey To = 3;
+ optional bool InclusiveTo = 4;
+}
+
+message TEvGetStoragePoolsResponse {
+ repeated TStoragePoolEntry Entries = 1;
+}
+
message TStorageStatsEntry {
reserved 1;
optional string PDiskFilter = 2;
@@ -330,216 +330,216 @@ message TEvGetStorageStatsResponse {
repeated TStorageStatsEntry Entries = 1;
}
-message TTabletEntry {
- optional fixed64 TabletId = 1;
+message TTabletEntry {
+ optional fixed64 TabletId = 1;
optional uint32 FollowerId = 2;
- optional string Type = 3;
- optional string State = 4;
- optional string VolatileState = 5;
- optional string BootState = 6;
- optional uint32 Generation = 7;
- optional uint32 NodeId = 8;
- optional uint64 CPU = 9;
- optional uint64 Memory = 10;
- optional uint64 Network = 11;
-}
-
-message TEvGetTabletIdsRequest {
- optional fixed64 From = 1;
- optional fixed64 To = 2;
-}
-
-message TEvGetTabletIdsResponse {
- repeated fixed64 TabletIds = 1;
-}
-
-message TEvGetTabletsRequest {
- repeated fixed64 TabletIds = 1;
- optional uint32 BatchSizeLimit = 2;
-}
-
-message TEvGetTabletsResponse {
- repeated TTabletEntry Entries = 1;
- optional fixed64 NextTabletId = 2;
-}
-
-// ---- Query metrics
-
-message TQueryMetrics {
- message TMetrics {
- optional uint64 Sum = 1;
- optional uint64 Min = 2;
- optional uint64 Max = 3;
- }
-
- optional fixed64 QueryTextHash = 1;
- optional uint64 Count = 2;
-
- optional TMetrics CpuTimeUs = 3;
- optional TMetrics DurationUs = 4;
- optional TMetrics ReadRows = 5;
- optional TMetrics ReadBytes = 6;
- optional TMetrics UpdateRows = 7;
- optional TMetrics UpdateBytes = 8;
- optional TMetrics DeleteRows = 9;
- optional TMetrics RequestUnits = 10;
-}
-
-// node -> sysview processor tablet
-// per node {hash -> cpu} mapping
-message TEvIntervalQuerySummary {
- optional string Database = 1;
- optional uint64 IntervalEndUs = 2;
- optional uint32 NodeId = 3;
-
- message TQuerySet {
- repeated fixed64 Hashes = 1; // hashes
- repeated uint64 Values = 2; // top by values
- }
-
- optional TQuerySet Metrics = 4;
- optional TQuerySet TopByDuration = 5;
- optional TQuerySet TopByReadBytes = 6;
- optional TQuerySet TopByCpuTime = 7;
- optional TQuerySet TopByRequestUnits = 8;
-}
-
-// sysview processor tablet -> node
-// only selected queries with top cpu
-message TEvGetIntervalMetricsRequest {
- optional string Database = 1;
- optional uint64 IntervalEndUs = 2;
- repeated fixed64 Metrics = 3; // request full query metrics
- repeated fixed64 QueryTextsToGet = 4; // request query texts
- repeated fixed64 TopByDuration = 5; // request interval top
- repeated fixed64 TopByReadBytes = 6; // request interval top
- repeated fixed64 TopByCpuTime = 7; // request interval top
- repeated fixed64 TopByRequestUnits = 8; // request interval top
-}
-
-// node -> sysview processor tablet
-// query metrics and texts
-message TEvGetIntervalMetricsResponse {
- message TQueryText {
- optional fixed64 Hash = 1;
- optional string Text = 2;
- }
-
- optional uint64 IntervalEndUs = 1;
- repeated TQueryMetrics Metrics = 2;
- repeated TQueryText QueryTexts = 3;
- repeated TQueryStats TopByDuration = 4;
- repeated TQueryStats TopByReadBytes = 5;
- repeated TQueryStats TopByCpuTime = 6;
- repeated TQueryStats TopByRequestUnits = 7;
-}
-
-message TEvConfigureProcessor {
- optional string Database = 1;
-}
-
-message TQueryMetricsKey {
- optional uint64 IntervalEndUs = 1;
- optional uint32 Rank = 2;
-}
-
-message TQueryMetricsEntry {
- optional TQueryMetricsKey Key = 1;
- optional TQueryMetrics Metrics = 2;
- optional string QueryText = 3;
-}
-
-message TQueryStatsEntry {
- optional TQueryMetricsKey Key = 1;
- optional TQueryStats Stats = 2;
-}
-
-message TEvGetQueryMetricsRequest {
- optional TQueryMetricsKey From = 1;
- optional bool InclusiveFrom = 2;
-
- optional TQueryMetricsKey To = 3;
- optional bool InclusiveTo = 4;
-
- optional EStatsType Type = 5;
-}
-
-// reply to TEvGetQueryMetricsRequest
-message TEvGetQueryMetricsResponse {
- repeated TQueryMetricsEntry Entries = 1;
- optional TQueryMetricsKey Next = 2;
- optional bool LastBatch = 3;
- optional bool Overloaded = 4;
-}
-
-// reply to TEvGetQueryMetricsRequest
-message TEvGetQueryStatsResponse {
- repeated TQueryStatsEntry Entries = 1;
- optional TQueryMetricsKey Next = 2;
- optional bool LastBatch = 3;
- optional bool Overloaded = 4;
-}
-
-// ---- Database counters
-
-// generic set of counters with stable enumeration order in each category
-// simple values are absolute
-// cumulative and histogram values may be absolute or diffs depending on the context
-// only index-value pairs with non-zero values are stored in diff mode
-message TDbCounters {
- message THistogram {
- repeated uint64 Buckets = 1;
- optional uint64 BucketsCount = 2; // total number of buckets in diff mode
- }
-
- repeated uint64 Simple = 1;
- repeated uint64 Cumulative = 2;
- optional uint64 CumulativeCount = 3; // total number of cumulative counters in diff mode
- repeated THistogram Histogram = 4;
-}
-
-message TDbTabletCounters {
- optional NKikimrTabletBase.TTabletTypes.EType Type = 1;
- optional TDbCounters ExecutorCounters = 2;
- optional TDbCounters AppCounters = 3;
- optional TDbCounters MaxExecutorCounters = 4;
- optional TDbCounters MaxAppCounters = 5;
-}
-
-message TDbGRpcCounters {
- optional string GRpcService = 1;
- optional string GRpcRequest = 2;
- optional TDbCounters RequestCounters = 3;
-}
-
-message TDbServiceCounters {
- optional TDbCounters Main = 1;
- repeated TDbTabletCounters TabletCounters = 2;
- repeated TDbGRpcCounters GRpcCounters = 3;
-}
-
-enum EDbCountersService {
- KQP = 1;
- TABLETS = 2;
- GRPC = 3;
- RESERVED_1 = 4;
- RESERVED_2 = 5;
-}
-
-// node -> sysview processor tablet
-// cumulative values are diffs since last confirmed state
-message TEvSendDbCountersRequest {
- message TServiceCounters {
- optional EDbCountersService Service = 1;
- optional TDbServiceCounters Counters = 2;
- }
- repeated TServiceCounters ServiceCounters = 1;
- optional uint64 NodeId = 2;
- optional uint64 Generation = 3;
-}
-
-message TEvSendDbCountersResponse {
- optional string Database = 1;
- optional uint64 Generation = 2; // confirmed generation
-}
-
+ optional string Type = 3;
+ optional string State = 4;
+ optional string VolatileState = 5;
+ optional string BootState = 6;
+ optional uint32 Generation = 7;
+ optional uint32 NodeId = 8;
+ optional uint64 CPU = 9;
+ optional uint64 Memory = 10;
+ optional uint64 Network = 11;
+}
+
+message TEvGetTabletIdsRequest {
+ optional fixed64 From = 1;
+ optional fixed64 To = 2;
+}
+
+message TEvGetTabletIdsResponse {
+ repeated fixed64 TabletIds = 1;
+}
+
+message TEvGetTabletsRequest {
+ repeated fixed64 TabletIds = 1;
+ optional uint32 BatchSizeLimit = 2;
+}
+
+message TEvGetTabletsResponse {
+ repeated TTabletEntry Entries = 1;
+ optional fixed64 NextTabletId = 2;
+}
+
+// ---- Query metrics
+
+message TQueryMetrics {
+ message TMetrics {
+ optional uint64 Sum = 1;
+ optional uint64 Min = 2;
+ optional uint64 Max = 3;
+ }
+
+ optional fixed64 QueryTextHash = 1;
+ optional uint64 Count = 2;
+
+ optional TMetrics CpuTimeUs = 3;
+ optional TMetrics DurationUs = 4;
+ optional TMetrics ReadRows = 5;
+ optional TMetrics ReadBytes = 6;
+ optional TMetrics UpdateRows = 7;
+ optional TMetrics UpdateBytes = 8;
+ optional TMetrics DeleteRows = 9;
+ optional TMetrics RequestUnits = 10;
+}
+
+// node -> sysview processor tablet
+// per node {hash -> cpu} mapping
+message TEvIntervalQuerySummary {
+ optional string Database = 1;
+ optional uint64 IntervalEndUs = 2;
+ optional uint32 NodeId = 3;
+
+ message TQuerySet {
+ repeated fixed64 Hashes = 1; // hashes
+ repeated uint64 Values = 2; // top by values
+ }
+
+ optional TQuerySet Metrics = 4;
+ optional TQuerySet TopByDuration = 5;
+ optional TQuerySet TopByReadBytes = 6;
+ optional TQuerySet TopByCpuTime = 7;
+ optional TQuerySet TopByRequestUnits = 8;
+}
+
+// sysview processor tablet -> node
+// only selected queries with top cpu
+message TEvGetIntervalMetricsRequest {
+ optional string Database = 1;
+ optional uint64 IntervalEndUs = 2;
+ repeated fixed64 Metrics = 3; // request full query metrics
+ repeated fixed64 QueryTextsToGet = 4; // request query texts
+ repeated fixed64 TopByDuration = 5; // request interval top
+ repeated fixed64 TopByReadBytes = 6; // request interval top
+ repeated fixed64 TopByCpuTime = 7; // request interval top
+ repeated fixed64 TopByRequestUnits = 8; // request interval top
+}
+
+// node -> sysview processor tablet
+// query metrics and texts
+message TEvGetIntervalMetricsResponse {
+ message TQueryText {
+ optional fixed64 Hash = 1;
+ optional string Text = 2;
+ }
+
+ optional uint64 IntervalEndUs = 1;
+ repeated TQueryMetrics Metrics = 2;
+ repeated TQueryText QueryTexts = 3;
+ repeated TQueryStats TopByDuration = 4;
+ repeated TQueryStats TopByReadBytes = 5;
+ repeated TQueryStats TopByCpuTime = 6;
+ repeated TQueryStats TopByRequestUnits = 7;
+}
+
+message TEvConfigureProcessor {
+ optional string Database = 1;
+}
+
+message TQueryMetricsKey {
+ optional uint64 IntervalEndUs = 1;
+ optional uint32 Rank = 2;
+}
+
+message TQueryMetricsEntry {
+ optional TQueryMetricsKey Key = 1;
+ optional TQueryMetrics Metrics = 2;
+ optional string QueryText = 3;
+}
+
+message TQueryStatsEntry {
+ optional TQueryMetricsKey Key = 1;
+ optional TQueryStats Stats = 2;
+}
+
+message TEvGetQueryMetricsRequest {
+ optional TQueryMetricsKey From = 1;
+ optional bool InclusiveFrom = 2;
+
+ optional TQueryMetricsKey To = 3;
+ optional bool InclusiveTo = 4;
+
+ optional EStatsType Type = 5;
+}
+
+// reply to TEvGetQueryMetricsRequest
+message TEvGetQueryMetricsResponse {
+ repeated TQueryMetricsEntry Entries = 1;
+ optional TQueryMetricsKey Next = 2;
+ optional bool LastBatch = 3;
+ optional bool Overloaded = 4;
+}
+
+// reply to TEvGetQueryMetricsRequest
+message TEvGetQueryStatsResponse {
+ repeated TQueryStatsEntry Entries = 1;
+ optional TQueryMetricsKey Next = 2;
+ optional bool LastBatch = 3;
+ optional bool Overloaded = 4;
+}
+
+// ---- Database counters
+
+// generic set of counters with stable enumeration order in each category
+// simple values are absolute
+// cumulative and histogram values may be absolute or diffs depending on the context
+// only index-value pairs with non-zero values are stored in diff mode
+message TDbCounters {
+ message THistogram {
+ repeated uint64 Buckets = 1;
+ optional uint64 BucketsCount = 2; // total number of buckets in diff mode
+ }
+
+ repeated uint64 Simple = 1;
+ repeated uint64 Cumulative = 2;
+ optional uint64 CumulativeCount = 3; // total number of cumulative counters in diff mode
+ repeated THistogram Histogram = 4;
+}
+
+message TDbTabletCounters {
+ optional NKikimrTabletBase.TTabletTypes.EType Type = 1;
+ optional TDbCounters ExecutorCounters = 2;
+ optional TDbCounters AppCounters = 3;
+ optional TDbCounters MaxExecutorCounters = 4;
+ optional TDbCounters MaxAppCounters = 5;
+}
+
+message TDbGRpcCounters {
+ optional string GRpcService = 1;
+ optional string GRpcRequest = 2;
+ optional TDbCounters RequestCounters = 3;
+}
+
+message TDbServiceCounters {
+ optional TDbCounters Main = 1;
+ repeated TDbTabletCounters TabletCounters = 2;
+ repeated TDbGRpcCounters GRpcCounters = 3;
+}
+
+enum EDbCountersService {
+ KQP = 1;
+ TABLETS = 2;
+ GRPC = 3;
+ RESERVED_1 = 4;
+ RESERVED_2 = 5;
+}
+
+// node -> sysview processor tablet
+// cumulative values are diffs since last confirmed state
+message TEvSendDbCountersRequest {
+ message TServiceCounters {
+ optional EDbCountersService Service = 1;
+ optional TDbServiceCounters Counters = 2;
+ }
+ repeated TServiceCounters ServiceCounters = 1;
+ optional uint64 NodeId = 2;
+ optional uint64 Generation = 3;
+}
+
+message TEvSendDbCountersResponse {
+ optional string Database = 1;
+ optional uint64 Generation = 2; // confirmed generation
+}
+
diff --git a/ydb/core/protos/tablet.proto b/ydb/core/protos/tablet.proto
index f0bec238e78..cc849726f1d 100644
--- a/ydb/core/protos/tablet.proto
+++ b/ydb/core/protos/tablet.proto
@@ -39,7 +39,7 @@ message TTabletTypes {
Kesus = 29;
BlockStorePartition2 = 30;
BlockStoreDiskRegistry = 31;
- SysViewProcessor = 32;
+ SysViewProcessor = 32;
FileStore = 33;
OlapShard = 34;
ColumnShard = 35;
diff --git a/ydb/core/protos/tablet_database.proto b/ydb/core/protos/tablet_database.proto
index a6658688a66..afa5d6899b7 100644
--- a/ydb/core/protos/tablet_database.proto
+++ b/ydb/core/protos/tablet_database.proto
@@ -43,7 +43,7 @@ message TExecutorSettings {
optional TCompactionPolicy CompactionPolicy = 10;
repeated TLocalityGroupPolicy LocalityGroupPolicy = 11;
- reserved 100; //optional bool ExposeCounters = 100;
+ reserved 100; //optional bool ExposeCounters = 100;
}
message TRuntimeSettings {
diff --git a/ydb/core/protos/tx_datashard.proto b/ydb/core/protos/tx_datashard.proto
index 89863b25400..11379e3fcee 100644
--- a/ydb/core/protos/tx_datashard.proto
+++ b/ydb/core/protos/tx_datashard.proto
@@ -749,9 +749,9 @@ message TEvPeriodicTableStats {
optional uint32 ShardState = 7;
repeated uint64 UserTablePartOwners = 8;
repeated uint64 SysTablesPartOwners = 9;
-
- optional uint32 NodeId = 10;
- optional uint64 StartTime = 11; // milliseconds since epoch
+
+ optional uint32 NodeId = 10;
+ optional uint64 StartTime = 11; // milliseconds since epoch
optional uint64 TableOwnerId = 12;
}
diff --git a/ydb/core/protos/ya.make b/ydb/core/protos/ya.make
index 70bb65514cd..09c5c38df32 100644
--- a/ydb/core/protos/ya.make
+++ b/ydb/core/protos/ya.make
@@ -49,7 +49,7 @@ SRCS(
counters_replication.proto
counters_schemeshard.proto
counters_sequenceshard.proto
- counters_sysview_processor.proto
+ counters_sysview_processor.proto
counters_testshard.proto
counters_tx_proxy.proto
counters_mediator.proto
@@ -128,7 +128,7 @@ SRCS(
ydb_table_impl.proto
scheme_board.proto
scheme_board_mon.proto
- sys_view.proto
+ sys_view.proto
)
GENERATE_ENUM_SERIALIZATION(blobstorage_pdisk_config.pb.h)
diff --git a/ydb/core/scheme/scheme_tabledefs.h b/ydb/core/scheme/scheme_tabledefs.h
index 190233fdeee..5ce1f7a1cf0 100644
--- a/ydb/core/scheme/scheme_tabledefs.h
+++ b/ydb/core/scheme/scheme_tabledefs.h
@@ -19,18 +19,18 @@ using TSchemaVersion = ui64;
// ident for table, must be unique in selected scope
// for global transactions ownerid is tabletid of owning schemeshard and tableid is counter designated by schemeshard
-// SysViewInfo is not empty for system views attached to corresponding table
-
+// SysViewInfo is not empty for system views attached to corresponding table
+
struct TTableId {
- TPathId PathId;
- TString SysViewInfo;
+ TPathId PathId;
+ TString SysViewInfo;
TSchemaVersion SchemaVersion = 0;
- TTableId() = default;
-
+ TTableId() = default;
+
// raw ctors
TTableId(ui64 ownerId, ui64 tableId, const TString& sysViewInfo, ui64 schemaVersion)
- : PathId(ownerId, tableId)
+ : PathId(ownerId, tableId)
, SysViewInfo(sysViewInfo)
, SchemaVersion(schemaVersion)
{}
@@ -65,17 +65,17 @@ struct TTableId {
{}
explicit operator bool() const noexcept {
- return bool(PathId);
+ return bool(PathId);
}
bool HasSamePath(const TTableId &x) const noexcept {
- return PathId == x.PathId && SysViewInfo == x.SysViewInfo;
- }
-
- bool IsSystemView() const noexcept {
- return !SysViewInfo.empty();
+ return PathId == x.PathId && SysViewInfo == x.SysViewInfo;
}
+ bool IsSystemView() const noexcept {
+ return !SysViewInfo.empty();
+ }
+
bool operator==(const TTableId& x) const {
return PathId == x.PathId && SysViewInfo == x.SysViewInfo && SchemaVersion == x.SchemaVersion;
}
@@ -97,12 +97,12 @@ struct TTableId {
}
ui64 PathHash() const noexcept {
- auto hash = PathId.Hash();
+ auto hash = PathId.Hash();
if (SysViewInfo) {
- hash = CombineHashes(hash, THash<TString>()(SysViewInfo));
+ hash = CombineHashes(hash, THash<TString>()(SysViewInfo));
}
- return hash;
+ return hash;
}
};
@@ -580,7 +580,7 @@ struct TSecurityObject : TAtomicRefCount<TSecurityObject>, NACLib::TSecurityObje
};
// key description of one minikql operation
-class TKeyDesc : TNonCopyable {
+class TKeyDesc : TNonCopyable {
public:
// what we do on row?
diff --git a/ydb/core/sys_view/common/common.h b/ydb/core/sys_view/common/common.h
index c88107063a1..2785e6b446f 100644
--- a/ydb/core/sys_view/common/common.h
+++ b/ydb/core/sys_view/common/common.h
@@ -1,32 +1,32 @@
-#pragma once
-
-#include <library/cpp/actors/core/log.h>
-
-#if defined SVLOG_T || \
- defined SVLOG_D || \
- defined SVLOG_I || \
- defined SVLOG_N || \
- defined SVLOG_W || \
- defined SVLOG_E || \
- defined SVLOG_CRIT
-#error log macro redefinition
-#endif
-
-#define SVLOG_T(stream) LOG_TRACE_S((TlsActivationContext->AsActorContext()), NKikimrServices::SYSTEM_VIEWS, stream)
-#define SVLOG_D(stream) LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::SYSTEM_VIEWS, stream)
-#define SVLOG_I(stream) LOG_INFO_S((TlsActivationContext->AsActorContext()), NKikimrServices::SYSTEM_VIEWS, stream)
-#define SVLOG_N(stream) LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::SYSTEM_VIEWS, stream)
-#define SVLOG_W(stream) LOG_WARN_S((TlsActivationContext->AsActorContext()), NKikimrServices::SYSTEM_VIEWS, stream)
-#define SVLOG_E(stream) LOG_ERROR_S((TlsActivationContext->AsActorContext()), NKikimrServices::SYSTEM_VIEWS, stream)
-#define SVLOG_CRIT(stream) LOG_CRIT_S((TlsActivationContext->AsActorContext()), NKikimrServices::SYSTEM_VIEWS, stream)
-
-namespace NKikimr {
-namespace NSysView {
-
-enum class EProcessorMode {
- MINUTE, // aggregate stats every minute
- FAST // fast mode for tests
-};
-
-} // NSysView
-} // NKikimr
+#pragma once
+
+#include <library/cpp/actors/core/log.h>
+
+#if defined SVLOG_T || \
+ defined SVLOG_D || \
+ defined SVLOG_I || \
+ defined SVLOG_N || \
+ defined SVLOG_W || \
+ defined SVLOG_E || \
+ defined SVLOG_CRIT
+#error log macro redefinition
+#endif
+
+#define SVLOG_T(stream) LOG_TRACE_S((TlsActivationContext->AsActorContext()), NKikimrServices::SYSTEM_VIEWS, stream)
+#define SVLOG_D(stream) LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::SYSTEM_VIEWS, stream)
+#define SVLOG_I(stream) LOG_INFO_S((TlsActivationContext->AsActorContext()), NKikimrServices::SYSTEM_VIEWS, stream)
+#define SVLOG_N(stream) LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::SYSTEM_VIEWS, stream)
+#define SVLOG_W(stream) LOG_WARN_S((TlsActivationContext->AsActorContext()), NKikimrServices::SYSTEM_VIEWS, stream)
+#define SVLOG_E(stream) LOG_ERROR_S((TlsActivationContext->AsActorContext()), NKikimrServices::SYSTEM_VIEWS, stream)
+#define SVLOG_CRIT(stream) LOG_CRIT_S((TlsActivationContext->AsActorContext()), NKikimrServices::SYSTEM_VIEWS, stream)
+
+namespace NKikimr {
+namespace NSysView {
+
+enum class EProcessorMode {
+ MINUTE, // aggregate stats every minute
+ FAST // fast mode for tests
+};
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/common/db_counters.h b/ydb/core/sys_view/common/db_counters.h
index 0a1850e78a4..3e00068a04b 100644
--- a/ydb/core/sys_view/common/db_counters.h
+++ b/ydb/core/sys_view/common/db_counters.h
@@ -1,79 +1,79 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/protos/sys_view.pb.h>
-
-#include <util/generic/hash.h>
-
-namespace NKikimr {
-namespace NSysView {
-
-class TDbServiceCounters {
- NKikimrSysView::TDbServiceCounters ProtoCounters;
-
- THashMap<NKikimrTabletBase::TTabletTypes::EType, NKikimrSysView::TDbTabletCounters*> ByTabletType;
-
- using TGRpcRequestDesc = std::pair<TString, TString>;
- THashMap<TGRpcRequestDesc, NKikimrSysView::TDbGRpcCounters*> ByGRpcRequest;
-
-public:
- void Swap(TDbServiceCounters& other) {
- ProtoCounters.Swap(&other.ProtoCounters);
- ByTabletType.swap(other.ByTabletType);
- ByGRpcRequest.swap(other.ByGRpcRequest);
- }
-
- NKikimrSysView::TDbServiceCounters& Proto() { return ProtoCounters; }
- const NKikimrSysView::TDbServiceCounters& Proto() const { return ProtoCounters; }
-
- NKikimrSysView::TDbTabletCounters* FindTabletCounters(
- NKikimrTabletBase::TTabletTypes::EType tabletType) const
- {
- if (auto it = ByTabletType.find(tabletType); it != ByTabletType.end()) {
- return it->second;
- }
- return {};
- }
-
- NKikimrSysView::TDbTabletCounters* FindOrAddTabletCounters(
- NKikimrTabletBase::TTabletTypes::EType tabletType)
- {
- if (auto it = ByTabletType.find(tabletType); it != ByTabletType.end()) {
- return it->second;
- }
-
- auto* counters = ProtoCounters.AddTabletCounters();
- counters->SetType(tabletType);
- ByTabletType[tabletType] = counters;
-
- return counters;
- }
-
- NKikimrSysView::TDbGRpcCounters* FindGRpcCounters(
- const TString& grpcService, const TString& grpcRequest) const
- {
- auto key = std::make_pair(grpcService, grpcRequest);
- if (auto it = ByGRpcRequest.find(key); it != ByGRpcRequest.end()) {
- return it->second;
- }
- return {};
- }
-
- NKikimrSysView::TDbGRpcCounters* FindOrAddGRpcCounters(
- const TString& grpcService, const TString& grpcRequest)
- {
- auto key = std::make_pair(grpcService, grpcRequest);
- if (auto it = ByGRpcRequest.find(key); it != ByGRpcRequest.end()) {
- return it->second;
- }
-
- auto* counters = ProtoCounters.AddGRpcCounters();
- counters->SetGRpcService(grpcService);
- counters->SetGRpcRequest(grpcRequest);
- ByGRpcRequest[key] = counters;
-
- return counters;
- }
-};
-
-} // NSysView
-} // NKikimr
+
+#include <util/generic/hash.h>
+
+namespace NKikimr {
+namespace NSysView {
+
+class TDbServiceCounters {
+ NKikimrSysView::TDbServiceCounters ProtoCounters;
+
+ THashMap<NKikimrTabletBase::TTabletTypes::EType, NKikimrSysView::TDbTabletCounters*> ByTabletType;
+
+ using TGRpcRequestDesc = std::pair<TString, TString>;
+ THashMap<TGRpcRequestDesc, NKikimrSysView::TDbGRpcCounters*> ByGRpcRequest;
+
+public:
+ void Swap(TDbServiceCounters& other) {
+ ProtoCounters.Swap(&other.ProtoCounters);
+ ByTabletType.swap(other.ByTabletType);
+ ByGRpcRequest.swap(other.ByGRpcRequest);
+ }
+
+ NKikimrSysView::TDbServiceCounters& Proto() { return ProtoCounters; }
+ const NKikimrSysView::TDbServiceCounters& Proto() const { return ProtoCounters; }
+
+ NKikimrSysView::TDbTabletCounters* FindTabletCounters(
+ NKikimrTabletBase::TTabletTypes::EType tabletType) const
+ {
+ if (auto it = ByTabletType.find(tabletType); it != ByTabletType.end()) {
+ return it->second;
+ }
+ return {};
+ }
+
+ NKikimrSysView::TDbTabletCounters* FindOrAddTabletCounters(
+ NKikimrTabletBase::TTabletTypes::EType tabletType)
+ {
+ if (auto it = ByTabletType.find(tabletType); it != ByTabletType.end()) {
+ return it->second;
+ }
+
+ auto* counters = ProtoCounters.AddTabletCounters();
+ counters->SetType(tabletType);
+ ByTabletType[tabletType] = counters;
+
+ return counters;
+ }
+
+ NKikimrSysView::TDbGRpcCounters* FindGRpcCounters(
+ const TString& grpcService, const TString& grpcRequest) const
+ {
+ auto key = std::make_pair(grpcService, grpcRequest);
+ if (auto it = ByGRpcRequest.find(key); it != ByGRpcRequest.end()) {
+ return it->second;
+ }
+ return {};
+ }
+
+ NKikimrSysView::TDbGRpcCounters* FindOrAddGRpcCounters(
+ const TString& grpcService, const TString& grpcRequest)
+ {
+ auto key = std::make_pair(grpcService, grpcRequest);
+ if (auto it = ByGRpcRequest.find(key); it != ByGRpcRequest.end()) {
+ return it->second;
+ }
+
+ auto* counters = ProtoCounters.AddGRpcCounters();
+ counters->SetGRpcService(grpcService);
+ counters->SetGRpcRequest(grpcRequest);
+ ByGRpcRequest[key] = counters;
+
+ return counters;
+ }
+};
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/common/events.h b/ydb/core/sys_view/common/events.h
index 6e19aadc2b7..b1b481451d2 100644
--- a/ydb/core/sys_view/common/events.h
+++ b/ydb/core/sys_view/common/events.h
@@ -1,341 +1,341 @@
-#pragma once
-
-#include "utils.h"
-#include "db_counters.h"
-
+#pragma once
+
+#include "utils.h"
+#include "db_counters.h"
+
#include <ydb/core/base/events.h>
#include <ydb/core/base/pathid.h>
#include <library/cpp/actors/core/events.h>
#include <ydb/core/protos/sys_view.pb.h>
-
-namespace NKikimr {
-namespace NSysView {
-
-using TShardIdx = std::pair<ui64, ui64>;
-
-class IDbCounters : public TThrRefBase {
-public:
- virtual void ToProto(NKikimr::NSysView::TDbServiceCounters& counters) = 0;
- virtual void FromProto(NKikimr::NSysView::TDbServiceCounters& counters) = 0;
-};
-
-struct TEvSysView {
- enum EEv {
- EvSendPartitionStats = EventSpaceBegin(TKikimrEvents::ES_SYSTEM_VIEW),
- EvSetPartitioning,
- EvRemoveTable,
- EvGetPartitionStats,
- EvGetPartitionStatsResult,
-
- EvCollectQueryStats,
- EvGetQueryStats,
- EvGetQueryStatsResult,
-
- EvGetScanLimiter,
- EvGetScanLimiterResult,
-
- EvGetPDisksRequest,
- EvGetPDisksResponse,
- EvGetVSlotsRequest,
- EvGetVSlotsResponse,
- EvGetGroupsRequest,
- EvGetGroupsResponse,
- EvGetStoragePoolsRequest,
- EvGetStoragePoolsResponse,
-
- EvGetTabletIdsRequest,
- EvGetTabletIdsResponse,
- EvGetTabletsRequest,
- EvGetTabletsResponse,
-
- EvConfigureProcessor,
- EvIntervalQuerySummary,
- EvGetIntervalMetricsRequest,
- EvGetIntervalMetricsResponse,
- EvGetQueryMetricsRequest,
- EvGetQueryMetricsResponse,
- EvGetQueryStatsResponse,
-
- EvRegisterDbCounters,
- EvSendDbCountersRequest,
- EvSendDbCountersResponse,
- EvWatchDatabase,
-
+
+namespace NKikimr {
+namespace NSysView {
+
+using TShardIdx = std::pair<ui64, ui64>;
+
+class IDbCounters : public TThrRefBase {
+public:
+ virtual void ToProto(NKikimr::NSysView::TDbServiceCounters& counters) = 0;
+ virtual void FromProto(NKikimr::NSysView::TDbServiceCounters& counters) = 0;
+};
+
+struct TEvSysView {
+ enum EEv {
+ EvSendPartitionStats = EventSpaceBegin(TKikimrEvents::ES_SYSTEM_VIEW),
+ EvSetPartitioning,
+ EvRemoveTable,
+ EvGetPartitionStats,
+ EvGetPartitionStatsResult,
+
+ EvCollectQueryStats,
+ EvGetQueryStats,
+ EvGetQueryStatsResult,
+
+ EvGetScanLimiter,
+ EvGetScanLimiterResult,
+
+ EvGetPDisksRequest,
+ EvGetPDisksResponse,
+ EvGetVSlotsRequest,
+ EvGetVSlotsResponse,
+ EvGetGroupsRequest,
+ EvGetGroupsResponse,
+ EvGetStoragePoolsRequest,
+ EvGetStoragePoolsResponse,
+
+ EvGetTabletIdsRequest,
+ EvGetTabletIdsResponse,
+ EvGetTabletsRequest,
+ EvGetTabletsResponse,
+
+ EvConfigureProcessor,
+ EvIntervalQuerySummary,
+ EvGetIntervalMetricsRequest,
+ EvGetIntervalMetricsResponse,
+ EvGetQueryMetricsRequest,
+ EvGetQueryMetricsResponse,
+ EvGetQueryStatsResponse,
+
+ EvRegisterDbCounters,
+ EvSendDbCountersRequest,
+ EvSendDbCountersResponse,
+ EvWatchDatabase,
+
EvUpdateTtlStats,
EvGetStorageStatsRequest,
EvGetStorageStatsResponse,
- EvEnd,
- };
-
- struct TEvSendPartitionStats : public TEventLocal<
- TEvSendPartitionStats,
- EvSendPartitionStats>
- {
- TPathId DomainKey;
- TPathId PathId;
- TShardIdx ShardIdx;
-
- NKikimrSysView::TPartitionStats Stats;
-
- TEvSendPartitionStats(TPathId domainKey, TPathId pathId, TShardIdx shardIdx)
- : DomainKey(domainKey)
- , PathId(pathId)
- , ShardIdx(shardIdx)
- {}
- };
-
- struct TEvSetPartitioning : public TEventLocal<
- TEvSetPartitioning,
- EvSetPartitioning>
- {
- TPathId DomainKey;
- TPathId PathId;
- TString Path;
- TVector<TShardIdx> ShardIndices;
-
- TEvSetPartitioning(TPathId domainKey, TPathId pathId, const TStringBuf path)
- : DomainKey(domainKey)
- , PathId(pathId)
- , Path(path)
- {}
- };
-
- struct TEvRemoveTable : public TEventLocal<
- TEvRemoveTable,
- EvRemoveTable>
- {
- TPathId DomainKey;
- TPathId PathId;
-
- TEvRemoveTable(TPathId domainKey, TPathId pathId)
- : DomainKey(domainKey)
- , PathId(pathId)
- {}
- };
-
- struct TEvGetPartitionStats : public TEventPB<
- TEvGetPartitionStats,
- NKikimrSysView::TEvGetPartitionStats,
- EvGetPartitionStats>
- {};
-
- struct TEvGetPartitionStatsResult : public TEventPB<
- TEvGetPartitionStatsResult,
- NKikimrSysView::TEvGetPartitionStatsResult,
- EvGetPartitionStatsResult>
- {};
-
- // KQP -> SysViewService
- struct TEvCollectQueryStats : public TEventLocal<
- TEvCollectQueryStats,
- EvCollectQueryStats>
- {
- NKikimrSysView::TQueryStats QueryStats;
- TString Database;
- };
-
- // top queries scan -> SysViewService
- struct TEvGetQueryStats : public TEventPB<
- TEvGetQueryStats,
- NKikimrSysView::TEvGetQueryStats,
- EvGetQueryStats>
- {};
-
- // SysViewService -> top queries scan
- struct TEvGetQueryStatsResult : public TEventPB<
- TEvGetQueryStatsResult,
- NKikimrSysView::TEvGetQueryStatsResult,
- EvGetQueryStatsResult>
- {};
-
- struct TEvGetScanLimiter : public TEventLocal<
- TEvGetScanLimiter,
- EvGetScanLimiter>
- {};
-
- struct TEvGetScanLimiterResult : public TEventLocal<
- TEvGetScanLimiterResult,
- EvGetScanLimiterResult>
- {
- TIntrusivePtr<TScanLimiter> ScanLimiter;
- };
-
- struct TEvGetPDisksRequest : public TEventPB<
- TEvGetPDisksRequest,
- NKikimrSysView::TEvGetPDisksRequest,
- EvGetPDisksRequest>
- {};
-
- struct TEvGetPDisksResponse : public TEventPB<
- TEvGetPDisksResponse,
- NKikimrSysView::TEvGetPDisksResponse,
- EvGetPDisksResponse>
- {};
-
- struct TEvGetVSlotsRequest : public TEventPB<
- TEvGetVSlotsRequest,
- NKikimrSysView::TEvGetVSlotsRequest,
- EvGetVSlotsRequest>
- {};
-
- struct TEvGetVSlotsResponse : public TEventPB<
- TEvGetVSlotsResponse,
- NKikimrSysView::TEvGetVSlotsResponse,
- EvGetVSlotsResponse>
- {};
-
- struct TEvGetGroupsRequest : public TEventPB<
- TEvGetGroupsRequest,
- NKikimrSysView::TEvGetGroupsRequest,
- EvGetGroupsRequest>
- {};
-
- struct TEvGetGroupsResponse : public TEventPB<
- TEvGetGroupsResponse,
- NKikimrSysView::TEvGetGroupsResponse,
- EvGetGroupsResponse>
- {};
-
- struct TEvGetStoragePoolsRequest : public TEventPB<
- TEvGetStoragePoolsRequest,
- NKikimrSysView::TEvGetStoragePoolsRequest,
- EvGetStoragePoolsRequest>
- {};
-
- struct TEvGetStoragePoolsResponse : public TEventPB<
- TEvGetStoragePoolsResponse,
- NKikimrSysView::TEvGetStoragePoolsResponse,
- EvGetStoragePoolsResponse>
- {};
-
- struct TEvGetTabletIdsRequest : public TEventPB<
- TEvGetTabletIdsRequest,
- NKikimrSysView::TEvGetTabletIdsRequest,
- EvGetTabletIdsRequest>
- {};
-
- struct TEvGetTabletIdsResponse : public TEventPB<
- TEvGetTabletIdsResponse,
- NKikimrSysView::TEvGetTabletIdsResponse,
- EvGetTabletIdsResponse>
- {};
-
- struct TEvGetTabletsRequest : public TEventPB<
- TEvGetTabletsRequest,
- NKikimrSysView::TEvGetTabletsRequest,
- EvGetTabletsRequest>
- {};
-
- struct TEvGetTabletsResponse : public TEventPB<
- TEvGetTabletsResponse,
- NKikimrSysView::TEvGetTabletsResponse,
- EvGetTabletsResponse>
- {};
-
- struct TEvConfigureProcessor : public TEventPB<
- TEvConfigureProcessor,
- NKikimrSysView::TEvConfigureProcessor,
- EvConfigureProcessor>
- {
- TEvConfigureProcessor() = default;
- explicit TEvConfigureProcessor(const TString& database) {
- Record.SetDatabase(database);
- }
- };
-
- struct TEvIntervalQuerySummary : public TEventPB<
- TEvIntervalQuerySummary,
- NKikimrSysView::TEvIntervalQuerySummary,
- EvIntervalQuerySummary>
- {};
-
- struct TEvGetIntervalMetricsRequest : public TEventPB<
- TEvGetIntervalMetricsRequest,
- NKikimrSysView::TEvGetIntervalMetricsRequest,
- EvGetIntervalMetricsRequest>
- {};
-
- struct TEvGetIntervalMetricsResponse : public TEventPB<
- TEvGetIntervalMetricsResponse,
- NKikimrSysView::TEvGetIntervalMetricsResponse,
- EvGetIntervalMetricsResponse>
- {};
-
- struct TEvGetQueryMetricsRequest : public TEventPB<
- TEvGetQueryMetricsRequest,
- NKikimrSysView::TEvGetQueryMetricsRequest,
- EvGetQueryMetricsRequest>
- {};
-
- struct TEvGetQueryMetricsResponse : public TEventPB<
- TEvGetQueryMetricsResponse,
- NKikimrSysView::TEvGetQueryMetricsResponse,
- EvGetQueryMetricsResponse>
- {};
-
- struct TEvGetQueryStatsResponse : public TEventPB<
- TEvGetQueryStatsResponse,
- NKikimrSysView::TEvGetQueryStatsResponse,
- EvGetQueryStatsResponse>
- {};
-
- struct TEvRegisterDbCounters : public TEventLocal<
- TEvRegisterDbCounters,
- EvRegisterDbCounters>
- {
- NKikimrSysView::EDbCountersService Service;
- TString Database;
- TPathId PathId; // database path id, for tablet counters
- TIntrusivePtr<IDbCounters> Counters;
-
- TEvRegisterDbCounters(
- NKikimrSysView::EDbCountersService service,
- const TString& database,
- TIntrusivePtr<IDbCounters> counters)
- : Service(service)
- , Database(database)
- , Counters(counters)
- {}
-
- TEvRegisterDbCounters(
- NKikimrSysView::EDbCountersService service,
- TPathId pathId,
- TIntrusivePtr<IDbCounters> counters)
- : Service(service)
- , PathId(pathId)
- , Counters(counters)
- {}
- };
-
- struct TEvSendDbCountersRequest : public TEventPB<
- TEvSendDbCountersRequest,
- NKikimrSysView::TEvSendDbCountersRequest,
- EvSendDbCountersRequest>
- {};
-
- struct TEvSendDbCountersResponse : public TEventPB<
- TEvSendDbCountersResponse,
- NKikimrSysView::TEvSendDbCountersResponse,
- EvSendDbCountersResponse>
- {};
-
- struct TEvWatchDatabase : public TEventLocal<
- TEvWatchDatabase,
- EvWatchDatabase>
- {
- TString Database;
- TPathId PathId;
-
- explicit TEvWatchDatabase(const TString& database)
- : Database(database)
- {}
-
- explicit TEvWatchDatabase(TPathId pathId)
- : PathId(pathId)
- {}
- };
+ EvEnd,
+ };
+
+ struct TEvSendPartitionStats : public TEventLocal<
+ TEvSendPartitionStats,
+ EvSendPartitionStats>
+ {
+ TPathId DomainKey;
+ TPathId PathId;
+ TShardIdx ShardIdx;
+
+ NKikimrSysView::TPartitionStats Stats;
+
+ TEvSendPartitionStats(TPathId domainKey, TPathId pathId, TShardIdx shardIdx)
+ : DomainKey(domainKey)
+ , PathId(pathId)
+ , ShardIdx(shardIdx)
+ {}
+ };
+
+ struct TEvSetPartitioning : public TEventLocal<
+ TEvSetPartitioning,
+ EvSetPartitioning>
+ {
+ TPathId DomainKey;
+ TPathId PathId;
+ TString Path;
+ TVector<TShardIdx> ShardIndices;
+
+ TEvSetPartitioning(TPathId domainKey, TPathId pathId, const TStringBuf path)
+ : DomainKey(domainKey)
+ , PathId(pathId)
+ , Path(path)
+ {}
+ };
+
+ struct TEvRemoveTable : public TEventLocal<
+ TEvRemoveTable,
+ EvRemoveTable>
+ {
+ TPathId DomainKey;
+ TPathId PathId;
+
+ TEvRemoveTable(TPathId domainKey, TPathId pathId)
+ : DomainKey(domainKey)
+ , PathId(pathId)
+ {}
+ };
+
+ struct TEvGetPartitionStats : public TEventPB<
+ TEvGetPartitionStats,
+ NKikimrSysView::TEvGetPartitionStats,
+ EvGetPartitionStats>
+ {};
+
+ struct TEvGetPartitionStatsResult : public TEventPB<
+ TEvGetPartitionStatsResult,
+ NKikimrSysView::TEvGetPartitionStatsResult,
+ EvGetPartitionStatsResult>
+ {};
+
+ // KQP -> SysViewService
+ struct TEvCollectQueryStats : public TEventLocal<
+ TEvCollectQueryStats,
+ EvCollectQueryStats>
+ {
+ NKikimrSysView::TQueryStats QueryStats;
+ TString Database;
+ };
+
+ // top queries scan -> SysViewService
+ struct TEvGetQueryStats : public TEventPB<
+ TEvGetQueryStats,
+ NKikimrSysView::TEvGetQueryStats,
+ EvGetQueryStats>
+ {};
+
+ // SysViewService -> top queries scan
+ struct TEvGetQueryStatsResult : public TEventPB<
+ TEvGetQueryStatsResult,
+ NKikimrSysView::TEvGetQueryStatsResult,
+ EvGetQueryStatsResult>
+ {};
+
+ struct TEvGetScanLimiter : public TEventLocal<
+ TEvGetScanLimiter,
+ EvGetScanLimiter>
+ {};
+
+ struct TEvGetScanLimiterResult : public TEventLocal<
+ TEvGetScanLimiterResult,
+ EvGetScanLimiterResult>
+ {
+ TIntrusivePtr<TScanLimiter> ScanLimiter;
+ };
+
+ struct TEvGetPDisksRequest : public TEventPB<
+ TEvGetPDisksRequest,
+ NKikimrSysView::TEvGetPDisksRequest,
+ EvGetPDisksRequest>
+ {};
+
+ struct TEvGetPDisksResponse : public TEventPB<
+ TEvGetPDisksResponse,
+ NKikimrSysView::TEvGetPDisksResponse,
+ EvGetPDisksResponse>
+ {};
+
+ struct TEvGetVSlotsRequest : public TEventPB<
+ TEvGetVSlotsRequest,
+ NKikimrSysView::TEvGetVSlotsRequest,
+ EvGetVSlotsRequest>
+ {};
+
+ struct TEvGetVSlotsResponse : public TEventPB<
+ TEvGetVSlotsResponse,
+ NKikimrSysView::TEvGetVSlotsResponse,
+ EvGetVSlotsResponse>
+ {};
+
+ struct TEvGetGroupsRequest : public TEventPB<
+ TEvGetGroupsRequest,
+ NKikimrSysView::TEvGetGroupsRequest,
+ EvGetGroupsRequest>
+ {};
+
+ struct TEvGetGroupsResponse : public TEventPB<
+ TEvGetGroupsResponse,
+ NKikimrSysView::TEvGetGroupsResponse,
+ EvGetGroupsResponse>
+ {};
+
+ struct TEvGetStoragePoolsRequest : public TEventPB<
+ TEvGetStoragePoolsRequest,
+ NKikimrSysView::TEvGetStoragePoolsRequest,
+ EvGetStoragePoolsRequest>
+ {};
+
+ struct TEvGetStoragePoolsResponse : public TEventPB<
+ TEvGetStoragePoolsResponse,
+ NKikimrSysView::TEvGetStoragePoolsResponse,
+ EvGetStoragePoolsResponse>
+ {};
+
+ struct TEvGetTabletIdsRequest : public TEventPB<
+ TEvGetTabletIdsRequest,
+ NKikimrSysView::TEvGetTabletIdsRequest,
+ EvGetTabletIdsRequest>
+ {};
+
+ struct TEvGetTabletIdsResponse : public TEventPB<
+ TEvGetTabletIdsResponse,
+ NKikimrSysView::TEvGetTabletIdsResponse,
+ EvGetTabletIdsResponse>
+ {};
+
+ struct TEvGetTabletsRequest : public TEventPB<
+ TEvGetTabletsRequest,
+ NKikimrSysView::TEvGetTabletsRequest,
+ EvGetTabletsRequest>
+ {};
+
+ struct TEvGetTabletsResponse : public TEventPB<
+ TEvGetTabletsResponse,
+ NKikimrSysView::TEvGetTabletsResponse,
+ EvGetTabletsResponse>
+ {};
+
+ struct TEvConfigureProcessor : public TEventPB<
+ TEvConfigureProcessor,
+ NKikimrSysView::TEvConfigureProcessor,
+ EvConfigureProcessor>
+ {
+ TEvConfigureProcessor() = default;
+ explicit TEvConfigureProcessor(const TString& database) {
+ Record.SetDatabase(database);
+ }
+ };
+
+ struct TEvIntervalQuerySummary : public TEventPB<
+ TEvIntervalQuerySummary,
+ NKikimrSysView::TEvIntervalQuerySummary,
+ EvIntervalQuerySummary>
+ {};
+
+ struct TEvGetIntervalMetricsRequest : public TEventPB<
+ TEvGetIntervalMetricsRequest,
+ NKikimrSysView::TEvGetIntervalMetricsRequest,
+ EvGetIntervalMetricsRequest>
+ {};
+
+ struct TEvGetIntervalMetricsResponse : public TEventPB<
+ TEvGetIntervalMetricsResponse,
+ NKikimrSysView::TEvGetIntervalMetricsResponse,
+ EvGetIntervalMetricsResponse>
+ {};
+
+ struct TEvGetQueryMetricsRequest : public TEventPB<
+ TEvGetQueryMetricsRequest,
+ NKikimrSysView::TEvGetQueryMetricsRequest,
+ EvGetQueryMetricsRequest>
+ {};
+
+ struct TEvGetQueryMetricsResponse : public TEventPB<
+ TEvGetQueryMetricsResponse,
+ NKikimrSysView::TEvGetQueryMetricsResponse,
+ EvGetQueryMetricsResponse>
+ {};
+
+ struct TEvGetQueryStatsResponse : public TEventPB<
+ TEvGetQueryStatsResponse,
+ NKikimrSysView::TEvGetQueryStatsResponse,
+ EvGetQueryStatsResponse>
+ {};
+
+ struct TEvRegisterDbCounters : public TEventLocal<
+ TEvRegisterDbCounters,
+ EvRegisterDbCounters>
+ {
+ NKikimrSysView::EDbCountersService Service;
+ TString Database;
+ TPathId PathId; // database path id, for tablet counters
+ TIntrusivePtr<IDbCounters> Counters;
+
+ TEvRegisterDbCounters(
+ NKikimrSysView::EDbCountersService service,
+ const TString& database,
+ TIntrusivePtr<IDbCounters> counters)
+ : Service(service)
+ , Database(database)
+ , Counters(counters)
+ {}
+
+ TEvRegisterDbCounters(
+ NKikimrSysView::EDbCountersService service,
+ TPathId pathId,
+ TIntrusivePtr<IDbCounters> counters)
+ : Service(service)
+ , PathId(pathId)
+ , Counters(counters)
+ {}
+ };
+
+ struct TEvSendDbCountersRequest : public TEventPB<
+ TEvSendDbCountersRequest,
+ NKikimrSysView::TEvSendDbCountersRequest,
+ EvSendDbCountersRequest>
+ {};
+
+ struct TEvSendDbCountersResponse : public TEventPB<
+ TEvSendDbCountersResponse,
+ NKikimrSysView::TEvSendDbCountersResponse,
+ EvSendDbCountersResponse>
+ {};
+
+ struct TEvWatchDatabase : public TEventLocal<
+ TEvWatchDatabase,
+ EvWatchDatabase>
+ {
+ TString Database;
+ TPathId PathId;
+
+ explicit TEvWatchDatabase(const TString& database)
+ : Database(database)
+ {}
+
+ explicit TEvWatchDatabase(TPathId pathId)
+ : PathId(pathId)
+ {}
+ };
struct TEvUpdateTtlStats : public TEventLocal<
TEvUpdateTtlStats,
@@ -361,7 +361,7 @@ struct TEvSysView {
struct TEvGetStorageStatsResponse
: TEventPB<TEvGetStorageStatsResponse, NKikimrSysView::TEvGetStorageStatsResponse, EvGetStorageStatsResponse>
{};
-};
-
-} // NSysView
-} // NKikimr
+};
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/common/keys.h b/ydb/core/sys_view/common/keys.h
index b200e7abb0f..96b420f70d5 100644
--- a/ydb/core/sys_view/common/keys.h
+++ b/ydb/core/sys_view/common/keys.h
@@ -1,48 +1,48 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/scheme/scheme_tabledefs.h>
-
-namespace NKikimr {
-namespace NSysView {
-
-template <size_t N, typename TKey, typename TField>
-void SetField(TKey&, TField);
-
-template <size_t N, typename TKey, typename TField>
-void ConvertField(TKey& key, const TConstArrayRef<TCell>& cells) {
- if (cells.size() > N) {
- if (!cells[N].IsNull()) {
- SetField<N>(key, cells[N].AsValue<TField>());
- } else {
- SetField<N>(key, Min<TField>());
- }
- } else {
- SetField<N>(key, Max<TField>());
- }
-}
-
-template <size_t N, typename TKey>
-void ConvertKey(TKey&, const TConstArrayRef<TCell>&) {
-}
-
-template <size_t N, typename TKey, typename TField, typename... TFields>
-void ConvertKey(TKey& key, const TConstArrayRef<TCell>& cells) {
- ConvertField<N, TKey, TField>(key, cells);
- ConvertKey<N + 1, TKey, TFields...>(key, cells);
-}
-
-template <typename TRange, typename... TFields>
-void ConvertKeyRange(TRange& protoRange, const TSerializedTableRange& tableRange) {
- ConvertKey<0, decltype(*protoRange.MutableFrom()), TFields...>(
- *protoRange.MutableFrom(), tableRange.From.GetCells());
-
- protoRange.SetInclusiveFrom(tableRange.FromInclusive);
-
- ConvertKey<0, decltype(*protoRange.MutableTo()), TFields...>(
- *protoRange.MutableTo(), tableRange.To.GetCells());
-
- protoRange.SetInclusiveTo(tableRange.To.GetCells().empty() ? true : tableRange.ToInclusive);
-}
-
-} // NSysView
-} // NKikimr
+
+namespace NKikimr {
+namespace NSysView {
+
+template <size_t N, typename TKey, typename TField>
+void SetField(TKey&, TField);
+
+template <size_t N, typename TKey, typename TField>
+void ConvertField(TKey& key, const TConstArrayRef<TCell>& cells) {
+ if (cells.size() > N) {
+ if (!cells[N].IsNull()) {
+ SetField<N>(key, cells[N].AsValue<TField>());
+ } else {
+ SetField<N>(key, Min<TField>());
+ }
+ } else {
+ SetField<N>(key, Max<TField>());
+ }
+}
+
+template <size_t N, typename TKey>
+void ConvertKey(TKey&, const TConstArrayRef<TCell>&) {
+}
+
+template <size_t N, typename TKey, typename TField, typename... TFields>
+void ConvertKey(TKey& key, const TConstArrayRef<TCell>& cells) {
+ ConvertField<N, TKey, TField>(key, cells);
+ ConvertKey<N + 1, TKey, TFields...>(key, cells);
+}
+
+template <typename TRange, typename... TFields>
+void ConvertKeyRange(TRange& protoRange, const TSerializedTableRange& tableRange) {
+ ConvertKey<0, decltype(*protoRange.MutableFrom()), TFields...>(
+ *protoRange.MutableFrom(), tableRange.From.GetCells());
+
+ protoRange.SetInclusiveFrom(tableRange.FromInclusive);
+
+ ConvertKey<0, decltype(*protoRange.MutableTo()), TFields...>(
+ *protoRange.MutableTo(), tableRange.To.GetCells());
+
+ protoRange.SetInclusiveTo(tableRange.To.GetCells().empty() ? true : tableRange.ToInclusive);
+}
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/common/path.h b/ydb/core/sys_view/common/path.h
index 5f17df91a4b..d917ca9038b 100644
--- a/ydb/core/sys_view/common/path.h
+++ b/ydb/core/sys_view/common/path.h
@@ -1,11 +1,11 @@
-#pragma once
-
-#include <util/generic/strbuf.h>
-
-namespace NKikimr {
-namespace NSysView {
-
+#pragma once
+
+#include <util/generic/strbuf.h>
+
+namespace NKikimr {
+namespace NSysView {
+
constexpr TStringBuf SysPathName = ".sys";
-
-} // NSysView
-} // NKikimr
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/common/scan_actor_base_impl.h b/ydb/core/sys_view/common/scan_actor_base_impl.h
index 298a8efbc03..ed0b3f4c126 100644
--- a/ydb/core/sys_view/common/scan_actor_base_impl.h
+++ b/ydb/core/sys_view/common/scan_actor_base_impl.h
@@ -1,72 +1,72 @@
-#pragma once
-
-#include "utils.h"
-
+#pragma once
+
+#include "utils.h"
+
#include <ydb/core/kqp/kqp_compute.h>
#include <ydb/core/tx/scheme_cache/scheme_cache.h>
#include <ydb/core/mind/tenant_node_enumeration.h>
#include <ydb/core/sys_view/service/sysview_service.h>
#include <ydb/core/base/appdata.h>
-
+
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h>
#include <ydb/library/yql/public/issue/yql_issue_message.h>
#include <library/cpp/actors/core/actor.h>
#include <library/cpp/actors/core/actor_bootstrapped.h>
#include <library/cpp/actors/core/hfunc.h>
-
+
#include <ydb/core/base/tablet_pipecache.h>
#include <ydb/core/tx/schemeshard/schemeshard.h>
-
-namespace NKikimr {
-namespace NSysView {
-
-template <typename TDerived>
-class TScanActorBase : public TActorBootstrapped<TDerived> {
-public:
- using TBase = TActorBootstrapped<TDerived>;
-
+
+namespace NKikimr {
+namespace NSysView {
+
+template <typename TDerived>
+class TScanActorBase : public TActorBootstrapped<TDerived> {
+public:
+ using TBase = TActorBootstrapped<TDerived>;
+
TScanActorBase(const TActorId& ownerId, ui32 scanId, const TTableId& tableId,
- const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns)
- : OwnerActorId(ownerId)
- , ScanId(scanId)
- , TableId(tableId)
- , TableRange(tableRange)
- , Columns(columns.begin(), columns.end())
- {}
-
- void Bootstrap(const TActorContext& ctx) {
- LOG_INFO_S(ctx, NKikimrServices::SYSTEM_VIEWS,
- "Scan started, actor: " << TBase::SelfId()
- << ", owner: " << OwnerActorId
- << ", scan id: " << ScanId
- << ", table id: " << TableId);
-
- auto sysViewServiceId = MakeSysViewServiceID(TBase::SelfId().NodeId());
- TBase::Send(sysViewServiceId, new TEvSysView::TEvGetScanLimiter());
-
- TBase::Schedule(Timeout, new TEvents::TEvWakeup());
- TBase::Become(&TDerived::StateLimiter);
- }
-
-protected:
- void SendBatch(THolder<NKqp::TEvKqpCompute::TEvScanData> batch) {
- LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::SYSTEM_VIEWS,
- "Sending scan batch, actor: " << TBase::SelfId()
- << ", row count: " << batch->Rows.size()
- << ", finished: " << batch->Finished);
-
- bool finished = batch->Finished;
- TBase::Send(OwnerActorId, batch.Release());
- if (finished) {
- this->PassAway();
- }
- }
-
- void HandleTimeout() {
- ReplyErrorAndDie(Ydb::StatusIds::TIMEOUT, "System view: timeout");
- }
-
+ const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns)
+ : OwnerActorId(ownerId)
+ , ScanId(scanId)
+ , TableId(tableId)
+ , TableRange(tableRange)
+ , Columns(columns.begin(), columns.end())
+ {}
+
+ void Bootstrap(const TActorContext& ctx) {
+ LOG_INFO_S(ctx, NKikimrServices::SYSTEM_VIEWS,
+ "Scan started, actor: " << TBase::SelfId()
+ << ", owner: " << OwnerActorId
+ << ", scan id: " << ScanId
+ << ", table id: " << TableId);
+
+ auto sysViewServiceId = MakeSysViewServiceID(TBase::SelfId().NodeId());
+ TBase::Send(sysViewServiceId, new TEvSysView::TEvGetScanLimiter());
+
+ TBase::Schedule(Timeout, new TEvents::TEvWakeup());
+ TBase::Become(&TDerived::StateLimiter);
+ }
+
+protected:
+ void SendBatch(THolder<NKqp::TEvKqpCompute::TEvScanData> batch) {
+ LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::SYSTEM_VIEWS,
+ "Sending scan batch, actor: " << TBase::SelfId()
+ << ", row count: " << batch->Rows.size()
+ << ", finished: " << batch->Finished);
+
+ bool finished = batch->Finished;
+ TBase::Send(OwnerActorId, batch.Release());
+ if (finished) {
+ this->PassAway();
+ }
+ }
+
+ void HandleTimeout() {
+ ReplyErrorAndDie(Ydb::StatusIds::TIMEOUT, "System view: timeout");
+ }
+
void HandleAbortExecution(NKqp::TEvKqp::TEvAbortExecution::TPtr& ev) {
LOG_ERROR_S(TlsActivationContext->AsActorContext(), NKikimrServices::SYSTEM_VIEWS,
"Got abort execution event, actor: " << TBase::SelfId()
@@ -79,273 +79,273 @@ protected:
this->PassAway();
}
- void ReplyErrorAndDie(Ydb::StatusIds::StatusCode status, const TString& message) {
- LOG_ERROR_S(TlsActivationContext->AsActorContext(), NKikimrServices::SYSTEM_VIEWS,
- "Scan error, actor: " << TBase::SelfId()
- << ", owner: " << OwnerActorId
- << ", scan id: " << ScanId
- << ", table id: " << TableId
- << ", error: " << message);
-
- auto error = MakeHolder<NKqp::TEvKqpCompute::TEvScanError>();
+ void ReplyErrorAndDie(Ydb::StatusIds::StatusCode status, const TString& message) {
+ LOG_ERROR_S(TlsActivationContext->AsActorContext(), NKikimrServices::SYSTEM_VIEWS,
+ "Scan error, actor: " << TBase::SelfId()
+ << ", owner: " << OwnerActorId
+ << ", scan id: " << ScanId
+ << ", table id: " << TableId
+ << ", error: " << message);
+
+ auto error = MakeHolder<NKqp::TEvKqpCompute::TEvScanError>();
error->Record.SetStatus(status);
IssueToMessage(NYql::TIssue(message), error->Record.MutableIssues()->Add());
-
- TBase::Send(OwnerActorId, error.Release());
-
- this->PassAway();
- }
-
- void ReplyEmptyAndDie() {
- auto batch = MakeHolder<NKqp::TEvKqpCompute::TEvScanData>(ScanId);
- batch->Finished = true;
- TBase::Send(OwnerActorId, batch.Release());
-
- this->PassAway();
- }
-
- void PassAway() override {
- LOG_INFO_S(TlsActivationContext->AsActorContext(), NKikimrServices::SYSTEM_VIEWS,
- "Scan finished, actor: " << TBase::SelfId()
- << ", owner: " << OwnerActorId
- << ", scan id: " << ScanId
- << ", table id: " << TableId);
-
- if (AllowedByLimiter) {
- ScanLimiter->Dec();
- }
-
- TBase::PassAway();
- }
-
- ui64 GetBSControllerId() {
- auto domainInfo = AppData()->DomainsInfo;
- if (domainInfo->Domains.empty()) {
- ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "Invalid domain info");
- return 0;
- }
-
- auto domain = domainInfo->Domains.begin()->second;
- auto defaultSSGroup = domainInfo->GetDefaultStateStorageGroup(domain->DomainUid);
- return MakeBSControllerID(defaultSSGroup);
- }
-
- template <typename TResponse, typename TEntry, typename TExtractorsMap, bool BatchSupport = false>
- void ReplyBatch(typename TResponse::TPtr& ev) {
- static TExtractorsMap extractors;
-
- const auto& record = ev->Get()->Record;
- auto batch = MakeHolder<NKqp::TEvKqpCompute::TEvScanData>(ScanId);
-
- TVector<TCell> cells;
- for (const auto& entry : record.GetEntries()) {
- for (auto column : Columns) {
- auto extractor = extractors.find(column.Tag);
- if (extractor == extractors.end()) {
- cells.push_back(TCell());
- } else {
- cells.push_back(extractor->second(entry));
- }
- }
-
- TArrayRef<const TCell> ref(cells);
+
+ TBase::Send(OwnerActorId, error.Release());
+
+ this->PassAway();
+ }
+
+ void ReplyEmptyAndDie() {
+ auto batch = MakeHolder<NKqp::TEvKqpCompute::TEvScanData>(ScanId);
+ batch->Finished = true;
+ TBase::Send(OwnerActorId, batch.Release());
+
+ this->PassAway();
+ }
+
+ void PassAway() override {
+ LOG_INFO_S(TlsActivationContext->AsActorContext(), NKikimrServices::SYSTEM_VIEWS,
+ "Scan finished, actor: " << TBase::SelfId()
+ << ", owner: " << OwnerActorId
+ << ", scan id: " << ScanId
+ << ", table id: " << TableId);
+
+ if (AllowedByLimiter) {
+ ScanLimiter->Dec();
+ }
+
+ TBase::PassAway();
+ }
+
+ ui64 GetBSControllerId() {
+ auto domainInfo = AppData()->DomainsInfo;
+ if (domainInfo->Domains.empty()) {
+ ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "Invalid domain info");
+ return 0;
+ }
+
+ auto domain = domainInfo->Domains.begin()->second;
+ auto defaultSSGroup = domainInfo->GetDefaultStateStorageGroup(domain->DomainUid);
+ return MakeBSControllerID(defaultSSGroup);
+ }
+
+ template <typename TResponse, typename TEntry, typename TExtractorsMap, bool BatchSupport = false>
+ void ReplyBatch(typename TResponse::TPtr& ev) {
+ static TExtractorsMap extractors;
+
+ const auto& record = ev->Get()->Record;
+ auto batch = MakeHolder<NKqp::TEvKqpCompute::TEvScanData>(ScanId);
+
+ TVector<TCell> cells;
+ for (const auto& entry : record.GetEntries()) {
+ for (auto column : Columns) {
+ auto extractor = extractors.find(column.Tag);
+ if (extractor == extractors.end()) {
+ cells.push_back(TCell());
+ } else {
+ cells.push_back(extractor->second(entry));
+ }
+ }
+
+ TArrayRef<const TCell> ref(cells);
batch->Rows.emplace_back(TOwnedCellVec::Make(ref));
- cells.clear();
- }
-
- if constexpr (BatchSupport) {
- batch->Finished = record.GetLastBatch();
- } else {
- batch->Finished = true;
- }
-
- SendBatch(std::move(batch));
- }
-
-private:
- virtual void ProceedToScan() = 0;
-
- void ReplyLimiterFailedAndDie() {
- ReplyErrorAndDie(Ydb::StatusIds::OVERLOADED, "System view: concurrent scans limit exceeded");
- }
-
- void ReplyNavigateFailedAndDie() {
- ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "System view: navigate failed");
- }
-
- void ReplyLookupFailedAndDie() {
- ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "System view: tenant nodes lookup failed");
- }
-
- void HandleScanAck(NKqp::TEvKqpCompute::TEvScanDataAck::TPtr&) {
- switch (FailState) {
- case LIMITER_FAILED:
- ReplyLimiterFailedAndDie();
- break;
- case NAVIGATE_FAILED:
- ReplyNavigateFailedAndDie();
- break;
- default:
- AckReceived = true;
- break;
- }
- }
-
- void HandleLimiter(TEvSysView::TEvGetScanLimiterResult::TPtr& ev) {
- ScanLimiter = ev->Get()->ScanLimiter;
-
- if (!ScanLimiter->Inc()) {
- FailState = LIMITER_FAILED;
- if (AckReceived) {
- ReplyLimiterFailedAndDie();
- }
- return;
- }
-
- AllowedByLimiter = true;
-
- using TNavigate = NSchemeCache::TSchemeCacheNavigate;
-
- auto request = MakeHolder<TNavigate>();
- request->ResultSet.push_back({});
- auto& entry = request->ResultSet.back();
-
- entry.TableId = TTableId(TableId.PathId.OwnerId, TableId.PathId.LocalPathId); // domain or subdomain
- entry.Operation = TNavigate::EOp::OpPath;
- entry.RequestType = TNavigate::TEntry::ERequestType::ByTableId;
-
- TBase::Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release()));
- TBase::Become(&TDerived::StateNavigate);
- }
-
- void HandleNavigate(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) {
- using TNavigate = NSchemeCache::TSchemeCacheNavigate;
-
+ cells.clear();
+ }
+
+ if constexpr (BatchSupport) {
+ batch->Finished = record.GetLastBatch();
+ } else {
+ batch->Finished = true;
+ }
+
+ SendBatch(std::move(batch));
+ }
+
+private:
+ virtual void ProceedToScan() = 0;
+
+ void ReplyLimiterFailedAndDie() {
+ ReplyErrorAndDie(Ydb::StatusIds::OVERLOADED, "System view: concurrent scans limit exceeded");
+ }
+
+ void ReplyNavigateFailedAndDie() {
+ ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "System view: navigate failed");
+ }
+
+ void ReplyLookupFailedAndDie() {
+ ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "System view: tenant nodes lookup failed");
+ }
+
+ void HandleScanAck(NKqp::TEvKqpCompute::TEvScanDataAck::TPtr&) {
+ switch (FailState) {
+ case LIMITER_FAILED:
+ ReplyLimiterFailedAndDie();
+ break;
+ case NAVIGATE_FAILED:
+ ReplyNavigateFailedAndDie();
+ break;
+ default:
+ AckReceived = true;
+ break;
+ }
+ }
+
+ void HandleLimiter(TEvSysView::TEvGetScanLimiterResult::TPtr& ev) {
+ ScanLimiter = ev->Get()->ScanLimiter;
+
+ if (!ScanLimiter->Inc()) {
+ FailState = LIMITER_FAILED;
+ if (AckReceived) {
+ ReplyLimiterFailedAndDie();
+ }
+ return;
+ }
+
+ AllowedByLimiter = true;
+
+ using TNavigate = NSchemeCache::TSchemeCacheNavigate;
+
+ auto request = MakeHolder<TNavigate>();
+ request->ResultSet.push_back({});
+ auto& entry = request->ResultSet.back();
+
+ entry.TableId = TTableId(TableId.PathId.OwnerId, TableId.PathId.LocalPathId); // domain or subdomain
+ entry.Operation = TNavigate::EOp::OpPath;
+ entry.RequestType = TNavigate::TEntry::ERequestType::ByTableId;
+
+ TBase::Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release()));
+ TBase::Become(&TDerived::StateNavigate);
+ }
+
+ void HandleNavigate(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) {
+ using TNavigate = NSchemeCache::TSchemeCacheNavigate;
+
THolder<NSchemeCache::TSchemeCacheNavigate> request(ev->Get()->Request.Release());
- Y_VERIFY(request->ResultSet.size() == 1);
-
- auto& entry = request->ResultSet.back();
- if (entry.Status != TNavigate::EStatus::Ok) {
- FailState = NAVIGATE_FAILED;
- if (AckReceived) {
- ReplyNavigateFailedAndDie();
- }
- return;
- }
-
+ Y_VERIFY(request->ResultSet.size() == 1);
+
+ auto& entry = request->ResultSet.back();
+ if (entry.Status != TNavigate::EStatus::Ok) {
+ FailState = NAVIGATE_FAILED;
+ if (AckReceived) {
+ ReplyNavigateFailedAndDie();
+ }
+ return;
+ }
+
SchemeShardId = entry.DomainInfo->ExtractSchemeShard();
-
- if (entry.DomainInfo->Params.HasSysViewProcessor()) {
- SysViewProcessorId = entry.DomainInfo->Params.GetSysViewProcessor();
- }
-
- if (entry.DomainInfo->Params.HasHive()) {
- HiveId = entry.DomainInfo->Params.GetHive();
- } else {
- auto domainInfo = AppData()->DomainsInfo;
- if (domainInfo->Domains.empty()) {
- ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "Invalid domain info");
- return;
- }
- auto domain = domainInfo->Domains.begin()->second;
- HiveId = domainInfo->GetHive(domain->DefaultHiveUid);
- }
-
- DomainKey = entry.DomainInfo->DomainKey;
-
- TenantName = CanonizePath(entry.Path);
-
- TBase::Register(CreateTenantNodeEnumerationLookup(TBase::SelfId(), TenantName));
- TBase::Become(&TDerived::StateLookup);
- }
-
- void HandleLookup(TEvTenantNodeEnumerator::TEvLookupResult::TPtr& ev) {
- if (ev->Get()->Success) {
- for (auto& node : ev->Get()->AssignedNodes) {
- TenantNodes.insert(node);
- }
- }
-
- LOG_INFO_S(TlsActivationContext->AsActorContext(), NKikimrServices::SYSTEM_VIEWS,
- "Scan prepared, actor: " << TBase::SelfId()
- << ", schemeshard id: " << SchemeShardId
- << ", hive id: " << HiveId
- << ", tenant name: " << TenantName
- << ", domain key: " << DomainKey
- << ", tenant node count: " << TenantNodes.size());
-
- ProceedToScan();
- }
-
- STFUNC(StateLimiter) {
- switch (ev->GetTypeRewrite()) {
- hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, HandleScanAck);
- hFunc(TEvSysView::TEvGetScanLimiterResult, HandleLimiter);
- cFunc(TEvents::TEvWakeup::EventType, HandleTimeout);
- cFunc(TEvents::TEvPoison::EventType, this->PassAway);
- default:
- LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS,
- "NSysView::TScanActorBase: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite());
- }
- }
-
- STFUNC(StateNavigate) {
- switch (ev->GetTypeRewrite()) {
- hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, HandleScanAck);
- hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleNavigate);
+
+ if (entry.DomainInfo->Params.HasSysViewProcessor()) {
+ SysViewProcessorId = entry.DomainInfo->Params.GetSysViewProcessor();
+ }
+
+ if (entry.DomainInfo->Params.HasHive()) {
+ HiveId = entry.DomainInfo->Params.GetHive();
+ } else {
+ auto domainInfo = AppData()->DomainsInfo;
+ if (domainInfo->Domains.empty()) {
+ ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "Invalid domain info");
+ return;
+ }
+ auto domain = domainInfo->Domains.begin()->second;
+ HiveId = domainInfo->GetHive(domain->DefaultHiveUid);
+ }
+
+ DomainKey = entry.DomainInfo->DomainKey;
+
+ TenantName = CanonizePath(entry.Path);
+
+ TBase::Register(CreateTenantNodeEnumerationLookup(TBase::SelfId(), TenantName));
+ TBase::Become(&TDerived::StateLookup);
+ }
+
+ void HandleLookup(TEvTenantNodeEnumerator::TEvLookupResult::TPtr& ev) {
+ if (ev->Get()->Success) {
+ for (auto& node : ev->Get()->AssignedNodes) {
+ TenantNodes.insert(node);
+ }
+ }
+
+ LOG_INFO_S(TlsActivationContext->AsActorContext(), NKikimrServices::SYSTEM_VIEWS,
+ "Scan prepared, actor: " << TBase::SelfId()
+ << ", schemeshard id: " << SchemeShardId
+ << ", hive id: " << HiveId
+ << ", tenant name: " << TenantName
+ << ", domain key: " << DomainKey
+ << ", tenant node count: " << TenantNodes.size());
+
+ ProceedToScan();
+ }
+
+ STFUNC(StateLimiter) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, HandleScanAck);
+ hFunc(TEvSysView::TEvGetScanLimiterResult, HandleLimiter);
+ cFunc(TEvents::TEvWakeup::EventType, HandleTimeout);
+ cFunc(TEvents::TEvPoison::EventType, this->PassAway);
+ default:
+ LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS,
+ "NSysView::TScanActorBase: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite());
+ }
+ }
+
+ STFUNC(StateNavigate) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, HandleScanAck);
+ hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleNavigate);
hFunc(NKqp::TEvKqp::TEvAbortExecution, HandleAbortExecution);
- cFunc(TEvents::TEvWakeup::EventType, HandleTimeout);
- cFunc(TEvents::TEvPoison::EventType, this->PassAway);
- default:
- LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS,
- "NSysView::TScanActorBase: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite());
- }
- }
-
- STFUNC(StateLookup) {
- switch (ev->GetTypeRewrite()) {
- hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, HandleScanAck);
- hFunc(TEvTenantNodeEnumerator::TEvLookupResult, HandleLookup);
+ cFunc(TEvents::TEvWakeup::EventType, HandleTimeout);
+ cFunc(TEvents::TEvPoison::EventType, this->PassAway);
+ default:
+ LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS,
+ "NSysView::TScanActorBase: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite());
+ }
+ }
+
+ STFUNC(StateLookup) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, HandleScanAck);
+ hFunc(TEvTenantNodeEnumerator::TEvLookupResult, HandleLookup);
hFunc(NKqp::TEvKqp::TEvAbortExecution, HandleAbortExecution);
- cFunc(TEvents::TEvWakeup::EventType, HandleTimeout);
- cFunc(TEvents::TEvPoison::EventType, this->PassAway);
- default:
- LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS,
- "NSysView::TScanActorBase: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite());
- }
- }
-
-protected:
- static constexpr TDuration Timeout = TDuration::Seconds(60);
-
+ cFunc(TEvents::TEvWakeup::EventType, HandleTimeout);
+ cFunc(TEvents::TEvPoison::EventType, this->PassAway);
+ default:
+ LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS,
+ "NSysView::TScanActorBase: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite());
+ }
+ }
+
+protected:
+ static constexpr TDuration Timeout = TDuration::Seconds(60);
+
const TActorId OwnerActorId;
- const ui32 ScanId;
- const TTableId TableId;
- TSerializedTableRange TableRange;
- TSmallVec<NMiniKQL::TKqpComputeContextBase::TColumn> Columns;
-
- ui64 SchemeShardId = 0;
- TPathId DomainKey;
- TString TenantName;
- THashSet<ui32> TenantNodes;
- ui64 HiveId = 0;
- ui64 SysViewProcessorId = 0;
-
- bool AckReceived = false;
-
- bool BatchRequestInFlight = false;
-
-private:
- enum EFailState {
- OK,
- LIMITER_FAILED,
- NAVIGATE_FAILED
- } FailState = OK;
-
- TIntrusivePtr<TScanLimiter> ScanLimiter;
- bool AllowedByLimiter = false;
-};
-
-
-} // NSysView
-} // NKikimr
+ const ui32 ScanId;
+ const TTableId TableId;
+ TSerializedTableRange TableRange;
+ TSmallVec<NMiniKQL::TKqpComputeContextBase::TColumn> Columns;
+
+ ui64 SchemeShardId = 0;
+ TPathId DomainKey;
+ TString TenantName;
+ THashSet<ui32> TenantNodes;
+ ui64 HiveId = 0;
+ ui64 SysViewProcessorId = 0;
+
+ bool AckReceived = false;
+
+ bool BatchRequestInFlight = false;
+
+private:
+ enum EFailState {
+ OK,
+ LIMITER_FAILED,
+ NAVIGATE_FAILED
+ } FailState = OK;
+
+ TIntrusivePtr<TScanLimiter> ScanLimiter;
+ bool AllowedByLimiter = false;
+};
+
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/common/schema.cpp b/ydb/core/sys_view/common/schema.cpp
index 1917ea35a49..a83113078fb 100644
--- a/ydb/core/sys_view/common/schema.cpp
+++ b/ydb/core/sys_view/common/schema.cpp
@@ -1,91 +1,91 @@
-#include "schema.h"
-
+#include "schema.h"
+
#include <ydb/core/base/appdata.h>
-
-namespace NKikimr {
-namespace NSysView {
-
-bool MaybeSystemViewPath(const TVector<TString>& path) {
- auto length = path.size();
- // minimal system view path should be /Root/.sys/view
- // only one level after ".sys" is allowed at the moment
- if (length < 3 || path[length - 2] != SysPathName) {
- return false;
- }
- return true;
-}
-
-bool MaybeSystemViewFolderPath(const TVector<TString>& path) {
- if (path.size() < 2 || path.back() != SysPathName) {
- return false;
- }
- return true;
-}
-
-class TSystemViewResolver : public ISystemViewResolver {
-public:
- TSystemViewResolver() {
- RegisterSystemViews();
- }
-
- bool IsSystemViewPath(const TVector<TString>& path, TSystemViewPath& sysViewPath) const override final {
- if (MaybeSystemViewFolderPath(path)) {
- TVector<TString> realPath(path.begin(), path.end() - 1);
- sysViewPath.Parent = std::move(realPath);
- sysViewPath.ViewName = SysPathName;
- return true;
-
- } else if (MaybeSystemViewPath(path)) {
- auto maybeSystemViewName = path.back();
- if (!DomainSystemViews.contains(maybeSystemViewName) &&
+
+namespace NKikimr {
+namespace NSysView {
+
+bool MaybeSystemViewPath(const TVector<TString>& path) {
+ auto length = path.size();
+ // minimal system view path should be /Root/.sys/view
+ // only one level after ".sys" is allowed at the moment
+ if (length < 3 || path[length - 2] != SysPathName) {
+ return false;
+ }
+ return true;
+}
+
+bool MaybeSystemViewFolderPath(const TVector<TString>& path) {
+ if (path.size() < 2 || path.back() != SysPathName) {
+ return false;
+ }
+ return true;
+}
+
+class TSystemViewResolver : public ISystemViewResolver {
+public:
+ TSystemViewResolver() {
+ RegisterSystemViews();
+ }
+
+ bool IsSystemViewPath(const TVector<TString>& path, TSystemViewPath& sysViewPath) const override final {
+ if (MaybeSystemViewFolderPath(path)) {
+ TVector<TString> realPath(path.begin(), path.end() - 1);
+ sysViewPath.Parent = std::move(realPath);
+ sysViewPath.ViewName = SysPathName;
+ return true;
+
+ } else if (MaybeSystemViewPath(path)) {
+ auto maybeSystemViewName = path.back();
+ if (!DomainSystemViews.contains(maybeSystemViewName) &&
!SubDomainSystemViews.contains(maybeSystemViewName) &&
!OlapStoreSystemViews.contains(maybeSystemViewName) &&
!OlapTableSystemViews.contains(maybeSystemViewName))
- {
- return false;
- }
- TVector<TString> realPath(path.begin(), path.end() - 2);
- sysViewPath.Parent = std::move(realPath);
- sysViewPath.ViewName = path.back();
- return true;
- }
- return false;
- }
-
- TMaybe<TSchema> GetSystemViewSchema(const TStringBuf viewName, ETarget target) const override final {
- const TSchema* view = nullptr;
- switch (target) {
- case ETarget::Domain:
- view = DomainSystemViews.FindPtr(viewName);
- break;
- case ETarget::SubDomain:
- view = SubDomainSystemViews.FindPtr(viewName);
- break;
+ {
+ return false;
+ }
+ TVector<TString> realPath(path.begin(), path.end() - 2);
+ sysViewPath.Parent = std::move(realPath);
+ sysViewPath.ViewName = path.back();
+ return true;
+ }
+ return false;
+ }
+
+ TMaybe<TSchema> GetSystemViewSchema(const TStringBuf viewName, ETarget target) const override final {
+ const TSchema* view = nullptr;
+ switch (target) {
+ case ETarget::Domain:
+ view = DomainSystemViews.FindPtr(viewName);
+ break;
+ case ETarget::SubDomain:
+ view = SubDomainSystemViews.FindPtr(viewName);
+ break;
case ETarget::OlapStore:
view = OlapStoreSystemViews.FindPtr(viewName);
break;
case ETarget::OlapTable:
view = OlapTableSystemViews.FindPtr(viewName);
break;
- }
- return view ? TMaybe<TSchema>(*view) : Nothing();
- }
-
- TVector<TString> GetSystemViewNames(ETarget target) const override {
- TVector<TString> result;
- switch (target) {
- case ETarget::Domain:
- result.reserve(DomainSystemViews.size());
- for (const auto& [name, _] : DomainSystemViews) {
- result.push_back(name);
- }
- break;
- case ETarget::SubDomain:
- result.reserve(SubDomainSystemViews.size());
- for (const auto& [name, _] : SubDomainSystemViews) {
- result.push_back(name);
- }
- break;
+ }
+ return view ? TMaybe<TSchema>(*view) : Nothing();
+ }
+
+ TVector<TString> GetSystemViewNames(ETarget target) const override {
+ TVector<TString> result;
+ switch (target) {
+ case ETarget::Domain:
+ result.reserve(DomainSystemViews.size());
+ for (const auto& [name, _] : DomainSystemViews) {
+ result.push_back(name);
+ }
+ break;
+ case ETarget::SubDomain:
+ result.reserve(SubDomainSystemViews.size());
+ for (const auto& [name, _] : SubDomainSystemViews) {
+ result.push_back(name);
+ }
+ break;
case ETarget::OlapStore:
result.reserve(OlapStoreSystemViews.size());
for (const auto& [name, _] : OlapStoreSystemViews) {
@@ -98,87 +98,87 @@ public:
result.push_back(name);
}
break;
- }
- return result;
- }
-
-private:
- template <typename Table>
- struct TSchemaFiller {
-
- template <typename...>
- struct TFiller;
-
- template <typename Column>
- struct TFiller<Column> {
- static void Fill(TSchema& schema) {
- schema.Columns[Column::ColumnId] = TSysTables::TTableColumnInfo(
- Table::template TableColumns<Column>::GetColumnName(),
- Column::ColumnId, Column::ColumnType, -1);
- }
- };
-
- template <typename Column, typename... Columns>
- struct TFiller<Column, Columns...> {
- static void Fill(TSchema& schema) {
- TFiller<Column>::Fill(schema);
- TFiller<Columns...>::Fill(schema);
- }
- };
-
- template <typename... Columns>
- using TColumnsType = typename Table::template TableColumns<Columns...>;
-
- template <typename... Columns>
- static void FillColumns(TSchema& schema, TColumnsType<Columns...>) {
- TFiller<Columns...>::Fill(schema);
- }
-
- template <typename...>
- struct TKeyFiller;
-
- template <typename Key>
- struct TKeyFiller<Key> {
- static void Fill(TSchema& schema, i32 index) {
- auto& column = schema.Columns[Key::ColumnId];
- column.KeyOrder = index;
- schema.KeyColumnTypes.push_back(column.PType);
- }
- };
-
- template <typename Key, typename... Keys>
- struct TKeyFiller<Key, Keys...> {
- static void Fill(TSchema& schema, i32 index) {
- TKeyFiller<Key>::Fill(schema, index);
- TKeyFiller<Keys...>::Fill(schema, index + 1);
- }
- };
-
- template <typename... Keys>
- using TKeysType = typename Table::template TableKey<Keys...>;
-
- template <typename... Keys>
- static void FillKeys(TSchema& schema, TKeysType<Keys...>) {
- TKeyFiller<Keys...>::Fill(schema, 0);
- }
-
- static void Fill(TSchema& schema) {
- FillColumns(schema, typename Table::TColumns());
- FillKeys(schema, typename Table::TKey());
- }
- };
-
- template <typename Table>
- void RegisterSystemView(const TStringBuf& name) {
- TSchemaFiller<Table>::Fill(DomainSystemViews[name]);
- TSchemaFiller<Table>::Fill(SubDomainSystemViews[name]);
- }
-
- template <typename Table>
- void RegisterDomainSystemView(const TStringBuf& name) {
- TSchemaFiller<Table>::Fill(DomainSystemViews[name]);
- }
-
+ }
+ return result;
+ }
+
+private:
+ template <typename Table>
+ struct TSchemaFiller {
+
+ template <typename...>
+ struct TFiller;
+
+ template <typename Column>
+ struct TFiller<Column> {
+ static void Fill(TSchema& schema) {
+ schema.Columns[Column::ColumnId] = TSysTables::TTableColumnInfo(
+ Table::template TableColumns<Column>::GetColumnName(),
+ Column::ColumnId, Column::ColumnType, -1);
+ }
+ };
+
+ template <typename Column, typename... Columns>
+ struct TFiller<Column, Columns...> {
+ static void Fill(TSchema& schema) {
+ TFiller<Column>::Fill(schema);
+ TFiller<Columns...>::Fill(schema);
+ }
+ };
+
+ template <typename... Columns>
+ using TColumnsType = typename Table::template TableColumns<Columns...>;
+
+ template <typename... Columns>
+ static void FillColumns(TSchema& schema, TColumnsType<Columns...>) {
+ TFiller<Columns...>::Fill(schema);
+ }
+
+ template <typename...>
+ struct TKeyFiller;
+
+ template <typename Key>
+ struct TKeyFiller<Key> {
+ static void Fill(TSchema& schema, i32 index) {
+ auto& column = schema.Columns[Key::ColumnId];
+ column.KeyOrder = index;
+ schema.KeyColumnTypes.push_back(column.PType);
+ }
+ };
+
+ template <typename Key, typename... Keys>
+ struct TKeyFiller<Key, Keys...> {
+ static void Fill(TSchema& schema, i32 index) {
+ TKeyFiller<Key>::Fill(schema, index);
+ TKeyFiller<Keys...>::Fill(schema, index + 1);
+ }
+ };
+
+ template <typename... Keys>
+ using TKeysType = typename Table::template TableKey<Keys...>;
+
+ template <typename... Keys>
+ static void FillKeys(TSchema& schema, TKeysType<Keys...>) {
+ TKeyFiller<Keys...>::Fill(schema, 0);
+ }
+
+ static void Fill(TSchema& schema) {
+ FillColumns(schema, typename Table::TColumns());
+ FillKeys(schema, typename Table::TKey());
+ }
+ };
+
+ template <typename Table>
+ void RegisterSystemView(const TStringBuf& name) {
+ TSchemaFiller<Table>::Fill(DomainSystemViews[name]);
+ TSchemaFiller<Table>::Fill(SubDomainSystemViews[name]);
+ }
+
+ template <typename Table>
+ void RegisterDomainSystemView(const TStringBuf& name) {
+ TSchemaFiller<Table>::Fill(DomainSystemViews[name]);
+ }
+
template <typename Table>
void RegisterOlapStoreSystemView(const TStringBuf& name) {
TSchemaFiller<Table>::Fill(OlapStoreSystemViews[name]);
@@ -189,45 +189,45 @@ private:
TSchemaFiller<Table>::Fill(OlapTableSystemViews[name]);
}
- void RegisterSystemViews() {
- RegisterSystemView<Schema::PartitionStats>(PartitionStatsName);
-
- // 'nodes' table is currently switched off
- // RegisterSystemView<Schema::Nodes>(NodesName);
-
- RegisterSystemView<Schema::QueryStats>(TopQueriesByDuration1MinuteName);
- RegisterSystemView<Schema::QueryStats>(TopQueriesByDuration1HourName);
- RegisterSystemView<Schema::QueryStats>(TopQueriesByReadBytes1MinuteName);
- RegisterSystemView<Schema::QueryStats>(TopQueriesByReadBytes1HourName);
- RegisterSystemView<Schema::QueryStats>(TopQueriesByCpuTime1MinuteName);
- RegisterSystemView<Schema::QueryStats>(TopQueriesByCpuTime1HourName);
- RegisterSystemView<Schema::QueryStats>(TopQueriesByRequestUnits1MinuteName);
- RegisterSystemView<Schema::QueryStats>(TopQueriesByRequestUnits1HourName);
-
- RegisterDomainSystemView<Schema::PDisks>(PDisksName);
- RegisterDomainSystemView<Schema::VSlots>(VSlotsName);
- RegisterDomainSystemView<Schema::Groups>(GroupsName);
- RegisterDomainSystemView<Schema::StoragePools>(StoragePoolsName);
+ void RegisterSystemViews() {
+ RegisterSystemView<Schema::PartitionStats>(PartitionStatsName);
+
+ // 'nodes' table is currently switched off
+ // RegisterSystemView<Schema::Nodes>(NodesName);
+
+ RegisterSystemView<Schema::QueryStats>(TopQueriesByDuration1MinuteName);
+ RegisterSystemView<Schema::QueryStats>(TopQueriesByDuration1HourName);
+ RegisterSystemView<Schema::QueryStats>(TopQueriesByReadBytes1MinuteName);
+ RegisterSystemView<Schema::QueryStats>(TopQueriesByReadBytes1HourName);
+ RegisterSystemView<Schema::QueryStats>(TopQueriesByCpuTime1MinuteName);
+ RegisterSystemView<Schema::QueryStats>(TopQueriesByCpuTime1HourName);
+ RegisterSystemView<Schema::QueryStats>(TopQueriesByRequestUnits1MinuteName);
+ RegisterSystemView<Schema::QueryStats>(TopQueriesByRequestUnits1HourName);
+
+ RegisterDomainSystemView<Schema::PDisks>(PDisksName);
+ RegisterDomainSystemView<Schema::VSlots>(VSlotsName);
+ RegisterDomainSystemView<Schema::Groups>(GroupsName);
+ RegisterDomainSystemView<Schema::StoragePools>(StoragePoolsName);
RegisterDomainSystemView<Schema::StorageStats>(StorageStatsName);
-
- RegisterDomainSystemView<Schema::Tablets>(TabletsName);
-
- RegisterSystemView<Schema::QueryMetrics>(QueryMetricsName);
+
+ RegisterDomainSystemView<Schema::Tablets>(TabletsName);
+
+ RegisterSystemView<Schema::QueryMetrics>(QueryMetricsName);
RegisterOlapStoreSystemView<Schema::PrimaryIndexStats>(StorePrimaryIndexStatsName);
RegisterOlapTableSystemView<Schema::PrimaryIndexStats>(TablePrimaryIndexStatsName);
- }
-
-private:
- THashMap<TString, TSchema> DomainSystemViews;
- THashMap<TString, TSchema> SubDomainSystemViews;
+ }
+
+private:
+ THashMap<TString, TSchema> DomainSystemViews;
+ THashMap<TString, TSchema> SubDomainSystemViews;
THashMap<TString, TSchema> OlapStoreSystemViews;
THashMap<TString, TSchema> OlapTableSystemViews;
-};
-
-ISystemViewResolver* CreateSystemViewResolver() {
- return new TSystemViewResolver();
-}
-
-} // NSysView
-} // NKikimr
+};
+
+ISystemViewResolver* CreateSystemViewResolver() {
+ return new TSystemViewResolver();
+}
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/common/schema.h b/ydb/core/sys_view/common/schema.h
index f7a4f191e13..96f15695a06 100644
--- a/ydb/core/sys_view/common/schema.h
+++ b/ydb/core/sys_view/common/schema.h
@@ -1,384 +1,384 @@
-#pragma once
-
-#include "path.h"
-
+#pragma once
+
+#include "path.h"
+
#include <ydb/core/tablet_flat/flat_cxx_database.h>
#include <ydb/core/tx/datashard/sys_tables.h>
-
-namespace NKikimr {
-namespace NSysView {
-
+
+namespace NKikimr {
+namespace NSysView {
+
constexpr TStringBuf PartitionStatsName = "partition_stats";
constexpr TStringBuf NodesName = "nodes";
-
+
constexpr TStringBuf TopQueriesByDuration1MinuteName = "top_queries_by_duration_one_minute";
constexpr TStringBuf TopQueriesByDuration1HourName = "top_queries_by_duration_one_hour";
constexpr TStringBuf TopQueriesByReadBytes1MinuteName = "top_queries_by_read_bytes_one_minute";
constexpr TStringBuf TopQueriesByReadBytes1HourName = "top_queries_by_read_bytes_one_hour";
constexpr TStringBuf TopQueriesByCpuTime1MinuteName = "top_queries_by_cpu_time_one_minute";
constexpr TStringBuf TopQueriesByCpuTime1HourName = "top_queries_by_cpu_time_one_hour";
-constexpr TStringBuf TopQueriesByRequestUnits1MinuteName = "top_queries_by_request_units_one_minute";
-constexpr TStringBuf TopQueriesByRequestUnits1HourName = "top_queries_by_request_units_one_hour";
-
+constexpr TStringBuf TopQueriesByRequestUnits1MinuteName = "top_queries_by_request_units_one_minute";
+constexpr TStringBuf TopQueriesByRequestUnits1HourName = "top_queries_by_request_units_one_hour";
+
constexpr TStringBuf PDisksName = "ds_pdisks";
constexpr TStringBuf VSlotsName = "ds_vslots";
constexpr TStringBuf GroupsName = "ds_groups";
constexpr TStringBuf StoragePoolsName = "ds_storage_pools";
constexpr TStringBuf StorageStatsName = "ds_storage_stats";
-
+
constexpr TStringBuf TabletsName = "hive_tablets";
-
+
constexpr TStringBuf QueryMetricsName = "query_metrics_one_minute";
-
+
constexpr TStringBuf StorePrimaryIndexStatsName = "store_primary_index_stats";
constexpr TStringBuf TablePrimaryIndexStatsName = "primary_index_stats";
-struct Schema : NIceDb::Schema {
- struct PartitionStats : Table<1> {
- struct OwnerId : Column<1, NScheme::NTypeIds::Uint64> {};
- struct PathId : Column<2, NScheme::NTypeIds::Uint64> {};
- struct PartIdx : Column<3, NScheme::NTypeIds::Uint64> {};
- struct DataSize : Column<4, NScheme::NTypeIds::Uint64> {};
- struct RowCount : Column<5, NScheme::NTypeIds::Uint64> {};
- struct IndexSize : Column<6, NScheme::NTypeIds::Uint64> {};
- struct CPUCores : Column<7, NScheme::NTypeIds::Double> {};
- struct TabletId : Column<8, NScheme::NTypeIds::Uint64> {};
- struct Path : Column<9, NScheme::NTypeIds::Utf8> {};
- struct NodeId : Column<10, NScheme::NTypeIds::Uint32> {};
- struct StartTime : Column<11, NScheme::NTypeIds::Timestamp> {};
- struct AccessTime : Column<12, NScheme::NTypeIds::Timestamp> {};
- struct UpdateTime : Column<13, NScheme::NTypeIds::Timestamp> {};
- struct InFlightTxCount : Column<14, NScheme::NTypeIds::Uint32> {};
- struct RowUpdates : Column<15, NScheme::NTypeIds::Uint64> {};
- struct RowDeletes : Column<16, NScheme::NTypeIds::Uint64> {};
- struct RowReads : Column<17, NScheme::NTypeIds::Uint64> {};
- struct RangeReads : Column<18, NScheme::NTypeIds::Uint64> {};
- struct RangeReadRows : Column<19, NScheme::NTypeIds::Uint64> {};
- struct ImmediateTxCompleted : Column<20, NScheme::NTypeIds::Uint64> {};
- struct CoordinatedTxCompleted : Column<21, NScheme::NTypeIds::Uint64> {};
- struct TxRejectedByOverload : Column<22, NScheme::NTypeIds::Uint64> {};
- struct TxRejectedByOutOfStorage : Column<23, NScheme::NTypeIds::Uint64> {};
+struct Schema : NIceDb::Schema {
+ struct PartitionStats : Table<1> {
+ struct OwnerId : Column<1, NScheme::NTypeIds::Uint64> {};
+ struct PathId : Column<2, NScheme::NTypeIds::Uint64> {};
+ struct PartIdx : Column<3, NScheme::NTypeIds::Uint64> {};
+ struct DataSize : Column<4, NScheme::NTypeIds::Uint64> {};
+ struct RowCount : Column<5, NScheme::NTypeIds::Uint64> {};
+ struct IndexSize : Column<6, NScheme::NTypeIds::Uint64> {};
+ struct CPUCores : Column<7, NScheme::NTypeIds::Double> {};
+ struct TabletId : Column<8, NScheme::NTypeIds::Uint64> {};
+ struct Path : Column<9, NScheme::NTypeIds::Utf8> {};
+ struct NodeId : Column<10, NScheme::NTypeIds::Uint32> {};
+ struct StartTime : Column<11, NScheme::NTypeIds::Timestamp> {};
+ struct AccessTime : Column<12, NScheme::NTypeIds::Timestamp> {};
+ struct UpdateTime : Column<13, NScheme::NTypeIds::Timestamp> {};
+ struct InFlightTxCount : Column<14, NScheme::NTypeIds::Uint32> {};
+ struct RowUpdates : Column<15, NScheme::NTypeIds::Uint64> {};
+ struct RowDeletes : Column<16, NScheme::NTypeIds::Uint64> {};
+ struct RowReads : Column<17, NScheme::NTypeIds::Uint64> {};
+ struct RangeReads : Column<18, NScheme::NTypeIds::Uint64> {};
+ struct RangeReadRows : Column<19, NScheme::NTypeIds::Uint64> {};
+ struct ImmediateTxCompleted : Column<20, NScheme::NTypeIds::Uint64> {};
+ struct CoordinatedTxCompleted : Column<21, NScheme::NTypeIds::Uint64> {};
+ struct TxRejectedByOverload : Column<22, NScheme::NTypeIds::Uint64> {};
+ struct TxRejectedByOutOfStorage : Column<23, NScheme::NTypeIds::Uint64> {};
struct LastTtlRunTime : Column<24, NScheme::NTypeIds::Timestamp> {};
struct LastTtlRowsProcessed : Column<25, NScheme::NTypeIds::Uint64> {};
struct LastTtlRowsErased : Column<26, NScheme::NTypeIds::Uint64> {};
-
- using TKey = TableKey<OwnerId, PathId, PartIdx>;
- using TColumns = TableColumns<
- OwnerId,
- PathId,
- PartIdx,
- DataSize,
- RowCount,
- IndexSize,
- CPUCores,
- TabletId,
- Path,
- NodeId,
- AccessTime,
- UpdateTime,
- StartTime,
- InFlightTxCount,
- RowUpdates,
- RowDeletes,
- RowReads,
- RangeReads,
- RangeReadRows,
- ImmediateTxCompleted,
- CoordinatedTxCompleted,
- TxRejectedByOverload,
+
+ using TKey = TableKey<OwnerId, PathId, PartIdx>;
+ using TColumns = TableColumns<
+ OwnerId,
+ PathId,
+ PartIdx,
+ DataSize,
+ RowCount,
+ IndexSize,
+ CPUCores,
+ TabletId,
+ Path,
+ NodeId,
+ AccessTime,
+ UpdateTime,
+ StartTime,
+ InFlightTxCount,
+ RowUpdates,
+ RowDeletes,
+ RowReads,
+ RangeReads,
+ RangeReadRows,
+ ImmediateTxCompleted,
+ CoordinatedTxCompleted,
+ TxRejectedByOverload,
TxRejectedByOutOfStorage,
LastTtlRunTime,
LastTtlRowsProcessed,
LastTtlRowsErased>;
- };
-
- struct Nodes : Table<2> {
- struct NodeId : Column<1, NScheme::NTypeIds::Uint32> {};
- struct Address : Column<2, NScheme::NTypeIds::String> {};
- struct Host : Column<3, NScheme::NTypeIds::String> {};
- struct Port : Column<4, NScheme::NTypeIds::Uint32> {};
- struct StartTime : Column<5, NScheme::NTypeIds::Timestamp> {};
- struct UpTime : Column<6, NScheme::NTypeIds::Interval> {};
-
- using TKey = TableKey<NodeId>;
- using TColumns = TableColumns<
- NodeId,
- Host,
- Address,
- Port,
- StartTime,
- UpTime>;
- };
-
- struct QueryStats : Table<3> {
- struct IntervalEnd : Column<1, NScheme::NTypeIds::Timestamp> {};
- struct Rank : Column<2, NScheme::NTypeIds::Uint32> {};
- struct QueryText : Column<3, NScheme::NTypeIds::Utf8> {};
- struct Duration : Column<4, NScheme::NTypeIds::Interval> {};
- struct EndTime : Column<5, NScheme::NTypeIds::Timestamp> {};
- struct ReadRows : Column<6, NScheme::NTypeIds::Uint64> {};
- struct ReadBytes : Column<7, NScheme::NTypeIds::Uint64> {};
- struct UpdateRows : Column<8, NScheme::NTypeIds::Uint64> {};
- struct UpdateBytes : Column<9, NScheme::NTypeIds::Uint64> {};
- struct DeleteRows : Column<10, NScheme::NTypeIds::Uint64> {};
- struct DeleteBytes : Column<11, NScheme::NTypeIds::Uint64> {}; // deprecated, always 0
- struct Partitions : Column<12, NScheme::NTypeIds::Uint64> {};
- struct UserSID : Column<13, NScheme::NTypeIds::String> {};
- struct ParametersSize : Column<14, NScheme::NTypeIds::Uint64> {};
- struct CompileDuration : Column<15, NScheme::NTypeIds::Interval> {};
- struct FromQueryCache : Column<16, NScheme::NTypeIds::Bool> {};
- struct CPUTime : Column<17, NScheme::NTypeIds::Uint64> {};
- struct ShardCount : Column<18, NScheme::NTypeIds::Uint64> {};
- struct SumShardCPUTime : Column<19, NScheme::NTypeIds::Uint64> {};
- struct MinShardCPUTime : Column<20, NScheme::NTypeIds::Uint64> {};
- struct MaxShardCPUTime : Column<21, NScheme::NTypeIds::Uint64> {};
- struct ComputeNodesCount : Column<22, NScheme::NTypeIds::Uint64> {};
- struct SumComputeCPUTime : Column<23, NScheme::NTypeIds::Uint64> {};
- struct MinComputeCPUTime : Column<24, NScheme::NTypeIds::Uint64> {};
- struct MaxComputeCPUTime : Column<25, NScheme::NTypeIds::Uint64> {};
- struct CompileCPUTime : Column<26, NScheme::NTypeIds::Uint64> {};
- struct ProcessCPUTime : Column<27, NScheme::NTypeIds::Uint64> {};
- struct TypeCol : Column<28, NScheme::NTypeIds::String> { static TString GetColumnName(const TString&) { return "Type"; } };
- struct RequestUnits : Column<29, NScheme::NTypeIds::Uint64> {};
-
- using TKey = TableKey<IntervalEnd, Rank>;
- using TColumns = TableColumns<
- IntervalEnd,
- Rank,
- QueryText,
- Duration,
- EndTime,
- ReadRows,
- ReadBytes,
- UpdateRows,
- UpdateBytes,
- DeleteRows,
- DeleteBytes,
- Partitions,
- UserSID,
- ParametersSize,
- CompileDuration,
- FromQueryCache,
- CPUTime,
- ShardCount,
- SumShardCPUTime,
- MinShardCPUTime,
- MaxShardCPUTime,
- ComputeNodesCount,
- SumComputeCPUTime,
- MinComputeCPUTime,
- MaxComputeCPUTime,
- CompileCPUTime,
- ProcessCPUTime,
- TypeCol,
- RequestUnits>;
- };
-
- struct PDisks : Table<4> {
- struct NodeId : Column<1, NScheme::NTypeIds::Uint32> {};
- struct PDiskId : Column<2, NScheme::NTypeIds::Uint32> {};
- struct TypeCol : Column<3, NScheme::NTypeIds::String> { static TString GetColumnName(const TString&) { return "Type"; } };
- struct Kind : Column<4, NScheme::NTypeIds::Uint64> {};
- struct Path : Column<5, NScheme::NTypeIds::String> {};
- struct Guid : Column<6, NScheme::NTypeIds::Uint64> {};
- struct BoxId : Column<7, NScheme::NTypeIds::Uint32> {};
- struct SharedWithOS : Column<8, NScheme::NTypeIds::Bool> {};
- struct ReadCentric : Column<9, NScheme::NTypeIds::Bool> {};
- struct AvailableSize : Column<10, NScheme::NTypeIds::Uint64> {};
- struct TotalSize : Column<11, NScheme::NTypeIds::Uint64> {};
- struct Status : Column<12, NScheme::NTypeIds::String> {};
+ };
+
+ struct Nodes : Table<2> {
+ struct NodeId : Column<1, NScheme::NTypeIds::Uint32> {};
+ struct Address : Column<2, NScheme::NTypeIds::String> {};
+ struct Host : Column<3, NScheme::NTypeIds::String> {};
+ struct Port : Column<4, NScheme::NTypeIds::Uint32> {};
+ struct StartTime : Column<5, NScheme::NTypeIds::Timestamp> {};
+ struct UpTime : Column<6, NScheme::NTypeIds::Interval> {};
+
+ using TKey = TableKey<NodeId>;
+ using TColumns = TableColumns<
+ NodeId,
+ Host,
+ Address,
+ Port,
+ StartTime,
+ UpTime>;
+ };
+
+ struct QueryStats : Table<3> {
+ struct IntervalEnd : Column<1, NScheme::NTypeIds::Timestamp> {};
+ struct Rank : Column<2, NScheme::NTypeIds::Uint32> {};
+ struct QueryText : Column<3, NScheme::NTypeIds::Utf8> {};
+ struct Duration : Column<4, NScheme::NTypeIds::Interval> {};
+ struct EndTime : Column<5, NScheme::NTypeIds::Timestamp> {};
+ struct ReadRows : Column<6, NScheme::NTypeIds::Uint64> {};
+ struct ReadBytes : Column<7, NScheme::NTypeIds::Uint64> {};
+ struct UpdateRows : Column<8, NScheme::NTypeIds::Uint64> {};
+ struct UpdateBytes : Column<9, NScheme::NTypeIds::Uint64> {};
+ struct DeleteRows : Column<10, NScheme::NTypeIds::Uint64> {};
+ struct DeleteBytes : Column<11, NScheme::NTypeIds::Uint64> {}; // deprecated, always 0
+ struct Partitions : Column<12, NScheme::NTypeIds::Uint64> {};
+ struct UserSID : Column<13, NScheme::NTypeIds::String> {};
+ struct ParametersSize : Column<14, NScheme::NTypeIds::Uint64> {};
+ struct CompileDuration : Column<15, NScheme::NTypeIds::Interval> {};
+ struct FromQueryCache : Column<16, NScheme::NTypeIds::Bool> {};
+ struct CPUTime : Column<17, NScheme::NTypeIds::Uint64> {};
+ struct ShardCount : Column<18, NScheme::NTypeIds::Uint64> {};
+ struct SumShardCPUTime : Column<19, NScheme::NTypeIds::Uint64> {};
+ struct MinShardCPUTime : Column<20, NScheme::NTypeIds::Uint64> {};
+ struct MaxShardCPUTime : Column<21, NScheme::NTypeIds::Uint64> {};
+ struct ComputeNodesCount : Column<22, NScheme::NTypeIds::Uint64> {};
+ struct SumComputeCPUTime : Column<23, NScheme::NTypeIds::Uint64> {};
+ struct MinComputeCPUTime : Column<24, NScheme::NTypeIds::Uint64> {};
+ struct MaxComputeCPUTime : Column<25, NScheme::NTypeIds::Uint64> {};
+ struct CompileCPUTime : Column<26, NScheme::NTypeIds::Uint64> {};
+ struct ProcessCPUTime : Column<27, NScheme::NTypeIds::Uint64> {};
+ struct TypeCol : Column<28, NScheme::NTypeIds::String> { static TString GetColumnName(const TString&) { return "Type"; } };
+ struct RequestUnits : Column<29, NScheme::NTypeIds::Uint64> {};
+
+ using TKey = TableKey<IntervalEnd, Rank>;
+ using TColumns = TableColumns<
+ IntervalEnd,
+ Rank,
+ QueryText,
+ Duration,
+ EndTime,
+ ReadRows,
+ ReadBytes,
+ UpdateRows,
+ UpdateBytes,
+ DeleteRows,
+ DeleteBytes,
+ Partitions,
+ UserSID,
+ ParametersSize,
+ CompileDuration,
+ FromQueryCache,
+ CPUTime,
+ ShardCount,
+ SumShardCPUTime,
+ MinShardCPUTime,
+ MaxShardCPUTime,
+ ComputeNodesCount,
+ SumComputeCPUTime,
+ MinComputeCPUTime,
+ MaxComputeCPUTime,
+ CompileCPUTime,
+ ProcessCPUTime,
+ TypeCol,
+ RequestUnits>;
+ };
+
+ struct PDisks : Table<4> {
+ struct NodeId : Column<1, NScheme::NTypeIds::Uint32> {};
+ struct PDiskId : Column<2, NScheme::NTypeIds::Uint32> {};
+ struct TypeCol : Column<3, NScheme::NTypeIds::String> { static TString GetColumnName(const TString&) { return "Type"; } };
+ struct Kind : Column<4, NScheme::NTypeIds::Uint64> {};
+ struct Path : Column<5, NScheme::NTypeIds::String> {};
+ struct Guid : Column<6, NScheme::NTypeIds::Uint64> {};
+ struct BoxId : Column<7, NScheme::NTypeIds::Uint32> {};
+ struct SharedWithOS : Column<8, NScheme::NTypeIds::Bool> {};
+ struct ReadCentric : Column<9, NScheme::NTypeIds::Bool> {};
+ struct AvailableSize : Column<10, NScheme::NTypeIds::Uint64> {};
+ struct TotalSize : Column<11, NScheme::NTypeIds::Uint64> {};
+ struct Status : Column<12, NScheme::NTypeIds::String> {};
//struct StopFactor : Column<13, NScheme::NTypeIds::Double> {};
struct StatusChangeTimestamp : Column<14, NScheme::NTypeIds::Timestamp> {};
struct ExpectedSlotCount : Column<15, NScheme::NTypeIds::Uint32> {};
struct NumActiveSlots : Column<16, NScheme::NTypeIds::Uint32> {};
-
- using TKey = TableKey<NodeId, PDiskId>;
- using TColumns = TableColumns<
- NodeId,
- PDiskId,
- TypeCol,
- Kind,
- Path,
- Guid,
- BoxId,
- SharedWithOS,
- ReadCentric,
- AvailableSize,
- TotalSize,
+
+ using TKey = TableKey<NodeId, PDiskId>;
+ using TColumns = TableColumns<
+ NodeId,
+ PDiskId,
+ TypeCol,
+ Kind,
+ Path,
+ Guid,
+ BoxId,
+ SharedWithOS,
+ ReadCentric,
+ AvailableSize,
+ TotalSize,
Status,
StatusChangeTimestamp,
ExpectedSlotCount,
NumActiveSlots>;
- };
-
- struct VSlots : Table<5> {
- struct NodeId : Column<1, NScheme::NTypeIds::Uint32> {};
- struct PDiskId : Column<2, NScheme::NTypeIds::Uint32> {};
- struct VSlotId : Column<3, NScheme::NTypeIds::Uint32> {};
- //struct Category : Column<4, NScheme::NTypeIds::Uint64> {};
- struct GroupId : Column<5, NScheme::NTypeIds::Uint32> {};
- struct GroupGeneration : Column<6, NScheme::NTypeIds::Uint32> {};
- //struct Ring : Column<7, NScheme::NTypeIds::Uint32> {};
- struct FailDomain : Column<8, NScheme::NTypeIds::Uint32> {};
- struct VDisk : Column<9, NScheme::NTypeIds::Uint32> {};
- struct AllocatedSize : Column<10, NScheme::NTypeIds::Uint64> {};
- struct AvailableSize : Column<11, NScheme::NTypeIds::Uint64> {};
- struct Status : Column<12, NScheme::NTypeIds::String> {};
+ };
+
+ struct VSlots : Table<5> {
+ struct NodeId : Column<1, NScheme::NTypeIds::Uint32> {};
+ struct PDiskId : Column<2, NScheme::NTypeIds::Uint32> {};
+ struct VSlotId : Column<3, NScheme::NTypeIds::Uint32> {};
+ //struct Category : Column<4, NScheme::NTypeIds::Uint64> {};
+ struct GroupId : Column<5, NScheme::NTypeIds::Uint32> {};
+ struct GroupGeneration : Column<6, NScheme::NTypeIds::Uint32> {};
+ //struct Ring : Column<7, NScheme::NTypeIds::Uint32> {};
+ struct FailDomain : Column<8, NScheme::NTypeIds::Uint32> {};
+ struct VDisk : Column<9, NScheme::NTypeIds::Uint32> {};
+ struct AllocatedSize : Column<10, NScheme::NTypeIds::Uint64> {};
+ struct AvailableSize : Column<11, NScheme::NTypeIds::Uint64> {};
+ struct Status : Column<12, NScheme::NTypeIds::String> {};
//struct StopFactor : Column<13, NScheme::NTypeIds::Double> {};
- struct Kind : Column<14, NScheme::NTypeIds::String> {};
- struct FailRealm : Column<15, NScheme::NTypeIds::Uint32> {};
-
- using TKey = TableKey<NodeId, PDiskId, VSlotId>;
- using TColumns = TableColumns<
- NodeId,
- PDiskId,
- VSlotId,
- GroupId,
- GroupGeneration,
- FailDomain,
- VDisk,
- AllocatedSize,
- AvailableSize,
- Status,
- Kind,
- FailRealm>;
- };
-
- struct Groups : Table<6> {
- struct GroupId : Column<1, NScheme::NTypeIds::Uint32> {};
- struct Generation : Column<2, NScheme::NTypeIds::Uint32> {};
- struct ErasureSpecies : Column<3, NScheme::NTypeIds::String> {};
- struct BoxId : Column<4, NScheme::NTypeIds::Uint64> {};
- struct StoragePoolId : Column<5, NScheme::NTypeIds::Uint64> {};
- struct EncryptionMode : Column<6, NScheme::NTypeIds::Uint32> {};
- struct LifeCyclePhase : Column<7, NScheme::NTypeIds::Uint32> {};
- struct AllocatedSize : Column<8, NScheme::NTypeIds::Uint64> {};
- struct AvailableSize : Column<9, NScheme::NTypeIds::Uint64> {};
+ struct Kind : Column<14, NScheme::NTypeIds::String> {};
+ struct FailRealm : Column<15, NScheme::NTypeIds::Uint32> {};
+
+ using TKey = TableKey<NodeId, PDiskId, VSlotId>;
+ using TColumns = TableColumns<
+ NodeId,
+ PDiskId,
+ VSlotId,
+ GroupId,
+ GroupGeneration,
+ FailDomain,
+ VDisk,
+ AllocatedSize,
+ AvailableSize,
+ Status,
+ Kind,
+ FailRealm>;
+ };
+
+ struct Groups : Table<6> {
+ struct GroupId : Column<1, NScheme::NTypeIds::Uint32> {};
+ struct Generation : Column<2, NScheme::NTypeIds::Uint32> {};
+ struct ErasureSpecies : Column<3, NScheme::NTypeIds::String> {};
+ struct BoxId : Column<4, NScheme::NTypeIds::Uint64> {};
+ struct StoragePoolId : Column<5, NScheme::NTypeIds::Uint64> {};
+ struct EncryptionMode : Column<6, NScheme::NTypeIds::Uint32> {};
+ struct LifeCyclePhase : Column<7, NScheme::NTypeIds::Uint32> {};
+ struct AllocatedSize : Column<8, NScheme::NTypeIds::Uint64> {};
+ struct AvailableSize : Column<9, NScheme::NTypeIds::Uint64> {};
//struct Usage : Column<10, NScheme::NTypeIds::Double> {};
//struct StopFactor : Column<11, NScheme::NTypeIds::Double> {};
- struct SeenOperational : Column<12, NScheme::NTypeIds::Bool> {};
- struct PutTabletLogLatency : Column<13, NScheme::NTypeIds::Interval> {};
- struct PutUserDataLatency : Column<14, NScheme::NTypeIds::Interval> {};
- struct GetFastLatency : Column<15, NScheme::NTypeIds::Interval> {};
-
- using TKey = TableKey<GroupId>;
- using TColumns = TableColumns<
- GroupId,
- Generation,
- ErasureSpecies,
- BoxId,
- StoragePoolId,
- EncryptionMode,
- LifeCyclePhase,
- AllocatedSize,
- AvailableSize,
- SeenOperational,
- PutTabletLogLatency,
- PutUserDataLatency,
- GetFastLatency>;
- };
-
- struct StoragePools : Table<7> {
- struct BoxId : Column<1, NScheme::NTypeIds::Uint64> {};
- struct StoragePoolId : Column<2, NScheme::NTypeIds::Uint64> {};
- struct Name : Column<3, NScheme::NTypeIds::String> {};
- struct Generation : Column<4, NScheme::NTypeIds::Uint64> {};
- struct ErasureSpecies : Column<5, NScheme::NTypeIds::String> {};
- struct VDiskKind : Column<6, NScheme::NTypeIds::String> {};
- struct Kind : Column<7, NScheme::NTypeIds::String> {};
- struct NumGroups : Column<8, NScheme::NTypeIds::Uint32> {};
- struct EncryptionMode : Column<9, NScheme::NTypeIds::Uint32> {};
- struct SchemeshardId : Column<10, NScheme::NTypeIds::Uint64> {};
- struct PathId : Column<11, NScheme::NTypeIds::Uint64> {};
-
- using TKey = TableKey<BoxId, StoragePoolId>;
- using TColumns = TableColumns<
- BoxId,
- StoragePoolId,
- Name,
- Generation,
- ErasureSpecies,
- VDiskKind,
- Kind,
- NumGroups,
- EncryptionMode,
- SchemeshardId,
- PathId>;
- };
-
- struct Tablets : Table<8> {
- struct TabletId : Column<1, NScheme::NTypeIds::Uint64> {};
+ struct SeenOperational : Column<12, NScheme::NTypeIds::Bool> {};
+ struct PutTabletLogLatency : Column<13, NScheme::NTypeIds::Interval> {};
+ struct PutUserDataLatency : Column<14, NScheme::NTypeIds::Interval> {};
+ struct GetFastLatency : Column<15, NScheme::NTypeIds::Interval> {};
+
+ using TKey = TableKey<GroupId>;
+ using TColumns = TableColumns<
+ GroupId,
+ Generation,
+ ErasureSpecies,
+ BoxId,
+ StoragePoolId,
+ EncryptionMode,
+ LifeCyclePhase,
+ AllocatedSize,
+ AvailableSize,
+ SeenOperational,
+ PutTabletLogLatency,
+ PutUserDataLatency,
+ GetFastLatency>;
+ };
+
+ struct StoragePools : Table<7> {
+ struct BoxId : Column<1, NScheme::NTypeIds::Uint64> {};
+ struct StoragePoolId : Column<2, NScheme::NTypeIds::Uint64> {};
+ struct Name : Column<3, NScheme::NTypeIds::String> {};
+ struct Generation : Column<4, NScheme::NTypeIds::Uint64> {};
+ struct ErasureSpecies : Column<5, NScheme::NTypeIds::String> {};
+ struct VDiskKind : Column<6, NScheme::NTypeIds::String> {};
+ struct Kind : Column<7, NScheme::NTypeIds::String> {};
+ struct NumGroups : Column<8, NScheme::NTypeIds::Uint32> {};
+ struct EncryptionMode : Column<9, NScheme::NTypeIds::Uint32> {};
+ struct SchemeshardId : Column<10, NScheme::NTypeIds::Uint64> {};
+ struct PathId : Column<11, NScheme::NTypeIds::Uint64> {};
+
+ using TKey = TableKey<BoxId, StoragePoolId>;
+ using TColumns = TableColumns<
+ BoxId,
+ StoragePoolId,
+ Name,
+ Generation,
+ ErasureSpecies,
+ VDiskKind,
+ Kind,
+ NumGroups,
+ EncryptionMode,
+ SchemeshardId,
+ PathId>;
+ };
+
+ struct Tablets : Table<8> {
+ struct TabletId : Column<1, NScheme::NTypeIds::Uint64> {};
struct FollowerId : Column<2, NScheme::NTypeIds::Uint32> {};
- struct TypeCol : Column<3, NScheme::NTypeIds::String> { static TString GetColumnName(const TString&) { return "Type"; } };
- struct State : Column<4, NScheme::NTypeIds::String> {};
- struct VolatileState : Column<5, NScheme::NTypeIds::String> {};
- struct BootState : Column<6, NScheme::NTypeIds::String> {};
- struct Generation : Column<7, NScheme::NTypeIds::Uint32> {};
- struct NodeId : Column<8, NScheme::NTypeIds::Uint32> {};
- struct CPU : Column<9, NScheme::NTypeIds::Double> {};
- struct Memory : Column<10, NScheme::NTypeIds::Uint64> {};
- struct Network : Column<11, NScheme::NTypeIds::Uint64> {};
-
+ struct TypeCol : Column<3, NScheme::NTypeIds::String> { static TString GetColumnName(const TString&) { return "Type"; } };
+ struct State : Column<4, NScheme::NTypeIds::String> {};
+ struct VolatileState : Column<5, NScheme::NTypeIds::String> {};
+ struct BootState : Column<6, NScheme::NTypeIds::String> {};
+ struct Generation : Column<7, NScheme::NTypeIds::Uint32> {};
+ struct NodeId : Column<8, NScheme::NTypeIds::Uint32> {};
+ struct CPU : Column<9, NScheme::NTypeIds::Double> {};
+ struct Memory : Column<10, NScheme::NTypeIds::Uint64> {};
+ struct Network : Column<11, NScheme::NTypeIds::Uint64> {};
+
using TKey = TableKey<TabletId, FollowerId>;
- using TColumns = TableColumns<
- TabletId,
+ using TColumns = TableColumns<
+ TabletId,
FollowerId,
- TypeCol,
- State,
- VolatileState,
- BootState,
- Generation,
- NodeId,
- CPU,
- Memory,
- Network>;
- };
-
- struct QueryMetrics : Table<9> {
- struct IntervalEnd : Column<1, NScheme::NTypeIds::Timestamp> {};
- struct Rank : Column<2, NScheme::NTypeIds::Uint32> {};
- struct QueryText : Column<3, NScheme::NTypeIds::Utf8> {};
- struct Count : Column<4, NScheme::NTypeIds::Uint64> {};
- struct SumCPUTime : Column<5, NScheme::NTypeIds::Uint64> {};
- struct MinCPUTime : Column<6, NScheme::NTypeIds::Uint64> {};
- struct MaxCPUTime : Column<7, NScheme::NTypeIds::Uint64> {};
- struct SumDuration : Column<8, NScheme::NTypeIds::Interval> {};
- struct MinDuration : Column<9, NScheme::NTypeIds::Interval> {};
- struct MaxDuration : Column<10, NScheme::NTypeIds::Interval> {};
- struct MinReadRows : Column<11, NScheme::NTypeIds::Uint64> {};
- struct MaxReadRows : Column<12, NScheme::NTypeIds::Uint64> {};
- struct SumReadRows : Column<13, NScheme::NTypeIds::Uint64> {};
- struct MinReadBytes : Column<14, NScheme::NTypeIds::Uint64> {};
- struct MaxReadBytes : Column<15, NScheme::NTypeIds::Uint64> {};
- struct SumReadBytes : Column<16, NScheme::NTypeIds::Uint64> {};
- struct MinUpdateRows : Column<17, NScheme::NTypeIds::Uint64> {};
- struct MaxUpdateRows : Column<18, NScheme::NTypeIds::Uint64> {};
- struct SumUpdateRows : Column<19, NScheme::NTypeIds::Uint64> {};
- struct MinUpdateBytes : Column<20, NScheme::NTypeIds::Uint64> {};
- struct MaxUpdateBytes : Column<21, NScheme::NTypeIds::Uint64> {};
- struct SumUpdateBytes : Column<22, NScheme::NTypeIds::Uint64> {};
- struct MinDeleteRows : Column<23, NScheme::NTypeIds::Uint64> {};
- struct MaxDeleteRows : Column<24, NScheme::NTypeIds::Uint64> {};
- struct SumDeleteRows : Column<25, NScheme::NTypeIds::Uint64> {};
- struct MinRequestUnits: Column<26, NScheme::NTypeIds::Uint64> {};
- struct MaxRequestUnits: Column<27, NScheme::NTypeIds::Uint64> {};
- struct SumRequestUnits: Column<28, NScheme::NTypeIds::Uint64> {};
-
- using TKey = TableKey<IntervalEnd, Rank>;
- using TColumns = TableColumns<
- IntervalEnd,
- Rank,
- QueryText,
- Count,
- SumCPUTime, MinCPUTime, MaxCPUTime,
- SumDuration, MinDuration, MaxDuration,
- SumReadRows, MinReadRows, MaxReadRows,
- SumReadBytes, MinReadBytes, MaxReadBytes,
- SumUpdateRows, MinUpdateRows, MaxUpdateRows,
- SumUpdateBytes, MinUpdateBytes, MaxUpdateBytes,
- SumDeleteRows, MinDeleteRows, MaxDeleteRows,
- SumRequestUnits, MinRequestUnits, MaxRequestUnits>;
- };
+ TypeCol,
+ State,
+ VolatileState,
+ BootState,
+ Generation,
+ NodeId,
+ CPU,
+ Memory,
+ Network>;
+ };
+
+ struct QueryMetrics : Table<9> {
+ struct IntervalEnd : Column<1, NScheme::NTypeIds::Timestamp> {};
+ struct Rank : Column<2, NScheme::NTypeIds::Uint32> {};
+ struct QueryText : Column<3, NScheme::NTypeIds::Utf8> {};
+ struct Count : Column<4, NScheme::NTypeIds::Uint64> {};
+ struct SumCPUTime : Column<5, NScheme::NTypeIds::Uint64> {};
+ struct MinCPUTime : Column<6, NScheme::NTypeIds::Uint64> {};
+ struct MaxCPUTime : Column<7, NScheme::NTypeIds::Uint64> {};
+ struct SumDuration : Column<8, NScheme::NTypeIds::Interval> {};
+ struct MinDuration : Column<9, NScheme::NTypeIds::Interval> {};
+ struct MaxDuration : Column<10, NScheme::NTypeIds::Interval> {};
+ struct MinReadRows : Column<11, NScheme::NTypeIds::Uint64> {};
+ struct MaxReadRows : Column<12, NScheme::NTypeIds::Uint64> {};
+ struct SumReadRows : Column<13, NScheme::NTypeIds::Uint64> {};
+ struct MinReadBytes : Column<14, NScheme::NTypeIds::Uint64> {};
+ struct MaxReadBytes : Column<15, NScheme::NTypeIds::Uint64> {};
+ struct SumReadBytes : Column<16, NScheme::NTypeIds::Uint64> {};
+ struct MinUpdateRows : Column<17, NScheme::NTypeIds::Uint64> {};
+ struct MaxUpdateRows : Column<18, NScheme::NTypeIds::Uint64> {};
+ struct SumUpdateRows : Column<19, NScheme::NTypeIds::Uint64> {};
+ struct MinUpdateBytes : Column<20, NScheme::NTypeIds::Uint64> {};
+ struct MaxUpdateBytes : Column<21, NScheme::NTypeIds::Uint64> {};
+ struct SumUpdateBytes : Column<22, NScheme::NTypeIds::Uint64> {};
+ struct MinDeleteRows : Column<23, NScheme::NTypeIds::Uint64> {};
+ struct MaxDeleteRows : Column<24, NScheme::NTypeIds::Uint64> {};
+ struct SumDeleteRows : Column<25, NScheme::NTypeIds::Uint64> {};
+ struct MinRequestUnits: Column<26, NScheme::NTypeIds::Uint64> {};
+ struct MaxRequestUnits: Column<27, NScheme::NTypeIds::Uint64> {};
+ struct SumRequestUnits: Column<28, NScheme::NTypeIds::Uint64> {};
+
+ using TKey = TableKey<IntervalEnd, Rank>;
+ using TColumns = TableColumns<
+ IntervalEnd,
+ Rank,
+ QueryText,
+ Count,
+ SumCPUTime, MinCPUTime, MaxCPUTime,
+ SumDuration, MinDuration, MaxDuration,
+ SumReadRows, MinReadRows, MaxReadRows,
+ SumReadBytes, MinReadBytes, MaxReadBytes,
+ SumUpdateRows, MinUpdateRows, MaxUpdateRows,
+ SumUpdateBytes, MinUpdateBytes, MaxUpdateBytes,
+ SumDeleteRows, MinDeleteRows, MaxDeleteRows,
+ SumRequestUnits, MinRequestUnits, MaxRequestUnits>;
+ };
struct PrimaryIndexStats : Table<10> {
struct PathId : Column<1, NScheme::NTypeIds::Uint64> {};
@@ -418,40 +418,40 @@ struct Schema : NIceDb::Schema {
using TColumns = TableColumns<PDiskFilter, ErasureSpecies, CurrentGroupsCreated, CurrentAllocatedSize,
CurrentAvailableSize, AvailableGroupsToCreate, AvailableSizeToCreate>;
};
-};
-
-bool MaybeSystemViewPath(const TVector<TString>& path);
-bool MaybeSystemViewFolderPath(const TVector<TString>& path);
-
-class ISystemViewResolver {
-public:
- virtual ~ISystemViewResolver() = default;
-
- enum class ETarget : ui8 {
- Domain,
+};
+
+bool MaybeSystemViewPath(const TVector<TString>& path);
+bool MaybeSystemViewFolderPath(const TVector<TString>& path);
+
+class ISystemViewResolver {
+public:
+ virtual ~ISystemViewResolver() = default;
+
+ enum class ETarget : ui8 {
+ Domain,
SubDomain,
OlapStore,
OlapTable
- };
-
- struct TSystemViewPath {
- TVector<TString> Parent;
- TString ViewName;
- };
-
- struct TSchema {
- THashMap<NTable::TTag, TSysTables::TTableColumnInfo> Columns;
- TVector<NScheme::TTypeId> KeyColumnTypes;
- };
-
- virtual bool IsSystemViewPath(const TVector<TString>& path, TSystemViewPath& sysViewPath) const = 0;
-
- virtual TMaybe<TSchema> GetSystemViewSchema(const TStringBuf viewName, ETarget target) const = 0;
-
- virtual TVector<TString> GetSystemViewNames(ETarget target) const = 0;
-};
-
-ISystemViewResolver* CreateSystemViewResolver();
-
-} // NSysView
-} // NKikimr
+ };
+
+ struct TSystemViewPath {
+ TVector<TString> Parent;
+ TString ViewName;
+ };
+
+ struct TSchema {
+ THashMap<NTable::TTag, TSysTables::TTableColumnInfo> Columns;
+ TVector<NScheme::TTypeId> KeyColumnTypes;
+ };
+
+ virtual bool IsSystemViewPath(const TVector<TString>& path, TSystemViewPath& sysViewPath) const = 0;
+
+ virtual TMaybe<TSchema> GetSystemViewSchema(const TStringBuf viewName, ETarget target) const = 0;
+
+ virtual TVector<TString> GetSystemViewNames(ETarget target) const = 0;
+};
+
+ISystemViewResolver* CreateSystemViewResolver();
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/common/utils.h b/ydb/core/sys_view/common/utils.h
index 47a367f3486..799aa160755 100644
--- a/ydb/core/sys_view/common/utils.h
+++ b/ydb/core/sys_view/common/utils.h
@@ -1,40 +1,40 @@
-#pragma once
-
-#include <util/generic/ptr.h>
-#include <util/stream/output.h>
-
-namespace NKikimr {
-namespace NSysView {
-
-class TScanLimiter : public TThrRefBase {
-public:
- explicit TScanLimiter(i64 limit)
- : Limit(limit)
- {}
-
- bool Inc() {
- i64 newValue;
- i64 prevValue;
- do {
- prevValue = AtomicGet(Count);
- Y_VERIFY(prevValue >= 0);
- if (Limit && prevValue >= Limit) {
- return false;
- }
- newValue = prevValue + 1;
- } while (!AtomicCas(&Count, newValue, prevValue));
- return true;
- }
-
- void Dec() {
- i64 newValue = AtomicDecrement(Count);
- Y_VERIFY(newValue >= 0);
- }
-
-private:
- const i64 Limit;
- TAtomic Count = 0;
-};
-
-} // NSysView
-} // NKikimr
+#pragma once
+
+#include <util/generic/ptr.h>
+#include <util/stream/output.h>
+
+namespace NKikimr {
+namespace NSysView {
+
+class TScanLimiter : public TThrRefBase {
+public:
+ explicit TScanLimiter(i64 limit)
+ : Limit(limit)
+ {}
+
+ bool Inc() {
+ i64 newValue;
+ i64 prevValue;
+ do {
+ prevValue = AtomicGet(Count);
+ Y_VERIFY(prevValue >= 0);
+ if (Limit && prevValue >= Limit) {
+ return false;
+ }
+ newValue = prevValue + 1;
+ } while (!AtomicCas(&Count, newValue, prevValue));
+ return true;
+ }
+
+ void Dec() {
+ i64 newValue = AtomicDecrement(Count);
+ Y_VERIFY(newValue >= 0);
+ }
+
+private:
+ const i64 Limit;
+ TAtomic Count = 0;
+};
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/common/ya.make b/ydb/core/sys_view/common/ya.make
index b4ec91b3d8e..b26b22c2785 100644
--- a/ydb/core/sys_view/common/ya.make
+++ b/ydb/core/sys_view/common/ya.make
@@ -1,26 +1,26 @@
-LIBRARY()
-
-OWNER(
- monster
- g:kikimr
-)
-
-SRCS(
- common.h
- events.h
- keys.h
- path.h
- scan_actor_base_impl.h
- schema.h
- schema.cpp
- utils.h
-)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(
+ monster
+ g:kikimr
+)
+
+SRCS(
+ common.h
+ events.h
+ keys.h
+ path.h
+ scan_actor_base_impl.h
+ schema.h
+ schema.cpp
+ utils.h
+)
+
+PEERDIR(
library/cpp/actors/core
ydb/core/base
ydb/core/protos
ydb/core/tablet_flat
-)
-
-END()
+)
+
+END()
diff --git a/ydb/core/sys_view/nodes/nodes.cpp b/ydb/core/sys_view/nodes/nodes.cpp
index a2322de9f96..d06091779b6 100644
--- a/ydb/core/sys_view/nodes/nodes.cpp
+++ b/ydb/core/sys_view/nodes/nodes.cpp
@@ -1,243 +1,243 @@
-#include "nodes.h"
-
+#include "nodes.h"
+
#include <ydb/core/sys_view/common/events.h>
#include <ydb/core/sys_view/common/schema.h>
#include <ydb/core/sys_view/common/scan_actor_base_impl.h>
#include <ydb/core/node_whiteboard/node_whiteboard.h>
-
+
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h>
#include <library/cpp/actors/core/interconnect.h>
#include <library/cpp/actors/interconnect/interconnect.h>
#include <library/cpp/actors/core/hfunc.h>
-
-namespace NKikimr {
-namespace NSysView {
-
-using namespace NActors;
-using namespace NNodeWhiteboard;
-
-class TNodesScan : public TScanActorBase<TNodesScan> {
-public:
- using TBase = TScanActorBase<TNodesScan>;
-
+
+namespace NKikimr {
+namespace NSysView {
+
+using namespace NActors;
+using namespace NNodeWhiteboard;
+
+class TNodesScan : public TScanActorBase<TNodesScan> {
+public:
+ using TBase = TScanActorBase<TNodesScan>;
+
static constexpr auto ActorActivityType() {
- return NKikimrServices::TActivity::KQP_SYSTEM_VIEW_SCAN;
- }
-
+ return NKikimrServices::TActivity::KQP_SYSTEM_VIEW_SCAN;
+ }
+
TNodesScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId,
- const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns)
- : TBase(ownerId, scanId, tableId, tableRange, columns)
- {
- const auto& cellsFrom = TableRange.From.GetCells();
- if (cellsFrom.size() == 1 && !cellsFrom[0].IsNull()) {
- NodeIdFrom = cellsFrom[0].AsValue<ui32>();
- if (!TableRange.FromInclusive) {
- if (NodeIdFrom < std::numeric_limits<ui32>::max()) {
- ++NodeIdFrom;
- } else {
- IsEmptyRange = true;
- }
- }
- }
-
- const auto& cellsTo = TableRange.To.GetCells();
- if (cellsTo.size() == 1 && !cellsTo[0].IsNull()) {
- NodeIdTo = cellsTo[0].AsValue<ui32>();
- if (!TableRange.ToInclusive) {
- if (NodeIdTo > 0) {
- --NodeIdTo;
- } else {
- IsEmptyRange = true;
- }
- }
- }
- }
-
- STFUNC(StateScan) {
- switch (ev->GetTypeRewrite()) {
- hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, Handle);
- hFunc(TEvInterconnect::TEvNodesInfo, Handle);
- hFunc(TEvWhiteboard::TEvSystemStateResponse, Handle);
- hFunc(TEvents::TEvUndelivered, Undelivered);
- hFunc(TEvInterconnect::TEvNodeConnected, Connected);
- hFunc(TEvInterconnect::TEvNodeDisconnected, Disconnected);
+ const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns)
+ : TBase(ownerId, scanId, tableId, tableRange, columns)
+ {
+ const auto& cellsFrom = TableRange.From.GetCells();
+ if (cellsFrom.size() == 1 && !cellsFrom[0].IsNull()) {
+ NodeIdFrom = cellsFrom[0].AsValue<ui32>();
+ if (!TableRange.FromInclusive) {
+ if (NodeIdFrom < std::numeric_limits<ui32>::max()) {
+ ++NodeIdFrom;
+ } else {
+ IsEmptyRange = true;
+ }
+ }
+ }
+
+ const auto& cellsTo = TableRange.To.GetCells();
+ if (cellsTo.size() == 1 && !cellsTo[0].IsNull()) {
+ NodeIdTo = cellsTo[0].AsValue<ui32>();
+ if (!TableRange.ToInclusive) {
+ if (NodeIdTo > 0) {
+ --NodeIdTo;
+ } else {
+ IsEmptyRange = true;
+ }
+ }
+ }
+ }
+
+ STFUNC(StateScan) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, Handle);
+ hFunc(TEvInterconnect::TEvNodesInfo, Handle);
+ hFunc(TEvWhiteboard::TEvSystemStateResponse, Handle);
+ hFunc(TEvents::TEvUndelivered, Undelivered);
+ hFunc(TEvInterconnect::TEvNodeConnected, Connected);
+ hFunc(TEvInterconnect::TEvNodeDisconnected, Disconnected);
hFunc(NKqp::TEvKqp::TEvAbortExecution, HandleAbortExecution);
- cFunc(TEvents::TEvWakeup::EventType, HandleTimeout);
- cFunc(TEvents::TEvPoison::EventType, PassAway);
- default:
- LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS,
- "NSysView::TNodesScan: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite());
- }
- }
-
-private:
- void ProceedToScan() override {
- Become(&TNodesScan::StateScan);
- if (AckReceived) {
- StartScan();
- }
- }
-
- void StartScan() {
- if (IsEmptyRange) {
- ReplyEmptyAndDie();
- return;
- }
-
+ cFunc(TEvents::TEvWakeup::EventType, HandleTimeout);
+ cFunc(TEvents::TEvPoison::EventType, PassAway);
+ default:
+ LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS,
+ "NSysView::TNodesScan: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite());
+ }
+ }
+
+private:
+ void ProceedToScan() override {
+ Become(&TNodesScan::StateScan);
+ if (AckReceived) {
+ StartScan();
+ }
+ }
+
+ void StartScan() {
+ if (IsEmptyRange) {
+ ReplyEmptyAndDie();
+ return;
+ }
+
const TActorId nameserviceId = GetNameserviceActorId();
- Send(nameserviceId, new TEvInterconnect::TEvListNodes());
- }
-
- void Handle(NKqp::TEvKqpCompute::TEvScanDataAck::TPtr&) {
- StartScan();
- }
-
- void Handle(TEvInterconnect::TEvNodesInfo::TPtr& ev) {
- THolder<TEvInterconnect::TEvNodesInfo> nodesInfo = ev->Release();
-
- for (const auto& info : nodesInfo->Nodes) {
- auto nodeId = info.NodeId;
-
- if (TenantNodes.find(nodeId) != TenantNodes.end() &&
- nodeId >= NodeIdFrom &&
- nodeId <= NodeIdTo)
- {
+ Send(nameserviceId, new TEvInterconnect::TEvListNodes());
+ }
+
+ void Handle(NKqp::TEvKqpCompute::TEvScanDataAck::TPtr&) {
+ StartScan();
+ }
+
+ void Handle(TEvInterconnect::TEvNodesInfo::TPtr& ev) {
+ THolder<TEvInterconnect::TEvNodesInfo> nodesInfo = ev->Release();
+
+ for (const auto& info : nodesInfo->Nodes) {
+ auto nodeId = info.NodeId;
+
+ if (TenantNodes.find(nodeId) != TenantNodes.end() &&
+ nodeId >= NodeIdFrom &&
+ nodeId <= NodeIdTo)
+ {
TActorId whiteboardId = MakeNodeWhiteboardServiceId(nodeId);
- auto request = MakeHolder<TEvWhiteboard::TEvSystemStateRequest>();
-
- Send(whiteboardId, request.Release(),
- IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession, nodeId);
-
- NodesInfo.emplace(nodeId, info);
- }
- }
-
- if (NodesInfo.empty()) {
- ReplyEmptyAndDie();
- }
- }
-
- void Handle(TEvWhiteboard::TEvSystemStateResponse::TPtr& ev) {
- ui32 nodeId = ev.Get()->Cookie;
- WBSystemInfo[nodeId] = ev->Release();
- RequestDone();
- }
-
- void Undelivered(TEvents::TEvUndelivered::TPtr& ev) {
- if (ev->Get()->SourceType == TEvWhiteboard::EvSystemStateRequest) {
- ui32 nodeId = ev.Get()->Cookie;
- WBSystemInfo[nodeId] = nullptr;
- RequestDone();
- }
- }
-
- void Connected(TEvInterconnect::TEvNodeConnected::TPtr&) {
- }
-
- void Disconnected(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) {
- ui32 nodeId = ev->Get()->NodeId;
- WBSystemInfo[nodeId] = nullptr;
- RequestDone();
- }
-
- void RequestDone() {
- if (NodesInfo.size() != WBSystemInfo.size()) {
- return;
- }
-
- using TNodeInfo = TEvInterconnect::TNodeInfo;
- using TWBInfo = TEvWhiteboard::TEvSystemStateResponse;
- using TExtractor = std::function<TCell(const TNodeInfo&, const TWBInfo*)>;
- using TSchema = Schema::Nodes;
-
- struct TExtractorsMap : public THashMap<NTable::TTag, TExtractor> {
- static TMaybe<ui64> GetStartTime(const TWBInfo* wbInfo) {
- if (!wbInfo || wbInfo->Record.SystemStateInfoSize() != 1) {
- return {};
- }
- const auto& systemState = wbInfo->Record.GetSystemStateInfo(0);
- if (!systemState.HasStartTime()) {
- return {};
- }
- return systemState.GetStartTime();
- };
-
- TExtractorsMap() {
- insert({TSchema::NodeId::ColumnId, [] (const TNodeInfo& info, const TWBInfo*) {
- return TCell::Make<ui32>(info.NodeId);
- }});
- insert({TSchema::Address::ColumnId, [] (const TNodeInfo& info, const TWBInfo*) {
- return TCell(info.Address.data(), info.Address.size());
- }});
- insert({TSchema::Host::ColumnId, [] (const TNodeInfo& info, const TWBInfo*) {
- return TCell(info.Host.data(), info.Host.size());
- }});
- insert({TSchema::Port::ColumnId, [] (const TNodeInfo& info, const TWBInfo*) {
- return TCell::Make<ui32>(info.Port);
- }});
- insert({TSchema::StartTime::ColumnId, [] (const TNodeInfo&, const TWBInfo* wbInfo) {
- auto time = TExtractorsMap::GetStartTime(wbInfo);
- return time ? TCell::Make<ui64>(*time * 1000u) : TCell();
- }});
- insert({TSchema::UpTime::ColumnId, [] (const TNodeInfo&, const TWBInfo* wbInfo) {
- auto time = TExtractorsMap::GetStartTime(wbInfo);
- if (!time) {
- return TCell();
- }
- auto interval = TInstant::Now().MicroSeconds() - *time * 1000;
- return TCell::Make<i64>(interval > 0 ? interval : 0);
- }});
- }
- };
- static TExtractorsMap extractors;
-
- auto batch = MakeHolder<NKqp::TEvKqpCompute::TEvScanData>(ScanId);
- batch->Finished = true;
-
- TVector<TCell> cells;
- for (const auto& info : NodesInfo) {
- auto nodeId = info.first;
- auto* wbInfo = WBSystemInfo[nodeId].Get();
-
- for (auto column : Columns) {
- auto extractor = extractors.find(column.Tag);
- if (extractor == extractors.end()) {
- cells.push_back(TCell());
- } else {
- cells.push_back(extractor->second(info.second, wbInfo));
- }
- }
-
- TArrayRef<const TCell> ref(cells);
+ auto request = MakeHolder<TEvWhiteboard::TEvSystemStateRequest>();
+
+ Send(whiteboardId, request.Release(),
+ IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession, nodeId);
+
+ NodesInfo.emplace(nodeId, info);
+ }
+ }
+
+ if (NodesInfo.empty()) {
+ ReplyEmptyAndDie();
+ }
+ }
+
+ void Handle(TEvWhiteboard::TEvSystemStateResponse::TPtr& ev) {
+ ui32 nodeId = ev.Get()->Cookie;
+ WBSystemInfo[nodeId] = ev->Release();
+ RequestDone();
+ }
+
+ void Undelivered(TEvents::TEvUndelivered::TPtr& ev) {
+ if (ev->Get()->SourceType == TEvWhiteboard::EvSystemStateRequest) {
+ ui32 nodeId = ev.Get()->Cookie;
+ WBSystemInfo[nodeId] = nullptr;
+ RequestDone();
+ }
+ }
+
+ void Connected(TEvInterconnect::TEvNodeConnected::TPtr&) {
+ }
+
+ void Disconnected(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) {
+ ui32 nodeId = ev->Get()->NodeId;
+ WBSystemInfo[nodeId] = nullptr;
+ RequestDone();
+ }
+
+ void RequestDone() {
+ if (NodesInfo.size() != WBSystemInfo.size()) {
+ return;
+ }
+
+ using TNodeInfo = TEvInterconnect::TNodeInfo;
+ using TWBInfo = TEvWhiteboard::TEvSystemStateResponse;
+ using TExtractor = std::function<TCell(const TNodeInfo&, const TWBInfo*)>;
+ using TSchema = Schema::Nodes;
+
+ struct TExtractorsMap : public THashMap<NTable::TTag, TExtractor> {
+ static TMaybe<ui64> GetStartTime(const TWBInfo* wbInfo) {
+ if (!wbInfo || wbInfo->Record.SystemStateInfoSize() != 1) {
+ return {};
+ }
+ const auto& systemState = wbInfo->Record.GetSystemStateInfo(0);
+ if (!systemState.HasStartTime()) {
+ return {};
+ }
+ return systemState.GetStartTime();
+ };
+
+ TExtractorsMap() {
+ insert({TSchema::NodeId::ColumnId, [] (const TNodeInfo& info, const TWBInfo*) {
+ return TCell::Make<ui32>(info.NodeId);
+ }});
+ insert({TSchema::Address::ColumnId, [] (const TNodeInfo& info, const TWBInfo*) {
+ return TCell(info.Address.data(), info.Address.size());
+ }});
+ insert({TSchema::Host::ColumnId, [] (const TNodeInfo& info, const TWBInfo*) {
+ return TCell(info.Host.data(), info.Host.size());
+ }});
+ insert({TSchema::Port::ColumnId, [] (const TNodeInfo& info, const TWBInfo*) {
+ return TCell::Make<ui32>(info.Port);
+ }});
+ insert({TSchema::StartTime::ColumnId, [] (const TNodeInfo&, const TWBInfo* wbInfo) {
+ auto time = TExtractorsMap::GetStartTime(wbInfo);
+ return time ? TCell::Make<ui64>(*time * 1000u) : TCell();
+ }});
+ insert({TSchema::UpTime::ColumnId, [] (const TNodeInfo&, const TWBInfo* wbInfo) {
+ auto time = TExtractorsMap::GetStartTime(wbInfo);
+ if (!time) {
+ return TCell();
+ }
+ auto interval = TInstant::Now().MicroSeconds() - *time * 1000;
+ return TCell::Make<i64>(interval > 0 ? interval : 0);
+ }});
+ }
+ };
+ static TExtractorsMap extractors;
+
+ auto batch = MakeHolder<NKqp::TEvKqpCompute::TEvScanData>(ScanId);
+ batch->Finished = true;
+
+ TVector<TCell> cells;
+ for (const auto& info : NodesInfo) {
+ auto nodeId = info.first;
+ auto* wbInfo = WBSystemInfo[nodeId].Get();
+
+ for (auto column : Columns) {
+ auto extractor = extractors.find(column.Tag);
+ if (extractor == extractors.end()) {
+ cells.push_back(TCell());
+ } else {
+ cells.push_back(extractor->second(info.second, wbInfo));
+ }
+ }
+
+ TArrayRef<const TCell> ref(cells);
batch->Rows.emplace_back(TOwnedCellVec::Make(ref));
- cells.clear();
- }
-
- SendBatch(std::move(batch));
- }
-
- void PassAway() override {
- for (const auto& info : NodesInfo) {
- Send(TActivationContext::InterconnectProxy(info.first), new TEvents::TEvUnsubscribe());
- }
- TBase::PassAway();
- }
-
-private:
- ui32 NodeIdFrom = 0;
- ui32 NodeIdTo = std::numeric_limits<ui32>::max();
- bool IsEmptyRange = false;
-
- TMap<ui32, TEvInterconnect::TNodeInfo> NodesInfo;
- THashMap<TNodeId, THolder<TEvWhiteboard::TEvSystemStateResponse>> WBSystemInfo;
-};
-
+ cells.clear();
+ }
+
+ SendBatch(std::move(batch));
+ }
+
+ void PassAway() override {
+ for (const auto& info : NodesInfo) {
+ Send(TActivationContext::InterconnectProxy(info.first), new TEvents::TEvUnsubscribe());
+ }
+ TBase::PassAway();
+ }
+
+private:
+ ui32 NodeIdFrom = 0;
+ ui32 NodeIdTo = std::numeric_limits<ui32>::max();
+ bool IsEmptyRange = false;
+
+ TMap<ui32, TEvInterconnect::TNodeInfo> NodesInfo;
+ THashMap<TNodeId, THolder<TEvWhiteboard::TEvSystemStateResponse>> WBSystemInfo;
+};
+
THolder<IActor> CreateNodesScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId,
- const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns)
-{
+ const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns)
+{
return MakeHolder<TNodesScan>(ownerId, scanId, tableId, tableRange, columns);
-}
-
-} // NSysView
-} // NKikimr
+}
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/nodes/nodes.h b/ydb/core/sys_view/nodes/nodes.h
index 8f1192250bd..e118555f281 100644
--- a/ydb/core/sys_view/nodes/nodes.h
+++ b/ydb/core/sys_view/nodes/nodes.h
@@ -1,12 +1,12 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/kqp/runtime/kqp_compute.h>
-
-namespace NKikimr {
-namespace NSysView {
-
+
+namespace NKikimr {
+namespace NSysView {
+
THolder<IActor> CreateNodesScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId,
- const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns);
-
-} // NSysView
-} // NKikimr
+ const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns);
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/nodes/ya.make b/ydb/core/sys_view/nodes/ya.make
index 3dc17dd4129..02276dd4f67 100644
--- a/ydb/core/sys_view/nodes/ya.make
+++ b/ydb/core/sys_view/nodes/ya.make
@@ -1,22 +1,22 @@
-LIBRARY()
-
-OWNER(
- monster
- g:kikimr
-)
-
-SRCS(
- nodes.h
- nodes.cpp
-)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(
+ monster
+ g:kikimr
+)
+
+SRCS(
+ nodes.h
+ nodes.cpp
+)
+
+PEERDIR(
library/cpp/actors/core
ydb/core/base
ydb/core/kqp/runtime
ydb/core/sys_view/common
-)
-
+)
+
YQL_LAST_ABI_VERSION()
-END()
+END()
diff --git a/ydb/core/sys_view/partition_stats/partition_stats.cpp b/ydb/core/sys_view/partition_stats/partition_stats.cpp
index d6ea9ff73bd..74e562f1859 100644
--- a/ydb/core/sys_view/partition_stats/partition_stats.cpp
+++ b/ydb/core/sys_view/partition_stats/partition_stats.cpp
@@ -1,104 +1,104 @@
-#include "partition_stats.h"
-
+#include "partition_stats.h"
+
#include <ydb/core/sys_view/common/events.h>
#include <ydb/core/sys_view/common/schema.h>
#include <ydb/core/sys_view/common/scan_actor_base_impl.h>
#include <ydb/core/base/tablet_pipecache.h>
-
+
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h>
#include <library/cpp/actors/core/hfunc.h>
-
-namespace NKikimr {
-namespace NSysView {
-
-using namespace NActors;
-
-class TPartitionStatsCollector : public TActor<TPartitionStatsCollector> {
-public:
+
+namespace NKikimr {
+namespace NSysView {
+
+using namespace NActors;
+
+class TPartitionStatsCollector : public TActor<TPartitionStatsCollector> {
+public:
static constexpr auto ActorActivityType() {
- return NKikimrServices::TActivity::SYSTEM_VIEW_PART_STATS_COLLECTOR;
- }
-
- explicit TPartitionStatsCollector(size_t batchSize, size_t pendingRequestsLimit)
- : TActor(&TPartitionStatsCollector::StateWork)
- , BatchSize(batchSize)
- , PendingRequestsLimit(pendingRequestsLimit)
- {}
-
- STFUNC(StateWork) {
- switch (ev->GetTypeRewrite()) {
- hFunc(TEvSysView::TEvSetPartitioning, Handle);
- hFunc(TEvSysView::TEvRemoveTable, Handle);
- hFunc(TEvSysView::TEvSendPartitionStats, Handle);
+ return NKikimrServices::TActivity::SYSTEM_VIEW_PART_STATS_COLLECTOR;
+ }
+
+ explicit TPartitionStatsCollector(size_t batchSize, size_t pendingRequestsLimit)
+ : TActor(&TPartitionStatsCollector::StateWork)
+ , BatchSize(batchSize)
+ , PendingRequestsLimit(pendingRequestsLimit)
+ {}
+
+ STFUNC(StateWork) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvSysView::TEvSetPartitioning, Handle);
+ hFunc(TEvSysView::TEvRemoveTable, Handle);
+ hFunc(TEvSysView::TEvSendPartitionStats, Handle);
hFunc(TEvSysView::TEvUpdateTtlStats, Handle);
- hFunc(TEvSysView::TEvGetPartitionStats, Handle);
- hFunc(TEvPrivate::TEvProcess, Handle);
- cFunc(TEvents::TEvPoison::EventType, PassAway);
- default:
- LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS,
- "NSysView::TPartitionStatsCollector: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite());
- }
- }
-
-private:
- struct TEvPrivate {
- enum EEv {
- EvProcess = EventSpaceBegin(TEvents::ES_PRIVATE),
-
- EvEnd
- };
-
- struct TEvProcess : public TEventLocal<TEvProcess, EvProcess> {};
- };
-
- void Handle(TEvSysView::TEvSetPartitioning::TPtr& ev) {
- const auto& domainKey = ev->Get()->DomainKey;
- const auto& pathId = ev->Get()->PathId;
-
- auto& tables = DomainTables[domainKey];
- auto it = tables.find(pathId);
- if (it != tables.end()) {
- auto& oldPartitions = it->second.Partitions;
- THashMap<TShardIdx, NKikimrSysView::TPartitionStats> newPartitions;
-
- for (auto shardIdx : ev->Get()->ShardIndices) {
- auto old = oldPartitions.find(shardIdx);
- if (old != oldPartitions.end()) {
- newPartitions[shardIdx] = old->second;
- }
- }
-
- oldPartitions.swap(newPartitions);
- it->second.ShardIndices.swap(ev->Get()->ShardIndices);
- it->second.Path = ev->Get()->Path;
-
- } else {
- auto& table = tables[pathId];
- table.ShardIndices.swap(ev->Get()->ShardIndices);
- table.Path = ev->Get()->Path;
- }
- }
-
- void Handle(TEvSysView::TEvRemoveTable::TPtr& ev) {
- const auto& domainKey = ev->Get()->DomainKey;
- const auto& pathId = ev->Get()->PathId;
-
- auto& tables = DomainTables[domainKey];
- tables.erase(pathId);
- }
-
- void Handle(TEvSysView::TEvSendPartitionStats::TPtr& ev) {
- const auto& domainKey = ev->Get()->DomainKey;
- const auto& pathId = ev->Get()->PathId;
- const auto& shardIdx = ev->Get()->ShardIdx;
-
- auto& tables = DomainTables[domainKey];
- auto it = tables.find(pathId);
- if (it == tables.end()) {
- return;
- }
-
+ hFunc(TEvSysView::TEvGetPartitionStats, Handle);
+ hFunc(TEvPrivate::TEvProcess, Handle);
+ cFunc(TEvents::TEvPoison::EventType, PassAway);
+ default:
+ LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS,
+ "NSysView::TPartitionStatsCollector: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite());
+ }
+ }
+
+private:
+ struct TEvPrivate {
+ enum EEv {
+ EvProcess = EventSpaceBegin(TEvents::ES_PRIVATE),
+
+ EvEnd
+ };
+
+ struct TEvProcess : public TEventLocal<TEvProcess, EvProcess> {};
+ };
+
+ void Handle(TEvSysView::TEvSetPartitioning::TPtr& ev) {
+ const auto& domainKey = ev->Get()->DomainKey;
+ const auto& pathId = ev->Get()->PathId;
+
+ auto& tables = DomainTables[domainKey];
+ auto it = tables.find(pathId);
+ if (it != tables.end()) {
+ auto& oldPartitions = it->second.Partitions;
+ THashMap<TShardIdx, NKikimrSysView::TPartitionStats> newPartitions;
+
+ for (auto shardIdx : ev->Get()->ShardIndices) {
+ auto old = oldPartitions.find(shardIdx);
+ if (old != oldPartitions.end()) {
+ newPartitions[shardIdx] = old->second;
+ }
+ }
+
+ oldPartitions.swap(newPartitions);
+ it->second.ShardIndices.swap(ev->Get()->ShardIndices);
+ it->second.Path = ev->Get()->Path;
+
+ } else {
+ auto& table = tables[pathId];
+ table.ShardIndices.swap(ev->Get()->ShardIndices);
+ table.Path = ev->Get()->Path;
+ }
+ }
+
+ void Handle(TEvSysView::TEvRemoveTable::TPtr& ev) {
+ const auto& domainKey = ev->Get()->DomainKey;
+ const auto& pathId = ev->Get()->PathId;
+
+ auto& tables = DomainTables[domainKey];
+ tables.erase(pathId);
+ }
+
+ void Handle(TEvSysView::TEvSendPartitionStats::TPtr& ev) {
+ const auto& domainKey = ev->Get()->DomainKey;
+ const auto& pathId = ev->Get()->PathId;
+ const auto& shardIdx = ev->Get()->ShardIdx;
+
+ auto& tables = DomainTables[domainKey];
+ auto it = tables.find(pathId);
+ if (it == tables.end()) {
+ return;
+ }
+
auto& oldStats = it->second.Partitions[shardIdx];
auto& newStats = ev->Get()->Stats;
@@ -107,8 +107,8 @@ private:
}
oldStats.Swap(&newStats);
- }
-
+ }
+
void Handle(TEvSysView::TEvUpdateTtlStats::TPtr& ev) {
const auto& domainKey = ev->Get()->DomainKey;
const auto& pathId = ev->Get()->PathId;
@@ -123,373 +123,373 @@ private:
it->second.Partitions[shardIdx].MutableTtlStats()->Swap(&ev->Get()->Stats);
}
- void Handle(TEvSysView::TEvGetPartitionStats::TPtr& ev) {
- if (PendingRequests.size() >= PendingRequestsLimit) {
- auto result = MakeHolder<TEvSysView::TEvGetPartitionStatsResult>();
- result->Record.SetOverloaded(true);
- Send(ev->Sender, std::move(result));
- return;
- }
-
- PendingRequests.push(std::move(ev));
-
- if (!ProcessInFly) {
- Send(SelfId(), new TEvPrivate::TEvProcess());
- ProcessInFly = true;
- }
- }
-
- void Handle(TEvPrivate::TEvProcess::TPtr&) {
- ProcessInFly = false;
-
- if (PendingRequests.empty()) {
- return;
- }
-
- TEvSysView::TEvGetPartitionStats::TPtr request = std::move(PendingRequests.front());
- PendingRequests.pop();
-
- if (!PendingRequests.empty()) {
- Send(SelfId(), new TEvPrivate::TEvProcess());
- ProcessInFly = true;
- }
-
- auto& record = request->Get()->Record;
-
- auto result = MakeHolder<TEvSysView::TEvGetPartitionStatsResult>();
- result->Record.SetLastBatch(true);
-
- if (!record.HasDomainKeyOwnerId() || !record.HasDomainKeyPathId()) {
- Send(request->Sender, std::move(result));
- return;
- }
-
- auto domainKey = TPathId(record.GetDomainKeyOwnerId(), record.GetDomainKeyPathId());
- auto itTables = DomainTables.find(domainKey);
- if (itTables == DomainTables.end()) {
- Send(request->Sender, std::move(result));
- return;
- }
- auto& tables = itTables->second;
-
- auto it = tables.begin();
- auto itEnd = tables.end();
-
- bool fromInclusive = record.HasFromInclusive() && record.GetFromInclusive();
- bool toInclusive = record.HasToInclusive() && record.GetToInclusive();
-
- TPathId fromPathId;
- TPathId toPathId;
-
- ui64 startPartIdx = 0;
- ui64 endPartIdx = 0;
-
- auto& from = record.GetFrom();
-
- if (from.HasOwnerId()) {
- if (from.HasPathId()) {
- fromPathId = TPathId(from.GetOwnerId(), from.GetPathId());
- if (fromInclusive || from.HasPartIdx()) {
- it = tables.lower_bound(fromPathId);
- if (it != tables.end() && it->first == fromPathId && from.GetPartIdx()) {
- startPartIdx = from.GetPartIdx();
- if (!fromInclusive) {
- ++startPartIdx;
- }
- }
- } else {
- it = tables.upper_bound(fromPathId);
- }
- } else {
- if (fromInclusive) {
- fromPathId = TPathId(from.GetOwnerId(), 0);
- it = tables.lower_bound(fromPathId);
- } else {
- fromPathId = TPathId(from.GetOwnerId(), std::numeric_limits<ui64>::max());
- it = tables.upper_bound(fromPathId);
- }
- }
- }
-
- auto& to = record.GetTo();
-
- if (to.HasOwnerId()) {
- if (to.HasPathId()) {
- toPathId = TPathId(to.GetOwnerId(), to.GetPathId());
- if (toInclusive || to.HasPartIdx()) {
- itEnd = tables.upper_bound(toPathId);
- if (to.HasPartIdx()) {
- endPartIdx = to.GetPartIdx();
- if (toInclusive) {
- ++endPartIdx;
- }
- }
- } else {
- itEnd = tables.lower_bound(toPathId);
- }
- } else {
- if (toInclusive) {
- toPathId = TPathId(to.GetOwnerId(), std::numeric_limits<ui64>::max());
- itEnd = tables.upper_bound(toPathId);
- } else {
- toPathId = TPathId(to.GetOwnerId(), 0);
- itEnd = tables.lower_bound(toPathId);
- }
- }
- }
-
- bool includePathColumn = !record.HasIncludePathColumn() || record.GetIncludePathColumn();
-
- for (size_t count = 0; count < BatchSize && it != itEnd; ++it) {
- auto& pathId = it->first;
- const auto& tableStats = it->second;
-
- ui64 end = tableStats.ShardIndices.size();
- if (to.HasPartIdx() && pathId == toPathId) {
- end = std::min(endPartIdx, end);
- }
-
- bool batchFinished = false;
-
- for (ui64 partIdx = startPartIdx; partIdx < end; ++partIdx) {
- auto* stats = result->Record.AddStats();
- auto* key = stats->MutableKey();
-
- key->SetOwnerId(pathId.OwnerId);
- key->SetPathId(pathId.LocalPathId);
- key->SetPartIdx(partIdx);
-
- if (includePathColumn) {
- stats->SetPath(tableStats.Path);
- }
-
- auto shardIdx = tableStats.ShardIndices[partIdx];
- auto part = tableStats.Partitions.find(shardIdx);
- if (part != tableStats.Partitions.end()) {
- *stats->MutableStats() = part->second;
- }
-
- if (++count == BatchSize) {
- auto* next = result->Record.MutableNext();
- next->SetOwnerId(pathId.OwnerId);
- next->SetPathId(pathId.LocalPathId);
- next->SetPartIdx(partIdx + 1);
- result->Record.SetLastBatch(false);
- batchFinished = true;
- break;
- }
- }
-
- if (batchFinished) {
- break;
- }
-
- startPartIdx = 0;
- }
-
- Send(request->Sender, std::move(result));
- }
-
-private:
- const size_t BatchSize;
- const size_t PendingRequestsLimit;
-
- struct TTableStats {
- THashMap<TShardIdx, NKikimrSysView::TPartitionStats> Partitions; // shardIdx -> stats
- TVector<TShardIdx> ShardIndices;
- TString Path;
- };
-
- using TDomainTables = TMap<TPathId, TTableStats>;
- THashMap<TPathId, TDomainTables> DomainTables;
-
- TQueue<TEvSysView::TEvGetPartitionStats::TPtr> PendingRequests;
- bool ProcessInFly = false;
-};
-
-THolder<IActor> CreatePartitionStatsCollector(size_t batchSize, size_t pendingRequestsLimit) {
+ void Handle(TEvSysView::TEvGetPartitionStats::TPtr& ev) {
+ if (PendingRequests.size() >= PendingRequestsLimit) {
+ auto result = MakeHolder<TEvSysView::TEvGetPartitionStatsResult>();
+ result->Record.SetOverloaded(true);
+ Send(ev->Sender, std::move(result));
+ return;
+ }
+
+ PendingRequests.push(std::move(ev));
+
+ if (!ProcessInFly) {
+ Send(SelfId(), new TEvPrivate::TEvProcess());
+ ProcessInFly = true;
+ }
+ }
+
+ void Handle(TEvPrivate::TEvProcess::TPtr&) {
+ ProcessInFly = false;
+
+ if (PendingRequests.empty()) {
+ return;
+ }
+
+ TEvSysView::TEvGetPartitionStats::TPtr request = std::move(PendingRequests.front());
+ PendingRequests.pop();
+
+ if (!PendingRequests.empty()) {
+ Send(SelfId(), new TEvPrivate::TEvProcess());
+ ProcessInFly = true;
+ }
+
+ auto& record = request->Get()->Record;
+
+ auto result = MakeHolder<TEvSysView::TEvGetPartitionStatsResult>();
+ result->Record.SetLastBatch(true);
+
+ if (!record.HasDomainKeyOwnerId() || !record.HasDomainKeyPathId()) {
+ Send(request->Sender, std::move(result));
+ return;
+ }
+
+ auto domainKey = TPathId(record.GetDomainKeyOwnerId(), record.GetDomainKeyPathId());
+ auto itTables = DomainTables.find(domainKey);
+ if (itTables == DomainTables.end()) {
+ Send(request->Sender, std::move(result));
+ return;
+ }
+ auto& tables = itTables->second;
+
+ auto it = tables.begin();
+ auto itEnd = tables.end();
+
+ bool fromInclusive = record.HasFromInclusive() && record.GetFromInclusive();
+ bool toInclusive = record.HasToInclusive() && record.GetToInclusive();
+
+ TPathId fromPathId;
+ TPathId toPathId;
+
+ ui64 startPartIdx = 0;
+ ui64 endPartIdx = 0;
+
+ auto& from = record.GetFrom();
+
+ if (from.HasOwnerId()) {
+ if (from.HasPathId()) {
+ fromPathId = TPathId(from.GetOwnerId(), from.GetPathId());
+ if (fromInclusive || from.HasPartIdx()) {
+ it = tables.lower_bound(fromPathId);
+ if (it != tables.end() && it->first == fromPathId && from.GetPartIdx()) {
+ startPartIdx = from.GetPartIdx();
+ if (!fromInclusive) {
+ ++startPartIdx;
+ }
+ }
+ } else {
+ it = tables.upper_bound(fromPathId);
+ }
+ } else {
+ if (fromInclusive) {
+ fromPathId = TPathId(from.GetOwnerId(), 0);
+ it = tables.lower_bound(fromPathId);
+ } else {
+ fromPathId = TPathId(from.GetOwnerId(), std::numeric_limits<ui64>::max());
+ it = tables.upper_bound(fromPathId);
+ }
+ }
+ }
+
+ auto& to = record.GetTo();
+
+ if (to.HasOwnerId()) {
+ if (to.HasPathId()) {
+ toPathId = TPathId(to.GetOwnerId(), to.GetPathId());
+ if (toInclusive || to.HasPartIdx()) {
+ itEnd = tables.upper_bound(toPathId);
+ if (to.HasPartIdx()) {
+ endPartIdx = to.GetPartIdx();
+ if (toInclusive) {
+ ++endPartIdx;
+ }
+ }
+ } else {
+ itEnd = tables.lower_bound(toPathId);
+ }
+ } else {
+ if (toInclusive) {
+ toPathId = TPathId(to.GetOwnerId(), std::numeric_limits<ui64>::max());
+ itEnd = tables.upper_bound(toPathId);
+ } else {
+ toPathId = TPathId(to.GetOwnerId(), 0);
+ itEnd = tables.lower_bound(toPathId);
+ }
+ }
+ }
+
+ bool includePathColumn = !record.HasIncludePathColumn() || record.GetIncludePathColumn();
+
+ for (size_t count = 0; count < BatchSize && it != itEnd; ++it) {
+ auto& pathId = it->first;
+ const auto& tableStats = it->second;
+
+ ui64 end = tableStats.ShardIndices.size();
+ if (to.HasPartIdx() && pathId == toPathId) {
+ end = std::min(endPartIdx, end);
+ }
+
+ bool batchFinished = false;
+
+ for (ui64 partIdx = startPartIdx; partIdx < end; ++partIdx) {
+ auto* stats = result->Record.AddStats();
+ auto* key = stats->MutableKey();
+
+ key->SetOwnerId(pathId.OwnerId);
+ key->SetPathId(pathId.LocalPathId);
+ key->SetPartIdx(partIdx);
+
+ if (includePathColumn) {
+ stats->SetPath(tableStats.Path);
+ }
+
+ auto shardIdx = tableStats.ShardIndices[partIdx];
+ auto part = tableStats.Partitions.find(shardIdx);
+ if (part != tableStats.Partitions.end()) {
+ *stats->MutableStats() = part->second;
+ }
+
+ if (++count == BatchSize) {
+ auto* next = result->Record.MutableNext();
+ next->SetOwnerId(pathId.OwnerId);
+ next->SetPathId(pathId.LocalPathId);
+ next->SetPartIdx(partIdx + 1);
+ result->Record.SetLastBatch(false);
+ batchFinished = true;
+ break;
+ }
+ }
+
+ if (batchFinished) {
+ break;
+ }
+
+ startPartIdx = 0;
+ }
+
+ Send(request->Sender, std::move(result));
+ }
+
+private:
+ const size_t BatchSize;
+ const size_t PendingRequestsLimit;
+
+ struct TTableStats {
+ THashMap<TShardIdx, NKikimrSysView::TPartitionStats> Partitions; // shardIdx -> stats
+ TVector<TShardIdx> ShardIndices;
+ TString Path;
+ };
+
+ using TDomainTables = TMap<TPathId, TTableStats>;
+ THashMap<TPathId, TDomainTables> DomainTables;
+
+ TQueue<TEvSysView::TEvGetPartitionStats::TPtr> PendingRequests;
+ bool ProcessInFly = false;
+};
+
+THolder<IActor> CreatePartitionStatsCollector(size_t batchSize, size_t pendingRequestsLimit) {
return MakeHolder<TPartitionStatsCollector>(batchSize, pendingRequestsLimit);
-}
-
-class TPartitionStatsScan : public TScanActorBase<TPartitionStatsScan> {
-public:
- using TBase = TScanActorBase<TPartitionStatsScan>;
-
+}
+
+class TPartitionStatsScan : public TScanActorBase<TPartitionStatsScan> {
+public:
+ using TBase = TScanActorBase<TPartitionStatsScan>;
+
static constexpr auto ActorActivityType() {
- return NKikimrServices::TActivity::KQP_SYSTEM_VIEW_SCAN;
- }
-
+ return NKikimrServices::TActivity::KQP_SYSTEM_VIEW_SCAN;
+ }
+
TPartitionStatsScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId,
- const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns)
- : TBase(ownerId, scanId, tableId, tableRange, columns)
- {
- auto extractKey = [] (NKikimrSysView::TPartitionStatsKey& key, const TConstArrayRef<TCell>& cells) {
- if (cells.size() > 0 && !cells[0].IsNull()) {
- key.SetOwnerId(cells[0].AsValue<ui64>());
- }
- if (cells.size() > 1 && !cells[1].IsNull()) {
- key.SetPathId(cells[1].AsValue<ui64>());
- }
- if (cells.size() > 2 && !cells[2].IsNull()) {
- key.SetPartIdx(cells[2].AsValue<ui64>());
- }
- };
-
- extractKey(From, TableRange.From.GetCells());
- FromInclusive = TableRange.FromInclusive;
-
- extractKey(To, TableRange.To.GetCells());
- ToInclusive = TableRange.ToInclusive;
-
- for (auto& column : columns) {
- if (column.Tag == Schema::PartitionStats::Path::ColumnId) {
- IncludePathColumn = true;
- break;
- }
- }
- }
-
- STFUNC(StateScan) {
- switch (ev->GetTypeRewrite()) {
- hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, Handle);
- hFunc(TEvSysView::TEvGetPartitionStatsResult, Handle);
- hFunc(TEvPipeCache::TEvDeliveryProblem, Handle);
+ const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns)
+ : TBase(ownerId, scanId, tableId, tableRange, columns)
+ {
+ auto extractKey = [] (NKikimrSysView::TPartitionStatsKey& key, const TConstArrayRef<TCell>& cells) {
+ if (cells.size() > 0 && !cells[0].IsNull()) {
+ key.SetOwnerId(cells[0].AsValue<ui64>());
+ }
+ if (cells.size() > 1 && !cells[1].IsNull()) {
+ key.SetPathId(cells[1].AsValue<ui64>());
+ }
+ if (cells.size() > 2 && !cells[2].IsNull()) {
+ key.SetPartIdx(cells[2].AsValue<ui64>());
+ }
+ };
+
+ extractKey(From, TableRange.From.GetCells());
+ FromInclusive = TableRange.FromInclusive;
+
+ extractKey(To, TableRange.To.GetCells());
+ ToInclusive = TableRange.ToInclusive;
+
+ for (auto& column : columns) {
+ if (column.Tag == Schema::PartitionStats::Path::ColumnId) {
+ IncludePathColumn = true;
+ break;
+ }
+ }
+ }
+
+ STFUNC(StateScan) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, Handle);
+ hFunc(TEvSysView::TEvGetPartitionStatsResult, Handle);
+ hFunc(TEvPipeCache::TEvDeliveryProblem, Handle);
hFunc(NKqp::TEvKqp::TEvAbortExecution, HandleAbortExecution);
- cFunc(TEvents::TEvWakeup::EventType, HandleTimeout);
- cFunc(TEvents::TEvPoison::EventType, PassAway);
- default:
- LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS,
- "NSysView::TPartitionStatsScan: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite());
- }
- }
-
-private:
- void ProceedToScan() override {
- Become(&TThis::StateScan);
- if (AckReceived) {
- RequestBatch();
- }
- }
-
- void Handle(NKqp::TEvKqpCompute::TEvScanDataAck::TPtr&) {
- RequestBatch();
- }
-
- void RequestBatch() {
- if (BatchRequestInFlight) {
- return;
- }
-
- auto request = MakeHolder<TEvSysView::TEvGetPartitionStats>();
-
- request->Record.SetDomainKeyOwnerId(DomainKey.OwnerId);
- request->Record.SetDomainKeyPathId(DomainKey.LocalPathId);
-
- request->Record.MutableFrom()->CopyFrom(From);
- request->Record.SetFromInclusive(FromInclusive);
- request->Record.MutableTo()->CopyFrom(To);
- request->Record.SetToInclusive(ToInclusive);
-
- request->Record.SetIncludePathColumn(IncludePathColumn);
-
- auto pipeCache = MakePipePeNodeCacheID(false);
- Send(pipeCache, new TEvPipeCache::TEvForward(request.Release(), SchemeShardId, true),
- IEventHandle::FlagTrackDelivery);
-
- BatchRequestInFlight = true;
- }
-
- void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr&) {
- ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "Delivery problem in partition stats scan");
- }
-
- void Handle(TEvSysView::TEvGetPartitionStatsResult::TPtr& ev) {
- auto& record = ev->Get()->Record;
-
- if (record.HasOverloaded() && record.GetOverloaded()) {
- ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "Partition stats collector is overloaded");
- return;
- }
-
- using TPartitionStats = NKikimrSysView::TPartitionStatsResult;
- using TExtractor = std::function<TCell(const TPartitionStats&)>;
- using TSchema = Schema::PartitionStats;
-
- struct TExtractorsMap : public THashMap<NTable::TTag, TExtractor> {
- TExtractorsMap() {
- insert({TSchema::OwnerId::ColumnId, [] (const TPartitionStats& s) {
- return TCell::Make<ui64>(s.GetKey().GetOwnerId());
- }});
- insert({TSchema::PathId::ColumnId, [] (const TPartitionStats& s) {
- return TCell::Make<ui64>(s.GetKey().GetPathId());
- }});
- insert({TSchema::PartIdx::ColumnId, [] (const TPartitionStats& s) {
- return TCell::Make<ui64>(s.GetKey().GetPartIdx());
- }});
- insert({TSchema::DataSize::ColumnId, [] (const TPartitionStats& s) {
- return TCell::Make<ui64>(s.GetStats().GetDataSize());
- }});
- insert({TSchema::RowCount::ColumnId, [] (const TPartitionStats& s) {
- return TCell::Make<ui64>(s.GetStats().GetRowCount());
- }});
- insert({TSchema::IndexSize::ColumnId, [] (const TPartitionStats& s) {
- return TCell::Make<ui64>(s.GetStats().GetIndexSize());
- }});
- insert({TSchema::CPUCores::ColumnId, [] (const TPartitionStats& s) {
- return TCell::Make<double>(s.GetStats().GetCPUCores());
- }});
- insert({TSchema::TabletId::ColumnId, [] (const TPartitionStats& s) {
- return TCell::Make<ui64>(s.GetStats().GetTabletId());
- }});
- insert({TSchema::Path::ColumnId, [] (const TPartitionStats& s) {
- if (!s.HasPath()) {
- return TCell();
- }
- auto& path = s.GetPath();
- return TCell(path.data(), path.size());
- }});
- insert({TSchema::NodeId::ColumnId, [] (const TPartitionStats& s) {
- return s.GetStats().HasNodeId() ? TCell::Make<ui32>(s.GetStats().GetNodeId()) : TCell();
- }});
- insert({TSchema::StartTime::ColumnId, [] (const TPartitionStats& s) {
- return s.GetStats().HasStartTime() ? TCell::Make<ui64>(s.GetStats().GetStartTime() * 1000) : TCell();
- }});
- insert({TSchema::AccessTime::ColumnId, [] (const TPartitionStats& s) {
- return s.GetStats().HasAccessTime() ? TCell::Make<ui64>(s.GetStats().GetAccessTime() * 1000) : TCell();
- }});
- insert({TSchema::UpdateTime::ColumnId, [] (const TPartitionStats& s) {
- return s.GetStats().HasUpdateTime() ? TCell::Make<ui64>(s.GetStats().GetUpdateTime() * 1000) : TCell();
- }});
- insert({TSchema::InFlightTxCount::ColumnId, [] (const TPartitionStats& s) {
+ cFunc(TEvents::TEvWakeup::EventType, HandleTimeout);
+ cFunc(TEvents::TEvPoison::EventType, PassAway);
+ default:
+ LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS,
+ "NSysView::TPartitionStatsScan: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite());
+ }
+ }
+
+private:
+ void ProceedToScan() override {
+ Become(&TThis::StateScan);
+ if (AckReceived) {
+ RequestBatch();
+ }
+ }
+
+ void Handle(NKqp::TEvKqpCompute::TEvScanDataAck::TPtr&) {
+ RequestBatch();
+ }
+
+ void RequestBatch() {
+ if (BatchRequestInFlight) {
+ return;
+ }
+
+ auto request = MakeHolder<TEvSysView::TEvGetPartitionStats>();
+
+ request->Record.SetDomainKeyOwnerId(DomainKey.OwnerId);
+ request->Record.SetDomainKeyPathId(DomainKey.LocalPathId);
+
+ request->Record.MutableFrom()->CopyFrom(From);
+ request->Record.SetFromInclusive(FromInclusive);
+ request->Record.MutableTo()->CopyFrom(To);
+ request->Record.SetToInclusive(ToInclusive);
+
+ request->Record.SetIncludePathColumn(IncludePathColumn);
+
+ auto pipeCache = MakePipePeNodeCacheID(false);
+ Send(pipeCache, new TEvPipeCache::TEvForward(request.Release(), SchemeShardId, true),
+ IEventHandle::FlagTrackDelivery);
+
+ BatchRequestInFlight = true;
+ }
+
+ void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr&) {
+ ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "Delivery problem in partition stats scan");
+ }
+
+ void Handle(TEvSysView::TEvGetPartitionStatsResult::TPtr& ev) {
+ auto& record = ev->Get()->Record;
+
+ if (record.HasOverloaded() && record.GetOverloaded()) {
+ ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "Partition stats collector is overloaded");
+ return;
+ }
+
+ using TPartitionStats = NKikimrSysView::TPartitionStatsResult;
+ using TExtractor = std::function<TCell(const TPartitionStats&)>;
+ using TSchema = Schema::PartitionStats;
+
+ struct TExtractorsMap : public THashMap<NTable::TTag, TExtractor> {
+ TExtractorsMap() {
+ insert({TSchema::OwnerId::ColumnId, [] (const TPartitionStats& s) {
+ return TCell::Make<ui64>(s.GetKey().GetOwnerId());
+ }});
+ insert({TSchema::PathId::ColumnId, [] (const TPartitionStats& s) {
+ return TCell::Make<ui64>(s.GetKey().GetPathId());
+ }});
+ insert({TSchema::PartIdx::ColumnId, [] (const TPartitionStats& s) {
+ return TCell::Make<ui64>(s.GetKey().GetPartIdx());
+ }});
+ insert({TSchema::DataSize::ColumnId, [] (const TPartitionStats& s) {
+ return TCell::Make<ui64>(s.GetStats().GetDataSize());
+ }});
+ insert({TSchema::RowCount::ColumnId, [] (const TPartitionStats& s) {
+ return TCell::Make<ui64>(s.GetStats().GetRowCount());
+ }});
+ insert({TSchema::IndexSize::ColumnId, [] (const TPartitionStats& s) {
+ return TCell::Make<ui64>(s.GetStats().GetIndexSize());
+ }});
+ insert({TSchema::CPUCores::ColumnId, [] (const TPartitionStats& s) {
+ return TCell::Make<double>(s.GetStats().GetCPUCores());
+ }});
+ insert({TSchema::TabletId::ColumnId, [] (const TPartitionStats& s) {
+ return TCell::Make<ui64>(s.GetStats().GetTabletId());
+ }});
+ insert({TSchema::Path::ColumnId, [] (const TPartitionStats& s) {
+ if (!s.HasPath()) {
+ return TCell();
+ }
+ auto& path = s.GetPath();
+ return TCell(path.data(), path.size());
+ }});
+ insert({TSchema::NodeId::ColumnId, [] (const TPartitionStats& s) {
+ return s.GetStats().HasNodeId() ? TCell::Make<ui32>(s.GetStats().GetNodeId()) : TCell();
+ }});
+ insert({TSchema::StartTime::ColumnId, [] (const TPartitionStats& s) {
+ return s.GetStats().HasStartTime() ? TCell::Make<ui64>(s.GetStats().GetStartTime() * 1000) : TCell();
+ }});
+ insert({TSchema::AccessTime::ColumnId, [] (const TPartitionStats& s) {
+ return s.GetStats().HasAccessTime() ? TCell::Make<ui64>(s.GetStats().GetAccessTime() * 1000) : TCell();
+ }});
+ insert({TSchema::UpdateTime::ColumnId, [] (const TPartitionStats& s) {
+ return s.GetStats().HasUpdateTime() ? TCell::Make<ui64>(s.GetStats().GetUpdateTime() * 1000) : TCell();
+ }});
+ insert({TSchema::InFlightTxCount::ColumnId, [] (const TPartitionStats& s) {
return TCell::Make<ui32>(s.GetStats().GetInFlightTxCount());
- }});
- insert({TSchema::RowUpdates::ColumnId, [] (const TPartitionStats& s) {
- return TCell::Make<ui64>(s.GetStats().GetRowUpdates());
- }});
- insert({TSchema::RowDeletes::ColumnId, [] (const TPartitionStats& s) {
- return TCell::Make<ui64>(s.GetStats().GetRowDeletes());
- }});
- insert({TSchema::RowReads::ColumnId, [] (const TPartitionStats& s) {
- return TCell::Make<ui64>(s.GetStats().GetRowReads());
- }});
- insert({TSchema::RangeReads::ColumnId, [] (const TPartitionStats& s) {
- return TCell::Make<ui64>(s.GetStats().GetRangeReads());
- }});
- insert({TSchema::RangeReadRows::ColumnId, [] (const TPartitionStats& s) {
- return TCell::Make<ui64>(s.GetStats().GetRangeReadRows());
- }});
- insert({TSchema::ImmediateTxCompleted::ColumnId, [] (const TPartitionStats& s) {
- return TCell::Make<ui64>(s.GetStats().GetImmediateTxCompleted());
- }});
- insert({TSchema::CoordinatedTxCompleted::ColumnId, [] (const TPartitionStats& s) {
- return TCell::Make<ui64>(s.GetStats().GetPlannedTxCompleted());
- }});
- insert({TSchema::TxRejectedByOverload::ColumnId, [] (const TPartitionStats& s) {
- return TCell::Make<ui64>(s.GetStats().GetTxRejectedByOverload());
- }});
- insert({TSchema::TxRejectedByOutOfStorage::ColumnId, [] (const TPartitionStats& s) {
- return TCell::Make<ui64>(s.GetStats().GetTxRejectedBySpace());
- }});
+ }});
+ insert({TSchema::RowUpdates::ColumnId, [] (const TPartitionStats& s) {
+ return TCell::Make<ui64>(s.GetStats().GetRowUpdates());
+ }});
+ insert({TSchema::RowDeletes::ColumnId, [] (const TPartitionStats& s) {
+ return TCell::Make<ui64>(s.GetStats().GetRowDeletes());
+ }});
+ insert({TSchema::RowReads::ColumnId, [] (const TPartitionStats& s) {
+ return TCell::Make<ui64>(s.GetStats().GetRowReads());
+ }});
+ insert({TSchema::RangeReads::ColumnId, [] (const TPartitionStats& s) {
+ return TCell::Make<ui64>(s.GetStats().GetRangeReads());
+ }});
+ insert({TSchema::RangeReadRows::ColumnId, [] (const TPartitionStats& s) {
+ return TCell::Make<ui64>(s.GetStats().GetRangeReadRows());
+ }});
+ insert({TSchema::ImmediateTxCompleted::ColumnId, [] (const TPartitionStats& s) {
+ return TCell::Make<ui64>(s.GetStats().GetImmediateTxCompleted());
+ }});
+ insert({TSchema::CoordinatedTxCompleted::ColumnId, [] (const TPartitionStats& s) {
+ return TCell::Make<ui64>(s.GetStats().GetPlannedTxCompleted());
+ }});
+ insert({TSchema::TxRejectedByOverload::ColumnId, [] (const TPartitionStats& s) {
+ return TCell::Make<ui64>(s.GetStats().GetTxRejectedByOverload());
+ }});
+ insert({TSchema::TxRejectedByOutOfStorage::ColumnId, [] (const TPartitionStats& s) {
+ return TCell::Make<ui64>(s.GetStats().GetTxRejectedBySpace());
+ }});
insert({TSchema::LastTtlRunTime::ColumnId, [] (const TPartitionStats& s) {
return s.GetStats().HasTtlStats() ? TCell::Make<ui64>(s.GetStats().GetTtlStats().GetLastRunTime() * 1000) : TCell();
}});
@@ -499,59 +499,59 @@ private:
insert({TSchema::LastTtlRowsErased::ColumnId, [] (const TPartitionStats& s) {
return s.GetStats().HasTtlStats() ? TCell::Make<ui64>(s.GetStats().GetTtlStats().GetLastRowsErased()) : TCell();
}});
- }
- };
- static TExtractorsMap extractors;
-
- auto batch = MakeHolder<NKqp::TEvKqpCompute::TEvScanData>(ScanId);
-
- TVector<TCell> cells;
- for (const auto& s : record.GetStats()) {
- for (auto& column : Columns) {
- auto extractor = extractors.find(column.Tag);
- if (extractor == extractors.end()) {
- cells.push_back(TCell());
- } else {
- cells.push_back(extractor->second(s));
- }
- }
- TArrayRef<const TCell> ref(cells);
+ }
+ };
+ static TExtractorsMap extractors;
+
+ auto batch = MakeHolder<NKqp::TEvKqpCompute::TEvScanData>(ScanId);
+
+ TVector<TCell> cells;
+ for (const auto& s : record.GetStats()) {
+ for (auto& column : Columns) {
+ auto extractor = extractors.find(column.Tag);
+ if (extractor == extractors.end()) {
+ cells.push_back(TCell());
+ } else {
+ cells.push_back(extractor->second(s));
+ }
+ }
+ TArrayRef<const TCell> ref(cells);
batch->Rows.emplace_back(TOwnedCellVec::Make(ref));
- cells.clear();
- }
-
- batch->Finished = record.GetLastBatch();
- if (!batch->Finished) {
- Y_VERIFY(record.HasNext());
- From = record.GetNext();
- FromInclusive = true;
- }
-
- SendBatch(std::move(batch));
-
- BatchRequestInFlight = false;
- }
-
- void PassAway() override {
- Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(0));
- TBase::PassAway();
- }
-
-private:
- NKikimrSysView::TPartitionStatsKey From;
- bool FromInclusive = false;
-
- NKikimrSysView::TPartitionStatsKey To;
- bool ToInclusive = false;
-
- bool IncludePathColumn = false;
-};
-
+ cells.clear();
+ }
+
+ batch->Finished = record.GetLastBatch();
+ if (!batch->Finished) {
+ Y_VERIFY(record.HasNext());
+ From = record.GetNext();
+ FromInclusive = true;
+ }
+
+ SendBatch(std::move(batch));
+
+ BatchRequestInFlight = false;
+ }
+
+ void PassAway() override {
+ Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(0));
+ TBase::PassAway();
+ }
+
+private:
+ NKikimrSysView::TPartitionStatsKey From;
+ bool FromInclusive = false;
+
+ NKikimrSysView::TPartitionStatsKey To;
+ bool ToInclusive = false;
+
+ bool IncludePathColumn = false;
+};
+
THolder<IActor> CreatePartitionStatsScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId,
- const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns)
-{
+ const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns)
+{
return MakeHolder<TPartitionStatsScan>(ownerId, scanId, tableId, tableRange, columns);
-}
-
-} // NSysView
-} // NKikimr
+}
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/partition_stats/partition_stats.h b/ydb/core/sys_view/partition_stats/partition_stats.h
index dedf62906fb..48348b061d4 100644
--- a/ydb/core/sys_view/partition_stats/partition_stats.h
+++ b/ydb/core/sys_view/partition_stats/partition_stats.h
@@ -1,19 +1,19 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/kqp/runtime/kqp_compute.h>
-
-namespace NKikimr {
-namespace NSysView {
-
-constexpr size_t STATS_COLLECTOR_BATCH_SIZE = 5000;
-constexpr size_t STATS_COLLECTOR_QUEUE_SIZE_LIMIT = 10;
-
-THolder<IActor> CreatePartitionStatsCollector(
- size_t batchSize = STATS_COLLECTOR_BATCH_SIZE,
- size_t pendingRequestsCount = STATS_COLLECTOR_QUEUE_SIZE_LIMIT);
-
+
+namespace NKikimr {
+namespace NSysView {
+
+constexpr size_t STATS_COLLECTOR_BATCH_SIZE = 5000;
+constexpr size_t STATS_COLLECTOR_QUEUE_SIZE_LIMIT = 10;
+
+THolder<IActor> CreatePartitionStatsCollector(
+ size_t batchSize = STATS_COLLECTOR_BATCH_SIZE,
+ size_t pendingRequestsCount = STATS_COLLECTOR_QUEUE_SIZE_LIMIT);
+
THolder<IActor> CreatePartitionStatsScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId,
- const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns);
-
-} // NSysView
-} // NKikimr
+ const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns);
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/partition_stats/partition_stats_ut.cpp b/ydb/core/sys_view/partition_stats/partition_stats_ut.cpp
index c1230129482..25baccca3fa 100644
--- a/ydb/core/sys_view/partition_stats/partition_stats_ut.cpp
+++ b/ydb/core/sys_view/partition_stats/partition_stats_ut.cpp
@@ -1,200 +1,200 @@
-#include "partition_stats.h"
-
+#include "partition_stats.h"
+
#include <ydb/core/testlib/actors/test_runtime.h>
#include <ydb/core/base/appdata.h>
#include <ydb/core/sys_view/common/events.h>
-
+
#include <library/cpp/testing/unittest/registar.h>
-
-namespace NKikimr {
-namespace NSysView {
-
-Y_UNIT_TEST_SUITE(PartitionStats) {
-
- TTestActorRuntime::TEgg MakeEgg()
- {
- return { new TAppData(0, 0, 0, 0, { }, nullptr, nullptr, nullptr, nullptr), nullptr, nullptr };
- }
-
- void TestCollector(size_t batchSize) {
- TTestActorRuntime runtime;
- runtime.Initialize(MakeEgg());
-
- auto collector = CreatePartitionStatsCollector(batchSize);
- auto collectorId = runtime.Register(collector.Release());
- auto sender = runtime.AllocateEdgeActor();
-
- auto domainKey = TPathId(1, 1);
-
- for (ui64 ownerId = 0; ownerId <= 1; ++ownerId) {
- for (ui64 pathId = 0; pathId <= 1; ++pathId) {
- auto id = TPathId(ownerId, pathId);
- auto setPartitioning = MakeHolder<TEvSysView::TEvSetPartitioning>(domainKey, id, "");
- setPartitioning->ShardIndices.push_back(TShardIdx(ownerId * 3 + pathId, 0));
- setPartitioning->ShardIndices.push_back(TShardIdx(ownerId * 3 + pathId, 1));
+
+namespace NKikimr {
+namespace NSysView {
+
+Y_UNIT_TEST_SUITE(PartitionStats) {
+
+ TTestActorRuntime::TEgg MakeEgg()
+ {
+ return { new TAppData(0, 0, 0, 0, { }, nullptr, nullptr, nullptr, nullptr), nullptr, nullptr };
+ }
+
+ void TestCollector(size_t batchSize) {
+ TTestActorRuntime runtime;
+ runtime.Initialize(MakeEgg());
+
+ auto collector = CreatePartitionStatsCollector(batchSize);
+ auto collectorId = runtime.Register(collector.Release());
+ auto sender = runtime.AllocateEdgeActor();
+
+ auto domainKey = TPathId(1, 1);
+
+ for (ui64 ownerId = 0; ownerId <= 1; ++ownerId) {
+ for (ui64 pathId = 0; pathId <= 1; ++pathId) {
+ auto id = TPathId(ownerId, pathId);
+ auto setPartitioning = MakeHolder<TEvSysView::TEvSetPartitioning>(domainKey, id, "");
+ setPartitioning->ShardIndices.push_back(TShardIdx(ownerId * 3 + pathId, 0));
+ setPartitioning->ShardIndices.push_back(TShardIdx(ownerId * 3 + pathId, 1));
runtime.Send(new IEventHandle(collectorId, TActorId(), setPartitioning.Release()));
- }
- }
-
- auto test = [&] (
- TMaybe<ui64> fromOwnerId, TMaybe<ui64> fromPathId, TMaybe<ui64> fromPartIdx, bool fromInc,
- TMaybe<ui64> toOwnerId, TMaybe<ui64> toPathId, TMaybe<ui64> toPartIdx, bool toInc,
- std::initializer_list<std::tuple<ui64, ui64, ui64>> check)
- {
- auto get = MakeHolder<TEvSysView::TEvGetPartitionStats>();
- auto& record = get->Record;
- record.SetDomainKeyOwnerId(domainKey.OwnerId);
- record.SetDomainKeyPathId(domainKey.LocalPathId);
-
- if (fromOwnerId) {
- record.MutableFrom()->SetOwnerId(*fromOwnerId);
- }
- if (fromPathId) {
- record.MutableFrom()->SetPathId(*fromPathId);
- }
- if (fromPartIdx) {
- record.MutableFrom()->SetPartIdx(*fromPartIdx);
- }
- record.SetFromInclusive(fromInc);
-
- if (toOwnerId) {
- record.MutableTo()->SetOwnerId(*toOwnerId);
- }
- if (toPathId) {
- record.MutableTo()->SetPathId(*toPathId);
- }
- if (toPartIdx) {
- record.MutableTo()->SetPartIdx(*toPartIdx);
- }
- record.SetToInclusive(toInc);
-
- auto checkIt = check.begin();
-
- while (true) {
- runtime.Send(new IEventHandle(collectorId, sender, get.Release()));
-
- TAutoPtr<IEventHandle> handle;
- auto result = runtime.GrabEdgeEvent<TEvSysView::TEvGetPartitionStatsResult>(handle);
-
- for (size_t i = 0; i < result->Record.StatsSize(); ++i, ++checkIt) {
- auto& stats = result->Record.GetStats(i);
- UNIT_ASSERT(checkIt != check.end());
- UNIT_ASSERT_VALUES_EQUAL(stats.GetKey().GetOwnerId(), std::get<0>(*checkIt));
- UNIT_ASSERT_VALUES_EQUAL(stats.GetKey().GetPathId(), std::get<1>(*checkIt));
- UNIT_ASSERT_VALUES_EQUAL(stats.GetKey().GetPartIdx(), std::get<2>(*checkIt));
- }
-
- if (result->Record.GetLastBatch()) {
- break;
- }
-
- get = MakeHolder<TEvSysView::TEvGetPartitionStats>();
- auto& record = get->Record;
- record.SetDomainKeyOwnerId(domainKey.OwnerId);
- record.SetDomainKeyPathId(domainKey.LocalPathId);
-
- record.MutableFrom()->CopyFrom(result->Record.GetNext());
- record.SetFromInclusive(true);
-
- if (toOwnerId) {
- record.MutableTo()->SetOwnerId(*toOwnerId);
- }
- if (toPathId) {
- record.MutableTo()->SetPathId(*toPathId);
- }
- if (toPartIdx) {
- record.MutableTo()->SetPartIdx(*toPartIdx);
- }
- record.SetToInclusive(toInc);
- }
-
- UNIT_ASSERT(checkIt == check.end());
- };
-
- test({}, {}, {}, false, {}, {}, {}, false, {
- {0, 0, 0},
- {0, 0, 1},
- {0, 1, 0},
- {0, 1, 1},
- {1, 0, 0},
- {1, 0, 1},
- {1, 1, 0},
- {1, 1, 1},
- });
-
- test(0, {}, {}, true, 1, {}, {}, false, {
- {0, 0, 0},
- {0, 0, 1},
- {0, 1, 0},
- {0, 1, 1},
- });
-
- test(0, {}, {}, false, 1, {}, {}, true, {
- {1, 0, 0},
- {1, 0, 1},
- {1, 1, 0},
- {1, 1, 1},
- });
-
- test(0, 1, {}, true, 1, 1, {}, false, {
- {0, 1, 0},
- {0, 1, 1},
- {1, 0, 0},
- {1, 0, 1},
- });
-
- test(0, 0, {}, false, 1, 0, {}, true, {
- {0, 1, 0},
- {0, 1, 1},
- {1, 0, 0},
- {1, 0, 1},
- });
-
- test(0, 0, 1, true, 1, 0, 1, false, {
- {0, 0, 1},
- {0, 1, 0},
- {0, 1, 1},
- {1, 0, 0},
- });
-
- test(0, 0, 1, false, 1, 0, 1, true, {
- {0, 1, 0},
- {0, 1, 1},
- {1, 0, 0},
- {1, 0, 1},
- });
- }
-
- Y_UNIT_TEST(Collector) {
- for (size_t batchSize = 1; batchSize < 9; ++batchSize) {
- TestCollector(batchSize);
- }
- }
-
- Y_UNIT_TEST(CollectorOverload) {
- TTestActorRuntime runtime;
- runtime.Initialize(MakeEgg());
-
- auto collector = CreatePartitionStatsCollector(1, 0);
- auto collectorId = runtime.Register(collector.Release());
- auto sender = runtime.AllocateEdgeActor();
-
- auto domainKey = TPathId(1, 1);
-
- auto get = MakeHolder<TEvSysView::TEvGetPartitionStats>();
- auto& record = get->Record;
- record.SetDomainKeyOwnerId(domainKey.OwnerId);
- record.SetDomainKeyPathId(domainKey.LocalPathId);
-
- runtime.Send(new IEventHandle(collectorId, sender, get.Release()));
-
- TAutoPtr<IEventHandle> handle;
- auto result = runtime.GrabEdgeEvent<TEvSysView::TEvGetPartitionStatsResult>(handle);
-
- UNIT_ASSERT_VALUES_EQUAL(result->Record.GetOverloaded(), true);
- }
-
-}
-
-} // NSysView
-} // NKikimr
+ }
+ }
+
+ auto test = [&] (
+ TMaybe<ui64> fromOwnerId, TMaybe<ui64> fromPathId, TMaybe<ui64> fromPartIdx, bool fromInc,
+ TMaybe<ui64> toOwnerId, TMaybe<ui64> toPathId, TMaybe<ui64> toPartIdx, bool toInc,
+ std::initializer_list<std::tuple<ui64, ui64, ui64>> check)
+ {
+ auto get = MakeHolder<TEvSysView::TEvGetPartitionStats>();
+ auto& record = get->Record;
+ record.SetDomainKeyOwnerId(domainKey.OwnerId);
+ record.SetDomainKeyPathId(domainKey.LocalPathId);
+
+ if (fromOwnerId) {
+ record.MutableFrom()->SetOwnerId(*fromOwnerId);
+ }
+ if (fromPathId) {
+ record.MutableFrom()->SetPathId(*fromPathId);
+ }
+ if (fromPartIdx) {
+ record.MutableFrom()->SetPartIdx(*fromPartIdx);
+ }
+ record.SetFromInclusive(fromInc);
+
+ if (toOwnerId) {
+ record.MutableTo()->SetOwnerId(*toOwnerId);
+ }
+ if (toPathId) {
+ record.MutableTo()->SetPathId(*toPathId);
+ }
+ if (toPartIdx) {
+ record.MutableTo()->SetPartIdx(*toPartIdx);
+ }
+ record.SetToInclusive(toInc);
+
+ auto checkIt = check.begin();
+
+ while (true) {
+ runtime.Send(new IEventHandle(collectorId, sender, get.Release()));
+
+ TAutoPtr<IEventHandle> handle;
+ auto result = runtime.GrabEdgeEvent<TEvSysView::TEvGetPartitionStatsResult>(handle);
+
+ for (size_t i = 0; i < result->Record.StatsSize(); ++i, ++checkIt) {
+ auto& stats = result->Record.GetStats(i);
+ UNIT_ASSERT(checkIt != check.end());
+ UNIT_ASSERT_VALUES_EQUAL(stats.GetKey().GetOwnerId(), std::get<0>(*checkIt));
+ UNIT_ASSERT_VALUES_EQUAL(stats.GetKey().GetPathId(), std::get<1>(*checkIt));
+ UNIT_ASSERT_VALUES_EQUAL(stats.GetKey().GetPartIdx(), std::get<2>(*checkIt));
+ }
+
+ if (result->Record.GetLastBatch()) {
+ break;
+ }
+
+ get = MakeHolder<TEvSysView::TEvGetPartitionStats>();
+ auto& record = get->Record;
+ record.SetDomainKeyOwnerId(domainKey.OwnerId);
+ record.SetDomainKeyPathId(domainKey.LocalPathId);
+
+ record.MutableFrom()->CopyFrom(result->Record.GetNext());
+ record.SetFromInclusive(true);
+
+ if (toOwnerId) {
+ record.MutableTo()->SetOwnerId(*toOwnerId);
+ }
+ if (toPathId) {
+ record.MutableTo()->SetPathId(*toPathId);
+ }
+ if (toPartIdx) {
+ record.MutableTo()->SetPartIdx(*toPartIdx);
+ }
+ record.SetToInclusive(toInc);
+ }
+
+ UNIT_ASSERT(checkIt == check.end());
+ };
+
+ test({}, {}, {}, false, {}, {}, {}, false, {
+ {0, 0, 0},
+ {0, 0, 1},
+ {0, 1, 0},
+ {0, 1, 1},
+ {1, 0, 0},
+ {1, 0, 1},
+ {1, 1, 0},
+ {1, 1, 1},
+ });
+
+ test(0, {}, {}, true, 1, {}, {}, false, {
+ {0, 0, 0},
+ {0, 0, 1},
+ {0, 1, 0},
+ {0, 1, 1},
+ });
+
+ test(0, {}, {}, false, 1, {}, {}, true, {
+ {1, 0, 0},
+ {1, 0, 1},
+ {1, 1, 0},
+ {1, 1, 1},
+ });
+
+ test(0, 1, {}, true, 1, 1, {}, false, {
+ {0, 1, 0},
+ {0, 1, 1},
+ {1, 0, 0},
+ {1, 0, 1},
+ });
+
+ test(0, 0, {}, false, 1, 0, {}, true, {
+ {0, 1, 0},
+ {0, 1, 1},
+ {1, 0, 0},
+ {1, 0, 1},
+ });
+
+ test(0, 0, 1, true, 1, 0, 1, false, {
+ {0, 0, 1},
+ {0, 1, 0},
+ {0, 1, 1},
+ {1, 0, 0},
+ });
+
+ test(0, 0, 1, false, 1, 0, 1, true, {
+ {0, 1, 0},
+ {0, 1, 1},
+ {1, 0, 0},
+ {1, 0, 1},
+ });
+ }
+
+ Y_UNIT_TEST(Collector) {
+ for (size_t batchSize = 1; batchSize < 9; ++batchSize) {
+ TestCollector(batchSize);
+ }
+ }
+
+ Y_UNIT_TEST(CollectorOverload) {
+ TTestActorRuntime runtime;
+ runtime.Initialize(MakeEgg());
+
+ auto collector = CreatePartitionStatsCollector(1, 0);
+ auto collectorId = runtime.Register(collector.Release());
+ auto sender = runtime.AllocateEdgeActor();
+
+ auto domainKey = TPathId(1, 1);
+
+ auto get = MakeHolder<TEvSysView::TEvGetPartitionStats>();
+ auto& record = get->Record;
+ record.SetDomainKeyOwnerId(domainKey.OwnerId);
+ record.SetDomainKeyPathId(domainKey.LocalPathId);
+
+ runtime.Send(new IEventHandle(collectorId, sender, get.Release()));
+
+ TAutoPtr<IEventHandle> handle;
+ auto result = runtime.GrabEdgeEvent<TEvSysView::TEvGetPartitionStatsResult>(handle);
+
+ UNIT_ASSERT_VALUES_EQUAL(result->Record.GetOverloaded(), true);
+ }
+
+}
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/partition_stats/ut/ya.make b/ydb/core/sys_view/partition_stats/ut/ya.make
index 01f5bbd1c28..41ddcb31657 100644
--- a/ydb/core/sys_view/partition_stats/ut/ya.make
+++ b/ydb/core/sys_view/partition_stats/ut/ya.make
@@ -1,25 +1,25 @@
UNITTEST_FOR(ydb/core/sys_view/partition_stats)
-
-OWNER(
- monster
- g:kikimr
-)
-
-FORK_SUBTESTS()
-
-SIZE(MEDIUM)
-
-TIMEOUT(600)
-
-PEERDIR(
+
+OWNER(
+ monster
+ g:kikimr
+)
+
+FORK_SUBTESTS()
+
+SIZE(MEDIUM)
+
+TIMEOUT(600)
+
+PEERDIR(
library/cpp/testing/unittest
ydb/core/testlib
-)
-
+)
+
YQL_LAST_ABI_VERSION()
-SRCS(
- partition_stats_ut.cpp
-)
-
-END()
+SRCS(
+ partition_stats_ut.cpp
+)
+
+END()
diff --git a/ydb/core/sys_view/partition_stats/ya.make b/ydb/core/sys_view/partition_stats/ya.make
index d24bf5c89dc..48c79c52e09 100644
--- a/ydb/core/sys_view/partition_stats/ya.make
+++ b/ydb/core/sys_view/partition_stats/ya.make
@@ -1,22 +1,22 @@
-LIBRARY()
-
-OWNER(
- monster
- g:kikimr
-)
-
-SRCS(
- partition_stats.h
- partition_stats.cpp
-)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(
+ monster
+ g:kikimr
+)
+
+SRCS(
+ partition_stats.h
+ partition_stats.cpp
+)
+
+PEERDIR(
library/cpp/actors/core
ydb/core/base
ydb/core/kqp/runtime
ydb/core/sys_view/common
-)
-
+)
+
YQL_LAST_ABI_VERSION()
END()
diff --git a/ydb/core/sys_view/processor/db_counters.cpp b/ydb/core/sys_view/processor/db_counters.cpp
index 42bfbd62691..573f5296604 100644
--- a/ydb/core/sys_view/processor/db_counters.cpp
+++ b/ydb/core/sys_view/processor/db_counters.cpp
@@ -1,483 +1,483 @@
-#include "processor_impl.h"
-
+#include "processor_impl.h"
+
#include <ydb/core/base/counters.h>
#include <ydb/core/base/path.h>
#include <ydb/core/grpc_services/grpc_helper.h>
#include <ydb/core/kqp/counters/kqp_counters.h>
#include <ydb/core/tablet/tablet_counters_aggregator.h>
#include <ydb/core/tablet_flat/flat_executor_counters.h>
-
-namespace NKikimr {
-namespace NSysView {
-
-template <bool IsMax>
-struct TAggregateCumulative {
- static void Apply(NKikimrSysView::TDbCounters* dst, const NKikimrSysView::TDbCounters& src) {
- auto cumulativeSize = src.GetCumulativeCount();
- auto histogramSize = src.HistogramSize();
-
- if (dst->CumulativeSize() < cumulativeSize) {
- dst->MutableCumulative()->Resize(cumulativeSize, 0);
- }
- if (dst->HistogramSize() < histogramSize) {
- auto missing = histogramSize - dst->HistogramSize();
- for (; missing > 0; --missing) {
- dst->AddHistogram();
- }
- }
-
- const auto& from = src.GetCumulative();
- auto* to = dst->MutableCumulative();
- auto doubleDiffSize = from.size() / 2 * 2;
- for (int i = 0; i < doubleDiffSize; ) {
- auto index = from[i++];
- auto value = from[i++];
- if (index >= cumulativeSize) {
- continue;
- }
- if constexpr (!IsMax) {
- (*to)[index] += value;
- } else {
- (*to)[index] = std::max(value, (*to)[index]);
- }
- }
- for (size_t i = 0; i < histogramSize; ++i) {
- const auto& histogram = src.GetHistogram(i);
- const auto& from = histogram.GetBuckets();
- auto* to = dst->MutableHistogram(i)->MutableBuckets();
- auto bucketCount = histogram.GetBucketsCount();
- if (to->size() < (int)bucketCount) {
- to->Resize(bucketCount, 0);
- }
- auto doubleDiffSize = from.size();
- for (int b = 0; b < doubleDiffSize; ) {
- auto index = from[b++];
- auto value = from[b++];
- if (index >= bucketCount) {
- continue;
- }
- if constexpr (!IsMax) {
- (*to)[index] += value;
- } else {
- (*to)[index] = std::max(value, (*to)[index]);
- }
- }
- }
- }
-};
-
-template <bool IsMax>
-struct TAggregateSimple {
- static void Apply(NKikimrSysView::TDbCounters* dst, const NKikimrSysView::TDbCounters& src) {
- auto simpleSize = src.SimpleSize();
- if (dst->SimpleSize() < simpleSize) {
- dst->MutableSimple()->Resize(simpleSize, 0);
- }
- const auto& from = src.GetSimple();
- auto* to = dst->MutableSimple();
- for (size_t i = 0; i < simpleSize; ++i) {
- if constexpr (!IsMax) {
- (*to)[i] += from[i];
- } else {
- (*to)[i] = std::max(from[i], (*to)[i]);
- }
- }
- }
-};
-
-struct TAggregateIncrementalSum {
- static void Apply(NKikimrSysView::TDbCounters* dst, const NKikimrSysView::TDbCounters& src) {
- TAggregateCumulative<false>::Apply(dst, src);
- }
-};
-
-struct TAggregateIncrementalMax {
- static void Apply(NKikimrSysView::TDbCounters* /*dst*/, const NKikimrSysView::TDbCounters& /*src*/) {
- }
-};
-
-struct TAggregateStatefulSum {
- static void Apply(NKikimrSysView::TDbCounters* dst, const NKikimrSysView::TDbCounters& src) {
- TAggregateSimple<false>::Apply(dst, src);
- }
-};
-
-struct TAggregateStatefulMax {
- static void Apply(NKikimrSysView::TDbCounters* dst, const NKikimrSysView::TDbCounters& src) {
- TAggregateSimple<true>::Apply(dst, src);
- TAggregateCumulative<true>::Apply(dst, src);
- }
-};
-
-static void SwapSimpleCounters(NKikimrSysView::TDbCounters* dst, NKikimrSysView::TDbCounters& src) {
- dst->MutableSimple()->Swap(src.MutableSimple());
-};
-
-static void SwapMaxCounters(NKikimrSysView::TDbCounters* dst, NKikimrSysView::TDbCounters& src) {
- SwapSimpleCounters(dst, src);
- dst->MutableCumulative()->Swap(src.MutableCumulative());
- dst->SetCumulativeCount(src.GetCumulativeCount());
-};
-
-static void ResetSimpleCounters(NKikimrSysView::TDbCounters* dst) {
- auto simpleSize = dst->SimpleSize();
- auto* to = dst->MutableSimple();
- for (size_t i = 0; i < simpleSize; ++i) {
- (*to)[i] = 0;
- }
-}
-
-static void ResetMaxCounters(NKikimrSysView::TDbCounters* dst) {
- ResetSimpleCounters(dst);
- auto cumulativeSize = dst->CumulativeSize();
- auto* to = dst->MutableCumulative();
- for (size_t i = 0; i < cumulativeSize; ++i) {
- (*to)[i] = 0;
- }
-}
-
-template <typename TAggrSum, typename TAggrMax>
-static void AggregateCounters(NKikimr::NSysView::TDbServiceCounters* dst,
- const NKikimrSysView::TDbServiceCounters& src)
-{
- if (src.HasMain()) {
- TAggrSum::Apply(dst->Proto().MutableMain(), src.GetMain());
- }
-
- for (const auto& srcTablet : src.GetTabletCounters()) {
- auto* dstTablet = dst->FindOrAddTabletCounters(srcTablet.GetType());
- TAggrSum::Apply(dstTablet->MutableExecutorCounters(), srcTablet.GetExecutorCounters());
- TAggrSum::Apply(dstTablet->MutableAppCounters(), srcTablet.GetAppCounters());
- TAggrMax::Apply(dstTablet->MutableMaxExecutorCounters(), srcTablet.GetMaxExecutorCounters());
- TAggrMax::Apply(dstTablet->MutableMaxAppCounters(), srcTablet.GetMaxAppCounters());
- }
-
- for (const auto& srcReq : src.GetGRpcCounters()) {
- auto* dstReq = dst->FindOrAddGRpcCounters(srcReq.GetGRpcService(), srcReq.GetGRpcRequest());
- TAggrSum::Apply(dstReq->MutableRequestCounters(), srcReq.GetRequestCounters());
- }
-}
-
-static void AggregateIncrementalCounters(NKikimr::NSysView::TDbServiceCounters* dst,
- const NKikimrSysView::TDbServiceCounters& src)
-{
- AggregateCounters<TAggregateIncrementalSum, TAggregateIncrementalMax>(dst, src);
-}
-
-static void AggregateStatefulCounters(NKikimr::NSysView::TDbServiceCounters* dst,
- const NKikimrSysView::TDbServiceCounters& src)
-{
- AggregateCounters<TAggregateStatefulSum, TAggregateStatefulMax>(dst, src);
-}
-
-static void SwapStatefulCounters(NKikimr::NSysView::TDbServiceCounters* dst,
- NKikimrSysView::TDbServiceCounters& src)
-{
- if (src.HasMain()) {
- SwapSimpleCounters(dst->Proto().MutableMain(), *src.MutableMain());
- }
-
- for (auto& srcTablet : *src.MutableTabletCounters()) {
- auto* dstTablet = dst->FindOrAddTabletCounters(srcTablet.GetType());
- SwapSimpleCounters(dstTablet->MutableExecutorCounters(), *srcTablet.MutableExecutorCounters());
- SwapSimpleCounters(dstTablet->MutableAppCounters(), *srcTablet.MutableAppCounters());
- SwapMaxCounters(dstTablet->MutableMaxExecutorCounters(), *srcTablet.MutableMaxExecutorCounters());
- SwapMaxCounters(dstTablet->MutableMaxAppCounters(), *srcTablet.MutableMaxAppCounters());
- }
-
- for (auto& srcReq : *src.MutableGRpcCounters()) {
- auto* dstReq = dst->FindOrAddGRpcCounters(srcReq.GetGRpcService(), srcReq.GetGRpcRequest());
- SwapSimpleCounters(dstReq->MutableRequestCounters(), *srcReq.MutableRequestCounters());
- }
-}
-
-static void ResetStatefulCounters(NKikimrSysView::TDbServiceCounters* dst) {
- if (dst->HasMain()) {
- ResetSimpleCounters(dst->MutableMain());
- }
- for (auto& dstTablet : *dst->MutableTabletCounters()) {
- ResetSimpleCounters(dstTablet.MutableExecutorCounters());
- ResetSimpleCounters(dstTablet.MutableAppCounters());
- ResetMaxCounters(dstTablet.MutableMaxExecutorCounters());
- ResetMaxCounters(dstTablet.MutableMaxAppCounters());
- }
- for (auto& dstReq : *dst->MutableGRpcCounters()) {
- ResetSimpleCounters(dstReq.MutableRequestCounters());
- }
-}
-
-void TSysViewProcessor::SendNavigate() {
- if (!Database) {
- ScheduleSendNavigate();
- return;
- }
-
- using TNavigate = NSchemeCache::TSchemeCacheNavigate;
- auto request = MakeHolder<TNavigate>();
- request->ResultSet.push_back({});
-
- auto& entry = request->ResultSet.back();
- entry.Path = SplitPath(Database);
- entry.Operation = TNavigate::EOp::OpPath;
- entry.RequestType = TNavigate::TEntry::ERequestType::ByPath;
- entry.RedirectRequired = false;
-
- Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release()));
-}
-
-TIntrusivePtr<IDbCounters> TSysViewProcessor::CreateCountersForService(
- NKikimrSysView::EDbCountersService service)
-{
- TIntrusivePtr<IDbCounters> result;
- switch (service) {
- case NKikimrSysView::KQP:
- result = MakeIntrusive<NKqp::TKqpDbCounters>(ExternalGroup, InternalGroup);
- break;
- case NKikimrSysView::TABLETS: {
- THolder<TTabletCountersBase> executorCounters(new NTabletFlatExecutor::TExecutorCounters);
- result = CreateTabletDbCounters(ExternalGroup, InternalGroup, std::move(executorCounters));
- break;
- }
- case NKikimrSysView::GRPC:
- result = NGRpcService::CreateGRpcDbCounters(ExternalGroup, InternalGroup);
- break;
- default:
- break;
- }
-
- if (result) {
- Counters[service] = result;
- }
- return result;
-}
-
-void TSysViewProcessor::AttachExternalCounters() {
- if (!Database) {
- return;
- }
-
- GetServiceCounters(AppData()->Counters, "ydb_serverless", false)
- ->GetSubgroup("database", Database)
- ->GetSubgroup("cloud_id", CloudId)
- ->GetSubgroup("folder_id", FolderId)
- ->GetSubgroup("database_id", DatabaseId)
- ->RegisterSubgroup("host", "", ExternalGroup);
-}
-
-void TSysViewProcessor::AttachInternalCounters() {
- if (!Database) {
- return;
- }
-
- GetServiceCounters(AppData()->Counters, "db", false)
- ->GetSubgroup("database", Database)
- ->RegisterSubgroup("host", "", InternalGroup);
-}
-
-void TSysViewProcessor::DetachExternalCounters() {
- if (!Database) {
- return;
- }
-
- GetServiceCounters(AppData()->Counters, "ydb_serverless", false)
- ->RemoveSubgroup("database", Database);
-}
-
-void TSysViewProcessor::DetachInternalCounters() {
- if (!Database) {
- return;
- }
-
- GetServiceCounters(AppData()->Counters, "db", false)
- ->RemoveSubgroup("database", Database);
-}
-
-void TSysViewProcessor::Handle(TEvSysView::TEvSendDbCountersRequest::TPtr& ev) {
- if (!AppData()->FeatureFlags.GetEnableDbCounters()) {
- return;
- }
-
- auto& record = ev->Get()->Record;
- auto nodeId = record.GetNodeId();
-
- auto& state = NodeCountersStates[nodeId];
- state.FreshCount = 0;
-
- if (state.Generation == record.GetGeneration()) {
- SVLOG_D("[" << TabletID() << "] TEvSendDbCountersRequest, known generation: "
- << "node id# " << nodeId
- << ", generation# " << record.GetGeneration());
-
- auto response = MakeHolder<TEvSysView::TEvSendDbCountersResponse>();
- response->Record.SetDatabase(Database);
- response->Record.SetGeneration(state.Generation);
- Send(ev->Sender, std::move(response));
- return;
- }
-
- state.Generation = record.GetGeneration();
-
- std::unordered_set<NKikimrSysView::EDbCountersService> incomingServicesSet;
-
- for (auto& serviceCounters : *record.MutableServiceCounters()) {
- auto service = serviceCounters.GetService();
- incomingServicesSet.insert(service);
-
- auto& simpleState = state.Simple[service];
- SwapStatefulCounters(&simpleState, *serviceCounters.MutableCounters());
-
- auto& aggrState = AggregatedCountersState[service];
- AggregateIncrementalCounters(&aggrState, serviceCounters.GetCounters());
- }
-
- for (auto it = state.Simple.begin(); it != state.Simple.end(); ) {
- if (incomingServicesSet.find(it->first) == incomingServicesSet.end()) {
- it = state.Simple.erase(it);
- } else {
- ++it;
- }
- }
-
- SVLOG_D("[" << TabletID() << "] TEvSendDbCountersRequest: "
- << "node id# " << nodeId
- << ", generation# " << state.Generation
- << ", services count# " << incomingServicesSet.size()
- << ", request size# " << record.ByteSize());
-
- auto response = MakeHolder<TEvSysView::TEvSendDbCountersResponse>();
- response->Record.SetDatabase(Database);
- response->Record.SetGeneration(state.Generation);
- Send(ev->Sender, std::move(response));
-}
-
-void TSysViewProcessor::Handle(TEvPrivate::TEvApplyCounters::TPtr&) {
- for (auto& [_, counters] : AggregatedCountersState) {
- ResetStatefulCounters(&counters.Proto());
- }
-
- for (auto it = NodeCountersStates.begin(); it != NodeCountersStates.end(); ) {
- auto& state = it->second;
- if (state.FreshCount > 1) {
- it = NodeCountersStates.erase(it);
- continue;
- }
- ++state.FreshCount;
- for (const auto& [service, counters] : state.Simple) {
- AggregateStatefulCounters(&AggregatedCountersState[service], counters.Proto());
- }
- ++it;
- }
-
- for (auto& [service, aggrCounters] : AggregatedCountersState) {
- TIntrusivePtr<IDbCounters> counters;
- if (auto it = Counters.find(service); it != Counters.end()) {
- counters = it->second;
- } else {
- counters = CreateCountersForService(service);
- }
- if (!counters) {
- continue;
- }
- counters->FromProto(aggrCounters);
- }
-
- SVLOG_D("[" << TabletID() << "] TEvApplyCounters: "
- << "services count# " << AggregatedCountersState.size());
-
- ScheduleApplyCounters();
-}
-
-void TSysViewProcessor::Handle(TEvPrivate::TEvSendNavigate::TPtr&) {
- SendNavigate();
-}
-
-void TSysViewProcessor::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) {
- using TNavigate = NSchemeCache::TSchemeCacheNavigate;
-
- THolder<TNavigate> request(ev->Get()->Request.Release());
- if (request->ResultSet.size() != 1) {
- SVLOG_CRIT("[" << TabletID() << "] TEvNavigateKeySetResult failed: "
- << "result set size# " << request->ResultSet.size());
- ScheduleSendNavigate();
- return;
- }
-
- auto& entry = request->ResultSet.back();
-
- if (entry.Status != TNavigate::EStatus::Ok) {
- SVLOG_W("[" << TabletID() << "] TEvNavigateKeySetResult failed: "
- << "status# " << entry.Status);
- ScheduleSendNavigate();
- return;
- }
-
- for (const auto& [key, value] : entry.Attributes) {
- if (key == "cloud_id") {
- CloudId = value;
- } else if (key == "folder_id") {
- FolderId = value;
- } else if (key == "database_id") {
- DatabaseId = value;
- }
- }
-
- AttachExternalCounters();
- AttachInternalCounters();
-
- Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvWatchPathId(entry.TableId.PathId));
-
- SVLOG_D("[" << TabletID() << "] TEvNavigateKeySetResult: "
- << "database# " << Database
- << ", pathId# " << entry.TableId.PathId
- << ", cloud_id# " << CloudId
- << ", folder_id# " << FolderId
- << ", database_id# " << DatabaseId);
-}
-
-void TSysViewProcessor::Handle(TEvTxProxySchemeCache::TEvWatchNotifyUpdated::TPtr& ev) {
- const auto& describeResult = ev->Get()->Result;
-
- TString path = describeResult->GetPath();
- if (path != Database) {
- SVLOG_W("[" << TabletID() << "] TEvWatchNotifyUpdated, invalid path: "
- << "database# " << Database
- << ", path# " << path);
- return;
- }
-
- const auto& pathDescription = describeResult->GetPathDescription();
- const auto& userAttrs = pathDescription.GetUserAttributes();
-
- TString cloudId, folderId, databaseId;
- for (const auto& attr : userAttrs) {
- if (attr.GetKey() == "cloud_id") {
- cloudId = attr.GetValue();
- } else if (attr.GetKey() == "folder_id") {
- folderId = attr.GetValue();
- } else if (attr.GetKey() == "database_id") {
- databaseId = attr.GetValue();
- }
- }
-
- SVLOG_D("[" << TabletID() << "] TEvWatchNotifyUpdated: "
- << "database# " << Database
- << ", cloud_id# " << cloudId
- << ", folder_id# " << folderId
- << ", database_id# " << databaseId);
-
- if (cloudId != CloudId || folderId != FolderId || databaseId != DatabaseId) {
- DetachExternalCounters();
- CloudId = cloudId;
- FolderId = folderId;
- DatabaseId = databaseId;
- AttachExternalCounters();
- }
-}
-
-void TSysViewProcessor::Handle(TEvTxProxySchemeCache::TEvWatchNotifyDeleted::TPtr& ev) {
- Y_UNUSED(ev);
-}
-
-} // NSysView
-} // NKikimr
-
+
+namespace NKikimr {
+namespace NSysView {
+
+template <bool IsMax>
+struct TAggregateCumulative {
+ static void Apply(NKikimrSysView::TDbCounters* dst, const NKikimrSysView::TDbCounters& src) {
+ auto cumulativeSize = src.GetCumulativeCount();
+ auto histogramSize = src.HistogramSize();
+
+ if (dst->CumulativeSize() < cumulativeSize) {
+ dst->MutableCumulative()->Resize(cumulativeSize, 0);
+ }
+ if (dst->HistogramSize() < histogramSize) {
+ auto missing = histogramSize - dst->HistogramSize();
+ for (; missing > 0; --missing) {
+ dst->AddHistogram();
+ }
+ }
+
+ const auto& from = src.GetCumulative();
+ auto* to = dst->MutableCumulative();
+ auto doubleDiffSize = from.size() / 2 * 2;
+ for (int i = 0; i < doubleDiffSize; ) {
+ auto index = from[i++];
+ auto value = from[i++];
+ if (index >= cumulativeSize) {
+ continue;
+ }
+ if constexpr (!IsMax) {
+ (*to)[index] += value;
+ } else {
+ (*to)[index] = std::max(value, (*to)[index]);
+ }
+ }
+ for (size_t i = 0; i < histogramSize; ++i) {
+ const auto& histogram = src.GetHistogram(i);
+ const auto& from = histogram.GetBuckets();
+ auto* to = dst->MutableHistogram(i)->MutableBuckets();
+ auto bucketCount = histogram.GetBucketsCount();
+ if (to->size() < (int)bucketCount) {
+ to->Resize(bucketCount, 0);
+ }
+ auto doubleDiffSize = from.size();
+ for (int b = 0; b < doubleDiffSize; ) {
+ auto index = from[b++];
+ auto value = from[b++];
+ if (index >= bucketCount) {
+ continue;
+ }
+ if constexpr (!IsMax) {
+ (*to)[index] += value;
+ } else {
+ (*to)[index] = std::max(value, (*to)[index]);
+ }
+ }
+ }
+ }
+};
+
+template <bool IsMax>
+struct TAggregateSimple {
+ static void Apply(NKikimrSysView::TDbCounters* dst, const NKikimrSysView::TDbCounters& src) {
+ auto simpleSize = src.SimpleSize();
+ if (dst->SimpleSize() < simpleSize) {
+ dst->MutableSimple()->Resize(simpleSize, 0);
+ }
+ const auto& from = src.GetSimple();
+ auto* to = dst->MutableSimple();
+ for (size_t i = 0; i < simpleSize; ++i) {
+ if constexpr (!IsMax) {
+ (*to)[i] += from[i];
+ } else {
+ (*to)[i] = std::max(from[i], (*to)[i]);
+ }
+ }
+ }
+};
+
+struct TAggregateIncrementalSum {
+ static void Apply(NKikimrSysView::TDbCounters* dst, const NKikimrSysView::TDbCounters& src) {
+ TAggregateCumulative<false>::Apply(dst, src);
+ }
+};
+
+struct TAggregateIncrementalMax {
+ static void Apply(NKikimrSysView::TDbCounters* /*dst*/, const NKikimrSysView::TDbCounters& /*src*/) {
+ }
+};
+
+struct TAggregateStatefulSum {
+ static void Apply(NKikimrSysView::TDbCounters* dst, const NKikimrSysView::TDbCounters& src) {
+ TAggregateSimple<false>::Apply(dst, src);
+ }
+};
+
+struct TAggregateStatefulMax {
+ static void Apply(NKikimrSysView::TDbCounters* dst, const NKikimrSysView::TDbCounters& src) {
+ TAggregateSimple<true>::Apply(dst, src);
+ TAggregateCumulative<true>::Apply(dst, src);
+ }
+};
+
+static void SwapSimpleCounters(NKikimrSysView::TDbCounters* dst, NKikimrSysView::TDbCounters& src) {
+ dst->MutableSimple()->Swap(src.MutableSimple());
+};
+
+static void SwapMaxCounters(NKikimrSysView::TDbCounters* dst, NKikimrSysView::TDbCounters& src) {
+ SwapSimpleCounters(dst, src);
+ dst->MutableCumulative()->Swap(src.MutableCumulative());
+ dst->SetCumulativeCount(src.GetCumulativeCount());
+};
+
+static void ResetSimpleCounters(NKikimrSysView::TDbCounters* dst) {
+ auto simpleSize = dst->SimpleSize();
+ auto* to = dst->MutableSimple();
+ for (size_t i = 0; i < simpleSize; ++i) {
+ (*to)[i] = 0;
+ }
+}
+
+static void ResetMaxCounters(NKikimrSysView::TDbCounters* dst) {
+ ResetSimpleCounters(dst);
+ auto cumulativeSize = dst->CumulativeSize();
+ auto* to = dst->MutableCumulative();
+ for (size_t i = 0; i < cumulativeSize; ++i) {
+ (*to)[i] = 0;
+ }
+}
+
+template <typename TAggrSum, typename TAggrMax>
+static void AggregateCounters(NKikimr::NSysView::TDbServiceCounters* dst,
+ const NKikimrSysView::TDbServiceCounters& src)
+{
+ if (src.HasMain()) {
+ TAggrSum::Apply(dst->Proto().MutableMain(), src.GetMain());
+ }
+
+ for (const auto& srcTablet : src.GetTabletCounters()) {
+ auto* dstTablet = dst->FindOrAddTabletCounters(srcTablet.GetType());
+ TAggrSum::Apply(dstTablet->MutableExecutorCounters(), srcTablet.GetExecutorCounters());
+ TAggrSum::Apply(dstTablet->MutableAppCounters(), srcTablet.GetAppCounters());
+ TAggrMax::Apply(dstTablet->MutableMaxExecutorCounters(), srcTablet.GetMaxExecutorCounters());
+ TAggrMax::Apply(dstTablet->MutableMaxAppCounters(), srcTablet.GetMaxAppCounters());
+ }
+
+ for (const auto& srcReq : src.GetGRpcCounters()) {
+ auto* dstReq = dst->FindOrAddGRpcCounters(srcReq.GetGRpcService(), srcReq.GetGRpcRequest());
+ TAggrSum::Apply(dstReq->MutableRequestCounters(), srcReq.GetRequestCounters());
+ }
+}
+
+static void AggregateIncrementalCounters(NKikimr::NSysView::TDbServiceCounters* dst,
+ const NKikimrSysView::TDbServiceCounters& src)
+{
+ AggregateCounters<TAggregateIncrementalSum, TAggregateIncrementalMax>(dst, src);
+}
+
+static void AggregateStatefulCounters(NKikimr::NSysView::TDbServiceCounters* dst,
+ const NKikimrSysView::TDbServiceCounters& src)
+{
+ AggregateCounters<TAggregateStatefulSum, TAggregateStatefulMax>(dst, src);
+}
+
+static void SwapStatefulCounters(NKikimr::NSysView::TDbServiceCounters* dst,
+ NKikimrSysView::TDbServiceCounters& src)
+{
+ if (src.HasMain()) {
+ SwapSimpleCounters(dst->Proto().MutableMain(), *src.MutableMain());
+ }
+
+ for (auto& srcTablet : *src.MutableTabletCounters()) {
+ auto* dstTablet = dst->FindOrAddTabletCounters(srcTablet.GetType());
+ SwapSimpleCounters(dstTablet->MutableExecutorCounters(), *srcTablet.MutableExecutorCounters());
+ SwapSimpleCounters(dstTablet->MutableAppCounters(), *srcTablet.MutableAppCounters());
+ SwapMaxCounters(dstTablet->MutableMaxExecutorCounters(), *srcTablet.MutableMaxExecutorCounters());
+ SwapMaxCounters(dstTablet->MutableMaxAppCounters(), *srcTablet.MutableMaxAppCounters());
+ }
+
+ for (auto& srcReq : *src.MutableGRpcCounters()) {
+ auto* dstReq = dst->FindOrAddGRpcCounters(srcReq.GetGRpcService(), srcReq.GetGRpcRequest());
+ SwapSimpleCounters(dstReq->MutableRequestCounters(), *srcReq.MutableRequestCounters());
+ }
+}
+
+static void ResetStatefulCounters(NKikimrSysView::TDbServiceCounters* dst) {
+ if (dst->HasMain()) {
+ ResetSimpleCounters(dst->MutableMain());
+ }
+ for (auto& dstTablet : *dst->MutableTabletCounters()) {
+ ResetSimpleCounters(dstTablet.MutableExecutorCounters());
+ ResetSimpleCounters(dstTablet.MutableAppCounters());
+ ResetMaxCounters(dstTablet.MutableMaxExecutorCounters());
+ ResetMaxCounters(dstTablet.MutableMaxAppCounters());
+ }
+ for (auto& dstReq : *dst->MutableGRpcCounters()) {
+ ResetSimpleCounters(dstReq.MutableRequestCounters());
+ }
+}
+
+void TSysViewProcessor::SendNavigate() {
+ if (!Database) {
+ ScheduleSendNavigate();
+ return;
+ }
+
+ using TNavigate = NSchemeCache::TSchemeCacheNavigate;
+ auto request = MakeHolder<TNavigate>();
+ request->ResultSet.push_back({});
+
+ auto& entry = request->ResultSet.back();
+ entry.Path = SplitPath(Database);
+ entry.Operation = TNavigate::EOp::OpPath;
+ entry.RequestType = TNavigate::TEntry::ERequestType::ByPath;
+ entry.RedirectRequired = false;
+
+ Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release()));
+}
+
+TIntrusivePtr<IDbCounters> TSysViewProcessor::CreateCountersForService(
+ NKikimrSysView::EDbCountersService service)
+{
+ TIntrusivePtr<IDbCounters> result;
+ switch (service) {
+ case NKikimrSysView::KQP:
+ result = MakeIntrusive<NKqp::TKqpDbCounters>(ExternalGroup, InternalGroup);
+ break;
+ case NKikimrSysView::TABLETS: {
+ THolder<TTabletCountersBase> executorCounters(new NTabletFlatExecutor::TExecutorCounters);
+ result = CreateTabletDbCounters(ExternalGroup, InternalGroup, std::move(executorCounters));
+ break;
+ }
+ case NKikimrSysView::GRPC:
+ result = NGRpcService::CreateGRpcDbCounters(ExternalGroup, InternalGroup);
+ break;
+ default:
+ break;
+ }
+
+ if (result) {
+ Counters[service] = result;
+ }
+ return result;
+}
+
+void TSysViewProcessor::AttachExternalCounters() {
+ if (!Database) {
+ return;
+ }
+
+ GetServiceCounters(AppData()->Counters, "ydb_serverless", false)
+ ->GetSubgroup("database", Database)
+ ->GetSubgroup("cloud_id", CloudId)
+ ->GetSubgroup("folder_id", FolderId)
+ ->GetSubgroup("database_id", DatabaseId)
+ ->RegisterSubgroup("host", "", ExternalGroup);
+}
+
+void TSysViewProcessor::AttachInternalCounters() {
+ if (!Database) {
+ return;
+ }
+
+ GetServiceCounters(AppData()->Counters, "db", false)
+ ->GetSubgroup("database", Database)
+ ->RegisterSubgroup("host", "", InternalGroup);
+}
+
+void TSysViewProcessor::DetachExternalCounters() {
+ if (!Database) {
+ return;
+ }
+
+ GetServiceCounters(AppData()->Counters, "ydb_serverless", false)
+ ->RemoveSubgroup("database", Database);
+}
+
+void TSysViewProcessor::DetachInternalCounters() {
+ if (!Database) {
+ return;
+ }
+
+ GetServiceCounters(AppData()->Counters, "db", false)
+ ->RemoveSubgroup("database", Database);
+}
+
+void TSysViewProcessor::Handle(TEvSysView::TEvSendDbCountersRequest::TPtr& ev) {
+ if (!AppData()->FeatureFlags.GetEnableDbCounters()) {
+ return;
+ }
+
+ auto& record = ev->Get()->Record;
+ auto nodeId = record.GetNodeId();
+
+ auto& state = NodeCountersStates[nodeId];
+ state.FreshCount = 0;
+
+ if (state.Generation == record.GetGeneration()) {
+ SVLOG_D("[" << TabletID() << "] TEvSendDbCountersRequest, known generation: "
+ << "node id# " << nodeId
+ << ", generation# " << record.GetGeneration());
+
+ auto response = MakeHolder<TEvSysView::TEvSendDbCountersResponse>();
+ response->Record.SetDatabase(Database);
+ response->Record.SetGeneration(state.Generation);
+ Send(ev->Sender, std::move(response));
+ return;
+ }
+
+ state.Generation = record.GetGeneration();
+
+ std::unordered_set<NKikimrSysView::EDbCountersService> incomingServicesSet;
+
+ for (auto& serviceCounters : *record.MutableServiceCounters()) {
+ auto service = serviceCounters.GetService();
+ incomingServicesSet.insert(service);
+
+ auto& simpleState = state.Simple[service];
+ SwapStatefulCounters(&simpleState, *serviceCounters.MutableCounters());
+
+ auto& aggrState = AggregatedCountersState[service];
+ AggregateIncrementalCounters(&aggrState, serviceCounters.GetCounters());
+ }
+
+ for (auto it = state.Simple.begin(); it != state.Simple.end(); ) {
+ if (incomingServicesSet.find(it->first) == incomingServicesSet.end()) {
+ it = state.Simple.erase(it);
+ } else {
+ ++it;
+ }
+ }
+
+ SVLOG_D("[" << TabletID() << "] TEvSendDbCountersRequest: "
+ << "node id# " << nodeId
+ << ", generation# " << state.Generation
+ << ", services count# " << incomingServicesSet.size()
+ << ", request size# " << record.ByteSize());
+
+ auto response = MakeHolder<TEvSysView::TEvSendDbCountersResponse>();
+ response->Record.SetDatabase(Database);
+ response->Record.SetGeneration(state.Generation);
+ Send(ev->Sender, std::move(response));
+}
+
+void TSysViewProcessor::Handle(TEvPrivate::TEvApplyCounters::TPtr&) {
+ for (auto& [_, counters] : AggregatedCountersState) {
+ ResetStatefulCounters(&counters.Proto());
+ }
+
+ for (auto it = NodeCountersStates.begin(); it != NodeCountersStates.end(); ) {
+ auto& state = it->second;
+ if (state.FreshCount > 1) {
+ it = NodeCountersStates.erase(it);
+ continue;
+ }
+ ++state.FreshCount;
+ for (const auto& [service, counters] : state.Simple) {
+ AggregateStatefulCounters(&AggregatedCountersState[service], counters.Proto());
+ }
+ ++it;
+ }
+
+ for (auto& [service, aggrCounters] : AggregatedCountersState) {
+ TIntrusivePtr<IDbCounters> counters;
+ if (auto it = Counters.find(service); it != Counters.end()) {
+ counters = it->second;
+ } else {
+ counters = CreateCountersForService(service);
+ }
+ if (!counters) {
+ continue;
+ }
+ counters->FromProto(aggrCounters);
+ }
+
+ SVLOG_D("[" << TabletID() << "] TEvApplyCounters: "
+ << "services count# " << AggregatedCountersState.size());
+
+ ScheduleApplyCounters();
+}
+
+void TSysViewProcessor::Handle(TEvPrivate::TEvSendNavigate::TPtr&) {
+ SendNavigate();
+}
+
+void TSysViewProcessor::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) {
+ using TNavigate = NSchemeCache::TSchemeCacheNavigate;
+
+ THolder<TNavigate> request(ev->Get()->Request.Release());
+ if (request->ResultSet.size() != 1) {
+ SVLOG_CRIT("[" << TabletID() << "] TEvNavigateKeySetResult failed: "
+ << "result set size# " << request->ResultSet.size());
+ ScheduleSendNavigate();
+ return;
+ }
+
+ auto& entry = request->ResultSet.back();
+
+ if (entry.Status != TNavigate::EStatus::Ok) {
+ SVLOG_W("[" << TabletID() << "] TEvNavigateKeySetResult failed: "
+ << "status# " << entry.Status);
+ ScheduleSendNavigate();
+ return;
+ }
+
+ for (const auto& [key, value] : entry.Attributes) {
+ if (key == "cloud_id") {
+ CloudId = value;
+ } else if (key == "folder_id") {
+ FolderId = value;
+ } else if (key == "database_id") {
+ DatabaseId = value;
+ }
+ }
+
+ AttachExternalCounters();
+ AttachInternalCounters();
+
+ Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvWatchPathId(entry.TableId.PathId));
+
+ SVLOG_D("[" << TabletID() << "] TEvNavigateKeySetResult: "
+ << "database# " << Database
+ << ", pathId# " << entry.TableId.PathId
+ << ", cloud_id# " << CloudId
+ << ", folder_id# " << FolderId
+ << ", database_id# " << DatabaseId);
+}
+
+void TSysViewProcessor::Handle(TEvTxProxySchemeCache::TEvWatchNotifyUpdated::TPtr& ev) {
+ const auto& describeResult = ev->Get()->Result;
+
+ TString path = describeResult->GetPath();
+ if (path != Database) {
+ SVLOG_W("[" << TabletID() << "] TEvWatchNotifyUpdated, invalid path: "
+ << "database# " << Database
+ << ", path# " << path);
+ return;
+ }
+
+ const auto& pathDescription = describeResult->GetPathDescription();
+ const auto& userAttrs = pathDescription.GetUserAttributes();
+
+ TString cloudId, folderId, databaseId;
+ for (const auto& attr : userAttrs) {
+ if (attr.GetKey() == "cloud_id") {
+ cloudId = attr.GetValue();
+ } else if (attr.GetKey() == "folder_id") {
+ folderId = attr.GetValue();
+ } else if (attr.GetKey() == "database_id") {
+ databaseId = attr.GetValue();
+ }
+ }
+
+ SVLOG_D("[" << TabletID() << "] TEvWatchNotifyUpdated: "
+ << "database# " << Database
+ << ", cloud_id# " << cloudId
+ << ", folder_id# " << folderId
+ << ", database_id# " << databaseId);
+
+ if (cloudId != CloudId || folderId != FolderId || databaseId != DatabaseId) {
+ DetachExternalCounters();
+ CloudId = cloudId;
+ FolderId = folderId;
+ DatabaseId = databaseId;
+ AttachExternalCounters();
+ }
+}
+
+void TSysViewProcessor::Handle(TEvTxProxySchemeCache::TEvWatchNotifyDeleted::TPtr& ev) {
+ Y_UNUSED(ev);
+}
+
+} // NSysView
+} // NKikimr
+
diff --git a/ydb/core/sys_view/processor/processor.cpp b/ydb/core/sys_view/processor/processor.cpp
index d0c09f8e7b7..64e715ea3b3 100644
--- a/ydb/core/sys_view/processor/processor.cpp
+++ b/ydb/core/sys_view/processor/processor.cpp
@@ -1,18 +1,18 @@
-#include "processor.h"
-
-#include "processor_impl.h"
-
-namespace NKikimr {
-namespace NSysView {
-
+#include "processor.h"
+
+#include "processor_impl.h"
+
+namespace NKikimr {
+namespace NSysView {
+
IActor* CreateSysViewProcessor(const TActorId& tablet, TTabletStorageInfo* info) {
- return new TSysViewProcessor(tablet, info, EProcessorMode::MINUTE);
-}
-
+ return new TSysViewProcessor(tablet, info, EProcessorMode::MINUTE);
+}
+
IActor* CreateSysViewProcessorForTests(const TActorId& tablet, TTabletStorageInfo* info) {
- return new TSysViewProcessor(tablet, info, EProcessorMode::FAST);
-}
-
-} // NSysView
-} // NKikimr
-
+ return new TSysViewProcessor(tablet, info, EProcessorMode::FAST);
+}
+
+} // NSysView
+} // NKikimr
+
diff --git a/ydb/core/sys_view/processor/processor.h b/ydb/core/sys_view/processor/processor.h
index ad086965974..10a47fee990 100644
--- a/ydb/core/sys_view/processor/processor.h
+++ b/ydb/core/sys_view/processor/processor.h
@@ -1,13 +1,13 @@
-#pragma once
-
-#include <library/cpp/actors/core/actor.h>
+#pragma once
+
+#include <library/cpp/actors/core/actor.h>
#include <ydb/core/base/blobstorage.h>
-
-namespace NKikimr {
-namespace NSysView {
-
+
+namespace NKikimr {
+namespace NSysView {
+
IActor* CreateSysViewProcessor(const TActorId& tablet, TTabletStorageInfo* info);
IActor* CreateSysViewProcessorForTests(const TActorId& tablet, TTabletStorageInfo* info);
-
-} // NSysView
-} // NKikimr
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/processor/processor_impl.cpp b/ydb/core/sys_view/processor/processor_impl.cpp
index 86ddbfd4238..49187f555eb 100644
--- a/ydb/core/sys_view/processor/processor_impl.cpp
+++ b/ydb/core/sys_view/processor/processor_impl.cpp
@@ -1,685 +1,685 @@
-#include "processor_impl.h"
-
+#include "processor_impl.h"
+
#include <ydb/core/sys_view/service/sysview_service.h>
#include <ydb/core/engine/minikql/flat_local_tx_factory.h>
-
-#include <library/cpp/monlib/service/pages/templates.h>
-
-namespace NKikimr {
-namespace NSysView {
-
+
+#include <library/cpp/monlib/service/pages/templates.h>
+
+namespace NKikimr {
+namespace NSysView {
+
TSysViewProcessor::TSysViewProcessor(const TActorId& tablet, TTabletStorageInfo* info, EProcessorMode processorMode)
- : TActor(&TThis::StateInit)
- , TTabletExecutedFlat(info, tablet, new NMiniKQL::TMiniKQLFactory)
- , TotalInterval(TDuration::Seconds(processorMode == EProcessorMode::FAST ? 6 : 60))
- , CollectInterval(TDuration::Seconds(processorMode == EProcessorMode::FAST ? 3 : 30))
- , ExternalGroup(new NMonitoring::TDynamicCounters)
- , InternalGroup(new NMonitoring::TDynamicCounters)
-{}
-
-void TSysViewProcessor::OnDetach(const TActorContext& ctx) {
- DetachExternalCounters();
- DetachInternalCounters();
-
- Die(ctx);
-}
-
-void TSysViewProcessor::OnTabletDead(TEvTablet::TEvTabletDead::TPtr&, const TActorContext& ctx) {
- DetachExternalCounters();
- DetachInternalCounters();
-
- Die(ctx);
-}
-
-void TSysViewProcessor::OnActivateExecutor(const TActorContext& ctx) {
- SVLOG_I("[" << TabletID() << "] OnActivateExecutor");
-
- // TODO: tablet counters
- Execute(CreateTxInitSchema(), ctx);
-}
-
-void TSysViewProcessor::DefaultSignalTabletActive(const TActorContext& ctx) {
- Y_UNUSED(ctx);
-}
-
-void TSysViewProcessor::Handle(TEvPrivate::TEvSendRequests::TPtr&) {
- SVLOG_D("[" << TabletID() << "] Handle TEvPrivate::TEvSendRequests");
- SendRequests();
-}
-
-void TSysViewProcessor::PersistSysParam(NIceDb::TNiceDb& db, ui64 id, const TString& value) {
- db.Table<Schema::SysParams>().Key(id).Update(
- NIceDb::TUpdate<Schema::SysParams::Value>(value));
-}
-
-void TSysViewProcessor::PersistDatabase(NIceDb::TNiceDb& db) {
- PersistSysParam(db, Schema::SysParam_Database, Database);
-}
-
-void TSysViewProcessor::PersistStage(NIceDb::TNiceDb& db) {
- ui64 stage = static_cast<ui64>(CurrentStage);
- PersistSysParam(db, Schema::SysParam_CurrentStage, ToString(stage));
-}
-
-void TSysViewProcessor::PersistIntervalEnd(NIceDb::TNiceDb& db) {
- ui64 intervalEndUs = IntervalEnd.MicroSeconds();
- PersistSysParam(db, Schema::SysParam_IntervalEnd, ToString(intervalEndUs));
-}
-
-template <typename TSchema>
-void TSysViewProcessor::PersistTopResults(NIceDb::TNiceDb& db,
- TTop& top, TResultStatsMap& results, TInstant intervalEnd)
-{
- ui64 intervalEndUs = intervalEnd.MicroSeconds();
- ui32 rank = 0;
-
- std::sort(top.begin(), top.end(), TopQueryCompare);
-
- for (const auto& entry : top) {
- if (entry.Stats) {
- auto key = std::make_pair(intervalEndUs, ++rank);
- auto& resultStats = results[key];
- resultStats = *entry.Stats;
-
- TString text;
- TString serialized;
- resultStats.MutableQueryText()->swap(text);
+ : TActor(&TThis::StateInit)
+ , TTabletExecutedFlat(info, tablet, new NMiniKQL::TMiniKQLFactory)
+ , TotalInterval(TDuration::Seconds(processorMode == EProcessorMode::FAST ? 6 : 60))
+ , CollectInterval(TDuration::Seconds(processorMode == EProcessorMode::FAST ? 3 : 30))
+ , ExternalGroup(new NMonitoring::TDynamicCounters)
+ , InternalGroup(new NMonitoring::TDynamicCounters)
+{}
+
+void TSysViewProcessor::OnDetach(const TActorContext& ctx) {
+ DetachExternalCounters();
+ DetachInternalCounters();
+
+ Die(ctx);
+}
+
+void TSysViewProcessor::OnTabletDead(TEvTablet::TEvTabletDead::TPtr&, const TActorContext& ctx) {
+ DetachExternalCounters();
+ DetachInternalCounters();
+
+ Die(ctx);
+}
+
+void TSysViewProcessor::OnActivateExecutor(const TActorContext& ctx) {
+ SVLOG_I("[" << TabletID() << "] OnActivateExecutor");
+
+ // TODO: tablet counters
+ Execute(CreateTxInitSchema(), ctx);
+}
+
+void TSysViewProcessor::DefaultSignalTabletActive(const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+}
+
+void TSysViewProcessor::Handle(TEvPrivate::TEvSendRequests::TPtr&) {
+ SVLOG_D("[" << TabletID() << "] Handle TEvPrivate::TEvSendRequests");
+ SendRequests();
+}
+
+void TSysViewProcessor::PersistSysParam(NIceDb::TNiceDb& db, ui64 id, const TString& value) {
+ db.Table<Schema::SysParams>().Key(id).Update(
+ NIceDb::TUpdate<Schema::SysParams::Value>(value));
+}
+
+void TSysViewProcessor::PersistDatabase(NIceDb::TNiceDb& db) {
+ PersistSysParam(db, Schema::SysParam_Database, Database);
+}
+
+void TSysViewProcessor::PersistStage(NIceDb::TNiceDb& db) {
+ ui64 stage = static_cast<ui64>(CurrentStage);
+ PersistSysParam(db, Schema::SysParam_CurrentStage, ToString(stage));
+}
+
+void TSysViewProcessor::PersistIntervalEnd(NIceDb::TNiceDb& db) {
+ ui64 intervalEndUs = IntervalEnd.MicroSeconds();
+ PersistSysParam(db, Schema::SysParam_IntervalEnd, ToString(intervalEndUs));
+}
+
+template <typename TSchema>
+void TSysViewProcessor::PersistTopResults(NIceDb::TNiceDb& db,
+ TTop& top, TResultStatsMap& results, TInstant intervalEnd)
+{
+ ui64 intervalEndUs = intervalEnd.MicroSeconds();
+ ui32 rank = 0;
+
+ std::sort(top.begin(), top.end(), TopQueryCompare);
+
+ for (const auto& entry : top) {
+ if (entry.Stats) {
+ auto key = std::make_pair(intervalEndUs, ++rank);
+ auto& resultStats = results[key];
+ resultStats = *entry.Stats;
+
+ TString text;
+ TString serialized;
+ resultStats.MutableQueryText()->swap(text);
Y_PROTOBUF_SUPPRESS_NODISCARD resultStats.SerializeToString(&serialized);
- db.Table<TSchema>().Key(key).Update(
- NIceDb::TUpdate<typename TSchema::Text>(text),
- NIceDb::TUpdate<typename TSchema::Data>(serialized));
- resultStats.MutableQueryText()->swap(text);
- }
- }
-
- SVLOG_D("[" << TabletID() << "] PersistTopResults: "
- << "table id# " << TSchema::TableId
- << ", interval end# " << intervalEnd
- << ", query count# " << top.size()
- << ", persisted# " << rank);
-}
-
-void TSysViewProcessor::PersistResults(NIceDb::TNiceDb& db) {
- std::vector<std::pair<ui64, TQueryHash>> sorted;
- sorted.reserve(QueryMetrics.size());
- for (const auto& [queryHash, metrics] : QueryMetrics) {
- sorted.emplace_back(metrics.Metrics.GetCpuTimeUs().GetSum(), queryHash);
- }
- std::sort(sorted.begin(), sorted.end(), [] (auto& l, auto& r) { return l.first > r.first; });
-
- ui64 intervalEndUs = IntervalEnd.MicroSeconds();
- ui32 rank = 0;
-
- for (const auto& entry : sorted) {
- auto key = std::make_pair(intervalEndUs, ++rank);
-
- auto& queryMetrics = QueryMetrics[entry.second];
- auto& resultMetrics = MetricsOneMinute[key];
- resultMetrics.Text = queryMetrics.Text;
- resultMetrics.Metrics = queryMetrics.Metrics;
-
- TString serialized;
+ db.Table<TSchema>().Key(key).Update(
+ NIceDb::TUpdate<typename TSchema::Text>(text),
+ NIceDb::TUpdate<typename TSchema::Data>(serialized));
+ resultStats.MutableQueryText()->swap(text);
+ }
+ }
+
+ SVLOG_D("[" << TabletID() << "] PersistTopResults: "
+ << "table id# " << TSchema::TableId
+ << ", interval end# " << intervalEnd
+ << ", query count# " << top.size()
+ << ", persisted# " << rank);
+}
+
+void TSysViewProcessor::PersistResults(NIceDb::TNiceDb& db) {
+ std::vector<std::pair<ui64, TQueryHash>> sorted;
+ sorted.reserve(QueryMetrics.size());
+ for (const auto& [queryHash, metrics] : QueryMetrics) {
+ sorted.emplace_back(metrics.Metrics.GetCpuTimeUs().GetSum(), queryHash);
+ }
+ std::sort(sorted.begin(), sorted.end(), [] (auto& l, auto& r) { return l.first > r.first; });
+
+ ui64 intervalEndUs = IntervalEnd.MicroSeconds();
+ ui32 rank = 0;
+
+ for (const auto& entry : sorted) {
+ auto key = std::make_pair(intervalEndUs, ++rank);
+
+ auto& queryMetrics = QueryMetrics[entry.second];
+ auto& resultMetrics = MetricsOneMinute[key];
+ resultMetrics.Text = queryMetrics.Text;
+ resultMetrics.Metrics = queryMetrics.Metrics;
+
+ TString serialized;
Y_PROTOBUF_SUPPRESS_NODISCARD resultMetrics.Metrics.SerializeToString(&serialized);
- db.Table<Schema::MetricsOneMinute>().Key(key).Update(
- NIceDb::TUpdate<Schema::MetricsOneMinute::Text>(resultMetrics.Text),
- NIceDb::TUpdate<Schema::MetricsOneMinute::Data>(serialized));
- }
-
- SVLOG_D("[" << TabletID() << "] PersistResults: "
- << "interval end# " << IntervalEnd
- << ", query count# " << sorted.size());
-
- // TODO: metrics one hour?
-
- PersistTopResults<Schema::TopByDurationOneMinute>(
- db, ByDurationMinute, TopByDurationOneMinute, IntervalEnd);
- PersistTopResults<Schema::TopByReadBytesOneMinute>(
- db, ByReadBytesMinute, TopByReadBytesOneMinute, IntervalEnd);
- PersistTopResults<Schema::TopByCpuTimeOneMinute>(
- db, ByCpuTimeMinute, TopByCpuTimeOneMinute, IntervalEnd);
- PersistTopResults<Schema::TopByRequestUnitsOneMinute>(
- db, ByRequestUnitsMinute, TopByRequestUnitsOneMinute, IntervalEnd);
-
- auto hourEnd = EndOfHourInterval(IntervalEnd);
-
- PersistTopResults<Schema::TopByDurationOneHour>(
- db, ByDurationHour, TopByDurationOneHour, hourEnd);
- PersistTopResults<Schema::TopByReadBytesOneHour>(
- db, ByReadBytesHour, TopByReadBytesOneHour, hourEnd);
- PersistTopResults<Schema::TopByCpuTimeOneHour>(
- db, ByCpuTimeHour, TopByCpuTimeOneHour, hourEnd);
- PersistTopResults<Schema::TopByRequestUnitsOneHour>(
- db, ByRequestUnitsHour, TopByRequestUnitsOneHour, hourEnd);
-}
-
-void TSysViewProcessor::ScheduleAggregate() {
- auto rangeUs = RandomNumber<ui64>(TotalInterval.MicroSeconds() / 12);
- auto deadline = IntervalEnd + CollectInterval + TDuration::MicroSeconds(rangeUs);
- Schedule(deadline, new TEvPrivate::TEvAggregate);
-}
-
-void TSysViewProcessor::ScheduleCollect() {
- auto rangeUs = RandomNumber<ui64>(TotalInterval.MicroSeconds() / 12);
- auto deadline = IntervalEnd + TotalInterval + TDuration::MicroSeconds(rangeUs);
- Schedule(deadline, new TEvPrivate::TEvCollect);
-}
-
-void TSysViewProcessor::ScheduleSendRequests() {
- auto intervalUs = TotalInterval.MicroSeconds() / 12;
- auto rangeUs = RandomNumber<ui64>(intervalUs);
- auto deadline = IntervalEnd + CollectInterval + TDuration::MicroSeconds(intervalUs + rangeUs);
- Schedule(deadline, new TEvPrivate::TEvSendRequests);
-}
-
-void TSysViewProcessor::ScheduleApplyCounters() {
- Schedule(ProcessCountersInterval, new TEvPrivate::TEvApplyCounters);
-}
-
-void TSysViewProcessor::ScheduleSendNavigate() {
- Schedule(SendNavigateInterval, new TEvPrivate::TEvSendNavigate);
-}
-
-template <typename TSchema, typename TEntry>
-void TSysViewProcessor::CutHistory(NIceDb::TNiceDb& db,
- TResultMap<TEntry>& metricsMap, TDuration historySize)
-{
- auto past = IntervalEnd - historySize;
- auto key = std::make_pair(past.MicroSeconds(), 0);
-
- auto bound = metricsMap.lower_bound(key);
- for (auto it = metricsMap.begin(); it != bound; ++it) {
- db.Table<TSchema>().Key(it->first).Delete();
- }
- metricsMap.erase(metricsMap.begin(), bound);
-}
-
-TInstant TSysViewProcessor::EndOfHourInterval(TInstant intervalEnd) {
- auto hourUs = ONE_HOUR_BUCKET_SIZE.MicroSeconds();
- auto hourEndUs = intervalEnd.MicroSeconds() / hourUs * hourUs;
- if (hourEndUs != intervalEnd.MicroSeconds()) {
- hourEndUs += hourUs;
- }
- return TInstant::MicroSeconds(hourEndUs);
-}
-
-void TSysViewProcessor::ClearIntervalSummaries(NIceDb::TNiceDb& db) {
- for (const auto& [queryHash, query] : Queries) {
- ui32 count = query.Nodes.size();
- for (ui32 i = 0; i < count; ++i) {
- db.Table<Schema::IntervalSummaries>().Key(queryHash, i).Delete();
- }
- }
- Queries.clear();
- ByCpu.clear();
- SummaryNodes.clear();
-}
-
-void TSysViewProcessor::Reset(NIceDb::TNiceDb& db, const TActorContext& ctx) {
- // TODO: efficient delete?
-
- ClearIntervalSummaries(db);
-
- for (const auto& [queryHash, _] : QueryMetrics) {
- db.Table<Schema::IntervalMetrics>().Key(queryHash).Delete();
- }
- QueryMetrics.clear();
-
- for (const auto& node : NodesToRequest) {
- db.Table<Schema::NodesToRequest>().Key(node.NodeId).Delete();
- }
- NodesToRequest.clear();
- NodesInFlight.clear();
-
- auto clearTop = [&] (NKikimrSysView::EStatsType type, TTop& top) {
- for (const auto& query : top) {
- db.Table<Schema::IntervalTops>().Key((ui32)type, query.Hash).Delete();
- }
- top.clear();
- };
-
- clearTop(NKikimrSysView::TOP_DURATION_ONE_MINUTE, ByDurationMinute);
- clearTop(NKikimrSysView::TOP_READ_BYTES_ONE_MINUTE, ByReadBytesMinute);
- clearTop(NKikimrSysView::TOP_CPU_TIME_ONE_MINUTE, ByCpuTimeMinute);
- clearTop(NKikimrSysView::TOP_REQUEST_UNITS_ONE_MINUTE, ByRequestUnitsMinute);
-
- CurrentStage = COLLECT;
- PersistStage(db);
-
- auto oldHourEnd = EndOfHourInterval(IntervalEnd);
-
- auto now = ctx.Now();
- auto intervalSize = TotalInterval.MicroSeconds();
- auto rounded = now.MicroSeconds() / intervalSize * intervalSize;
- IntervalEnd = TInstant::MicroSeconds(rounded);
- PersistIntervalEnd(db);
-
- auto newHourEnd = EndOfHourInterval(IntervalEnd);
-
- if (oldHourEnd != newHourEnd) {
- clearTop(NKikimrSysView::TOP_DURATION_ONE_HOUR, ByDurationHour);
- clearTop(NKikimrSysView::TOP_READ_BYTES_ONE_HOUR, ByReadBytesHour);
- clearTop(NKikimrSysView::TOP_CPU_TIME_ONE_HOUR, ByCpuTimeHour);
- clearTop(NKikimrSysView::TOP_REQUEST_UNITS_ONE_HOUR, ByRequestUnitsHour);
- }
-
- SVLOG_D("[" << TabletID() << "] Reset: interval end# " << IntervalEnd);
-
- const auto minuteHistorySize = TotalInterval * ONE_MINUTE_BUCKET_COUNT;
- const auto hourHistorySize = ONE_HOUR_BUCKET_SIZE * ONE_HOUR_BUCKET_COUNT;
-
- CutHistory<Schema::MetricsOneMinute, TQueryToMetrics>(db, MetricsOneMinute, minuteHistorySize);
- CutHistory<Schema::MetricsOneHour, TQueryToMetrics>(db, MetricsOneHour, hourHistorySize);
-
- using TStats = NKikimrSysView::TQueryStats;
- CutHistory<Schema::TopByDurationOneMinute, TStats>(db, TopByDurationOneMinute, minuteHistorySize);
- CutHistory<Schema::TopByDurationOneHour, TStats>(db, TopByDurationOneHour, hourHistorySize);
- CutHistory<Schema::TopByReadBytesOneMinute, TStats>(db, TopByReadBytesOneMinute, minuteHistorySize);
- CutHistory<Schema::TopByReadBytesOneHour, TStats>(db, TopByReadBytesOneHour, hourHistorySize);
- CutHistory<Schema::TopByCpuTimeOneMinute, TStats>(db, TopByCpuTimeOneMinute, minuteHistorySize);
- CutHistory<Schema::TopByCpuTimeOneHour, TStats>(db, TopByCpuTimeOneHour, hourHistorySize);
- CutHistory<Schema::TopByRequestUnitsOneMinute, TStats>(db, TopByRequestUnitsOneMinute, minuteHistorySize);
- CutHistory<Schema::TopByRequestUnitsOneHour, TStats>(db, TopByRequestUnitsOneHour, hourHistorySize);
-}
-
-void TSysViewProcessor::SendRequests() {
- while (!NodesToRequest.empty() && NodesInFlight.size() < MaxInFlightRequests) {
- auto& req = NodesToRequest.back();
-
- auto request = MakeHolder<TEvSysView::TEvGetIntervalMetricsRequest>();
- auto& record = request->Record;
- record.SetIntervalEndUs(IntervalEnd.MicroSeconds());
- record.SetDatabase(Database);
-
- auto fillHashes = [&] (const THashVector& hashes, NProtoBuf::RepeatedField<ui64>& result) {
- result.Reserve(hashes.size());
- for (auto queryHash : hashes) {
- result.Add(queryHash);
- }
- };
-
- fillHashes(req.Hashes, *record.MutableMetrics());
- fillHashes(req.TextsToGet, *record.MutableQueryTextsToGet());
- fillHashes(req.ByDuration, *record.MutableTopByDuration());
- fillHashes(req.ByReadBytes, *record.MutableTopByReadBytes());
- fillHashes(req.ByCpuTime, *record.MutableTopByCpuTime());
- fillHashes(req.ByRequestUnits, *record.MutableTopByRequestUnits());
-
- SVLOG_D("[" << TabletID() << "] Send TEvGetIntervalMetricsRequest: "
- << "node id# " << req.NodeId
- << ", hashes# " << req.Hashes.size()
- << ", texts# " << req.TextsToGet.size()
- << ", by duration# " << req.ByDuration.size()
- << ", by read bytes# " << req.ByReadBytes.size()
- << ", by cpu time# " << req.ByCpuTime.size()
- << ", by request units# " << req.ByRequestUnits.size());
-
- Send(MakeSysViewServiceID(req.NodeId),
- std::move(request),
- IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession,
- req.NodeId);
-
- NodesInFlight[req.NodeId] = std::move(req);
- NodesToRequest.pop_back();
- }
-}
-
-void TSysViewProcessor::IgnoreFailure(TNodeId nodeId) {
- NodesInFlight.erase(nodeId);
-}
-
-void TSysViewProcessor::Handle(TEvents::TEvPoisonPill::TPtr&) {
- Become(&TThis::StateBroken);
- Send(Tablet(), new TEvents::TEvPoisonPill);
-}
-
-void TSysViewProcessor::Handle(TEvents::TEvUndelivered::TPtr& ev) {
- auto nodeId = (TNodeId)ev.Get()->Cookie;
- SVLOG_W("[" << TabletID() << "] TEvUndelivered: node id# " << nodeId);
- IgnoreFailure(nodeId);
-}
-
-void TSysViewProcessor::Handle(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) {
- auto nodeId = ev->Get()->NodeId;
- SVLOG_W("[" << TabletID() << "] TEvNodeDisconnected: node id# " << nodeId);
- IgnoreFailure(nodeId);
-}
-
-void TSysViewProcessor::Handle(TEvSysView::TEvGetQueryMetricsRequest::TPtr& ev) {
- const auto& record = ev->Get()->Record;
- auto type = record.GetType();
-
- if (PendingRequests.size() >= PendingRequestsLimit) {
- if (type == NKikimrSysView::METRICS_ONE_MINUTE || type == NKikimrSysView::METRICS_ONE_HOUR) {
- ReplyOverloaded<TEvSysView::TEvGetQueryMetricsResponse>(ev);
- } else {
- ReplyOverloaded<TEvSysView::TEvGetQueryStatsResponse>(ev);
- }
- return;
- }
-
- PendingRequests.push(std::move(ev));
-
- if (!ProcessInFly) {
- Send(SelfId(), new TEvPrivate::TEvProcess());
- ProcessInFly = true;
- }
-}
-
-void TSysViewProcessor::Handle(TEvPrivate::TEvProcess::TPtr&) {
- ProcessInFly = false;
-
- if (PendingRequests.empty()) {
- return;
- }
-
- TEvSysView::TEvGetQueryMetricsRequest::TPtr request = std::move(PendingRequests.front());
- PendingRequests.pop();
-
- if (!PendingRequests.empty()) {
- Send(SelfId(), new TEvPrivate::TEvProcess);
- ProcessInFly = true;
- }
-
- const auto& record = request->Get()->Record;
- auto type = record.GetType();
-
- if (type == NKikimrSysView::METRICS_ONE_MINUTE || type == NKikimrSysView::METRICS_ONE_HOUR) {
- Reply<TQueryToMetrics, TEvSysView::TEvGetQueryMetricsResponse>(request);
- } else {
- Reply<NKikimrSysView::TQueryStats, TEvSysView::TEvGetQueryStatsResponse>(request);
- }
-}
-
-template <typename TResponse>
-void TSysViewProcessor::ReplyOverloaded(TEvSysView::TEvGetQueryMetricsRequest::TPtr& ev) {
- auto response = MakeHolder<TResponse>();
- response->Record.SetOverloaded(true);
- Send(ev->Sender, std::move(response));
-}
-
-template <typename TEntry, typename TResponse>
-void TSysViewProcessor::Reply(TEvSysView::TEvGetQueryMetricsRequest::TPtr& ev) {
- const auto& record = ev->Get()->Record;
- auto response = MakeHolder<TResponse>();
- response->Record.SetLastBatch(true);
-
- TResultMap<TEntry>* entries = nullptr;
- if constexpr (std::is_same<TEntry, TQueryToMetrics>::value) {
- switch (record.GetType()) {
- case NKikimrSysView::METRICS_ONE_MINUTE:
- entries = &MetricsOneMinute;
- break;
- case NKikimrSysView::METRICS_ONE_HOUR:
- entries = &MetricsOneHour;
- break;
- default:
- SVLOG_CRIT("[" << TabletID() << "] unexpected stats type: " << (size_t)record.GetType());
- Send(ev->Sender, std::move(response));
- return;
- }
- } else {
- switch (record.GetType()) {
- case NKikimrSysView::TOP_DURATION_ONE_MINUTE:
- entries = &TopByDurationOneMinute;
- break;
- case NKikimrSysView::TOP_DURATION_ONE_HOUR:
- entries = &TopByDurationOneHour;
- break;
- case NKikimrSysView::TOP_READ_BYTES_ONE_MINUTE:
- entries = &TopByReadBytesOneMinute;
- break;
- case NKikimrSysView::TOP_READ_BYTES_ONE_HOUR:
- entries = &TopByReadBytesOneHour;
- break;
- case NKikimrSysView::TOP_CPU_TIME_ONE_MINUTE:
- entries = &TopByCpuTimeOneMinute;
- break;
- case NKikimrSysView::TOP_CPU_TIME_ONE_HOUR:
- entries = &TopByCpuTimeOneHour;
- break;
- case NKikimrSysView::TOP_REQUEST_UNITS_ONE_MINUTE:
- entries = &TopByRequestUnitsOneMinute;
- break;
- case NKikimrSysView::TOP_REQUEST_UNITS_ONE_HOUR:
- entries = &TopByRequestUnitsOneHour;
- break;
- default:
- SVLOG_CRIT("[" << TabletID() << "] unexpected stats type: " << (size_t)record.GetType());
- Send(ev->Sender, std::move(response));
- return;
- }
- }
-
- Y_VERIFY(entries);
-
- auto from = entries->begin();
- auto to = entries->end();
-
- TString fromStr("[]");
- if (record.HasFrom()) {
- auto key = std::make_pair(record.GetFrom().GetIntervalEndUs(), record.GetFrom().GetRank());
- if (!record.HasInclusiveFrom() || record.GetInclusiveFrom()) {
- from = entries->lower_bound(key);
- } else {
- from = entries->upper_bound(key);
- }
- TStringBuilder str;
- str << "[" << record.GetFrom().GetIntervalEndUs()
- << ", " << record.GetFrom().GetRank()
- << ", " << (record.GetInclusiveFrom() ? "inc]" : "exc]");
- fromStr = str;
- }
-
- TString toStr("[]");
- if (record.HasTo()) {
- auto key = std::make_pair(record.GetTo().GetIntervalEndUs(), record.GetTo().GetRank());
- if (!record.HasInclusiveTo() || !record.GetInclusiveTo()) {
- to = entries->lower_bound(key);
- } else {
- to = entries->upper_bound(key);
- }
- TStringBuilder str;
- str << "[" << record.GetTo().GetIntervalEndUs()
- << ", " << record.GetTo().GetRank()
- << ", " << (record.GetInclusiveTo() ? "inc]" : "exc]");
- toStr = str;
- }
-
- TString nextStr("[]");
- size_t size = 0;
- size_t count = 0;
- for (auto it = from; it != to; ++it) {
- const auto& key = it->first;
-
- auto& entry = *response->Record.AddEntries();
-
- auto& entryKey = *entry.MutableKey();
- entryKey.SetIntervalEndUs(key.first);
- entryKey.SetRank(key.second);
-
- if constexpr (std::is_same<TEntry, TQueryToMetrics>::value) {
- const auto& metrics = it->second;
- entry.MutableMetrics()->CopyFrom(metrics.Metrics);
- entry.SetQueryText(metrics.Text);
- } else {
- const auto& stats = it->second;
- entry.MutableStats()->CopyFrom(stats);
- }
-
- size += entry.ByteSizeLong();
- ++count;
-
- if (size >= BatchSizeLimit) {
- auto* next = response->Record.MutableNext();
- next->SetIntervalEndUs(key.first);
- next->SetRank(key.second + 1);
- response->Record.SetLastBatch(false);
-
- TStringBuilder str;
- str << "[" << next->GetIntervalEndUs()
- << ", " << next->GetRank()
- << "]";
- nextStr = str;
- break;
- }
- }
-
- SVLOG_D("[" << TabletID() << "] Reply to TEvGetQueryMetricsRequest: "
- << "from# " << fromStr
- << ", to# " << toStr
- << ", rows# " << count
- << ", bytes# " << size
- << ", next# " << nextStr);
-
- Send(ev->Sender, std::move(response));
-}
-
-bool TSysViewProcessor::OnRenderAppHtmlPage(NMon::TEvRemoteHttpInfo::TPtr ev,
- const TActorContext& ctx)
-{
- if (!ev) {
- return true;
- }
-
- TStringStream str;
- HTML(str) {
- PRE() {
- str << "---- SysViewProcessor ----" << Endl << Endl;
- str << "Database: " << Database << Endl;
- str << "IntervalEnd: " << IntervalEnd << Endl;
- str << "CurrentStage: " << (CurrentStage == COLLECT ? "Collect" : "Aggregate")
- << Endl << Endl;
- {
- str << "IntervalSummaries" << Endl;
- str << " QueryCount: " << Queries.size() << Endl;
-
- auto it = ByCpu.rbegin();
- static constexpr size_t queriesLimit = 32;
- for (size_t q = 0;
- it != ByCpu.rend() && q < queriesLimit;
- ++it, ++q)
- {
- const auto queryHash = it->second;
- auto queryIt = Queries.find(queryHash);
- Y_VERIFY(queryIt != Queries.end());
- const auto& query = queryIt->second;
-
- str << " Hash: " << queryHash
- << ", Cpu: " << query.Cpu
- << ", NodeCount: " << query.Nodes.size()
- << ", Nodes: ";
-
- static constexpr size_t nodesLimit = 4;
- auto nodeIt = query.Nodes.begin();
- for (size_t n = 0;
- nodeIt != query.Nodes.end() && n < nodesLimit;
- ++nodeIt, ++n)
- {
- str << "{ " << nodeIt->first << ", Cpu: " << nodeIt->second << " } ";
- }
- if (nodeIt != query.Nodes.end()) {
- str << "...";
- }
- str << Endl;
- }
- if (it != ByCpu.rend()) {
- str << " ..." << Endl;
- }
- str << Endl;
- }
- {
- str << "IntervalMetrics" << Endl;
- for (const auto& [queryHash, metrics] : QueryMetrics) {
- str << " Hash: " << queryHash
- << ", Count: " << metrics.Metrics.GetCount()
- << ", SumCpuTime: " << metrics.Metrics.GetCpuTimeUs().GetSum() << Endl;
- }
- str << Endl;
- }
- {
- auto dumpNode = [&str] (const TNodeToQueries& node) {
- str << " NodeId: " << node.NodeId
- << ", Hashes: " << node.Hashes.size()
- << ", TextsToGet: " << node.TextsToGet.size()
- << ", ByDuration: " << node.ByDuration.size()
- << ", ByReadBytes: " << node.ByReadBytes.size()
- << ", ByCpuTime: " << node.ByCpuTime.size()
- << ", ByRequestUnits: " << node.ByRequestUnits.size()
- << Endl;
- };
- str << "NodesToRequest" << Endl;
- for (const auto& node : NodesToRequest) {
- dumpNode(node);
- }
- str << Endl;
- str << "NodesInFlight" << Endl;
- for (const auto& [_, node] : NodesInFlight) {
- dumpNode(node);
- }
- str << Endl;
- }
- {
- auto printTop = [&str] (const TTop& top) {
- for (const auto& query : top) {
- str << " Hash: " << query.Hash
- << ", Value: " << query.Value
- << ", NodeId: " << query.NodeId << Endl;
- }
- };
- str << "ByDurationMinute" << Endl;
- printTop(ByDurationMinute);
- str << Endl;
- str << "ByDurationHour" << Endl;
- printTop(ByDurationHour);
- str << Endl;
- str << "ByReadBytesMinute" << Endl;
- printTop(ByReadBytesMinute);
- str << Endl;
- str << "ByReadBytesHour" << Endl;
- printTop(ByReadBytesHour);
- str << Endl;
- str << "ByCpuTimeMinute" << Endl;
- printTop(ByCpuTimeMinute);
- str << Endl;
- str << "ByCpuTimeHour" << Endl;
- printTop(ByCpuTimeHour);
- str << Endl;
- str << "ByRequestUnitsMinute" << Endl;
- printTop(ByRequestUnitsMinute);
- str << Endl;
- str << "ByRequestUnitsHour" << Endl;
- printTop(ByRequestUnitsHour);
- str << Endl;
- }
- {
- str << "MetricsOneMinute" << Endl
- << " Count: " << MetricsOneMinute.size() << Endl << Endl;
- str << "MetricsOneHour" << Endl
- << " Count: " << MetricsOneHour.size() << Endl << Endl;
- str << "TopByDurationOneMinute" << Endl
- << " Count: " << TopByDurationOneMinute.size() << Endl << Endl;
- str << "TopByDurationOneHour" << Endl
- << " Count: " << TopByDurationOneHour.size() << Endl << Endl;
- str << "TopByReadBytesOneMinute" << Endl
- << " Count: " << TopByReadBytesOneMinute.size() << Endl << Endl;
- str << "TopByReadBytesOneHour" << Endl
- << " Count: " << TopByReadBytesOneHour.size() << Endl << Endl;
- str << "TopByCpuTimeOneMinute" << Endl
- << " Count: " << TopByCpuTimeOneMinute.size() << Endl << Endl;
- str << "TopByCpuTimeOneHour" << Endl
- << " Count: " << TopByCpuTimeOneHour.size() << Endl << Endl;
- str << "TopByRequestUnitsOneMinute" << Endl
- << " Count: " << TopByRequestUnitsOneMinute.size() << Endl << Endl;
- str << "TopByRequestUnitsOneHour" << Endl
- << " Count: " << TopByRequestUnitsOneHour.size() << Endl << Endl;
- }
- }
- }
-
- ctx.Send(ev->Sender, new NMon::TEvRemoteHttpInfoRes(str.Str()));
- return true;
-}
-
-} // NSysView
-} // NKikimr
-
+ db.Table<Schema::MetricsOneMinute>().Key(key).Update(
+ NIceDb::TUpdate<Schema::MetricsOneMinute::Text>(resultMetrics.Text),
+ NIceDb::TUpdate<Schema::MetricsOneMinute::Data>(serialized));
+ }
+
+ SVLOG_D("[" << TabletID() << "] PersistResults: "
+ << "interval end# " << IntervalEnd
+ << ", query count# " << sorted.size());
+
+ // TODO: metrics one hour?
+
+ PersistTopResults<Schema::TopByDurationOneMinute>(
+ db, ByDurationMinute, TopByDurationOneMinute, IntervalEnd);
+ PersistTopResults<Schema::TopByReadBytesOneMinute>(
+ db, ByReadBytesMinute, TopByReadBytesOneMinute, IntervalEnd);
+ PersistTopResults<Schema::TopByCpuTimeOneMinute>(
+ db, ByCpuTimeMinute, TopByCpuTimeOneMinute, IntervalEnd);
+ PersistTopResults<Schema::TopByRequestUnitsOneMinute>(
+ db, ByRequestUnitsMinute, TopByRequestUnitsOneMinute, IntervalEnd);
+
+ auto hourEnd = EndOfHourInterval(IntervalEnd);
+
+ PersistTopResults<Schema::TopByDurationOneHour>(
+ db, ByDurationHour, TopByDurationOneHour, hourEnd);
+ PersistTopResults<Schema::TopByReadBytesOneHour>(
+ db, ByReadBytesHour, TopByReadBytesOneHour, hourEnd);
+ PersistTopResults<Schema::TopByCpuTimeOneHour>(
+ db, ByCpuTimeHour, TopByCpuTimeOneHour, hourEnd);
+ PersistTopResults<Schema::TopByRequestUnitsOneHour>(
+ db, ByRequestUnitsHour, TopByRequestUnitsOneHour, hourEnd);
+}
+
+void TSysViewProcessor::ScheduleAggregate() {
+ auto rangeUs = RandomNumber<ui64>(TotalInterval.MicroSeconds() / 12);
+ auto deadline = IntervalEnd + CollectInterval + TDuration::MicroSeconds(rangeUs);
+ Schedule(deadline, new TEvPrivate::TEvAggregate);
+}
+
+void TSysViewProcessor::ScheduleCollect() {
+ auto rangeUs = RandomNumber<ui64>(TotalInterval.MicroSeconds() / 12);
+ auto deadline = IntervalEnd + TotalInterval + TDuration::MicroSeconds(rangeUs);
+ Schedule(deadline, new TEvPrivate::TEvCollect);
+}
+
+void TSysViewProcessor::ScheduleSendRequests() {
+ auto intervalUs = TotalInterval.MicroSeconds() / 12;
+ auto rangeUs = RandomNumber<ui64>(intervalUs);
+ auto deadline = IntervalEnd + CollectInterval + TDuration::MicroSeconds(intervalUs + rangeUs);
+ Schedule(deadline, new TEvPrivate::TEvSendRequests);
+}
+
+void TSysViewProcessor::ScheduleApplyCounters() {
+ Schedule(ProcessCountersInterval, new TEvPrivate::TEvApplyCounters);
+}
+
+void TSysViewProcessor::ScheduleSendNavigate() {
+ Schedule(SendNavigateInterval, new TEvPrivate::TEvSendNavigate);
+}
+
+template <typename TSchema, typename TEntry>
+void TSysViewProcessor::CutHistory(NIceDb::TNiceDb& db,
+ TResultMap<TEntry>& metricsMap, TDuration historySize)
+{
+ auto past = IntervalEnd - historySize;
+ auto key = std::make_pair(past.MicroSeconds(), 0);
+
+ auto bound = metricsMap.lower_bound(key);
+ for (auto it = metricsMap.begin(); it != bound; ++it) {
+ db.Table<TSchema>().Key(it->first).Delete();
+ }
+ metricsMap.erase(metricsMap.begin(), bound);
+}
+
+TInstant TSysViewProcessor::EndOfHourInterval(TInstant intervalEnd) {
+ auto hourUs = ONE_HOUR_BUCKET_SIZE.MicroSeconds();
+ auto hourEndUs = intervalEnd.MicroSeconds() / hourUs * hourUs;
+ if (hourEndUs != intervalEnd.MicroSeconds()) {
+ hourEndUs += hourUs;
+ }
+ return TInstant::MicroSeconds(hourEndUs);
+}
+
+void TSysViewProcessor::ClearIntervalSummaries(NIceDb::TNiceDb& db) {
+ for (const auto& [queryHash, query] : Queries) {
+ ui32 count = query.Nodes.size();
+ for (ui32 i = 0; i < count; ++i) {
+ db.Table<Schema::IntervalSummaries>().Key(queryHash, i).Delete();
+ }
+ }
+ Queries.clear();
+ ByCpu.clear();
+ SummaryNodes.clear();
+}
+
+void TSysViewProcessor::Reset(NIceDb::TNiceDb& db, const TActorContext& ctx) {
+ // TODO: efficient delete?
+
+ ClearIntervalSummaries(db);
+
+ for (const auto& [queryHash, _] : QueryMetrics) {
+ db.Table<Schema::IntervalMetrics>().Key(queryHash).Delete();
+ }
+ QueryMetrics.clear();
+
+ for (const auto& node : NodesToRequest) {
+ db.Table<Schema::NodesToRequest>().Key(node.NodeId).Delete();
+ }
+ NodesToRequest.clear();
+ NodesInFlight.clear();
+
+ auto clearTop = [&] (NKikimrSysView::EStatsType type, TTop& top) {
+ for (const auto& query : top) {
+ db.Table<Schema::IntervalTops>().Key((ui32)type, query.Hash).Delete();
+ }
+ top.clear();
+ };
+
+ clearTop(NKikimrSysView::TOP_DURATION_ONE_MINUTE, ByDurationMinute);
+ clearTop(NKikimrSysView::TOP_READ_BYTES_ONE_MINUTE, ByReadBytesMinute);
+ clearTop(NKikimrSysView::TOP_CPU_TIME_ONE_MINUTE, ByCpuTimeMinute);
+ clearTop(NKikimrSysView::TOP_REQUEST_UNITS_ONE_MINUTE, ByRequestUnitsMinute);
+
+ CurrentStage = COLLECT;
+ PersistStage(db);
+
+ auto oldHourEnd = EndOfHourInterval(IntervalEnd);
+
+ auto now = ctx.Now();
+ auto intervalSize = TotalInterval.MicroSeconds();
+ auto rounded = now.MicroSeconds() / intervalSize * intervalSize;
+ IntervalEnd = TInstant::MicroSeconds(rounded);
+ PersistIntervalEnd(db);
+
+ auto newHourEnd = EndOfHourInterval(IntervalEnd);
+
+ if (oldHourEnd != newHourEnd) {
+ clearTop(NKikimrSysView::TOP_DURATION_ONE_HOUR, ByDurationHour);
+ clearTop(NKikimrSysView::TOP_READ_BYTES_ONE_HOUR, ByReadBytesHour);
+ clearTop(NKikimrSysView::TOP_CPU_TIME_ONE_HOUR, ByCpuTimeHour);
+ clearTop(NKikimrSysView::TOP_REQUEST_UNITS_ONE_HOUR, ByRequestUnitsHour);
+ }
+
+ SVLOG_D("[" << TabletID() << "] Reset: interval end# " << IntervalEnd);
+
+ const auto minuteHistorySize = TotalInterval * ONE_MINUTE_BUCKET_COUNT;
+ const auto hourHistorySize = ONE_HOUR_BUCKET_SIZE * ONE_HOUR_BUCKET_COUNT;
+
+ CutHistory<Schema::MetricsOneMinute, TQueryToMetrics>(db, MetricsOneMinute, minuteHistorySize);
+ CutHistory<Schema::MetricsOneHour, TQueryToMetrics>(db, MetricsOneHour, hourHistorySize);
+
+ using TStats = NKikimrSysView::TQueryStats;
+ CutHistory<Schema::TopByDurationOneMinute, TStats>(db, TopByDurationOneMinute, minuteHistorySize);
+ CutHistory<Schema::TopByDurationOneHour, TStats>(db, TopByDurationOneHour, hourHistorySize);
+ CutHistory<Schema::TopByReadBytesOneMinute, TStats>(db, TopByReadBytesOneMinute, minuteHistorySize);
+ CutHistory<Schema::TopByReadBytesOneHour, TStats>(db, TopByReadBytesOneHour, hourHistorySize);
+ CutHistory<Schema::TopByCpuTimeOneMinute, TStats>(db, TopByCpuTimeOneMinute, minuteHistorySize);
+ CutHistory<Schema::TopByCpuTimeOneHour, TStats>(db, TopByCpuTimeOneHour, hourHistorySize);
+ CutHistory<Schema::TopByRequestUnitsOneMinute, TStats>(db, TopByRequestUnitsOneMinute, minuteHistorySize);
+ CutHistory<Schema::TopByRequestUnitsOneHour, TStats>(db, TopByRequestUnitsOneHour, hourHistorySize);
+}
+
+void TSysViewProcessor::SendRequests() {
+ while (!NodesToRequest.empty() && NodesInFlight.size() < MaxInFlightRequests) {
+ auto& req = NodesToRequest.back();
+
+ auto request = MakeHolder<TEvSysView::TEvGetIntervalMetricsRequest>();
+ auto& record = request->Record;
+ record.SetIntervalEndUs(IntervalEnd.MicroSeconds());
+ record.SetDatabase(Database);
+
+ auto fillHashes = [&] (const THashVector& hashes, NProtoBuf::RepeatedField<ui64>& result) {
+ result.Reserve(hashes.size());
+ for (auto queryHash : hashes) {
+ result.Add(queryHash);
+ }
+ };
+
+ fillHashes(req.Hashes, *record.MutableMetrics());
+ fillHashes(req.TextsToGet, *record.MutableQueryTextsToGet());
+ fillHashes(req.ByDuration, *record.MutableTopByDuration());
+ fillHashes(req.ByReadBytes, *record.MutableTopByReadBytes());
+ fillHashes(req.ByCpuTime, *record.MutableTopByCpuTime());
+ fillHashes(req.ByRequestUnits, *record.MutableTopByRequestUnits());
+
+ SVLOG_D("[" << TabletID() << "] Send TEvGetIntervalMetricsRequest: "
+ << "node id# " << req.NodeId
+ << ", hashes# " << req.Hashes.size()
+ << ", texts# " << req.TextsToGet.size()
+ << ", by duration# " << req.ByDuration.size()
+ << ", by read bytes# " << req.ByReadBytes.size()
+ << ", by cpu time# " << req.ByCpuTime.size()
+ << ", by request units# " << req.ByRequestUnits.size());
+
+ Send(MakeSysViewServiceID(req.NodeId),
+ std::move(request),
+ IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession,
+ req.NodeId);
+
+ NodesInFlight[req.NodeId] = std::move(req);
+ NodesToRequest.pop_back();
+ }
+}
+
+void TSysViewProcessor::IgnoreFailure(TNodeId nodeId) {
+ NodesInFlight.erase(nodeId);
+}
+
+void TSysViewProcessor::Handle(TEvents::TEvPoisonPill::TPtr&) {
+ Become(&TThis::StateBroken);
+ Send(Tablet(), new TEvents::TEvPoisonPill);
+}
+
+void TSysViewProcessor::Handle(TEvents::TEvUndelivered::TPtr& ev) {
+ auto nodeId = (TNodeId)ev.Get()->Cookie;
+ SVLOG_W("[" << TabletID() << "] TEvUndelivered: node id# " << nodeId);
+ IgnoreFailure(nodeId);
+}
+
+void TSysViewProcessor::Handle(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) {
+ auto nodeId = ev->Get()->NodeId;
+ SVLOG_W("[" << TabletID() << "] TEvNodeDisconnected: node id# " << nodeId);
+ IgnoreFailure(nodeId);
+}
+
+void TSysViewProcessor::Handle(TEvSysView::TEvGetQueryMetricsRequest::TPtr& ev) {
+ const auto& record = ev->Get()->Record;
+ auto type = record.GetType();
+
+ if (PendingRequests.size() >= PendingRequestsLimit) {
+ if (type == NKikimrSysView::METRICS_ONE_MINUTE || type == NKikimrSysView::METRICS_ONE_HOUR) {
+ ReplyOverloaded<TEvSysView::TEvGetQueryMetricsResponse>(ev);
+ } else {
+ ReplyOverloaded<TEvSysView::TEvGetQueryStatsResponse>(ev);
+ }
+ return;
+ }
+
+ PendingRequests.push(std::move(ev));
+
+ if (!ProcessInFly) {
+ Send(SelfId(), new TEvPrivate::TEvProcess());
+ ProcessInFly = true;
+ }
+}
+
+void TSysViewProcessor::Handle(TEvPrivate::TEvProcess::TPtr&) {
+ ProcessInFly = false;
+
+ if (PendingRequests.empty()) {
+ return;
+ }
+
+ TEvSysView::TEvGetQueryMetricsRequest::TPtr request = std::move(PendingRequests.front());
+ PendingRequests.pop();
+
+ if (!PendingRequests.empty()) {
+ Send(SelfId(), new TEvPrivate::TEvProcess);
+ ProcessInFly = true;
+ }
+
+ const auto& record = request->Get()->Record;
+ auto type = record.GetType();
+
+ if (type == NKikimrSysView::METRICS_ONE_MINUTE || type == NKikimrSysView::METRICS_ONE_HOUR) {
+ Reply<TQueryToMetrics, TEvSysView::TEvGetQueryMetricsResponse>(request);
+ } else {
+ Reply<NKikimrSysView::TQueryStats, TEvSysView::TEvGetQueryStatsResponse>(request);
+ }
+}
+
+template <typename TResponse>
+void TSysViewProcessor::ReplyOverloaded(TEvSysView::TEvGetQueryMetricsRequest::TPtr& ev) {
+ auto response = MakeHolder<TResponse>();
+ response->Record.SetOverloaded(true);
+ Send(ev->Sender, std::move(response));
+}
+
+template <typename TEntry, typename TResponse>
+void TSysViewProcessor::Reply(TEvSysView::TEvGetQueryMetricsRequest::TPtr& ev) {
+ const auto& record = ev->Get()->Record;
+ auto response = MakeHolder<TResponse>();
+ response->Record.SetLastBatch(true);
+
+ TResultMap<TEntry>* entries = nullptr;
+ if constexpr (std::is_same<TEntry, TQueryToMetrics>::value) {
+ switch (record.GetType()) {
+ case NKikimrSysView::METRICS_ONE_MINUTE:
+ entries = &MetricsOneMinute;
+ break;
+ case NKikimrSysView::METRICS_ONE_HOUR:
+ entries = &MetricsOneHour;
+ break;
+ default:
+ SVLOG_CRIT("[" << TabletID() << "] unexpected stats type: " << (size_t)record.GetType());
+ Send(ev->Sender, std::move(response));
+ return;
+ }
+ } else {
+ switch (record.GetType()) {
+ case NKikimrSysView::TOP_DURATION_ONE_MINUTE:
+ entries = &TopByDurationOneMinute;
+ break;
+ case NKikimrSysView::TOP_DURATION_ONE_HOUR:
+ entries = &TopByDurationOneHour;
+ break;
+ case NKikimrSysView::TOP_READ_BYTES_ONE_MINUTE:
+ entries = &TopByReadBytesOneMinute;
+ break;
+ case NKikimrSysView::TOP_READ_BYTES_ONE_HOUR:
+ entries = &TopByReadBytesOneHour;
+ break;
+ case NKikimrSysView::TOP_CPU_TIME_ONE_MINUTE:
+ entries = &TopByCpuTimeOneMinute;
+ break;
+ case NKikimrSysView::TOP_CPU_TIME_ONE_HOUR:
+ entries = &TopByCpuTimeOneHour;
+ break;
+ case NKikimrSysView::TOP_REQUEST_UNITS_ONE_MINUTE:
+ entries = &TopByRequestUnitsOneMinute;
+ break;
+ case NKikimrSysView::TOP_REQUEST_UNITS_ONE_HOUR:
+ entries = &TopByRequestUnitsOneHour;
+ break;
+ default:
+ SVLOG_CRIT("[" << TabletID() << "] unexpected stats type: " << (size_t)record.GetType());
+ Send(ev->Sender, std::move(response));
+ return;
+ }
+ }
+
+ Y_VERIFY(entries);
+
+ auto from = entries->begin();
+ auto to = entries->end();
+
+ TString fromStr("[]");
+ if (record.HasFrom()) {
+ auto key = std::make_pair(record.GetFrom().GetIntervalEndUs(), record.GetFrom().GetRank());
+ if (!record.HasInclusiveFrom() || record.GetInclusiveFrom()) {
+ from = entries->lower_bound(key);
+ } else {
+ from = entries->upper_bound(key);
+ }
+ TStringBuilder str;
+ str << "[" << record.GetFrom().GetIntervalEndUs()
+ << ", " << record.GetFrom().GetRank()
+ << ", " << (record.GetInclusiveFrom() ? "inc]" : "exc]");
+ fromStr = str;
+ }
+
+ TString toStr("[]");
+ if (record.HasTo()) {
+ auto key = std::make_pair(record.GetTo().GetIntervalEndUs(), record.GetTo().GetRank());
+ if (!record.HasInclusiveTo() || !record.GetInclusiveTo()) {
+ to = entries->lower_bound(key);
+ } else {
+ to = entries->upper_bound(key);
+ }
+ TStringBuilder str;
+ str << "[" << record.GetTo().GetIntervalEndUs()
+ << ", " << record.GetTo().GetRank()
+ << ", " << (record.GetInclusiveTo() ? "inc]" : "exc]");
+ toStr = str;
+ }
+
+ TString nextStr("[]");
+ size_t size = 0;
+ size_t count = 0;
+ for (auto it = from; it != to; ++it) {
+ const auto& key = it->first;
+
+ auto& entry = *response->Record.AddEntries();
+
+ auto& entryKey = *entry.MutableKey();
+ entryKey.SetIntervalEndUs(key.first);
+ entryKey.SetRank(key.second);
+
+ if constexpr (std::is_same<TEntry, TQueryToMetrics>::value) {
+ const auto& metrics = it->second;
+ entry.MutableMetrics()->CopyFrom(metrics.Metrics);
+ entry.SetQueryText(metrics.Text);
+ } else {
+ const auto& stats = it->second;
+ entry.MutableStats()->CopyFrom(stats);
+ }
+
+ size += entry.ByteSizeLong();
+ ++count;
+
+ if (size >= BatchSizeLimit) {
+ auto* next = response->Record.MutableNext();
+ next->SetIntervalEndUs(key.first);
+ next->SetRank(key.second + 1);
+ response->Record.SetLastBatch(false);
+
+ TStringBuilder str;
+ str << "[" << next->GetIntervalEndUs()
+ << ", " << next->GetRank()
+ << "]";
+ nextStr = str;
+ break;
+ }
+ }
+
+ SVLOG_D("[" << TabletID() << "] Reply to TEvGetQueryMetricsRequest: "
+ << "from# " << fromStr
+ << ", to# " << toStr
+ << ", rows# " << count
+ << ", bytes# " << size
+ << ", next# " << nextStr);
+
+ Send(ev->Sender, std::move(response));
+}
+
+bool TSysViewProcessor::OnRenderAppHtmlPage(NMon::TEvRemoteHttpInfo::TPtr ev,
+ const TActorContext& ctx)
+{
+ if (!ev) {
+ return true;
+ }
+
+ TStringStream str;
+ HTML(str) {
+ PRE() {
+ str << "---- SysViewProcessor ----" << Endl << Endl;
+ str << "Database: " << Database << Endl;
+ str << "IntervalEnd: " << IntervalEnd << Endl;
+ str << "CurrentStage: " << (CurrentStage == COLLECT ? "Collect" : "Aggregate")
+ << Endl << Endl;
+ {
+ str << "IntervalSummaries" << Endl;
+ str << " QueryCount: " << Queries.size() << Endl;
+
+ auto it = ByCpu.rbegin();
+ static constexpr size_t queriesLimit = 32;
+ for (size_t q = 0;
+ it != ByCpu.rend() && q < queriesLimit;
+ ++it, ++q)
+ {
+ const auto queryHash = it->second;
+ auto queryIt = Queries.find(queryHash);
+ Y_VERIFY(queryIt != Queries.end());
+ const auto& query = queryIt->second;
+
+ str << " Hash: " << queryHash
+ << ", Cpu: " << query.Cpu
+ << ", NodeCount: " << query.Nodes.size()
+ << ", Nodes: ";
+
+ static constexpr size_t nodesLimit = 4;
+ auto nodeIt = query.Nodes.begin();
+ for (size_t n = 0;
+ nodeIt != query.Nodes.end() && n < nodesLimit;
+ ++nodeIt, ++n)
+ {
+ str << "{ " << nodeIt->first << ", Cpu: " << nodeIt->second << " } ";
+ }
+ if (nodeIt != query.Nodes.end()) {
+ str << "...";
+ }
+ str << Endl;
+ }
+ if (it != ByCpu.rend()) {
+ str << " ..." << Endl;
+ }
+ str << Endl;
+ }
+ {
+ str << "IntervalMetrics" << Endl;
+ for (const auto& [queryHash, metrics] : QueryMetrics) {
+ str << " Hash: " << queryHash
+ << ", Count: " << metrics.Metrics.GetCount()
+ << ", SumCpuTime: " << metrics.Metrics.GetCpuTimeUs().GetSum() << Endl;
+ }
+ str << Endl;
+ }
+ {
+ auto dumpNode = [&str] (const TNodeToQueries& node) {
+ str << " NodeId: " << node.NodeId
+ << ", Hashes: " << node.Hashes.size()
+ << ", TextsToGet: " << node.TextsToGet.size()
+ << ", ByDuration: " << node.ByDuration.size()
+ << ", ByReadBytes: " << node.ByReadBytes.size()
+ << ", ByCpuTime: " << node.ByCpuTime.size()
+ << ", ByRequestUnits: " << node.ByRequestUnits.size()
+ << Endl;
+ };
+ str << "NodesToRequest" << Endl;
+ for (const auto& node : NodesToRequest) {
+ dumpNode(node);
+ }
+ str << Endl;
+ str << "NodesInFlight" << Endl;
+ for (const auto& [_, node] : NodesInFlight) {
+ dumpNode(node);
+ }
+ str << Endl;
+ }
+ {
+ auto printTop = [&str] (const TTop& top) {
+ for (const auto& query : top) {
+ str << " Hash: " << query.Hash
+ << ", Value: " << query.Value
+ << ", NodeId: " << query.NodeId << Endl;
+ }
+ };
+ str << "ByDurationMinute" << Endl;
+ printTop(ByDurationMinute);
+ str << Endl;
+ str << "ByDurationHour" << Endl;
+ printTop(ByDurationHour);
+ str << Endl;
+ str << "ByReadBytesMinute" << Endl;
+ printTop(ByReadBytesMinute);
+ str << Endl;
+ str << "ByReadBytesHour" << Endl;
+ printTop(ByReadBytesHour);
+ str << Endl;
+ str << "ByCpuTimeMinute" << Endl;
+ printTop(ByCpuTimeMinute);
+ str << Endl;
+ str << "ByCpuTimeHour" << Endl;
+ printTop(ByCpuTimeHour);
+ str << Endl;
+ str << "ByRequestUnitsMinute" << Endl;
+ printTop(ByRequestUnitsMinute);
+ str << Endl;
+ str << "ByRequestUnitsHour" << Endl;
+ printTop(ByRequestUnitsHour);
+ str << Endl;
+ }
+ {
+ str << "MetricsOneMinute" << Endl
+ << " Count: " << MetricsOneMinute.size() << Endl << Endl;
+ str << "MetricsOneHour" << Endl
+ << " Count: " << MetricsOneHour.size() << Endl << Endl;
+ str << "TopByDurationOneMinute" << Endl
+ << " Count: " << TopByDurationOneMinute.size() << Endl << Endl;
+ str << "TopByDurationOneHour" << Endl
+ << " Count: " << TopByDurationOneHour.size() << Endl << Endl;
+ str << "TopByReadBytesOneMinute" << Endl
+ << " Count: " << TopByReadBytesOneMinute.size() << Endl << Endl;
+ str << "TopByReadBytesOneHour" << Endl
+ << " Count: " << TopByReadBytesOneHour.size() << Endl << Endl;
+ str << "TopByCpuTimeOneMinute" << Endl
+ << " Count: " << TopByCpuTimeOneMinute.size() << Endl << Endl;
+ str << "TopByCpuTimeOneHour" << Endl
+ << " Count: " << TopByCpuTimeOneHour.size() << Endl << Endl;
+ str << "TopByRequestUnitsOneMinute" << Endl
+ << " Count: " << TopByRequestUnitsOneMinute.size() << Endl << Endl;
+ str << "TopByRequestUnitsOneHour" << Endl
+ << " Count: " << TopByRequestUnitsOneHour.size() << Endl << Endl;
+ }
+ }
+ }
+
+ ctx.Send(ev->Sender, new NMon::TEvRemoteHttpInfoRes(str.Str()));
+ return true;
+}
+
+} // NSysView
+} // NKikimr
+
diff --git a/ydb/core/sys_view/processor/processor_impl.h b/ydb/core/sys_view/processor/processor_impl.h
index 4b49ecbb93c..78e8ba78561 100644
--- a/ydb/core/sys_view/processor/processor_impl.h
+++ b/ydb/core/sys_view/processor/processor_impl.h
@@ -1,10 +1,10 @@
-#pragma once
-
-#include "schema.h"
-
+#pragma once
+
+#include "schema.h"
+
#include <ydb/core/protos/sys_view.pb.h>
#include <ydb/core/protos/counters_sysview_processor.pb.h>
-
+
#include <ydb/core/base/tablet_pipe.h>
#include <ydb/core/sys_view/common/common.h>
#include <ydb/core/sys_view/common/events.h>
@@ -13,327 +13,327 @@
#include <ydb/core/tablet_flat/tablet_flat_executed.h>
#include <ydb/core/tx/scheme_cache/scheme_cache.h>
#include <ydb/core/tx/tx.h>
-
-#include <library/cpp/actors/core/interconnect.h>
-#include <library/cpp/actors/core/memory_track.h>
-
-namespace NKikimr {
-namespace NSysView {
-
-static constexpr char MemoryLabelResults[] = "SysViewProcessor/Results";
-
-class TSysViewProcessor : public TActor<TSysViewProcessor>, public NTabletFlatExecutor::TTabletExecutedFlat {
-public:
- static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
- return NKikimrServices::TActivity::SYSTEM_VIEW_PROCESSOR;
- }
-
+
+#include <library/cpp/actors/core/interconnect.h>
+#include <library/cpp/actors/core/memory_track.h>
+
+namespace NKikimr {
+namespace NSysView {
+
+static constexpr char MemoryLabelResults[] = "SysViewProcessor/Results";
+
+class TSysViewProcessor : public TActor<TSysViewProcessor>, public NTabletFlatExecutor::TTabletExecutedFlat {
+public:
+ static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
+ return NKikimrServices::TActivity::SYSTEM_VIEW_PROCESSOR;
+ }
+
TSysViewProcessor(const TActorId& tablet, TTabletStorageInfo* info, EProcessorMode processorMode);
-
-private:
- using Schema = TProcessorSchema;
- using TTxBase = NTabletFlatExecutor::TTransactionBase<TSysViewProcessor>;
-
- struct TTxInitSchema;
- struct TTxInit;
- struct TTxConfigure;
- struct TTxCollect;
- struct TTxAggregate;
- struct TTxIntervalSummary;
- struct TTxIntervalMetrics;
-
- struct TEvPrivate {
- enum EEv {
- EvCollect = EventSpaceBegin(TEvents::ES_PRIVATE),
- EvAggregate,
- EvSendRequests,
- EvProcess,
- EvApplyCounters,
- EvSendNavigate,
- EvEnd
- };
-
- struct TEvCollect : public TEventLocal<TEvCollect, EvCollect> {};
-
- struct TEvAggregate : public TEventLocal<TEvAggregate, EvAggregate> {};
-
- struct TEvSendRequests : public TEventLocal<TEvSendRequests, EvSendRequests> {};
-
- struct TEvProcess : public TEventLocal<TEvProcess, EvProcess> {};
-
- struct TEvApplyCounters : public TEventLocal<TEvApplyCounters, EvApplyCounters> {};
-
- struct TEvSendNavigate : public TEventLocal<TEvSendNavigate, EvSendNavigate> {};
- };
-
- struct TTopQuery {
- TQueryHash Hash;
- ui64 Value;
- TNodeId NodeId;
- THolder<NKikimrSysView::TQueryStats> Stats;
- };
- using TTop = std::vector<TTopQuery>;
-
- using TResultKey = std::pair<ui64, ui32>;
-
- template <typename TEntry>
- using TResultMap = std::map<TResultKey, TEntry, std::less<TResultKey>,
- NActors::NMemory::TAlloc<std::pair<const TResultKey, TEntry>, MemoryLabelResults>>;
- using TResultStatsMap = TResultMap<NKikimrSysView::TQueryStats>;
-
-private:
- static bool TopQueryCompare(const TTopQuery& l, const TTopQuery& r) {
- return l.Value == r.Value ? l.Hash > r.Hash : l.Value > r.Value;
- }
-
- void OnDetach(const TActorContext& ctx) override;
- void OnTabletDead(TEvTablet::TEvTabletDead::TPtr& ev, const TActorContext& ctx) override;
- void OnActivateExecutor(const TActorContext& ctx) override;
- void DefaultSignalTabletActive(const TActorContext& ctx) override;
- bool OnRenderAppHtmlPage(NMon::TEvRemoteHttpInfo::TPtr ev, const TActorContext &ctx) override;
-
- NTabletFlatExecutor::ITransaction* CreateTxInitSchema();
- NTabletFlatExecutor::ITransaction* CreateTxInit();
-
- void Handle(TEvSysView::TEvConfigureProcessor::TPtr& ev);
- void Handle(TEvPrivate::TEvCollect::TPtr& ev);
- void Handle(TEvPrivate::TEvAggregate::TPtr& ev);
- void Handle(TEvPrivate::TEvSendRequests::TPtr& ev);
- void Handle(TEvPrivate::TEvProcess::TPtr& ev);
- void Handle(TEvSysView::TEvIntervalQuerySummary::TPtr& ev);
- void Handle(TEvSysView::TEvGetIntervalMetricsResponse::TPtr& ev);
- void Handle(TEvSysView::TEvGetQueryMetricsRequest::TPtr& ev);
-
- void Handle(TEvSysView::TEvSendDbCountersRequest::TPtr& ev);
- void Handle(TEvPrivate::TEvApplyCounters::TPtr& ev);
- void Handle(TEvPrivate::TEvSendNavigate::TPtr& ev);
- void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev);
- void Handle(TEvTxProxySchemeCache::TEvWatchNotifyUpdated::TPtr& ev);
- void Handle(TEvTxProxySchemeCache::TEvWatchNotifyDeleted::TPtr& ev);
-
- void Handle(TEvents::TEvPoisonPill::TPtr& ev);
- void Handle(TEvents::TEvUndelivered::TPtr& ev);
- void Handle(TEvInterconnect::TEvNodeDisconnected::TPtr& ev);
-
- void PersistSysParam(NIceDb::TNiceDb& db, ui64 id, const TString& value);
- void PersistDatabase(NIceDb::TNiceDb& db);
- void PersistStage(NIceDb::TNiceDb& db);
- void PersistIntervalEnd(NIceDb::TNiceDb& db);
-
- template <typename TSchema>
- void PersistTopResults(NIceDb::TNiceDb& db,
- TTop& top, TResultStatsMap& results, TInstant intervalEnd);
-
- void PersistResults(NIceDb::TNiceDb& db);
-
- void ScheduleAggregate();
- void ScheduleCollect();
- void ScheduleSendRequests();
- void ScheduleApplyCounters();
- void ScheduleSendNavigate();
-
- template <typename TSchema, typename TEntry>
- void CutHistory(NIceDb::TNiceDb& db, TResultMap<TEntry>& results,
- TDuration historySize);
-
- static TInstant EndOfHourInterval(TInstant intervalEnd);
-
- void ClearIntervalSummaries(NIceDb::TNiceDb& db);
-
- void Reset(NIceDb::TNiceDb& db, const TActorContext& ctx);
-
- void SendRequests();
- void IgnoreFailure(TNodeId nodeId);
-
- template <typename TResponse>
- void ReplyOverloaded(TEvSysView::TEvGetQueryMetricsRequest::TPtr& ev);
-
- template <typename TEntry, typename TResponse>
- void Reply(TEvSysView::TEvGetQueryMetricsRequest::TPtr& ev);
-
- TIntrusivePtr<IDbCounters> CreateCountersForService(NKikimrSysView::EDbCountersService service);
- void AttachExternalCounters();
- void AttachInternalCounters();
- void DetachExternalCounters();
- void DetachInternalCounters();
- void SendNavigate();
-
- STFUNC(StateInit) {
- switch(ev->GetTypeRewrite()) {
- hFunc(TEvSysView::TEvConfigureProcessor, Handle);
- hFunc(TEvents::TEvPoisonPill, Handle);
- IgnoreFunc(TEvSysView::TEvIntervalQuerySummary);
- IgnoreFunc(TEvSysView::TEvGetIntervalMetricsResponse);
- IgnoreFunc(TEvSysView::TEvGetQueryMetricsRequest);
- IgnoreFunc(TEvSysView::TEvSendDbCountersRequest);
- default:
- if (!HandleDefaultEvents(ev, ctx)) {
- LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS,
- "TSysViewProcessor StateInit unexpected event 0x%08" PRIx32, ev->GetTypeRewrite());
- }
- }
- }
-
- STFUNC(StateOffline) {
- switch(ev->GetTypeRewrite()) {
- hFunc(TEvents::TEvPoisonPill, Handle);
- hFunc(TEvSysView::TEvConfigureProcessor, Handle);
- IgnoreFunc(TEvSysView::TEvIntervalQuerySummary);
- IgnoreFunc(TEvSysView::TEvGetIntervalMetricsResponse);
- IgnoreFunc(TEvSysView::TEvGetQueryMetricsRequest);
- IgnoreFunc(TEvSysView::TEvSendDbCountersRequest);
- default:
- if (!HandleDefaultEvents(ev, ctx)) {
- LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS,
- "TSysViewProcessor StateOffline unexpected event 0x%08" PRIx32, ev->GetTypeRewrite());
- }
- }
- }
-
- STFUNC(StateWork) {
- switch(ev->GetTypeRewrite()) {
- hFunc(TEvSysView::TEvConfigureProcessor, Handle);
- hFunc(TEvPrivate::TEvCollect, Handle);
- hFunc(TEvPrivate::TEvAggregate, Handle);
- hFunc(TEvPrivate::TEvSendRequests, Handle);
- hFunc(TEvPrivate::TEvProcess, Handle);
- hFunc(TEvSysView::TEvIntervalQuerySummary, Handle);
- hFunc(TEvSysView::TEvGetIntervalMetricsResponse, Handle);
- hFunc(TEvSysView::TEvGetQueryMetricsRequest, Handle);
- hFunc(TEvSysView::TEvSendDbCountersRequest, Handle);
- hFunc(TEvPrivate::TEvApplyCounters, Handle);
- hFunc(TEvPrivate::TEvSendNavigate, Handle);
- hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle);
- hFunc(TEvTxProxySchemeCache::TEvWatchNotifyUpdated, Handle);
- hFunc(TEvTxProxySchemeCache::TEvWatchNotifyDeleted, Handle);
- hFunc(TEvents::TEvPoisonPill, Handle);
- hFunc(TEvents::TEvUndelivered, Handle);
- hFunc(TEvInterconnect::TEvNodeDisconnected, Handle);
- IgnoreFunc(TEvInterconnect::TEvNodeConnected);
- IgnoreFunc(TEvTabletPipe::TEvServerConnected);
- IgnoreFunc(TEvTabletPipe::TEvServerDisconnected);
- default:
- if (!HandleDefaultEvents(ev, ctx)) {
- LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS,
- "TSysViewProcessor StateWork unexpected event 0x%08" PRIx32, ev->GetTypeRewrite());
- }
- }
- }
-
- STFUNC(StateBroken) {
- HandleDefaultEvents(ev, ctx);
- }
-
-private:
- // limit on number of distinct queries when gathering summaries
- static constexpr size_t DistinctQueriesLimit = 1024;
- // limit on number of queries to aggregate metrics
- static constexpr size_t TopCountLimit = 256;
- // limit on number of concurrent metrics requests from services
- static constexpr size_t MaxInFlightRequests = 16;
- // limit on scan batch size
- static constexpr size_t BatchSizeLimit = 4 << 20;
- // interval of db counters processing
- static constexpr TDuration ProcessCountersInterval = TDuration::Seconds(5);
- // interval of sending next navigate request
- static constexpr TDuration SendNavigateInterval = TDuration::Seconds(5);
-
- const TDuration TotalInterval;
- const TDuration CollectInterval;
-
- TString Database;
-
- TInstant IntervalEnd;
-
- enum EStage {
- COLLECT,
- AGGREGATE
- };
- EStage CurrentStage = COLLECT;
-
- // IntervalSummaries
- struct TQueryToNodes {
- ui64 Cpu = 0;
- std::vector<std::pair<TNodeId, ui64>> Nodes; // nodeId, cpu
- };
- std::unordered_map<TQueryHash, TQueryToNodes> Queries;
- std::multimap<ui64, TQueryHash> ByCpu;
- std::unordered_set<TNodeId> SummaryNodes;
-
- // IntervalMetrics
- struct TQueryToMetrics {
- NKikimrSysView::TQueryMetrics Metrics;
- TString Text;
- };
- std::unordered_map<TQueryHash, TQueryToMetrics> QueryMetrics;
-
- // NodesToRequest
- using THashVector = std::vector<TQueryHash>;
- struct TNodeToQueries {
- TNodeId NodeId = 0;
- THashVector Hashes;
- THashVector TextsToGet;
- THashVector ByDuration;
- THashVector ByReadBytes;
- THashVector ByCpuTime;
- THashVector ByRequestUnits;
- };
- std::vector<TNodeToQueries> NodesToRequest;
- std::unordered_map<TNodeId, TNodeToQueries> NodesInFlight;
-
- // IntervalTops
- TTop ByDurationMinute;
- TTop ByReadBytesMinute;
- TTop ByCpuTimeMinute;
- TTop ByRequestUnitsMinute;
- TTop ByDurationHour;
- TTop ByReadBytesHour;
- TTop ByCpuTimeHour;
- TTop ByRequestUnitsHour;
-
- // Metrics...
- using TResultMetricsMap = TResultMap<TQueryToMetrics>;
-
- TResultMetricsMap MetricsOneMinute;
- TResultMetricsMap MetricsOneHour;
-
- // TopBy...
- TResultStatsMap TopByDurationOneMinute;
- TResultStatsMap TopByDurationOneHour;
- TResultStatsMap TopByReadBytesOneMinute;
- TResultStatsMap TopByReadBytesOneHour;
- TResultStatsMap TopByCpuTimeOneMinute;
- TResultStatsMap TopByCpuTimeOneHour;
- TResultStatsMap TopByRequestUnitsOneMinute;
- TResultStatsMap TopByRequestUnitsOneHour;
-
- // limited queue of user requests
- static constexpr size_t PendingRequestsLimit = 5;
- std::queue<TEvSysView::TEvGetQueryMetricsRequest::TPtr> PendingRequests;
- bool ProcessInFly = false;
-
- // db counters
- TString CloudId;
- TString FolderId;
- TString DatabaseId;
-
- NMonitoring::TDynamicCounterPtr ExternalGroup;
- NMonitoring::TDynamicCounterPtr InternalGroup;
-
- using TDbCountersServiceMap = std::unordered_map<NKikimrSysView::EDbCountersService,
- NKikimr::NSysView::TDbServiceCounters>;
-
- struct TNodeCountersState {
- TDbCountersServiceMap Simple; // only simple counters
- ui64 Generation = 0;
- size_t FreshCount = 0;
- };
- std::unordered_map<TNodeId, TNodeCountersState> NodeCountersStates;
- TDbCountersServiceMap AggregatedCountersState;
-
- std::unordered_map<NKikimrSysView::EDbCountersService, TIntrusivePtr<IDbCounters>> Counters;
-};
-
-} // NSysView
-} // NKikimr
-
+
+private:
+ using Schema = TProcessorSchema;
+ using TTxBase = NTabletFlatExecutor::TTransactionBase<TSysViewProcessor>;
+
+ struct TTxInitSchema;
+ struct TTxInit;
+ struct TTxConfigure;
+ struct TTxCollect;
+ struct TTxAggregate;
+ struct TTxIntervalSummary;
+ struct TTxIntervalMetrics;
+
+ struct TEvPrivate {
+ enum EEv {
+ EvCollect = EventSpaceBegin(TEvents::ES_PRIVATE),
+ EvAggregate,
+ EvSendRequests,
+ EvProcess,
+ EvApplyCounters,
+ EvSendNavigate,
+ EvEnd
+ };
+
+ struct TEvCollect : public TEventLocal<TEvCollect, EvCollect> {};
+
+ struct TEvAggregate : public TEventLocal<TEvAggregate, EvAggregate> {};
+
+ struct TEvSendRequests : public TEventLocal<TEvSendRequests, EvSendRequests> {};
+
+ struct TEvProcess : public TEventLocal<TEvProcess, EvProcess> {};
+
+ struct TEvApplyCounters : public TEventLocal<TEvApplyCounters, EvApplyCounters> {};
+
+ struct TEvSendNavigate : public TEventLocal<TEvSendNavigate, EvSendNavigate> {};
+ };
+
+ struct TTopQuery {
+ TQueryHash Hash;
+ ui64 Value;
+ TNodeId NodeId;
+ THolder<NKikimrSysView::TQueryStats> Stats;
+ };
+ using TTop = std::vector<TTopQuery>;
+
+ using TResultKey = std::pair<ui64, ui32>;
+
+ template <typename TEntry>
+ using TResultMap = std::map<TResultKey, TEntry, std::less<TResultKey>,
+ NActors::NMemory::TAlloc<std::pair<const TResultKey, TEntry>, MemoryLabelResults>>;
+ using TResultStatsMap = TResultMap<NKikimrSysView::TQueryStats>;
+
+private:
+ static bool TopQueryCompare(const TTopQuery& l, const TTopQuery& r) {
+ return l.Value == r.Value ? l.Hash > r.Hash : l.Value > r.Value;
+ }
+
+ void OnDetach(const TActorContext& ctx) override;
+ void OnTabletDead(TEvTablet::TEvTabletDead::TPtr& ev, const TActorContext& ctx) override;
+ void OnActivateExecutor(const TActorContext& ctx) override;
+ void DefaultSignalTabletActive(const TActorContext& ctx) override;
+ bool OnRenderAppHtmlPage(NMon::TEvRemoteHttpInfo::TPtr ev, const TActorContext &ctx) override;
+
+ NTabletFlatExecutor::ITransaction* CreateTxInitSchema();
+ NTabletFlatExecutor::ITransaction* CreateTxInit();
+
+ void Handle(TEvSysView::TEvConfigureProcessor::TPtr& ev);
+ void Handle(TEvPrivate::TEvCollect::TPtr& ev);
+ void Handle(TEvPrivate::TEvAggregate::TPtr& ev);
+ void Handle(TEvPrivate::TEvSendRequests::TPtr& ev);
+ void Handle(TEvPrivate::TEvProcess::TPtr& ev);
+ void Handle(TEvSysView::TEvIntervalQuerySummary::TPtr& ev);
+ void Handle(TEvSysView::TEvGetIntervalMetricsResponse::TPtr& ev);
+ void Handle(TEvSysView::TEvGetQueryMetricsRequest::TPtr& ev);
+
+ void Handle(TEvSysView::TEvSendDbCountersRequest::TPtr& ev);
+ void Handle(TEvPrivate::TEvApplyCounters::TPtr& ev);
+ void Handle(TEvPrivate::TEvSendNavigate::TPtr& ev);
+ void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev);
+ void Handle(TEvTxProxySchemeCache::TEvWatchNotifyUpdated::TPtr& ev);
+ void Handle(TEvTxProxySchemeCache::TEvWatchNotifyDeleted::TPtr& ev);
+
+ void Handle(TEvents::TEvPoisonPill::TPtr& ev);
+ void Handle(TEvents::TEvUndelivered::TPtr& ev);
+ void Handle(TEvInterconnect::TEvNodeDisconnected::TPtr& ev);
+
+ void PersistSysParam(NIceDb::TNiceDb& db, ui64 id, const TString& value);
+ void PersistDatabase(NIceDb::TNiceDb& db);
+ void PersistStage(NIceDb::TNiceDb& db);
+ void PersistIntervalEnd(NIceDb::TNiceDb& db);
+
+ template <typename TSchema>
+ void PersistTopResults(NIceDb::TNiceDb& db,
+ TTop& top, TResultStatsMap& results, TInstant intervalEnd);
+
+ void PersistResults(NIceDb::TNiceDb& db);
+
+ void ScheduleAggregate();
+ void ScheduleCollect();
+ void ScheduleSendRequests();
+ void ScheduleApplyCounters();
+ void ScheduleSendNavigate();
+
+ template <typename TSchema, typename TEntry>
+ void CutHistory(NIceDb::TNiceDb& db, TResultMap<TEntry>& results,
+ TDuration historySize);
+
+ static TInstant EndOfHourInterval(TInstant intervalEnd);
+
+ void ClearIntervalSummaries(NIceDb::TNiceDb& db);
+
+ void Reset(NIceDb::TNiceDb& db, const TActorContext& ctx);
+
+ void SendRequests();
+ void IgnoreFailure(TNodeId nodeId);
+
+ template <typename TResponse>
+ void ReplyOverloaded(TEvSysView::TEvGetQueryMetricsRequest::TPtr& ev);
+
+ template <typename TEntry, typename TResponse>
+ void Reply(TEvSysView::TEvGetQueryMetricsRequest::TPtr& ev);
+
+ TIntrusivePtr<IDbCounters> CreateCountersForService(NKikimrSysView::EDbCountersService service);
+ void AttachExternalCounters();
+ void AttachInternalCounters();
+ void DetachExternalCounters();
+ void DetachInternalCounters();
+ void SendNavigate();
+
+ STFUNC(StateInit) {
+ switch(ev->GetTypeRewrite()) {
+ hFunc(TEvSysView::TEvConfigureProcessor, Handle);
+ hFunc(TEvents::TEvPoisonPill, Handle);
+ IgnoreFunc(TEvSysView::TEvIntervalQuerySummary);
+ IgnoreFunc(TEvSysView::TEvGetIntervalMetricsResponse);
+ IgnoreFunc(TEvSysView::TEvGetQueryMetricsRequest);
+ IgnoreFunc(TEvSysView::TEvSendDbCountersRequest);
+ default:
+ if (!HandleDefaultEvents(ev, ctx)) {
+ LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS,
+ "TSysViewProcessor StateInit unexpected event 0x%08" PRIx32, ev->GetTypeRewrite());
+ }
+ }
+ }
+
+ STFUNC(StateOffline) {
+ switch(ev->GetTypeRewrite()) {
+ hFunc(TEvents::TEvPoisonPill, Handle);
+ hFunc(TEvSysView::TEvConfigureProcessor, Handle);
+ IgnoreFunc(TEvSysView::TEvIntervalQuerySummary);
+ IgnoreFunc(TEvSysView::TEvGetIntervalMetricsResponse);
+ IgnoreFunc(TEvSysView::TEvGetQueryMetricsRequest);
+ IgnoreFunc(TEvSysView::TEvSendDbCountersRequest);
+ default:
+ if (!HandleDefaultEvents(ev, ctx)) {
+ LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS,
+ "TSysViewProcessor StateOffline unexpected event 0x%08" PRIx32, ev->GetTypeRewrite());
+ }
+ }
+ }
+
+ STFUNC(StateWork) {
+ switch(ev->GetTypeRewrite()) {
+ hFunc(TEvSysView::TEvConfigureProcessor, Handle);
+ hFunc(TEvPrivate::TEvCollect, Handle);
+ hFunc(TEvPrivate::TEvAggregate, Handle);
+ hFunc(TEvPrivate::TEvSendRequests, Handle);
+ hFunc(TEvPrivate::TEvProcess, Handle);
+ hFunc(TEvSysView::TEvIntervalQuerySummary, Handle);
+ hFunc(TEvSysView::TEvGetIntervalMetricsResponse, Handle);
+ hFunc(TEvSysView::TEvGetQueryMetricsRequest, Handle);
+ hFunc(TEvSysView::TEvSendDbCountersRequest, Handle);
+ hFunc(TEvPrivate::TEvApplyCounters, Handle);
+ hFunc(TEvPrivate::TEvSendNavigate, Handle);
+ hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle);
+ hFunc(TEvTxProxySchemeCache::TEvWatchNotifyUpdated, Handle);
+ hFunc(TEvTxProxySchemeCache::TEvWatchNotifyDeleted, Handle);
+ hFunc(TEvents::TEvPoisonPill, Handle);
+ hFunc(TEvents::TEvUndelivered, Handle);
+ hFunc(TEvInterconnect::TEvNodeDisconnected, Handle);
+ IgnoreFunc(TEvInterconnect::TEvNodeConnected);
+ IgnoreFunc(TEvTabletPipe::TEvServerConnected);
+ IgnoreFunc(TEvTabletPipe::TEvServerDisconnected);
+ default:
+ if (!HandleDefaultEvents(ev, ctx)) {
+ LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS,
+ "TSysViewProcessor StateWork unexpected event 0x%08" PRIx32, ev->GetTypeRewrite());
+ }
+ }
+ }
+
+ STFUNC(StateBroken) {
+ HandleDefaultEvents(ev, ctx);
+ }
+
+private:
+ // limit on number of distinct queries when gathering summaries
+ static constexpr size_t DistinctQueriesLimit = 1024;
+ // limit on number of queries to aggregate metrics
+ static constexpr size_t TopCountLimit = 256;
+ // limit on number of concurrent metrics requests from services
+ static constexpr size_t MaxInFlightRequests = 16;
+ // limit on scan batch size
+ static constexpr size_t BatchSizeLimit = 4 << 20;
+ // interval of db counters processing
+ static constexpr TDuration ProcessCountersInterval = TDuration::Seconds(5);
+ // interval of sending next navigate request
+ static constexpr TDuration SendNavigateInterval = TDuration::Seconds(5);
+
+ const TDuration TotalInterval;
+ const TDuration CollectInterval;
+
+ TString Database;
+
+ TInstant IntervalEnd;
+
+ enum EStage {
+ COLLECT,
+ AGGREGATE
+ };
+ EStage CurrentStage = COLLECT;
+
+ // IntervalSummaries
+ struct TQueryToNodes {
+ ui64 Cpu = 0;
+ std::vector<std::pair<TNodeId, ui64>> Nodes; // nodeId, cpu
+ };
+ std::unordered_map<TQueryHash, TQueryToNodes> Queries;
+ std::multimap<ui64, TQueryHash> ByCpu;
+ std::unordered_set<TNodeId> SummaryNodes;
+
+ // IntervalMetrics
+ struct TQueryToMetrics {
+ NKikimrSysView::TQueryMetrics Metrics;
+ TString Text;
+ };
+ std::unordered_map<TQueryHash, TQueryToMetrics> QueryMetrics;
+
+ // NodesToRequest
+ using THashVector = std::vector<TQueryHash>;
+ struct TNodeToQueries {
+ TNodeId NodeId = 0;
+ THashVector Hashes;
+ THashVector TextsToGet;
+ THashVector ByDuration;
+ THashVector ByReadBytes;
+ THashVector ByCpuTime;
+ THashVector ByRequestUnits;
+ };
+ std::vector<TNodeToQueries> NodesToRequest;
+ std::unordered_map<TNodeId, TNodeToQueries> NodesInFlight;
+
+ // IntervalTops
+ TTop ByDurationMinute;
+ TTop ByReadBytesMinute;
+ TTop ByCpuTimeMinute;
+ TTop ByRequestUnitsMinute;
+ TTop ByDurationHour;
+ TTop ByReadBytesHour;
+ TTop ByCpuTimeHour;
+ TTop ByRequestUnitsHour;
+
+ // Metrics...
+ using TResultMetricsMap = TResultMap<TQueryToMetrics>;
+
+ TResultMetricsMap MetricsOneMinute;
+ TResultMetricsMap MetricsOneHour;
+
+ // TopBy...
+ TResultStatsMap TopByDurationOneMinute;
+ TResultStatsMap TopByDurationOneHour;
+ TResultStatsMap TopByReadBytesOneMinute;
+ TResultStatsMap TopByReadBytesOneHour;
+ TResultStatsMap TopByCpuTimeOneMinute;
+ TResultStatsMap TopByCpuTimeOneHour;
+ TResultStatsMap TopByRequestUnitsOneMinute;
+ TResultStatsMap TopByRequestUnitsOneHour;
+
+ // limited queue of user requests
+ static constexpr size_t PendingRequestsLimit = 5;
+ std::queue<TEvSysView::TEvGetQueryMetricsRequest::TPtr> PendingRequests;
+ bool ProcessInFly = false;
+
+ // db counters
+ TString CloudId;
+ TString FolderId;
+ TString DatabaseId;
+
+ NMonitoring::TDynamicCounterPtr ExternalGroup;
+ NMonitoring::TDynamicCounterPtr InternalGroup;
+
+ using TDbCountersServiceMap = std::unordered_map<NKikimrSysView::EDbCountersService,
+ NKikimr::NSysView::TDbServiceCounters>;
+
+ struct TNodeCountersState {
+ TDbCountersServiceMap Simple; // only simple counters
+ ui64 Generation = 0;
+ size_t FreshCount = 0;
+ };
+ std::unordered_map<TNodeId, TNodeCountersState> NodeCountersStates;
+ TDbCountersServiceMap AggregatedCountersState;
+
+ std::unordered_map<NKikimrSysView::EDbCountersService, TIntrusivePtr<IDbCounters>> Counters;
+};
+
+} // NSysView
+} // NKikimr
+
diff --git a/ydb/core/sys_view/processor/schema.cpp b/ydb/core/sys_view/processor/schema.cpp
index e9fd222405c..5239455e0c5 100644
--- a/ydb/core/sys_view/processor/schema.cpp
+++ b/ydb/core/sys_view/processor/schema.cpp
@@ -1 +1 @@
-#include "schema.h"
+#include "schema.h"
diff --git a/ydb/core/sys_view/processor/schema.h b/ydb/core/sys_view/processor/schema.h
index 159ad84f581..b9ae04dff16 100644
--- a/ydb/core/sys_view/processor/schema.h
+++ b/ydb/core/sys_view/processor/schema.h
@@ -1,116 +1,116 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/tablet_flat/flat_cxx_database.h>
-
-namespace NKikimr {
-namespace NSysView {
-
-struct TProcessorSchema : NIceDb::Schema {
- struct SysParams : Table<1> {
- struct Id : Column<1, NScheme::NTypeIds::Uint64> {};
- struct Value : Column<2, NScheme::NTypeIds::Utf8> {};
-
- using TKey = TableKey<Id>;
- using TColumns = TableColumns<Id, Value>;
- };
-
- struct IntervalSummaries : Table<2> {
- struct QueryHash : Column<1, NScheme::NTypeIds::Uint64> {};
- struct Index : Column<2, NScheme::NTypeIds::Uint32> {};
- struct CPU : Column<3, NScheme::NTypeIds::Uint64> {};
- struct NodeId : Column<4, NScheme::NTypeIds::Uint32> {};
-
- using TKey = TableKey<QueryHash, Index>;
- using TColumns = TableColumns<QueryHash, Index, CPU, NodeId>;
- };
-
- struct IntervalMetrics : Table<3> {
- struct QueryHash : Column<1, NScheme::NTypeIds::Uint64> {};
- struct Metrics : Column<2, NScheme::NTypeIds::String> {};
- struct Text : Column<3, NScheme::NTypeIds::Utf8> {};
-
- using TKey = TableKey<QueryHash>;
- using TColumns = TableColumns<QueryHash, Metrics, Text>;
- };
-
- struct IntervalTops : Table<4> {
- struct TypeCol : Column<1, NScheme::NTypeIds::Uint32> {
- static TString GetColumnName(const TString&) { return "Type"; }
- };
- struct QueryHash : Column<2, NScheme::NTypeIds::Uint64> {};
- struct Value : Column<3, NScheme::NTypeIds::Uint64> {};
- struct NodeId : Column<4, NScheme::NTypeIds::Uint32> {};
- struct Stats : Column<5, NScheme::NTypeIds::String> {};
-
- using TKey = TableKey<TypeCol, QueryHash>;
- using TColumns = TableColumns<TypeCol, QueryHash, Value, NodeId, Stats>;
- };
-
- struct NodesToRequest : Table<5> {
- struct NodeId : Column<1, NScheme::NTypeIds::Uint32> {};
- struct QueryHashes : Column<2, NScheme::NTypeIds::String> {};
- struct TextsToGet : Column<3, NScheme::NTypeIds::String> {};
- struct ByDuration : Column<4, NScheme::NTypeIds::String> {};
- struct ByReadBytes : Column<5, NScheme::NTypeIds::String> {};
- struct ByCpuTime : Column<6, NScheme::NTypeIds::String> {};
- struct ByRequestUnits : Column<7, NScheme::NTypeIds::String> {};
-
- using TKey = TableKey<NodeId>;
- using TColumns = TableColumns<NodeId, QueryHashes, TextsToGet,
- ByDuration, ByReadBytes, ByCpuTime, ByRequestUnits>;
- };
-
-#define RESULT_TABLE(TableName, TableID) \
- struct TableName : Table<TableID> { \
- struct IntervalEnd : Column<1, NScheme::NTypeIds::Timestamp> {}; \
- struct Rank : Column<2, NScheme::NTypeIds::Uint32> {}; \
- struct Text : Column<3, NScheme::NTypeIds::Utf8> {}; \
- struct Data : Column<4, NScheme::NTypeIds::String> {}; \
- \
- using TKey = TableKey<IntervalEnd, Rank>; \
- using TColumns = TableColumns<IntervalEnd, Rank, Text, Data>; \
- };
-
- RESULT_TABLE(MetricsOneMinute, 6)
- RESULT_TABLE(MetricsOneHour, 7)
- RESULT_TABLE(TopByDurationOneMinute, 8)
- RESULT_TABLE(TopByDurationOneHour, 9)
- RESULT_TABLE(TopByReadBytesOneMinute, 10)
- RESULT_TABLE(TopByReadBytesOneHour, 11)
- RESULT_TABLE(TopByCpuTimeOneMinute, 12)
- RESULT_TABLE(TopByCpuTimeOneHour, 13)
- RESULT_TABLE(TopByRequestUnitsOneMinute, 14)
- RESULT_TABLE(TopByRequestUnitsOneHour, 15)
-
-#undef RESULT_TABLE
-
- using TTables = SchemaTables<
- SysParams,
- IntervalSummaries,
- IntervalMetrics,
- NodesToRequest,
- IntervalTops,
- MetricsOneMinute,
- MetricsOneHour,
- TopByDurationOneMinute,
- TopByDurationOneHour,
- TopByReadBytesOneMinute,
- TopByReadBytesOneHour,
- TopByCpuTimeOneMinute,
- TopByCpuTimeOneHour,
- TopByRequestUnitsOneMinute,
- TopByRequestUnitsOneHour
- >;
-
- using TSettings = SchemaSettings<
- ExecutorLogBatching<true>,
- ExecutorLogFlushPeriod<512>
- >;
-
- static constexpr ui64 SysParam_Database = 1;
- static constexpr ui64 SysParam_CurrentStage = 2;
- static constexpr ui64 SysParam_IntervalEnd = 3;
-};
-
-}
-}
+
+namespace NKikimr {
+namespace NSysView {
+
+struct TProcessorSchema : NIceDb::Schema {
+ struct SysParams : Table<1> {
+ struct Id : Column<1, NScheme::NTypeIds::Uint64> {};
+ struct Value : Column<2, NScheme::NTypeIds::Utf8> {};
+
+ using TKey = TableKey<Id>;
+ using TColumns = TableColumns<Id, Value>;
+ };
+
+ struct IntervalSummaries : Table<2> {
+ struct QueryHash : Column<1, NScheme::NTypeIds::Uint64> {};
+ struct Index : Column<2, NScheme::NTypeIds::Uint32> {};
+ struct CPU : Column<3, NScheme::NTypeIds::Uint64> {};
+ struct NodeId : Column<4, NScheme::NTypeIds::Uint32> {};
+
+ using TKey = TableKey<QueryHash, Index>;
+ using TColumns = TableColumns<QueryHash, Index, CPU, NodeId>;
+ };
+
+ struct IntervalMetrics : Table<3> {
+ struct QueryHash : Column<1, NScheme::NTypeIds::Uint64> {};
+ struct Metrics : Column<2, NScheme::NTypeIds::String> {};
+ struct Text : Column<3, NScheme::NTypeIds::Utf8> {};
+
+ using TKey = TableKey<QueryHash>;
+ using TColumns = TableColumns<QueryHash, Metrics, Text>;
+ };
+
+ struct IntervalTops : Table<4> {
+ struct TypeCol : Column<1, NScheme::NTypeIds::Uint32> {
+ static TString GetColumnName(const TString&) { return "Type"; }
+ };
+ struct QueryHash : Column<2, NScheme::NTypeIds::Uint64> {};
+ struct Value : Column<3, NScheme::NTypeIds::Uint64> {};
+ struct NodeId : Column<4, NScheme::NTypeIds::Uint32> {};
+ struct Stats : Column<5, NScheme::NTypeIds::String> {};
+
+ using TKey = TableKey<TypeCol, QueryHash>;
+ using TColumns = TableColumns<TypeCol, QueryHash, Value, NodeId, Stats>;
+ };
+
+ struct NodesToRequest : Table<5> {
+ struct NodeId : Column<1, NScheme::NTypeIds::Uint32> {};
+ struct QueryHashes : Column<2, NScheme::NTypeIds::String> {};
+ struct TextsToGet : Column<3, NScheme::NTypeIds::String> {};
+ struct ByDuration : Column<4, NScheme::NTypeIds::String> {};
+ struct ByReadBytes : Column<5, NScheme::NTypeIds::String> {};
+ struct ByCpuTime : Column<6, NScheme::NTypeIds::String> {};
+ struct ByRequestUnits : Column<7, NScheme::NTypeIds::String> {};
+
+ using TKey = TableKey<NodeId>;
+ using TColumns = TableColumns<NodeId, QueryHashes, TextsToGet,
+ ByDuration, ByReadBytes, ByCpuTime, ByRequestUnits>;
+ };
+
+#define RESULT_TABLE(TableName, TableID) \
+ struct TableName : Table<TableID> { \
+ struct IntervalEnd : Column<1, NScheme::NTypeIds::Timestamp> {}; \
+ struct Rank : Column<2, NScheme::NTypeIds::Uint32> {}; \
+ struct Text : Column<3, NScheme::NTypeIds::Utf8> {}; \
+ struct Data : Column<4, NScheme::NTypeIds::String> {}; \
+ \
+ using TKey = TableKey<IntervalEnd, Rank>; \
+ using TColumns = TableColumns<IntervalEnd, Rank, Text, Data>; \
+ };
+
+ RESULT_TABLE(MetricsOneMinute, 6)
+ RESULT_TABLE(MetricsOneHour, 7)
+ RESULT_TABLE(TopByDurationOneMinute, 8)
+ RESULT_TABLE(TopByDurationOneHour, 9)
+ RESULT_TABLE(TopByReadBytesOneMinute, 10)
+ RESULT_TABLE(TopByReadBytesOneHour, 11)
+ RESULT_TABLE(TopByCpuTimeOneMinute, 12)
+ RESULT_TABLE(TopByCpuTimeOneHour, 13)
+ RESULT_TABLE(TopByRequestUnitsOneMinute, 14)
+ RESULT_TABLE(TopByRequestUnitsOneHour, 15)
+
+#undef RESULT_TABLE
+
+ using TTables = SchemaTables<
+ SysParams,
+ IntervalSummaries,
+ IntervalMetrics,
+ NodesToRequest,
+ IntervalTops,
+ MetricsOneMinute,
+ MetricsOneHour,
+ TopByDurationOneMinute,
+ TopByDurationOneHour,
+ TopByReadBytesOneMinute,
+ TopByReadBytesOneHour,
+ TopByCpuTimeOneMinute,
+ TopByCpuTimeOneHour,
+ TopByRequestUnitsOneMinute,
+ TopByRequestUnitsOneHour
+ >;
+
+ using TSettings = SchemaSettings<
+ ExecutorLogBatching<true>,
+ ExecutorLogFlushPeriod<512>
+ >;
+
+ static constexpr ui64 SysParam_Database = 1;
+ static constexpr ui64 SysParam_CurrentStage = 2;
+ static constexpr ui64 SysParam_IntervalEnd = 3;
+};
+
+}
+}
diff --git a/ydb/core/sys_view/processor/tx_aggregate.cpp b/ydb/core/sys_view/processor/tx_aggregate.cpp
index 654b47a689b..fe351733cc7 100644
--- a/ydb/core/sys_view/processor/tx_aggregate.cpp
+++ b/ydb/core/sys_view/processor/tx_aggregate.cpp
@@ -1,128 +1,128 @@
-#include "processor_impl.h"
-
-namespace NKikimr {
-namespace NSysView {
-
-struct TSysViewProcessor::TTxAggregate : public TTxBase {
- explicit TTxAggregate(TSelf* self)
- : TTxBase(self)
- {}
-
- TTxType GetTxType() const override { return TXTYPE_AGGREGATE; }
-
- bool Execute(TTransactionContext& txc, const TActorContext& ctx) override {
- SVLOG_D("[" << Self->TabletID() << "] TTxAggregate::Execute");
-
- NIceDb::TNiceDb db(txc.DB);
-
- auto deadline = Self->IntervalEnd + Self->TotalInterval;
- if (ctx.Now() >= deadline) {
- Self->Reset(db, ctx);
- return true;
- }
-
- std::unordered_map<TNodeId, TNodeToQueries> nodesToRequest;
- size_t count = 0;
- for (auto it = Self->ByCpu.rbegin();
- it != Self->ByCpu.rend() && count < Self->TopCountLimit;
- ++it, ++count)
- {
- auto queryHash = it->second;
- const auto& nodes = Self->Queries[queryHash].Nodes;
- for (const auto& node : nodes) {
- nodesToRequest[node.first].Hashes.emplace_back(queryHash);
- }
-
- static constexpr size_t textReqsCount = 3;
- const auto nodesCount = nodes.size();
- if (nodesCount <= textReqsCount) {
- for (const auto& node : nodes) {
- nodesToRequest[node.first].TextsToGet.emplace_back(queryHash);
- }
- } else {
- std::unordered_set<TNodeId> used;
- for (size_t count = 0; count < textReqsCount; ) {
- auto nodeId = nodes[RandomNumber<ui64>(nodesCount)].first;
- if (used.find(nodeId) != used.end()) {
- continue;
- }
- nodesToRequest[nodeId].TextsToGet.emplace_back(queryHash);
- used.insert(nodeId);
- ++count;
- }
- }
- }
-
- for (auto& entry : Self->ByDurationMinute) {
- nodesToRequest[entry.NodeId].ByDuration.emplace_back(entry.Hash);
- }
- for (auto& entry : Self->ByReadBytesMinute) {
- nodesToRequest[entry.NodeId].ByReadBytes.emplace_back(entry.Hash);
- }
- for (auto& entry : Self->ByCpuTimeMinute) {
- nodesToRequest[entry.NodeId].ByCpuTime.emplace_back(entry.Hash);
- }
- for (auto& entry : Self->ByRequestUnitsMinute) {
- nodesToRequest[entry.NodeId].ByRequestUnits.emplace_back(entry.Hash);
- }
-
- Self->NodesToRequest.reserve(nodesToRequest.size());
-
- for (auto& [nodeId, queries] : nodesToRequest) {
- queries.NodeId = nodeId;
- auto& hashes = queries.Hashes;
- auto& texts = queries.TextsToGet;
- auto& byDuration = queries.ByDuration;
- auto& byReadBytes = queries.ByReadBytes;
- auto& byCpuTime = queries.ByCpuTime;
- auto& byRequestUnits = queries.ByRequestUnits;
-
- db.Table<Schema::NodesToRequest>().Key(nodeId).Update(
- NIceDb::TUpdate<Schema::NodesToRequest::QueryHashes>(
- TString((char*)hashes.data(), hashes.size() * sizeof(TQueryHash))),
- NIceDb::TUpdate<Schema::NodesToRequest::TextsToGet>(
- TString((char*)texts.data(), texts.size() * sizeof(TQueryHash))),
- NIceDb::TUpdate<Schema::NodesToRequest::ByDuration>(
- TString((char*)byDuration.data(), byDuration.size() * sizeof(TQueryHash))),
- NIceDb::TUpdate<Schema::NodesToRequest::ByReadBytes>(
- TString((char*)byReadBytes.data(), byReadBytes.size() * sizeof(TQueryHash))),
- NIceDb::TUpdate<Schema::NodesToRequest::ByCpuTime>(
- TString((char*)byCpuTime.data(), byCpuTime.size() * sizeof(TQueryHash))),
- NIceDb::TUpdate<Schema::NodesToRequest::ByRequestUnits>(
- TString((char*)byRequestUnits.data(), byRequestUnits.size() * sizeof(TQueryHash))));
-
- Self->NodesToRequest.emplace_back(std::move(queries));
- }
-
- Self->ClearIntervalSummaries(db);
-
- if (Self->NodesToRequest.empty()) {
- Self->PersistResults(db);
- }
-
- Self->CurrentStage = AGGREGATE;
- Self->PersistStage(db);
-
- return true;
- }
-
- void Complete(const TActorContext&) override {
- SVLOG_D("[" << Self->TabletID() << "] TTxAggregate::Complete");
-
- if (Self->CurrentStage == COLLECT) {
- Self->ScheduleAggregate();
- } else {
- Self->ScheduleCollect();
- if (!Self->NodesToRequest.empty()) {
- Self->ScheduleSendRequests();
- }
- }
- }
-};
-
-void TSysViewProcessor::Handle(TEvPrivate::TEvAggregate::TPtr&) {
- Execute(new TTxAggregate(this), TActivationContext::AsActorContext());
-}
-
-} // NSysView
-} // NKikimr
+#include "processor_impl.h"
+
+namespace NKikimr {
+namespace NSysView {
+
+struct TSysViewProcessor::TTxAggregate : public TTxBase {
+ explicit TTxAggregate(TSelf* self)
+ : TTxBase(self)
+ {}
+
+ TTxType GetTxType() const override { return TXTYPE_AGGREGATE; }
+
+ bool Execute(TTransactionContext& txc, const TActorContext& ctx) override {
+ SVLOG_D("[" << Self->TabletID() << "] TTxAggregate::Execute");
+
+ NIceDb::TNiceDb db(txc.DB);
+
+ auto deadline = Self->IntervalEnd + Self->TotalInterval;
+ if (ctx.Now() >= deadline) {
+ Self->Reset(db, ctx);
+ return true;
+ }
+
+ std::unordered_map<TNodeId, TNodeToQueries> nodesToRequest;
+ size_t count = 0;
+ for (auto it = Self->ByCpu.rbegin();
+ it != Self->ByCpu.rend() && count < Self->TopCountLimit;
+ ++it, ++count)
+ {
+ auto queryHash = it->second;
+ const auto& nodes = Self->Queries[queryHash].Nodes;
+ for (const auto& node : nodes) {
+ nodesToRequest[node.first].Hashes.emplace_back(queryHash);
+ }
+
+ static constexpr size_t textReqsCount = 3;
+ const auto nodesCount = nodes.size();
+ if (nodesCount <= textReqsCount) {
+ for (const auto& node : nodes) {
+ nodesToRequest[node.first].TextsToGet.emplace_back(queryHash);
+ }
+ } else {
+ std::unordered_set<TNodeId> used;
+ for (size_t count = 0; count < textReqsCount; ) {
+ auto nodeId = nodes[RandomNumber<ui64>(nodesCount)].first;
+ if (used.find(nodeId) != used.end()) {
+ continue;
+ }
+ nodesToRequest[nodeId].TextsToGet.emplace_back(queryHash);
+ used.insert(nodeId);
+ ++count;
+ }
+ }
+ }
+
+ for (auto& entry : Self->ByDurationMinute) {
+ nodesToRequest[entry.NodeId].ByDuration.emplace_back(entry.Hash);
+ }
+ for (auto& entry : Self->ByReadBytesMinute) {
+ nodesToRequest[entry.NodeId].ByReadBytes.emplace_back(entry.Hash);
+ }
+ for (auto& entry : Self->ByCpuTimeMinute) {
+ nodesToRequest[entry.NodeId].ByCpuTime.emplace_back(entry.Hash);
+ }
+ for (auto& entry : Self->ByRequestUnitsMinute) {
+ nodesToRequest[entry.NodeId].ByRequestUnits.emplace_back(entry.Hash);
+ }
+
+ Self->NodesToRequest.reserve(nodesToRequest.size());
+
+ for (auto& [nodeId, queries] : nodesToRequest) {
+ queries.NodeId = nodeId;
+ auto& hashes = queries.Hashes;
+ auto& texts = queries.TextsToGet;
+ auto& byDuration = queries.ByDuration;
+ auto& byReadBytes = queries.ByReadBytes;
+ auto& byCpuTime = queries.ByCpuTime;
+ auto& byRequestUnits = queries.ByRequestUnits;
+
+ db.Table<Schema::NodesToRequest>().Key(nodeId).Update(
+ NIceDb::TUpdate<Schema::NodesToRequest::QueryHashes>(
+ TString((char*)hashes.data(), hashes.size() * sizeof(TQueryHash))),
+ NIceDb::TUpdate<Schema::NodesToRequest::TextsToGet>(
+ TString((char*)texts.data(), texts.size() * sizeof(TQueryHash))),
+ NIceDb::TUpdate<Schema::NodesToRequest::ByDuration>(
+ TString((char*)byDuration.data(), byDuration.size() * sizeof(TQueryHash))),
+ NIceDb::TUpdate<Schema::NodesToRequest::ByReadBytes>(
+ TString((char*)byReadBytes.data(), byReadBytes.size() * sizeof(TQueryHash))),
+ NIceDb::TUpdate<Schema::NodesToRequest::ByCpuTime>(
+ TString((char*)byCpuTime.data(), byCpuTime.size() * sizeof(TQueryHash))),
+ NIceDb::TUpdate<Schema::NodesToRequest::ByRequestUnits>(
+ TString((char*)byRequestUnits.data(), byRequestUnits.size() * sizeof(TQueryHash))));
+
+ Self->NodesToRequest.emplace_back(std::move(queries));
+ }
+
+ Self->ClearIntervalSummaries(db);
+
+ if (Self->NodesToRequest.empty()) {
+ Self->PersistResults(db);
+ }
+
+ Self->CurrentStage = AGGREGATE;
+ Self->PersistStage(db);
+
+ return true;
+ }
+
+ void Complete(const TActorContext&) override {
+ SVLOG_D("[" << Self->TabletID() << "] TTxAggregate::Complete");
+
+ if (Self->CurrentStage == COLLECT) {
+ Self->ScheduleAggregate();
+ } else {
+ Self->ScheduleCollect();
+ if (!Self->NodesToRequest.empty()) {
+ Self->ScheduleSendRequests();
+ }
+ }
+ }
+};
+
+void TSysViewProcessor::Handle(TEvPrivate::TEvAggregate::TPtr&) {
+ Execute(new TTxAggregate(this), TActivationContext::AsActorContext());
+}
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/processor/tx_collect.cpp b/ydb/core/sys_view/processor/tx_collect.cpp
index 67a197c366a..fd282e18e9e 100644
--- a/ydb/core/sys_view/processor/tx_collect.cpp
+++ b/ydb/core/sys_view/processor/tx_collect.cpp
@@ -1,39 +1,39 @@
-#include "processor_impl.h"
-
-namespace NKikimr {
-namespace NSysView {
-
-struct TSysViewProcessor::TTxCollect : public TTxBase {
- explicit TTxCollect(TSelf* self)
- : TTxBase(self)
- {}
-
- TTxType GetTxType() const override { return TXTYPE_COLLECT; }
-
- bool Execute(TTransactionContext& txc, const TActorContext& ctx) override {
- SVLOG_D("[" << Self->TabletID() << "] TTxCollect::Execute");
-
- NIceDb::TNiceDb db(txc.DB);
-
- if (!Self->NodesInFlight.empty() || !Self->NodesToRequest.empty()) {
- Self->PersistResults(db);
- }
-
- Self->Reset(db, ctx);
-
- return true;
- }
-
- void Complete(const TActorContext&) override {
- SVLOG_D("[" << Self->TabletID() << "] TTxCollect::Complete");
-
- Self->ScheduleAggregate();
- }
-};
-
-void TSysViewProcessor::Handle(TEvPrivate::TEvCollect::TPtr&) {
- Execute(new TTxCollect(this), TActivationContext::AsActorContext());
-}
-
-} // NSysView
-} // NKikimr
+#include "processor_impl.h"
+
+namespace NKikimr {
+namespace NSysView {
+
+struct TSysViewProcessor::TTxCollect : public TTxBase {
+ explicit TTxCollect(TSelf* self)
+ : TTxBase(self)
+ {}
+
+ TTxType GetTxType() const override { return TXTYPE_COLLECT; }
+
+ bool Execute(TTransactionContext& txc, const TActorContext& ctx) override {
+ SVLOG_D("[" << Self->TabletID() << "] TTxCollect::Execute");
+
+ NIceDb::TNiceDb db(txc.DB);
+
+ if (!Self->NodesInFlight.empty() || !Self->NodesToRequest.empty()) {
+ Self->PersistResults(db);
+ }
+
+ Self->Reset(db, ctx);
+
+ return true;
+ }
+
+ void Complete(const TActorContext&) override {
+ SVLOG_D("[" << Self->TabletID() << "] TTxCollect::Complete");
+
+ Self->ScheduleAggregate();
+ }
+};
+
+void TSysViewProcessor::Handle(TEvPrivate::TEvCollect::TPtr&) {
+ Execute(new TTxCollect(this), TActivationContext::AsActorContext());
+}
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/processor/tx_configure.cpp b/ydb/core/sys_view/processor/tx_configure.cpp
index 68a2a133e55..4ddc8c65727 100644
--- a/ydb/core/sys_view/processor/tx_configure.cpp
+++ b/ydb/core/sys_view/processor/tx_configure.cpp
@@ -1,44 +1,44 @@
-#include "processor_impl.h"
-
-namespace NKikimr {
-namespace NSysView {
-
-struct TSysViewProcessor::TTxConfigure : public TTxBase {
- NKikimrSysView::TEvConfigureProcessor Record;
+#include "processor_impl.h"
+
+namespace NKikimr {
+namespace NSysView {
+
+struct TSysViewProcessor::TTxConfigure : public TTxBase {
+ NKikimrSysView::TEvConfigureProcessor Record;
TActorId Sender;
-
+
TTxConfigure(TSelf* self, NKikimrSysView::TEvConfigureProcessor&& record, const TActorId& sender)
- : TTxBase(self)
- , Record(std::move(record))
- , Sender(sender)
- {}
-
- TTxType GetTxType() const override { return TXTYPE_CONFIGURE; }
-
- bool Execute(TTransactionContext& txc, const TActorContext&) override {
- SVLOG_D("[" << Self->TabletID() << "] TTxConfigure::Execute: "
- << "database# " << Record.GetDatabase());
-
- NIceDb::TNiceDb db(txc.DB);
-
- Self->Database = Record.GetDatabase();
- Self->PersistDatabase(db);
- return true;
- }
-
- void Complete(const TActorContext& ctx) override {
- SVLOG_D("[" << Self->TabletID() << "] TTxConfigure::Complete");
-
- ctx.Send(Sender, new TEvSubDomain::TEvConfigureStatus(
- NKikimrTx::TEvSubDomainConfigurationAck::SUCCESS, Self->TabletID()));
- }
-};
-
-void TSysViewProcessor::Handle(TEvSysView::TEvConfigureProcessor::TPtr& ev) {
- auto& record = ev->Get()->Record;
- Execute(new TTxConfigure(this, std::move(record), ev->Sender),
- TActivationContext::AsActorContext());
-}
-
-} // NSysView
-} // NKikimr
+ : TTxBase(self)
+ , Record(std::move(record))
+ , Sender(sender)
+ {}
+
+ TTxType GetTxType() const override { return TXTYPE_CONFIGURE; }
+
+ bool Execute(TTransactionContext& txc, const TActorContext&) override {
+ SVLOG_D("[" << Self->TabletID() << "] TTxConfigure::Execute: "
+ << "database# " << Record.GetDatabase());
+
+ NIceDb::TNiceDb db(txc.DB);
+
+ Self->Database = Record.GetDatabase();
+ Self->PersistDatabase(db);
+ return true;
+ }
+
+ void Complete(const TActorContext& ctx) override {
+ SVLOG_D("[" << Self->TabletID() << "] TTxConfigure::Complete");
+
+ ctx.Send(Sender, new TEvSubDomain::TEvConfigureStatus(
+ NKikimrTx::TEvSubDomainConfigurationAck::SUCCESS, Self->TabletID()));
+ }
+};
+
+void TSysViewProcessor::Handle(TEvSysView::TEvConfigureProcessor::TPtr& ev) {
+ auto& record = ev->Get()->Record;
+ Execute(new TTxConfigure(this, std::move(record), ev->Sender),
+ TActivationContext::AsActorContext());
+}
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/processor/tx_init.cpp b/ydb/core/sys_view/processor/tx_init.cpp
index 89c7fd637f9..7c78d3da53f 100644
--- a/ydb/core/sys_view/processor/tx_init.cpp
+++ b/ydb/core/sys_view/processor/tx_init.cpp
@@ -1,382 +1,382 @@
-#include "processor_impl.h"
-
-namespace NKikimr {
-namespace NSysView {
-
-struct TSysViewProcessor::TTxInit : public TTxBase {
- explicit TTxInit(TSelf* self)
- : TTxBase(self)
- {}
-
- TTxType GetTxType() const override { return TXTYPE_INIT; }
-
- template <typename TSchema, typename TEntry>
- bool LoadResults(NIceDb::TNiceDb& db, TResultMap<TEntry>& results) {
- results.clear();
-
- auto rowset = db.Table<TSchema>().Range().Select();
- if (!rowset.IsReady()) {
- return false;
- }
-
- while (!rowset.EndOfSet()) {
- ui64 intervalEnd = rowset.template GetValue<typename TSchema::IntervalEnd>();
- ui32 rank = rowset.template GetValue<typename TSchema::Rank>();
- TString text = rowset.template GetValue<typename TSchema::Text>();
- TString data = rowset.template GetValue<typename TSchema::Data>();
-
- auto key = std::make_pair(intervalEnd, rank);
- auto& result = results[key];
- if constexpr (std::is_same<TEntry, TQueryToMetrics>::value) {
- result.Text = std::move(text);
- if (data) {
+#include "processor_impl.h"
+
+namespace NKikimr {
+namespace NSysView {
+
+struct TSysViewProcessor::TTxInit : public TTxBase {
+ explicit TTxInit(TSelf* self)
+ : TTxBase(self)
+ {}
+
+ TTxType GetTxType() const override { return TXTYPE_INIT; }
+
+ template <typename TSchema, typename TEntry>
+ bool LoadResults(NIceDb::TNiceDb& db, TResultMap<TEntry>& results) {
+ results.clear();
+
+ auto rowset = db.Table<TSchema>().Range().Select();
+ if (!rowset.IsReady()) {
+ return false;
+ }
+
+ while (!rowset.EndOfSet()) {
+ ui64 intervalEnd = rowset.template GetValue<typename TSchema::IntervalEnd>();
+ ui32 rank = rowset.template GetValue<typename TSchema::Rank>();
+ TString text = rowset.template GetValue<typename TSchema::Text>();
+ TString data = rowset.template GetValue<typename TSchema::Data>();
+
+ auto key = std::make_pair(intervalEnd, rank);
+ auto& result = results[key];
+ if constexpr (std::is_same<TEntry, TQueryToMetrics>::value) {
+ result.Text = std::move(text);
+ if (data) {
Y_PROTOBUF_SUPPRESS_NODISCARD result.Metrics.ParseFromString(data);
- }
- } else {
- if (data) {
+ }
+ } else {
+ if (data) {
Y_PROTOBUF_SUPPRESS_NODISCARD result.ParseFromString(data);
- }
- result.SetQueryText(std::move(text));
- }
-
- if (!rowset.Next()) {
- return false;
- }
- }
-
- SVLOG_D("[" << Self->TabletID() << "] Loading results: "
- << "table# " << TSchema::TableId
- << ", results count# " << results.size());
-
- return true;
- };
-
- bool Execute(TTransactionContext& txc, const TActorContext& ctx) override {
- SVLOG_D("[" << Self->TabletID() << "] TTxInit::Execute");
-
- NIceDb::TNiceDb db(txc.DB);
-
- { // precharge
- auto sysParamsRowset = db.Table<Schema::SysParams>().Range().Select();
- auto intervalSummariesRowset = db.Table<Schema::IntervalSummaries>().Range().Select();
- auto intervalMetricsRowset = db.Table<Schema::IntervalMetrics>().Range().Select();
- auto intervalTopsRowset = db.Table<Schema::IntervalTops>().Range().Select();
- auto nodesToRequestRowset = db.Table<Schema::NodesToRequest>().Range().Select();
- auto metricsOneMinuteRowset = db.Table<Schema::MetricsOneMinute>().Range().Select();
- auto metricsOneHourRowset = db.Table<Schema::MetricsOneHour>().Range().Select();
- auto durationOneMinuteRowset = db.Table<Schema::TopByDurationOneMinute>().Range().Select();
- auto durationOneHourRowset = db.Table<Schema::TopByDurationOneHour>().Range().Select();
- auto readBytesOneMinuteRowset = db.Table<Schema::TopByDurationOneMinute>().Range().Select();
- auto readBytesOneHourRowset = db.Table<Schema::TopByReadBytesOneHour>().Range().Select();
- auto cpuTimeOneMinuteRowset = db.Table<Schema::TopByCpuTimeOneMinute>().Range().Select();
- auto cpuTimeOneHourRowset = db.Table<Schema::TopByCpuTimeOneHour>().Range().Select();
- auto reqUnitsOneMinuteRowset = db.Table<Schema::TopByRequestUnitsOneMinute>().Range().Select();
- auto reqUnitsOneHourRowset = db.Table<Schema::TopByRequestUnitsOneHour>().Range().Select();
-
- if (!sysParamsRowset.IsReady() ||
- !intervalSummariesRowset.IsReady() ||
- !intervalMetricsRowset.IsReady() ||
- !intervalTopsRowset.IsReady() ||
- !nodesToRequestRowset.IsReady() ||
- !metricsOneMinuteRowset.IsReady() ||
- !metricsOneHourRowset.IsReady() ||
- !durationOneMinuteRowset.IsReady() ||
- !durationOneHourRowset.IsReady() ||
- !readBytesOneMinuteRowset.IsReady() ||
- !readBytesOneHourRowset.IsReady() ||
- !cpuTimeOneMinuteRowset.IsReady() ||
- !cpuTimeOneHourRowset.IsReady() ||
- !reqUnitsOneMinuteRowset.IsReady() ||
- !reqUnitsOneHourRowset.IsReady())
- {
- return false;
- }
- }
-
- // SysParams
- {
- auto rowset = db.Table<Schema::SysParams>().Range().Select();
- if (!rowset.IsReady()) {
- return false;
- }
-
- while (!rowset.EndOfSet()) {
- ui64 id = rowset.GetValue<Schema::SysParams::Id>();
- TString value = rowset.GetValue<Schema::SysParams::Value>();
-
- switch (id) {
- case Schema::SysParam_Database:
- Self->Database = value;
- SVLOG_D("[" << Self->TabletID() << "] Loading database: " << Self->Database);
- break;
- case Schema::SysParam_CurrentStage: {
- auto stage = FromString<ui64>(value);
- Self->CurrentStage = static_cast<EStage>(stage);
- SVLOG_D("[" << Self->TabletID() << "] Loading stage: " << stage);
- break;
- }
- case Schema::SysParam_IntervalEnd:
- Self->IntervalEnd = TInstant::MicroSeconds(FromString<ui64>(value));
- SVLOG_D("[" << Self->TabletID() << "] Loading interval end: " << Self->IntervalEnd);
- break;
- default:
- SVLOG_CRIT("[" << Self->TabletID() << "] Unexpected SysParam id: " << id);
- }
-
- if (!rowset.Next()) {
- return false;
- }
- }
- }
-
- // IntervalSummaries
- {
- Self->Queries.clear();
- Self->ByCpu.clear();
- Self->SummaryNodes.clear();
-
- auto rowset = db.Table<Schema::IntervalSummaries>().Range().Select();
- if (!rowset.IsReady()) {
- return false;
- }
-
- size_t totalNodeIdsCount = 0;
- while (!rowset.EndOfSet()) {
- TQueryHash queryHash = rowset.GetValue<Schema::IntervalSummaries::QueryHash>();
- ui64 cpu = rowset.GetValue<Schema::IntervalSummaries::CPU>();
- TNodeId nodeId = rowset.GetValue<Schema::IntervalSummaries::NodeId>();
-
- auto& query = Self->Queries[queryHash];
- query.Cpu += cpu;
- query.Nodes.emplace_back(nodeId, cpu);
- ++totalNodeIdsCount;
-
- Self->SummaryNodes.insert(nodeId);
-
- if (!rowset.Next()) {
- return false;
- }
- }
-
- for (const auto& [queryHash, query] : Self->Queries) {
- Self->ByCpu.emplace(query.Cpu, queryHash);
- }
-
- SVLOG_D("[" << Self->TabletID() << "] Loading interval summaries: "
- << "query count# " << Self->Queries.size()
- << ", node ids count# " << Self->SummaryNodes.size()
- << ", total count# " << totalNodeIdsCount);
- }
-
- // IntervalMetrics
- {
- Self->QueryMetrics.clear();
-
- auto rowset = db.Table<Schema::IntervalMetrics>().Range().Select();
- if (!rowset.IsReady()) {
- return false;
- }
- while (!rowset.EndOfSet()) {
- TQueryHash queryHash = rowset.GetValue<Schema::IntervalMetrics::QueryHash>();
- TString metrics = rowset.GetValue<Schema::IntervalMetrics::Metrics>();
- TString text = rowset.GetValue<Schema::IntervalMetrics::Text>();
-
- auto& queryMetrics = Self->QueryMetrics[queryHash];
- queryMetrics.Text = text;
- if (metrics) {
+ }
+ result.SetQueryText(std::move(text));
+ }
+
+ if (!rowset.Next()) {
+ return false;
+ }
+ }
+
+ SVLOG_D("[" << Self->TabletID() << "] Loading results: "
+ << "table# " << TSchema::TableId
+ << ", results count# " << results.size());
+
+ return true;
+ };
+
+ bool Execute(TTransactionContext& txc, const TActorContext& ctx) override {
+ SVLOG_D("[" << Self->TabletID() << "] TTxInit::Execute");
+
+ NIceDb::TNiceDb db(txc.DB);
+
+ { // precharge
+ auto sysParamsRowset = db.Table<Schema::SysParams>().Range().Select();
+ auto intervalSummariesRowset = db.Table<Schema::IntervalSummaries>().Range().Select();
+ auto intervalMetricsRowset = db.Table<Schema::IntervalMetrics>().Range().Select();
+ auto intervalTopsRowset = db.Table<Schema::IntervalTops>().Range().Select();
+ auto nodesToRequestRowset = db.Table<Schema::NodesToRequest>().Range().Select();
+ auto metricsOneMinuteRowset = db.Table<Schema::MetricsOneMinute>().Range().Select();
+ auto metricsOneHourRowset = db.Table<Schema::MetricsOneHour>().Range().Select();
+ auto durationOneMinuteRowset = db.Table<Schema::TopByDurationOneMinute>().Range().Select();
+ auto durationOneHourRowset = db.Table<Schema::TopByDurationOneHour>().Range().Select();
+ auto readBytesOneMinuteRowset = db.Table<Schema::TopByDurationOneMinute>().Range().Select();
+ auto readBytesOneHourRowset = db.Table<Schema::TopByReadBytesOneHour>().Range().Select();
+ auto cpuTimeOneMinuteRowset = db.Table<Schema::TopByCpuTimeOneMinute>().Range().Select();
+ auto cpuTimeOneHourRowset = db.Table<Schema::TopByCpuTimeOneHour>().Range().Select();
+ auto reqUnitsOneMinuteRowset = db.Table<Schema::TopByRequestUnitsOneMinute>().Range().Select();
+ auto reqUnitsOneHourRowset = db.Table<Schema::TopByRequestUnitsOneHour>().Range().Select();
+
+ if (!sysParamsRowset.IsReady() ||
+ !intervalSummariesRowset.IsReady() ||
+ !intervalMetricsRowset.IsReady() ||
+ !intervalTopsRowset.IsReady() ||
+ !nodesToRequestRowset.IsReady() ||
+ !metricsOneMinuteRowset.IsReady() ||
+ !metricsOneHourRowset.IsReady() ||
+ !durationOneMinuteRowset.IsReady() ||
+ !durationOneHourRowset.IsReady() ||
+ !readBytesOneMinuteRowset.IsReady() ||
+ !readBytesOneHourRowset.IsReady() ||
+ !cpuTimeOneMinuteRowset.IsReady() ||
+ !cpuTimeOneHourRowset.IsReady() ||
+ !reqUnitsOneMinuteRowset.IsReady() ||
+ !reqUnitsOneHourRowset.IsReady())
+ {
+ return false;
+ }
+ }
+
+ // SysParams
+ {
+ auto rowset = db.Table<Schema::SysParams>().Range().Select();
+ if (!rowset.IsReady()) {
+ return false;
+ }
+
+ while (!rowset.EndOfSet()) {
+ ui64 id = rowset.GetValue<Schema::SysParams::Id>();
+ TString value = rowset.GetValue<Schema::SysParams::Value>();
+
+ switch (id) {
+ case Schema::SysParam_Database:
+ Self->Database = value;
+ SVLOG_D("[" << Self->TabletID() << "] Loading database: " << Self->Database);
+ break;
+ case Schema::SysParam_CurrentStage: {
+ auto stage = FromString<ui64>(value);
+ Self->CurrentStage = static_cast<EStage>(stage);
+ SVLOG_D("[" << Self->TabletID() << "] Loading stage: " << stage);
+ break;
+ }
+ case Schema::SysParam_IntervalEnd:
+ Self->IntervalEnd = TInstant::MicroSeconds(FromString<ui64>(value));
+ SVLOG_D("[" << Self->TabletID() << "] Loading interval end: " << Self->IntervalEnd);
+ break;
+ default:
+ SVLOG_CRIT("[" << Self->TabletID() << "] Unexpected SysParam id: " << id);
+ }
+
+ if (!rowset.Next()) {
+ return false;
+ }
+ }
+ }
+
+ // IntervalSummaries
+ {
+ Self->Queries.clear();
+ Self->ByCpu.clear();
+ Self->SummaryNodes.clear();
+
+ auto rowset = db.Table<Schema::IntervalSummaries>().Range().Select();
+ if (!rowset.IsReady()) {
+ return false;
+ }
+
+ size_t totalNodeIdsCount = 0;
+ while (!rowset.EndOfSet()) {
+ TQueryHash queryHash = rowset.GetValue<Schema::IntervalSummaries::QueryHash>();
+ ui64 cpu = rowset.GetValue<Schema::IntervalSummaries::CPU>();
+ TNodeId nodeId = rowset.GetValue<Schema::IntervalSummaries::NodeId>();
+
+ auto& query = Self->Queries[queryHash];
+ query.Cpu += cpu;
+ query.Nodes.emplace_back(nodeId, cpu);
+ ++totalNodeIdsCount;
+
+ Self->SummaryNodes.insert(nodeId);
+
+ if (!rowset.Next()) {
+ return false;
+ }
+ }
+
+ for (const auto& [queryHash, query] : Self->Queries) {
+ Self->ByCpu.emplace(query.Cpu, queryHash);
+ }
+
+ SVLOG_D("[" << Self->TabletID() << "] Loading interval summaries: "
+ << "query count# " << Self->Queries.size()
+ << ", node ids count# " << Self->SummaryNodes.size()
+ << ", total count# " << totalNodeIdsCount);
+ }
+
+ // IntervalMetrics
+ {
+ Self->QueryMetrics.clear();
+
+ auto rowset = db.Table<Schema::IntervalMetrics>().Range().Select();
+ if (!rowset.IsReady()) {
+ return false;
+ }
+ while (!rowset.EndOfSet()) {
+ TQueryHash queryHash = rowset.GetValue<Schema::IntervalMetrics::QueryHash>();
+ TString metrics = rowset.GetValue<Schema::IntervalMetrics::Metrics>();
+ TString text = rowset.GetValue<Schema::IntervalMetrics::Text>();
+
+ auto& queryMetrics = Self->QueryMetrics[queryHash];
+ queryMetrics.Text = text;
+ if (metrics) {
Y_PROTOBUF_SUPPRESS_NODISCARD queryMetrics.Metrics.ParseFromString(metrics);
- }
-
- if (!rowset.Next()) {
- return false;
- }
- }
- SVLOG_D("[" << Self->TabletID() << "] Loading interval metrics: "
- << "query count# " << Self->QueryMetrics.size());
- }
-
- // IntervalTops
- {
- Self->ByDurationMinute.clear();
- Self->ByDurationHour.clear();
- Self->ByReadBytesMinute.clear();
- Self->ByReadBytesHour.clear();
- Self->ByCpuTimeMinute.clear();
- Self->ByCpuTimeHour.clear();
- Self->ByRequestUnitsMinute.clear();
- Self->ByRequestUnitsHour.clear();
-
- auto rowset = db.Table<Schema::IntervalTops>().Range().Select();
- if (!rowset.IsReady()) {
- return false;
- }
-
- size_t queryCount = 0;
- while (!rowset.EndOfSet()) {
- ui32 type = rowset.GetValue<Schema::IntervalTops::TypeCol>();
- TQueryHash queryHash = rowset.GetValue<Schema::IntervalTops::QueryHash>();
- ui64 value = rowset.GetValue<Schema::IntervalTops::Value>();
- TNodeId nodeId = rowset.GetValue<Schema::IntervalTops::NodeId>();
- TString stats = rowset.GetValue<Schema::IntervalTops::Stats>();
-
- TTopQuery query{queryHash, value, nodeId, {}};
- if (stats) {
- query.Stats = MakeHolder<NKikimrSysView::TQueryStats>();
+ }
+
+ if (!rowset.Next()) {
+ return false;
+ }
+ }
+ SVLOG_D("[" << Self->TabletID() << "] Loading interval metrics: "
+ << "query count# " << Self->QueryMetrics.size());
+ }
+
+ // IntervalTops
+ {
+ Self->ByDurationMinute.clear();
+ Self->ByDurationHour.clear();
+ Self->ByReadBytesMinute.clear();
+ Self->ByReadBytesHour.clear();
+ Self->ByCpuTimeMinute.clear();
+ Self->ByCpuTimeHour.clear();
+ Self->ByRequestUnitsMinute.clear();
+ Self->ByRequestUnitsHour.clear();
+
+ auto rowset = db.Table<Schema::IntervalTops>().Range().Select();
+ if (!rowset.IsReady()) {
+ return false;
+ }
+
+ size_t queryCount = 0;
+ while (!rowset.EndOfSet()) {
+ ui32 type = rowset.GetValue<Schema::IntervalTops::TypeCol>();
+ TQueryHash queryHash = rowset.GetValue<Schema::IntervalTops::QueryHash>();
+ ui64 value = rowset.GetValue<Schema::IntervalTops::Value>();
+ TNodeId nodeId = rowset.GetValue<Schema::IntervalTops::NodeId>();
+ TString stats = rowset.GetValue<Schema::IntervalTops::Stats>();
+
+ TTopQuery query{queryHash, value, nodeId, {}};
+ if (stats) {
+ query.Stats = MakeHolder<NKikimrSysView::TQueryStats>();
Y_PROTOBUF_SUPPRESS_NODISCARD query.Stats->ParseFromString(stats);
- }
-
- switch ((NKikimrSysView::EStatsType)type) {
- case NKikimrSysView::TOP_DURATION_ONE_MINUTE:
- Self->ByDurationMinute.emplace_back(std::move(query));
- break;
- case NKikimrSysView::TOP_DURATION_ONE_HOUR:
- Self->ByDurationHour.emplace_back(std::move(query));
- break;
- case NKikimrSysView::TOP_READ_BYTES_ONE_MINUTE:
- Self->ByReadBytesMinute.emplace_back(std::move(query));
- break;
- case NKikimrSysView::TOP_READ_BYTES_ONE_HOUR:
- Self->ByReadBytesHour.emplace_back(std::move(query));
- break;
- case NKikimrSysView::TOP_CPU_TIME_ONE_MINUTE:
- Self->ByCpuTimeMinute.emplace_back(std::move(query));
- break;
- case NKikimrSysView::TOP_CPU_TIME_ONE_HOUR:
- Self->ByCpuTimeHour.emplace_back(std::move(query));
- break;
- case NKikimrSysView::TOP_REQUEST_UNITS_ONE_MINUTE:
- Self->ByRequestUnitsMinute.emplace_back(std::move(query));
- break;
- case NKikimrSysView::TOP_REQUEST_UNITS_ONE_HOUR:
- Self->ByRequestUnitsHour.emplace_back(std::move(query));
- break;
- default:
- SVLOG_CRIT("[" << Self->TabletID() << "] ignoring unexpected stats type: " << type);
- }
-
- ++queryCount;
- if (!rowset.Next()) {
- return false;
- }
- }
-
- std::sort(Self->ByDurationMinute.begin(), Self->ByDurationMinute.end(), TopQueryCompare);
- std::sort(Self->ByDurationHour.begin(), Self->ByDurationHour.end(), TopQueryCompare);
- std::sort(Self->ByReadBytesMinute.begin(), Self->ByReadBytesMinute.end(), TopQueryCompare);
- std::sort(Self->ByReadBytesHour.begin(), Self->ByReadBytesHour.end(), TopQueryCompare);
- std::sort(Self->ByCpuTimeMinute.begin(), Self->ByCpuTimeMinute.end(), TopQueryCompare);
- std::sort(Self->ByCpuTimeHour.begin(), Self->ByCpuTimeHour.end(), TopQueryCompare);
- std::sort(Self->ByRequestUnitsMinute.begin(), Self->ByRequestUnitsMinute.end(), TopQueryCompare);
- std::sort(Self->ByRequestUnitsHour.begin(), Self->ByRequestUnitsHour.end(), TopQueryCompare);
-
- SVLOG_D("[" << Self->TabletID() << "] Loading interval tops: "
- << "total query count# " << queryCount);
- }
-
- // NodesToRequest
- {
- Self->NodesToRequest.clear();
- Self->NodesInFlight.clear();
-
- auto rowset = db.Table<Schema::NodesToRequest>().Range().Select();
- if (!rowset.IsReady()) {
- return false;
- }
-
- size_t totalHashesCount = 0;
- while (!rowset.EndOfSet()) {
- TNodeId nodeId = rowset.GetValue<Schema::NodesToRequest::NodeId>();
- TString hashes = rowset.GetValue<Schema::NodesToRequest::QueryHashes>();
- TString textsToGet = rowset.GetValue<Schema::NodesToRequest::TextsToGet>();
- TString byDuration = rowset.GetValue<Schema::NodesToRequest::ByDuration>();
- TString byReadBytes = rowset.GetValue<Schema::NodesToRequest::ByReadBytes>();
- TString byCpuTime = rowset.GetValue<Schema::NodesToRequest::ByCpuTime>();
- TString byRequestUnits = rowset.GetValue<Schema::NodesToRequest::ByRequestUnits>();
-
- auto loadHashes = [&totalHashesCount] (const TString& from, THashVector& to) {
- auto size = from.size() / sizeof(TQueryHash);
- to.resize(size);
- std::memcpy(to.data(), from.data(), size * sizeof(TQueryHash));
- totalHashesCount += size;
- };
-
- TNodeToQueries entry;
- entry.NodeId = nodeId;
- loadHashes(hashes, entry.Hashes);
- loadHashes(textsToGet, entry.TextsToGet);
- loadHashes(byDuration, entry.ByDuration);
- loadHashes(byReadBytes, entry.ByReadBytes);
- loadHashes(byCpuTime, entry.ByCpuTime);
- loadHashes(byRequestUnits, entry.ByRequestUnits);
- Self->NodesToRequest.emplace_back(std::move(entry));
-
- if (!rowset.Next()) {
- return false;
- }
- }
- SVLOG_D("[" << Self->TabletID() << "] Loading nodes to request: "
- << "nodes count# " << Self->NodesToRequest.size()
- << ", hashes count# " << totalHashesCount);
- }
-
- // Metrics...
- if (!LoadResults<Schema::MetricsOneMinute, TQueryToMetrics>(db, Self->MetricsOneMinute))
- return false;
- if (!LoadResults<Schema::MetricsOneHour, TQueryToMetrics>(db, Self->MetricsOneHour))
- return false;
-
- // TopBy...
- using TStats = NKikimrSysView::TQueryStats;
- if (!LoadResults<Schema::TopByDurationOneMinute, TStats>(db, Self->TopByDurationOneMinute))
- return false;
- if (!LoadResults<Schema::TopByDurationOneHour, TStats>(db, Self->TopByDurationOneHour))
- return false;
- if (!LoadResults<Schema::TopByReadBytesOneMinute, TStats>(db, Self->TopByReadBytesOneMinute))
- return false;
- if (!LoadResults<Schema::TopByReadBytesOneHour, TStats>(db, Self->TopByReadBytesOneHour))
- return false;
- if (!LoadResults<Schema::TopByCpuTimeOneMinute, TStats>(db, Self->TopByCpuTimeOneMinute))
- return false;
- if (!LoadResults<Schema::TopByCpuTimeOneHour, TStats>(db, Self->TopByCpuTimeOneHour))
- return false;
- if (!LoadResults<Schema::TopByRequestUnitsOneMinute, TStats>(db, Self->TopByRequestUnitsOneMinute))
- return false;
- if (!LoadResults<Schema::TopByRequestUnitsOneHour, TStats>(db, Self->TopByRequestUnitsOneHour))
- return false;
-
- auto deadline = Self->IntervalEnd + Self->TotalInterval;
- if (ctx.Now() >= deadline) {
- Self->Reset(db, ctx);
- }
-
- return true;
- }
-
- void Complete(const TActorContext& ctx) override {
- SVLOG_D("[" << Self->TabletID() << "] TTxInit::Complete");
-
- if (Self->CurrentStage == COLLECT) {
- Self->ScheduleAggregate();
- } else {
- Self->ScheduleCollect();
- if (!Self->NodesToRequest.empty()) {
- Self->ScheduleSendRequests();
- }
- }
-
- if (AppData()->FeatureFlags.GetEnableDbCounters()) {
- Self->ScheduleApplyCounters();
- Self->SendNavigate();
- }
-
- Self->SignalTabletActive(ctx);
- Self->Become(&TThis::StateWork);
- }
-};
-
-NTabletFlatExecutor::ITransaction* TSysViewProcessor::CreateTxInit() {
- return new TTxInit(this);
-}
-
-} // NSysView
-} // NKikimr
+ }
+
+ switch ((NKikimrSysView::EStatsType)type) {
+ case NKikimrSysView::TOP_DURATION_ONE_MINUTE:
+ Self->ByDurationMinute.emplace_back(std::move(query));
+ break;
+ case NKikimrSysView::TOP_DURATION_ONE_HOUR:
+ Self->ByDurationHour.emplace_back(std::move(query));
+ break;
+ case NKikimrSysView::TOP_READ_BYTES_ONE_MINUTE:
+ Self->ByReadBytesMinute.emplace_back(std::move(query));
+ break;
+ case NKikimrSysView::TOP_READ_BYTES_ONE_HOUR:
+ Self->ByReadBytesHour.emplace_back(std::move(query));
+ break;
+ case NKikimrSysView::TOP_CPU_TIME_ONE_MINUTE:
+ Self->ByCpuTimeMinute.emplace_back(std::move(query));
+ break;
+ case NKikimrSysView::TOP_CPU_TIME_ONE_HOUR:
+ Self->ByCpuTimeHour.emplace_back(std::move(query));
+ break;
+ case NKikimrSysView::TOP_REQUEST_UNITS_ONE_MINUTE:
+ Self->ByRequestUnitsMinute.emplace_back(std::move(query));
+ break;
+ case NKikimrSysView::TOP_REQUEST_UNITS_ONE_HOUR:
+ Self->ByRequestUnitsHour.emplace_back(std::move(query));
+ break;
+ default:
+ SVLOG_CRIT("[" << Self->TabletID() << "] ignoring unexpected stats type: " << type);
+ }
+
+ ++queryCount;
+ if (!rowset.Next()) {
+ return false;
+ }
+ }
+
+ std::sort(Self->ByDurationMinute.begin(), Self->ByDurationMinute.end(), TopQueryCompare);
+ std::sort(Self->ByDurationHour.begin(), Self->ByDurationHour.end(), TopQueryCompare);
+ std::sort(Self->ByReadBytesMinute.begin(), Self->ByReadBytesMinute.end(), TopQueryCompare);
+ std::sort(Self->ByReadBytesHour.begin(), Self->ByReadBytesHour.end(), TopQueryCompare);
+ std::sort(Self->ByCpuTimeMinute.begin(), Self->ByCpuTimeMinute.end(), TopQueryCompare);
+ std::sort(Self->ByCpuTimeHour.begin(), Self->ByCpuTimeHour.end(), TopQueryCompare);
+ std::sort(Self->ByRequestUnitsMinute.begin(), Self->ByRequestUnitsMinute.end(), TopQueryCompare);
+ std::sort(Self->ByRequestUnitsHour.begin(), Self->ByRequestUnitsHour.end(), TopQueryCompare);
+
+ SVLOG_D("[" << Self->TabletID() << "] Loading interval tops: "
+ << "total query count# " << queryCount);
+ }
+
+ // NodesToRequest
+ {
+ Self->NodesToRequest.clear();
+ Self->NodesInFlight.clear();
+
+ auto rowset = db.Table<Schema::NodesToRequest>().Range().Select();
+ if (!rowset.IsReady()) {
+ return false;
+ }
+
+ size_t totalHashesCount = 0;
+ while (!rowset.EndOfSet()) {
+ TNodeId nodeId = rowset.GetValue<Schema::NodesToRequest::NodeId>();
+ TString hashes = rowset.GetValue<Schema::NodesToRequest::QueryHashes>();
+ TString textsToGet = rowset.GetValue<Schema::NodesToRequest::TextsToGet>();
+ TString byDuration = rowset.GetValue<Schema::NodesToRequest::ByDuration>();
+ TString byReadBytes = rowset.GetValue<Schema::NodesToRequest::ByReadBytes>();
+ TString byCpuTime = rowset.GetValue<Schema::NodesToRequest::ByCpuTime>();
+ TString byRequestUnits = rowset.GetValue<Schema::NodesToRequest::ByRequestUnits>();
+
+ auto loadHashes = [&totalHashesCount] (const TString& from, THashVector& to) {
+ auto size = from.size() / sizeof(TQueryHash);
+ to.resize(size);
+ std::memcpy(to.data(), from.data(), size * sizeof(TQueryHash));
+ totalHashesCount += size;
+ };
+
+ TNodeToQueries entry;
+ entry.NodeId = nodeId;
+ loadHashes(hashes, entry.Hashes);
+ loadHashes(textsToGet, entry.TextsToGet);
+ loadHashes(byDuration, entry.ByDuration);
+ loadHashes(byReadBytes, entry.ByReadBytes);
+ loadHashes(byCpuTime, entry.ByCpuTime);
+ loadHashes(byRequestUnits, entry.ByRequestUnits);
+ Self->NodesToRequest.emplace_back(std::move(entry));
+
+ if (!rowset.Next()) {
+ return false;
+ }
+ }
+ SVLOG_D("[" << Self->TabletID() << "] Loading nodes to request: "
+ << "nodes count# " << Self->NodesToRequest.size()
+ << ", hashes count# " << totalHashesCount);
+ }
+
+ // Metrics...
+ if (!LoadResults<Schema::MetricsOneMinute, TQueryToMetrics>(db, Self->MetricsOneMinute))
+ return false;
+ if (!LoadResults<Schema::MetricsOneHour, TQueryToMetrics>(db, Self->MetricsOneHour))
+ return false;
+
+ // TopBy...
+ using TStats = NKikimrSysView::TQueryStats;
+ if (!LoadResults<Schema::TopByDurationOneMinute, TStats>(db, Self->TopByDurationOneMinute))
+ return false;
+ if (!LoadResults<Schema::TopByDurationOneHour, TStats>(db, Self->TopByDurationOneHour))
+ return false;
+ if (!LoadResults<Schema::TopByReadBytesOneMinute, TStats>(db, Self->TopByReadBytesOneMinute))
+ return false;
+ if (!LoadResults<Schema::TopByReadBytesOneHour, TStats>(db, Self->TopByReadBytesOneHour))
+ return false;
+ if (!LoadResults<Schema::TopByCpuTimeOneMinute, TStats>(db, Self->TopByCpuTimeOneMinute))
+ return false;
+ if (!LoadResults<Schema::TopByCpuTimeOneHour, TStats>(db, Self->TopByCpuTimeOneHour))
+ return false;
+ if (!LoadResults<Schema::TopByRequestUnitsOneMinute, TStats>(db, Self->TopByRequestUnitsOneMinute))
+ return false;
+ if (!LoadResults<Schema::TopByRequestUnitsOneHour, TStats>(db, Self->TopByRequestUnitsOneHour))
+ return false;
+
+ auto deadline = Self->IntervalEnd + Self->TotalInterval;
+ if (ctx.Now() >= deadline) {
+ Self->Reset(db, ctx);
+ }
+
+ return true;
+ }
+
+ void Complete(const TActorContext& ctx) override {
+ SVLOG_D("[" << Self->TabletID() << "] TTxInit::Complete");
+
+ if (Self->CurrentStage == COLLECT) {
+ Self->ScheduleAggregate();
+ } else {
+ Self->ScheduleCollect();
+ if (!Self->NodesToRequest.empty()) {
+ Self->ScheduleSendRequests();
+ }
+ }
+
+ if (AppData()->FeatureFlags.GetEnableDbCounters()) {
+ Self->ScheduleApplyCounters();
+ Self->SendNavigate();
+ }
+
+ Self->SignalTabletActive(ctx);
+ Self->Become(&TThis::StateWork);
+ }
+};
+
+NTabletFlatExecutor::ITransaction* TSysViewProcessor::CreateTxInit() {
+ return new TTxInit(this);
+}
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/processor/tx_init_schema.cpp b/ydb/core/sys_view/processor/tx_init_schema.cpp
index e0485d741f1..b7701f41678 100644
--- a/ydb/core/sys_view/processor/tx_init_schema.cpp
+++ b/ydb/core/sys_view/processor/tx_init_schema.cpp
@@ -1,57 +1,57 @@
-#include "processor_impl.h"
-
-namespace NKikimr {
-namespace NSysView {
-
-struct TSysViewProcessor::TTxInitSchema : public TTxBase {
- explicit TTxInitSchema(TSelf* self)
- : TTxBase(self)
- {}
-
- TTxType GetTxType() const override { return TXTYPE_INIT_SCHEMA; }
-
- bool Execute(TTransactionContext& txc, const TActorContext&) override {
- SVLOG_D("[" << Self->TabletID() << "] TTxInitSchema::Execute");
-
- NIceDb::TNiceDb(txc.DB).Materialize<Schema>();
-
- static constexpr NIceDb::TTableId resultTableIds[] = {
- Schema::MetricsOneMinute::TableId,
- Schema::MetricsOneHour::TableId,
- Schema::TopByDurationOneMinute::TableId,
- Schema::TopByDurationOneHour::TableId,
- Schema::TopByReadBytesOneMinute::TableId,
- Schema::TopByReadBytesOneHour::TableId,
- Schema::TopByCpuTimeOneMinute::TableId,
- Schema::TopByCpuTimeOneHour::TableId,
- Schema::TopByRequestUnitsOneMinute::TableId,
- Schema::TopByRequestUnitsOneHour::TableId
- };
-
- for (auto id : resultTableIds) {
- txc.DB.Alter().SetCompactionPolicy(id, *NLocalDb::CreateDefaultUserTablePolicy());
- }
-
- return true;
- }
-
- void Complete(const TActorContext& ctx) override {
- SVLOG_D("[" << Self->TabletID() << "] TTxInitSchema::Complete");
-
+#include "processor_impl.h"
+
+namespace NKikimr {
+namespace NSysView {
+
+struct TSysViewProcessor::TTxInitSchema : public TTxBase {
+ explicit TTxInitSchema(TSelf* self)
+ : TTxBase(self)
+ {}
+
+ TTxType GetTxType() const override { return TXTYPE_INIT_SCHEMA; }
+
+ bool Execute(TTransactionContext& txc, const TActorContext&) override {
+ SVLOG_D("[" << Self->TabletID() << "] TTxInitSchema::Execute");
+
+ NIceDb::TNiceDb(txc.DB).Materialize<Schema>();
+
+ static constexpr NIceDb::TTableId resultTableIds[] = {
+ Schema::MetricsOneMinute::TableId,
+ Schema::MetricsOneHour::TableId,
+ Schema::TopByDurationOneMinute::TableId,
+ Schema::TopByDurationOneHour::TableId,
+ Schema::TopByReadBytesOneMinute::TableId,
+ Schema::TopByReadBytesOneHour::TableId,
+ Schema::TopByCpuTimeOneMinute::TableId,
+ Schema::TopByCpuTimeOneHour::TableId,
+ Schema::TopByRequestUnitsOneMinute::TableId,
+ Schema::TopByRequestUnitsOneHour::TableId
+ };
+
+ for (auto id : resultTableIds) {
+ txc.DB.Alter().SetCompactionPolicy(id, *NLocalDb::CreateDefaultUserTablePolicy());
+ }
+
+ return true;
+ }
+
+ void Complete(const TActorContext& ctx) override {
+ SVLOG_D("[" << Self->TabletID() << "] TTxInitSchema::Complete");
+
if (!AppData()->FeatureFlags.GetEnablePersistentQueryStats()) {
- SVLOG_D("[" << Self->TabletID() << "] tablet is offline");
- Self->SignalTabletActive(ctx);
- Self->Become(&TThis::StateOffline);
- return;
- }
-
- Self->Execute(Self->CreateTxInit(), ctx);
- }
-};
-
-NTabletFlatExecutor::ITransaction* TSysViewProcessor::CreateTxInitSchema() {
- return new TTxInitSchema(this);
-}
-
-} // NSysView
-} // NKikimr
+ SVLOG_D("[" << Self->TabletID() << "] tablet is offline");
+ Self->SignalTabletActive(ctx);
+ Self->Become(&TThis::StateOffline);
+ return;
+ }
+
+ Self->Execute(Self->CreateTxInit(), ctx);
+ }
+};
+
+NTabletFlatExecutor::ITransaction* TSysViewProcessor::CreateTxInitSchema() {
+ return new TTxInitSchema(this);
+}
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/processor/tx_interval_metrics.cpp b/ydb/core/sys_view/processor/tx_interval_metrics.cpp
index 4c6d9924877..e19ebd96fa4 100644
--- a/ydb/core/sys_view/processor/tx_interval_metrics.cpp
+++ b/ydb/core/sys_view/processor/tx_interval_metrics.cpp
@@ -1,140 +1,140 @@
-#include "processor_impl.h"
-
+#include "processor_impl.h"
+
#include <ydb/core/sys_view/service/query_interval.h>
-
-namespace NKikimr {
-namespace NSysView {
-
-struct TSysViewProcessor::TTxIntervalMetrics : public TTxBase {
- TNodeId NodeId;
- NKikimrSysView::TEvGetIntervalMetricsResponse Record;
-
- TTxIntervalMetrics(TSelf* self, TNodeId nodeId,
- NKikimrSysView::TEvGetIntervalMetricsResponse&& record)
- : TTxBase(self)
- , NodeId(nodeId)
- , Record(std::move(record))
- {}
-
- TTxType GetTxType() const override { return TXTYPE_INTERVAL_METRICS; }
-
- bool Execute(TTransactionContext& txc, const TActorContext&) override {
- SVLOG_D("[" << Self->TabletID() << "] TTxIntervalMetrics::Execute: "
- << "node id# " << NodeId
- << ", metrics count# " << Record.MetricsSize()
- << ", texts count# " << Record.QueryTextsSize());
-
- NIceDb::TNiceDb db(txc.DB);
-
- for (auto& queryText : *Record.MutableQueryTexts()) {
- auto queryHash = queryText.GetHash();
- auto& text = *queryText.MutableText();
-
- db.Table<Schema::IntervalMetrics>().Key(queryHash).Update(
- NIceDb::TUpdate<Schema::IntervalMetrics::Text>(text));
-
- Self->QueryMetrics[queryHash].Text = std::move(text);
- }
-
- for (auto& metrics : *Record.MutableMetrics()) {
- auto queryHash = metrics.GetQueryTextHash();
- auto& newMetrics = Self->QueryMetrics[queryHash].Metrics;
-
- if (!newMetrics.GetCount()) {
- newMetrics.Swap(&metrics);
- } else {
- Aggregate(newMetrics, metrics);
- }
-
- TString serialized;
+
+namespace NKikimr {
+namespace NSysView {
+
+struct TSysViewProcessor::TTxIntervalMetrics : public TTxBase {
+ TNodeId NodeId;
+ NKikimrSysView::TEvGetIntervalMetricsResponse Record;
+
+ TTxIntervalMetrics(TSelf* self, TNodeId nodeId,
+ NKikimrSysView::TEvGetIntervalMetricsResponse&& record)
+ : TTxBase(self)
+ , NodeId(nodeId)
+ , Record(std::move(record))
+ {}
+
+ TTxType GetTxType() const override { return TXTYPE_INTERVAL_METRICS; }
+
+ bool Execute(TTransactionContext& txc, const TActorContext&) override {
+ SVLOG_D("[" << Self->TabletID() << "] TTxIntervalMetrics::Execute: "
+ << "node id# " << NodeId
+ << ", metrics count# " << Record.MetricsSize()
+ << ", texts count# " << Record.QueryTextsSize());
+
+ NIceDb::TNiceDb db(txc.DB);
+
+ for (auto& queryText : *Record.MutableQueryTexts()) {
+ auto queryHash = queryText.GetHash();
+ auto& text = *queryText.MutableText();
+
+ db.Table<Schema::IntervalMetrics>().Key(queryHash).Update(
+ NIceDb::TUpdate<Schema::IntervalMetrics::Text>(text));
+
+ Self->QueryMetrics[queryHash].Text = std::move(text);
+ }
+
+ for (auto& metrics : *Record.MutableMetrics()) {
+ auto queryHash = metrics.GetQueryTextHash();
+ auto& newMetrics = Self->QueryMetrics[queryHash].Metrics;
+
+ if (!newMetrics.GetCount()) {
+ newMetrics.Swap(&metrics);
+ } else {
+ Aggregate(newMetrics, metrics);
+ }
+
+ TString serialized;
Y_PROTOBUF_SUPPRESS_NODISCARD newMetrics.SerializeToString(&serialized);
- db.Table<Schema::IntervalMetrics>().Key(queryHash).Update(
- NIceDb::TUpdate<Schema::IntervalMetrics::Metrics>(serialized));
- }
-
- auto fillTops = [&] (TTop& minuteTop, TTop& hourTop,
- NKikimrSysView::EStatsType minuteType, NKikimrSysView::EStatsType hourType,
- const NProtoBuf::RepeatedPtrField<NKikimrSysView::TQueryStats>& queryStats)
- {
- for (auto& stats : queryStats) {
- auto queryHash = stats.GetQueryTextHash();
- TString serialized;
- for (auto& query : minuteTop) {
- if (query.Hash == queryHash) {
- query.Stats = MakeHolder<NKikimrSysView::TQueryStats>();
- query.Stats->CopyFrom(stats);
+ db.Table<Schema::IntervalMetrics>().Key(queryHash).Update(
+ NIceDb::TUpdate<Schema::IntervalMetrics::Metrics>(serialized));
+ }
+
+ auto fillTops = [&] (TTop& minuteTop, TTop& hourTop,
+ NKikimrSysView::EStatsType minuteType, NKikimrSysView::EStatsType hourType,
+ const NProtoBuf::RepeatedPtrField<NKikimrSysView::TQueryStats>& queryStats)
+ {
+ for (auto& stats : queryStats) {
+ auto queryHash = stats.GetQueryTextHash();
+ TString serialized;
+ for (auto& query : minuteTop) {
+ if (query.Hash == queryHash) {
+ query.Stats = MakeHolder<NKikimrSysView::TQueryStats>();
+ query.Stats->CopyFrom(stats);
Y_PROTOBUF_SUPPRESS_NODISCARD query.Stats->SerializeToString(&serialized);
- db.Table<Schema::IntervalTops>().Key((ui32)minuteType, queryHash).Update(
- NIceDb::TUpdate<Schema::IntervalTops::Stats>(serialized));
- break;
- }
- }
- for (auto& query : hourTop) {
- if (!query.Stats && query.Hash == queryHash) {
- query.Stats = MakeHolder<NKikimrSysView::TQueryStats>();
- query.Stats->CopyFrom(stats);
- // hash must be in a minute top as well
- db.Table<Schema::IntervalTops>().Key((ui32)hourType, queryHash).Update(
- NIceDb::TUpdate<Schema::IntervalTops::Stats>(serialized));
- break;
- }
- }
- }
- };
-
- fillTops(Self->ByDurationMinute, Self->ByDurationHour,
- NKikimrSysView::TOP_DURATION_ONE_MINUTE, NKikimrSysView::TOP_DURATION_ONE_HOUR,
- *Record.MutableTopByDuration());
-
- fillTops(Self->ByReadBytesMinute, Self->ByReadBytesHour,
- NKikimrSysView::TOP_READ_BYTES_ONE_MINUTE, NKikimrSysView::TOP_READ_BYTES_ONE_HOUR,
- *Record.MutableTopByReadBytes());
-
- fillTops(Self->ByCpuTimeMinute, Self->ByCpuTimeHour,
- NKikimrSysView::TOP_CPU_TIME_ONE_MINUTE, NKikimrSysView::TOP_CPU_TIME_ONE_HOUR,
- *Record.MutableTopByCpuTime());
-
- fillTops(Self->ByRequestUnitsMinute, Self->ByRequestUnitsHour,
- NKikimrSysView::TOP_REQUEST_UNITS_ONE_MINUTE, NKikimrSysView::TOP_REQUEST_UNITS_ONE_HOUR,
- *Record.MutableTopByRequestUnits());
-
- Self->NodesInFlight.erase(NodeId);
- db.Table<Schema::NodesToRequest>().Key(NodeId).Delete();
-
- if (Self->NodesInFlight.empty() && Self->NodesToRequest.empty()) {
- Self->PersistResults(db);
- } else {
- Self->SendRequests();
- }
- return true;
- }
-
- void Complete(const TActorContext&) override {
- SVLOG_D("[" << Self->TabletID() << "] TTxIntervalMetrics::Complete");
- }
-};
-
-void TSysViewProcessor::Handle(TEvSysView::TEvGetIntervalMetricsResponse::TPtr& ev) {
- auto& record = ev->Get()->Record;
- TNodeId nodeId = ev.Get()->Cookie;
-
- if (CurrentStage != AGGREGATE) {
- SVLOG_W("[" << TabletID() << "] TEvGetIntervalMetricsResponse, wrong stage: "
- << "node id# " << nodeId);
- return;
- }
-
- if (record.GetIntervalEndUs() != IntervalEnd.MicroSeconds()) {
- SVLOG_W("[" << TabletID() << "] TEvGetIntervalMetricsResponse, time mismatch: "
- << "node id# " << nodeId
- << "interval end# " << IntervalEnd
- << "event interval end# " << TInstant::MicroSeconds(record.GetIntervalEndUs()));
- return;
- }
-
- Execute(new TTxIntervalMetrics(this, nodeId, std::move(record)),
- TActivationContext::AsActorContext());
-}
-
-} // NSysView
-} // NKikimr
+ db.Table<Schema::IntervalTops>().Key((ui32)minuteType, queryHash).Update(
+ NIceDb::TUpdate<Schema::IntervalTops::Stats>(serialized));
+ break;
+ }
+ }
+ for (auto& query : hourTop) {
+ if (!query.Stats && query.Hash == queryHash) {
+ query.Stats = MakeHolder<NKikimrSysView::TQueryStats>();
+ query.Stats->CopyFrom(stats);
+ // hash must be in a minute top as well
+ db.Table<Schema::IntervalTops>().Key((ui32)hourType, queryHash).Update(
+ NIceDb::TUpdate<Schema::IntervalTops::Stats>(serialized));
+ break;
+ }
+ }
+ }
+ };
+
+ fillTops(Self->ByDurationMinute, Self->ByDurationHour,
+ NKikimrSysView::TOP_DURATION_ONE_MINUTE, NKikimrSysView::TOP_DURATION_ONE_HOUR,
+ *Record.MutableTopByDuration());
+
+ fillTops(Self->ByReadBytesMinute, Self->ByReadBytesHour,
+ NKikimrSysView::TOP_READ_BYTES_ONE_MINUTE, NKikimrSysView::TOP_READ_BYTES_ONE_HOUR,
+ *Record.MutableTopByReadBytes());
+
+ fillTops(Self->ByCpuTimeMinute, Self->ByCpuTimeHour,
+ NKikimrSysView::TOP_CPU_TIME_ONE_MINUTE, NKikimrSysView::TOP_CPU_TIME_ONE_HOUR,
+ *Record.MutableTopByCpuTime());
+
+ fillTops(Self->ByRequestUnitsMinute, Self->ByRequestUnitsHour,
+ NKikimrSysView::TOP_REQUEST_UNITS_ONE_MINUTE, NKikimrSysView::TOP_REQUEST_UNITS_ONE_HOUR,
+ *Record.MutableTopByRequestUnits());
+
+ Self->NodesInFlight.erase(NodeId);
+ db.Table<Schema::NodesToRequest>().Key(NodeId).Delete();
+
+ if (Self->NodesInFlight.empty() && Self->NodesToRequest.empty()) {
+ Self->PersistResults(db);
+ } else {
+ Self->SendRequests();
+ }
+ return true;
+ }
+
+ void Complete(const TActorContext&) override {
+ SVLOG_D("[" << Self->TabletID() << "] TTxIntervalMetrics::Complete");
+ }
+};
+
+void TSysViewProcessor::Handle(TEvSysView::TEvGetIntervalMetricsResponse::TPtr& ev) {
+ auto& record = ev->Get()->Record;
+ TNodeId nodeId = ev.Get()->Cookie;
+
+ if (CurrentStage != AGGREGATE) {
+ SVLOG_W("[" << TabletID() << "] TEvGetIntervalMetricsResponse, wrong stage: "
+ << "node id# " << nodeId);
+ return;
+ }
+
+ if (record.GetIntervalEndUs() != IntervalEnd.MicroSeconds()) {
+ SVLOG_W("[" << TabletID() << "] TEvGetIntervalMetricsResponse, time mismatch: "
+ << "node id# " << nodeId
+ << "interval end# " << IntervalEnd
+ << "event interval end# " << TInstant::MicroSeconds(record.GetIntervalEndUs()));
+ return;
+ }
+
+ Execute(new TTxIntervalMetrics(this, nodeId, std::move(record)),
+ TActivationContext::AsActorContext());
+}
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/processor/tx_interval_summary.cpp b/ydb/core/sys_view/processor/tx_interval_summary.cpp
index 35fe2789e1d..4124a0aa770 100644
--- a/ydb/core/sys_view/processor/tx_interval_summary.cpp
+++ b/ydb/core/sys_view/processor/tx_interval_summary.cpp
@@ -1,228 +1,228 @@
-#include "processor_impl.h"
-
-namespace NKikimr {
-namespace NSysView {
-
-struct TSysViewProcessor::TTxIntervalSummary : public TTxBase {
- NKikimrSysView::TEvIntervalQuerySummary Record;
-
- TTxIntervalSummary(TSelf* self, NKikimrSysView::TEvIntervalQuerySummary&& record)
- : TTxBase(self)
- , Record(std::move(record))
- {}
-
- TTxType GetTxType() const override { return TXTYPE_INTERVAL_SUMMARY; }
-
- void AddSummary(NIceDb::TNiceDb& db, ui64 queryHash, ui64 cpu, TNodeId nodeId) {
- if (auto queryIt = Self->Queries.find(queryHash); queryIt != Self->Queries.end()) {
- auto& query = queryIt->second;
-
- auto range = Self->ByCpu.equal_range(query.Cpu);
- auto it = range.first;
- for (; it != range.second; ++it) {
- if (it->second == queryHash) {
- break;
- }
- }
- Y_VERIFY(it != range.second);
- Self->ByCpu.erase(it);
-
- query.Cpu += cpu;
- Self->ByCpu.emplace(query.Cpu, queryHash);
-
- auto& nodes = query.Nodes;
- size_t nodeIndex = nodes.size();
- nodes.emplace_back(nodeId, cpu);
-
- db.Table<Schema::IntervalSummaries>().Key(queryHash, nodeIndex).Update(
- NIceDb::TUpdate<Schema::IntervalSummaries::CPU>(cpu),
- NIceDb::TUpdate<Schema::IntervalSummaries::NodeId>(nodeId));
-
- } else {
- if (Self->ByCpu.size() == DistinctQueriesLimit) {
- auto it = Self->ByCpu.begin();
- if (it->first >= cpu) {
- return;
- }
- auto removeHash = it->second;
- const auto& removeQuery = Self->Queries[removeHash];
- for (ui32 i = 0; i < removeQuery.Nodes.size(); ++i) {
- db.Table<Schema::IntervalSummaries>().Key(removeHash, i).Delete();
- }
- Self->Queries.erase(removeHash);
- Self->ByCpu.erase(it);
- }
-
- TQueryToNodes query{cpu, {{nodeId, cpu}}};
-
- Self->Queries.emplace(queryHash, std::move(query));
- Self->ByCpu.emplace(cpu, queryHash);
-
- db.Table<Schema::IntervalSummaries>().Key(queryHash, 0).Update(
- NIceDb::TUpdate<Schema::IntervalSummaries::CPU>(cpu),
- NIceDb::TUpdate<Schema::IntervalSummaries::NodeId>(nodeId));
- }
- }
-
- void ProcessTop(NIceDb::TNiceDb& db,
- TNodeId nodeId,
- NKikimrSysView::EStatsType statsType,
- const NKikimrSysView::TEvIntervalQuerySummary::TQuerySet& queries,
- TTop& top)
- {
- TTop result;
- std::unordered_set<TQueryHash> seenHashes;
- size_t queryIndex = 0;
- auto topIt = top.begin();
-
- auto copyNewQuery = [&] () {
- auto queryHash = queries.GetHashes(queryIndex);
- auto value = queries.GetValues(queryIndex);
-
- TTopQuery topQuery{queryHash, value, nodeId, {}};
- result.emplace_back(std::move(topQuery));
-
- db.Table<Schema::IntervalTops>().Key((ui32)statsType, queryHash).Update(
- NIceDb::TUpdate<Schema::IntervalTops::Value>(value),
- NIceDb::TUpdate<Schema::IntervalTops::NodeId>(nodeId));
-
- seenHashes.insert(queryHash);
- ++queryIndex;
- };
-
- while (result.size() < TOP_QUERIES_COUNT) {
- if (topIt == top.end()) {
- if (queryIndex == queries.HashesSize()) {
- break;
- }
- auto queryHash = queries.GetHashes(queryIndex);
- if (seenHashes.find(queryHash) != seenHashes.end()) {
- ++queryIndex;
- continue;
- }
- copyNewQuery();
- } else {
- auto topHash = topIt->Hash;
- if (seenHashes.find(topHash) != seenHashes.end()) {
- ++topIt;
- continue;
- }
- if (queryIndex == queries.HashesSize()) {
- result.emplace_back(std::move(*topIt++));
- seenHashes.insert(topHash);
- continue;
- }
- auto queryHash = queries.GetHashes(queryIndex);
- if (seenHashes.find(queryHash) != seenHashes.end()) {
- ++queryIndex;
- continue;
- }
- if (topIt->Value >= queries.GetValues(queryIndex)) {
- result.emplace_back(std::move(*topIt++));
- seenHashes.insert(topHash);
- } else {
- copyNewQuery();
- }
- }
- }
-
- for (; topIt != top.end(); ++topIt) {
- auto topHash = topIt->Hash;
- if (seenHashes.find(topHash) != seenHashes.end()) {
- continue;
- }
- db.Table<Schema::IntervalTops>().Key((ui32)statsType, topHash).Delete();
- }
-
- top.swap(result);
- }
-
- bool Execute(TTransactionContext& txc, const TActorContext&) override {
- auto nodeId = Record.GetNodeId();
-
- if (Record.GetMetrics().ValuesSize() != Record.GetMetrics().HashesSize() ||
- Record.GetTopByDuration().ValuesSize() != Record.GetTopByDuration().HashesSize() ||
- Record.GetTopByReadBytes().ValuesSize() != Record.GetTopByReadBytes().HashesSize() ||
- Record.GetTopByCpuTime().ValuesSize() != Record.GetTopByCpuTime().HashesSize() ||
- Record.GetTopByRequestUnits().ValuesSize() != Record.GetTopByRequestUnits().HashesSize())
- {
- SVLOG_W("[" << Self->TabletID() << "] TTxIntervalSummary::Execute, malformed summary: "
- << "node id# " << nodeId);
- return true;
- }
-
- if (Self->SummaryNodes.find(nodeId) != Self->SummaryNodes.end()) {
- SVLOG_W("[" << Self->TabletID() << "] TTxIntervalSummary::Execute, duplicate summary: "
- << "node id# " << nodeId);
- return true;
- }
- Self->SummaryNodes.insert(nodeId);
-
- const auto& metrics = Record.GetMetrics();
- auto count = metrics.HashesSize();
-
- SVLOG_D("[" << Self->TabletID() << "] TTxIntervalSummary::Execute: "
- << "node id# " << nodeId
- << ", query count# " << count);
-
- NIceDb::TNiceDb db(txc.DB);
- for (size_t i = 0; i < count; ++i) {
- AddSummary(db, metrics.GetHashes(i), metrics.GetValues(i), nodeId);
- }
-
- ProcessTop(db, nodeId, NKikimrSysView::TOP_DURATION_ONE_MINUTE,
- Record.GetTopByDuration(), Self->ByDurationMinute);
- ProcessTop(db, nodeId, NKikimrSysView::TOP_DURATION_ONE_HOUR,
- Record.GetTopByDuration(), Self->ByDurationHour);
- ProcessTop(db, nodeId, NKikimrSysView::TOP_READ_BYTES_ONE_MINUTE,
- Record.GetTopByReadBytes(), Self->ByReadBytesMinute);
- ProcessTop(db, nodeId, NKikimrSysView::TOP_READ_BYTES_ONE_HOUR,
- Record.GetTopByReadBytes(), Self->ByReadBytesHour);
- ProcessTop(db, nodeId, NKikimrSysView::TOP_CPU_TIME_ONE_MINUTE,
- Record.GetTopByCpuTime(), Self->ByCpuTimeMinute);
- ProcessTop(db, nodeId, NKikimrSysView::TOP_CPU_TIME_ONE_HOUR,
- Record.GetTopByCpuTime(), Self->ByCpuTimeHour);
- ProcessTop(db, nodeId, NKikimrSysView::TOP_REQUEST_UNITS_ONE_MINUTE,
- Record.GetTopByRequestUnits(), Self->ByRequestUnitsMinute);
- ProcessTop(db, nodeId, NKikimrSysView::TOP_REQUEST_UNITS_ONE_HOUR,
- Record.GetTopByRequestUnits(), Self->ByRequestUnitsHour);
-
- return true;
- }
-
- void Complete(const TActorContext&) override {
- SVLOG_D("[" << Self->TabletID() << "] TTxIntervalSummary::Complete");
- }
-};
-
-void TSysViewProcessor::Handle(TEvSysView::TEvIntervalQuerySummary::TPtr& ev) {
- auto& record = ev->Get()->Record;
- auto nodeId = record.GetNodeId();
-
- if (CurrentStage != COLLECT) {
- SVLOG_W("[" << TabletID() << "] TEvIntervalQuerySummary, wrong stage: "
- << "node id# " << nodeId);
- return;
- }
-
- if (record.GetIntervalEndUs() != IntervalEnd.MicroSeconds()) {
- SVLOG_W("[" << TabletID() << "] TEvIntervalQuerySummary, time mismath: "
- << "node id# " << nodeId
- << ", interval end# " << IntervalEnd
- << ", event interval end# " << TInstant::MicroSeconds(record.GetIntervalEndUs()));
- return;
- }
-
- if (record.GetDatabase() != Database) {
- SVLOG_W("[" << TabletID() << "] TEvIntervalQuerySummary, db mismatch: "
- << "node id# " << nodeId
- << ", database# " << Database
- << ", event database# " << record.GetDatabase());
- return;
- }
-
- Execute(new TTxIntervalSummary(this, std::move(record)), TActivationContext::AsActorContext());
-}
-
-} // NSysView
-} // NKikimr
+#include "processor_impl.h"
+
+namespace NKikimr {
+namespace NSysView {
+
+struct TSysViewProcessor::TTxIntervalSummary : public TTxBase {
+ NKikimrSysView::TEvIntervalQuerySummary Record;
+
+ TTxIntervalSummary(TSelf* self, NKikimrSysView::TEvIntervalQuerySummary&& record)
+ : TTxBase(self)
+ , Record(std::move(record))
+ {}
+
+ TTxType GetTxType() const override { return TXTYPE_INTERVAL_SUMMARY; }
+
+ void AddSummary(NIceDb::TNiceDb& db, ui64 queryHash, ui64 cpu, TNodeId nodeId) {
+ if (auto queryIt = Self->Queries.find(queryHash); queryIt != Self->Queries.end()) {
+ auto& query = queryIt->second;
+
+ auto range = Self->ByCpu.equal_range(query.Cpu);
+ auto it = range.first;
+ for (; it != range.second; ++it) {
+ if (it->second == queryHash) {
+ break;
+ }
+ }
+ Y_VERIFY(it != range.second);
+ Self->ByCpu.erase(it);
+
+ query.Cpu += cpu;
+ Self->ByCpu.emplace(query.Cpu, queryHash);
+
+ auto& nodes = query.Nodes;
+ size_t nodeIndex = nodes.size();
+ nodes.emplace_back(nodeId, cpu);
+
+ db.Table<Schema::IntervalSummaries>().Key(queryHash, nodeIndex).Update(
+ NIceDb::TUpdate<Schema::IntervalSummaries::CPU>(cpu),
+ NIceDb::TUpdate<Schema::IntervalSummaries::NodeId>(nodeId));
+
+ } else {
+ if (Self->ByCpu.size() == DistinctQueriesLimit) {
+ auto it = Self->ByCpu.begin();
+ if (it->first >= cpu) {
+ return;
+ }
+ auto removeHash = it->second;
+ const auto& removeQuery = Self->Queries[removeHash];
+ for (ui32 i = 0; i < removeQuery.Nodes.size(); ++i) {
+ db.Table<Schema::IntervalSummaries>().Key(removeHash, i).Delete();
+ }
+ Self->Queries.erase(removeHash);
+ Self->ByCpu.erase(it);
+ }
+
+ TQueryToNodes query{cpu, {{nodeId, cpu}}};
+
+ Self->Queries.emplace(queryHash, std::move(query));
+ Self->ByCpu.emplace(cpu, queryHash);
+
+ db.Table<Schema::IntervalSummaries>().Key(queryHash, 0).Update(
+ NIceDb::TUpdate<Schema::IntervalSummaries::CPU>(cpu),
+ NIceDb::TUpdate<Schema::IntervalSummaries::NodeId>(nodeId));
+ }
+ }
+
+ void ProcessTop(NIceDb::TNiceDb& db,
+ TNodeId nodeId,
+ NKikimrSysView::EStatsType statsType,
+ const NKikimrSysView::TEvIntervalQuerySummary::TQuerySet& queries,
+ TTop& top)
+ {
+ TTop result;
+ std::unordered_set<TQueryHash> seenHashes;
+ size_t queryIndex = 0;
+ auto topIt = top.begin();
+
+ auto copyNewQuery = [&] () {
+ auto queryHash = queries.GetHashes(queryIndex);
+ auto value = queries.GetValues(queryIndex);
+
+ TTopQuery topQuery{queryHash, value, nodeId, {}};
+ result.emplace_back(std::move(topQuery));
+
+ db.Table<Schema::IntervalTops>().Key((ui32)statsType, queryHash).Update(
+ NIceDb::TUpdate<Schema::IntervalTops::Value>(value),
+ NIceDb::TUpdate<Schema::IntervalTops::NodeId>(nodeId));
+
+ seenHashes.insert(queryHash);
+ ++queryIndex;
+ };
+
+ while (result.size() < TOP_QUERIES_COUNT) {
+ if (topIt == top.end()) {
+ if (queryIndex == queries.HashesSize()) {
+ break;
+ }
+ auto queryHash = queries.GetHashes(queryIndex);
+ if (seenHashes.find(queryHash) != seenHashes.end()) {
+ ++queryIndex;
+ continue;
+ }
+ copyNewQuery();
+ } else {
+ auto topHash = topIt->Hash;
+ if (seenHashes.find(topHash) != seenHashes.end()) {
+ ++topIt;
+ continue;
+ }
+ if (queryIndex == queries.HashesSize()) {
+ result.emplace_back(std::move(*topIt++));
+ seenHashes.insert(topHash);
+ continue;
+ }
+ auto queryHash = queries.GetHashes(queryIndex);
+ if (seenHashes.find(queryHash) != seenHashes.end()) {
+ ++queryIndex;
+ continue;
+ }
+ if (topIt->Value >= queries.GetValues(queryIndex)) {
+ result.emplace_back(std::move(*topIt++));
+ seenHashes.insert(topHash);
+ } else {
+ copyNewQuery();
+ }
+ }
+ }
+
+ for (; topIt != top.end(); ++topIt) {
+ auto topHash = topIt->Hash;
+ if (seenHashes.find(topHash) != seenHashes.end()) {
+ continue;
+ }
+ db.Table<Schema::IntervalTops>().Key((ui32)statsType, topHash).Delete();
+ }
+
+ top.swap(result);
+ }
+
+ bool Execute(TTransactionContext& txc, const TActorContext&) override {
+ auto nodeId = Record.GetNodeId();
+
+ if (Record.GetMetrics().ValuesSize() != Record.GetMetrics().HashesSize() ||
+ Record.GetTopByDuration().ValuesSize() != Record.GetTopByDuration().HashesSize() ||
+ Record.GetTopByReadBytes().ValuesSize() != Record.GetTopByReadBytes().HashesSize() ||
+ Record.GetTopByCpuTime().ValuesSize() != Record.GetTopByCpuTime().HashesSize() ||
+ Record.GetTopByRequestUnits().ValuesSize() != Record.GetTopByRequestUnits().HashesSize())
+ {
+ SVLOG_W("[" << Self->TabletID() << "] TTxIntervalSummary::Execute, malformed summary: "
+ << "node id# " << nodeId);
+ return true;
+ }
+
+ if (Self->SummaryNodes.find(nodeId) != Self->SummaryNodes.end()) {
+ SVLOG_W("[" << Self->TabletID() << "] TTxIntervalSummary::Execute, duplicate summary: "
+ << "node id# " << nodeId);
+ return true;
+ }
+ Self->SummaryNodes.insert(nodeId);
+
+ const auto& metrics = Record.GetMetrics();
+ auto count = metrics.HashesSize();
+
+ SVLOG_D("[" << Self->TabletID() << "] TTxIntervalSummary::Execute: "
+ << "node id# " << nodeId
+ << ", query count# " << count);
+
+ NIceDb::TNiceDb db(txc.DB);
+ for (size_t i = 0; i < count; ++i) {
+ AddSummary(db, metrics.GetHashes(i), metrics.GetValues(i), nodeId);
+ }
+
+ ProcessTop(db, nodeId, NKikimrSysView::TOP_DURATION_ONE_MINUTE,
+ Record.GetTopByDuration(), Self->ByDurationMinute);
+ ProcessTop(db, nodeId, NKikimrSysView::TOP_DURATION_ONE_HOUR,
+ Record.GetTopByDuration(), Self->ByDurationHour);
+ ProcessTop(db, nodeId, NKikimrSysView::TOP_READ_BYTES_ONE_MINUTE,
+ Record.GetTopByReadBytes(), Self->ByReadBytesMinute);
+ ProcessTop(db, nodeId, NKikimrSysView::TOP_READ_BYTES_ONE_HOUR,
+ Record.GetTopByReadBytes(), Self->ByReadBytesHour);
+ ProcessTop(db, nodeId, NKikimrSysView::TOP_CPU_TIME_ONE_MINUTE,
+ Record.GetTopByCpuTime(), Self->ByCpuTimeMinute);
+ ProcessTop(db, nodeId, NKikimrSysView::TOP_CPU_TIME_ONE_HOUR,
+ Record.GetTopByCpuTime(), Self->ByCpuTimeHour);
+ ProcessTop(db, nodeId, NKikimrSysView::TOP_REQUEST_UNITS_ONE_MINUTE,
+ Record.GetTopByRequestUnits(), Self->ByRequestUnitsMinute);
+ ProcessTop(db, nodeId, NKikimrSysView::TOP_REQUEST_UNITS_ONE_HOUR,
+ Record.GetTopByRequestUnits(), Self->ByRequestUnitsHour);
+
+ return true;
+ }
+
+ void Complete(const TActorContext&) override {
+ SVLOG_D("[" << Self->TabletID() << "] TTxIntervalSummary::Complete");
+ }
+};
+
+void TSysViewProcessor::Handle(TEvSysView::TEvIntervalQuerySummary::TPtr& ev) {
+ auto& record = ev->Get()->Record;
+ auto nodeId = record.GetNodeId();
+
+ if (CurrentStage != COLLECT) {
+ SVLOG_W("[" << TabletID() << "] TEvIntervalQuerySummary, wrong stage: "
+ << "node id# " << nodeId);
+ return;
+ }
+
+ if (record.GetIntervalEndUs() != IntervalEnd.MicroSeconds()) {
+ SVLOG_W("[" << TabletID() << "] TEvIntervalQuerySummary, time mismath: "
+ << "node id# " << nodeId
+ << ", interval end# " << IntervalEnd
+ << ", event interval end# " << TInstant::MicroSeconds(record.GetIntervalEndUs()));
+ return;
+ }
+
+ if (record.GetDatabase() != Database) {
+ SVLOG_W("[" << TabletID() << "] TEvIntervalQuerySummary, db mismatch: "
+ << "node id# " << nodeId
+ << ", database# " << Database
+ << ", event database# " << record.GetDatabase());
+ return;
+ }
+
+ Execute(new TTxIntervalSummary(this, std::move(record)), TActivationContext::AsActorContext());
+}
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/processor/ya.make b/ydb/core/sys_view/processor/ya.make
index ddf4664cfac..16bda2c8be9 100644
--- a/ydb/core/sys_view/processor/ya.make
+++ b/ydb/core/sys_view/processor/ya.make
@@ -1,28 +1,28 @@
-LIBRARY()
-
-OWNER(
- monster
- g:kikimr
-)
-
-SRCS(
- processor.h
- processor.cpp
- processor_impl.h
- processor_impl.cpp
- schema.h
- schema.cpp
- db_counters.cpp
- tx_init.cpp
- tx_init_schema.cpp
- tx_configure.cpp
- tx_collect.cpp
- tx_aggregate.cpp
- tx_interval_summary.cpp
- tx_interval_metrics.cpp
-)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(
+ monster
+ g:kikimr
+)
+
+SRCS(
+ processor.h
+ processor.cpp
+ processor_impl.h
+ processor_impl.cpp
+ schema.h
+ schema.cpp
+ db_counters.cpp
+ tx_init.cpp
+ tx_init_schema.cpp
+ tx_configure.cpp
+ tx_collect.cpp
+ tx_aggregate.cpp
+ tx_interval_summary.cpp
+ tx_interval_metrics.cpp
+)
+
+PEERDIR(
ydb/core/base
ydb/core/engine/minikql
ydb/core/grpc_services/counters
@@ -31,8 +31,8 @@ PEERDIR(
ydb/core/tablet
ydb/core/tablet_flat
ydb/core/tx/scheme_cache
-)
-
+)
+
YQL_LAST_ABI_VERSION()
-
-END()
+
+END()
diff --git a/ydb/core/sys_view/query_stats/query_metrics.cpp b/ydb/core/sys_view/query_stats/query_metrics.cpp
index 5f2acb5c23e..79af63e46f8 100644
--- a/ydb/core/sys_view/query_stats/query_metrics.cpp
+++ b/ydb/core/sys_view/query_stats/query_metrics.cpp
@@ -1,173 +1,173 @@
-#include "query_metrics.h"
-
+#include "query_metrics.h"
+
#include <ydb/core/sys_view/common/common.h>
#include <ydb/core/sys_view/common/events.h>
#include <ydb/core/sys_view/common/keys.h>
#include <ydb/core/sys_view/common/schema.h>
#include <ydb/core/sys_view/common/scan_actor_base_impl.h>
-
+
#include <ydb/core/base/tablet_pipecache.h>
-
-#include <library/cpp/actors/core/hfunc.h>
-
-namespace NKikimr {
-namespace NSysView {
-
-using namespace NActors;
-
-template <>
-void SetField<0>(NKikimrSysView::TQueryMetricsKey& key, ui64 value) {
- key.SetIntervalEndUs(value);
-}
-
-template <>
-void SetField<1>(NKikimrSysView::TQueryMetricsKey& key, ui32 value) {
- key.SetRank(value);
-}
-
-class TQueryMetricsScan : public TScanActorBase<TQueryMetricsScan> {
-public:
- using TBase = TScanActorBase<TQueryMetricsScan>;
-
+
+#include <library/cpp/actors/core/hfunc.h>
+
+namespace NKikimr {
+namespace NSysView {
+
+using namespace NActors;
+
+template <>
+void SetField<0>(NKikimrSysView::TQueryMetricsKey& key, ui64 value) {
+ key.SetIntervalEndUs(value);
+}
+
+template <>
+void SetField<1>(NKikimrSysView::TQueryMetricsKey& key, ui32 value) {
+ key.SetRank(value);
+}
+
+class TQueryMetricsScan : public TScanActorBase<TQueryMetricsScan> {
+public:
+ using TBase = TScanActorBase<TQueryMetricsScan>;
+
static constexpr auto ActorActivityType() {
- return NKikimrServices::TActivity::KQP_SYSTEM_VIEW_SCAN;
- }
-
+ return NKikimrServices::TActivity::KQP_SYSTEM_VIEW_SCAN;
+ }
+
TQueryMetricsScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId,
- const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns)
- : TBase(ownerId, scanId, tableId, tableRange, columns)
- {
- ConvertKeyRange<NKikimrSysView::TEvGetQueryMetricsRequest, ui64, ui32>(Request, TableRange);
- Request.SetType(NKikimrSysView::METRICS_ONE_MINUTE);
- }
-
- STFUNC(StateScan) {
- switch (ev->GetTypeRewrite()) {
- hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, Handle);
- hFunc(TEvSysView::TEvGetQueryMetricsResponse, Handle);
- hFunc(TEvPipeCache::TEvDeliveryProblem, Handle);
- hFunc(NKqp::TEvKqp::TEvAbortExecution, HandleAbortExecution);
- cFunc(TEvents::TEvWakeup::EventType, HandleTimeout);
- cFunc(TEvents::TEvPoison::EventType, PassAway);
- default:
- LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS,
- "NSysView::TQueryMetricsScan: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite());
- }
- }
-
-private:
- void ProceedToScan() override {
- Become(&TThis::StateScan);
- if (AckReceived) {
- RequestBatch();
- }
- }
-
- void RequestBatch() {
- if (!SysViewProcessorId) {
- SVLOG_W("No sysview processor for database " << TenantName
- << ", sending empty response");
- ReplyEmptyAndDie();
- return;
- }
-
- if (BatchRequestInFlight) {
- return;
- }
-
- auto request = MakeHolder<TEvSysView::TEvGetQueryMetricsRequest>();
- request->Record.CopyFrom(Request);
-
- Send(MakePipePeNodeCacheID(false),
- new TEvPipeCache::TEvForward(request.Release(), SysViewProcessorId, true),
- IEventHandle::FlagTrackDelivery);
-
- BatchRequestInFlight = true;
- }
-
- void Handle(NKqp::TEvKqpCompute::TEvScanDataAck::TPtr&) {
- RequestBatch();
- }
-
- void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr&) {
- ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "Delivery problem in query metrics scan");
- }
-
- void Handle(TEvSysView::TEvGetQueryMetricsResponse::TPtr& ev) {
- using TEntry = NKikimrSysView::TQueryMetricsEntry;
- using TExtractor = std::function<TCell(const TEntry&)>;
- using TSchema = Schema::QueryMetrics;
-
- struct TExtractorsMap : public THashMap<NTable::TTag, TExtractor> {
- TExtractorsMap() {
- insert({TSchema::IntervalEnd::ColumnId, [] (const TEntry& entry) {
- return TCell::Make<ui64>(entry.GetKey().GetIntervalEndUs());
- }});
- insert({TSchema::Rank::ColumnId, [] (const TEntry& entry) {
- return TCell::Make<ui32>(entry.GetKey().GetRank());
- }});
- insert({TSchema::QueryText::ColumnId, [] (const TEntry& entry) {
- const auto& text = entry.GetQueryText();
- return TCell(text.data(), text.size());
- }});
- insert({TSchema::Count::ColumnId, [] (const TEntry& entry) {
- return TCell::Make<ui64>(entry.GetMetrics().GetCount());
- }});
-
-#define ADD_METRICS(FieldName, MetricsName) \
- insert({TSchema::Sum ## FieldName::ColumnId, [] (const TEntry& entry) { \
- return TCell::Make<ui64>(entry.GetMetrics().Get ## MetricsName().GetSum()); \
- }}); \
- insert({TSchema::Min ## FieldName::ColumnId, [] (const TEntry& entry) { \
- return TCell::Make<ui64>(entry.GetMetrics().Get ## MetricsName().GetMin()); \
- }}); \
- insert({TSchema::Max ## FieldName::ColumnId, [] (const TEntry& entry) { \
- return TCell::Make<ui64>(entry.GetMetrics().Get ## MetricsName().GetMax()); \
- }});
-
- ADD_METRICS(CPUTime, CpuTimeUs);
- ADD_METRICS(Duration, DurationUs);
- ADD_METRICS(ReadRows, ReadRows);
- ADD_METRICS(ReadBytes, ReadBytes);
- ADD_METRICS(UpdateRows, UpdateRows);
- ADD_METRICS(UpdateBytes, UpdateBytes);
- ADD_METRICS(DeleteRows, DeleteRows);
- ADD_METRICS(RequestUnits, RequestUnits);
-#undef ADD_METRICS
- }
- };
-
- const auto& record = ev->Get()->Record;
- if (record.HasOverloaded() && record.GetOverloaded()) {
- ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "SysViewProcessor is overloaded");
- return;
- }
-
- ReplyBatch<TEvSysView::TEvGetQueryMetricsResponse, TEntry, TExtractorsMap, true>(ev);
-
- if (!record.GetLastBatch()) {
- Y_VERIFY(record.HasNext());
- Request.MutableFrom()->CopyFrom(record.GetNext());
- Request.SetInclusiveFrom(true);
- }
-
- BatchRequestInFlight = false;
- }
-
- void PassAway() override {
- Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(0));
- TBase::PassAway();
- }
-
-private:
- NKikimrSysView::TEvGetQueryMetricsRequest Request;
-};
-
+ const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns)
+ : TBase(ownerId, scanId, tableId, tableRange, columns)
+ {
+ ConvertKeyRange<NKikimrSysView::TEvGetQueryMetricsRequest, ui64, ui32>(Request, TableRange);
+ Request.SetType(NKikimrSysView::METRICS_ONE_MINUTE);
+ }
+
+ STFUNC(StateScan) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, Handle);
+ hFunc(TEvSysView::TEvGetQueryMetricsResponse, Handle);
+ hFunc(TEvPipeCache::TEvDeliveryProblem, Handle);
+ hFunc(NKqp::TEvKqp::TEvAbortExecution, HandleAbortExecution);
+ cFunc(TEvents::TEvWakeup::EventType, HandleTimeout);
+ cFunc(TEvents::TEvPoison::EventType, PassAway);
+ default:
+ LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS,
+ "NSysView::TQueryMetricsScan: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite());
+ }
+ }
+
+private:
+ void ProceedToScan() override {
+ Become(&TThis::StateScan);
+ if (AckReceived) {
+ RequestBatch();
+ }
+ }
+
+ void RequestBatch() {
+ if (!SysViewProcessorId) {
+ SVLOG_W("No sysview processor for database " << TenantName
+ << ", sending empty response");
+ ReplyEmptyAndDie();
+ return;
+ }
+
+ if (BatchRequestInFlight) {
+ return;
+ }
+
+ auto request = MakeHolder<TEvSysView::TEvGetQueryMetricsRequest>();
+ request->Record.CopyFrom(Request);
+
+ Send(MakePipePeNodeCacheID(false),
+ new TEvPipeCache::TEvForward(request.Release(), SysViewProcessorId, true),
+ IEventHandle::FlagTrackDelivery);
+
+ BatchRequestInFlight = true;
+ }
+
+ void Handle(NKqp::TEvKqpCompute::TEvScanDataAck::TPtr&) {
+ RequestBatch();
+ }
+
+ void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr&) {
+ ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "Delivery problem in query metrics scan");
+ }
+
+ void Handle(TEvSysView::TEvGetQueryMetricsResponse::TPtr& ev) {
+ using TEntry = NKikimrSysView::TQueryMetricsEntry;
+ using TExtractor = std::function<TCell(const TEntry&)>;
+ using TSchema = Schema::QueryMetrics;
+
+ struct TExtractorsMap : public THashMap<NTable::TTag, TExtractor> {
+ TExtractorsMap() {
+ insert({TSchema::IntervalEnd::ColumnId, [] (const TEntry& entry) {
+ return TCell::Make<ui64>(entry.GetKey().GetIntervalEndUs());
+ }});
+ insert({TSchema::Rank::ColumnId, [] (const TEntry& entry) {
+ return TCell::Make<ui32>(entry.GetKey().GetRank());
+ }});
+ insert({TSchema::QueryText::ColumnId, [] (const TEntry& entry) {
+ const auto& text = entry.GetQueryText();
+ return TCell(text.data(), text.size());
+ }});
+ insert({TSchema::Count::ColumnId, [] (const TEntry& entry) {
+ return TCell::Make<ui64>(entry.GetMetrics().GetCount());
+ }});
+
+#define ADD_METRICS(FieldName, MetricsName) \
+ insert({TSchema::Sum ## FieldName::ColumnId, [] (const TEntry& entry) { \
+ return TCell::Make<ui64>(entry.GetMetrics().Get ## MetricsName().GetSum()); \
+ }}); \
+ insert({TSchema::Min ## FieldName::ColumnId, [] (const TEntry& entry) { \
+ return TCell::Make<ui64>(entry.GetMetrics().Get ## MetricsName().GetMin()); \
+ }}); \
+ insert({TSchema::Max ## FieldName::ColumnId, [] (const TEntry& entry) { \
+ return TCell::Make<ui64>(entry.GetMetrics().Get ## MetricsName().GetMax()); \
+ }});
+
+ ADD_METRICS(CPUTime, CpuTimeUs);
+ ADD_METRICS(Duration, DurationUs);
+ ADD_METRICS(ReadRows, ReadRows);
+ ADD_METRICS(ReadBytes, ReadBytes);
+ ADD_METRICS(UpdateRows, UpdateRows);
+ ADD_METRICS(UpdateBytes, UpdateBytes);
+ ADD_METRICS(DeleteRows, DeleteRows);
+ ADD_METRICS(RequestUnits, RequestUnits);
+#undef ADD_METRICS
+ }
+ };
+
+ const auto& record = ev->Get()->Record;
+ if (record.HasOverloaded() && record.GetOverloaded()) {
+ ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "SysViewProcessor is overloaded");
+ return;
+ }
+
+ ReplyBatch<TEvSysView::TEvGetQueryMetricsResponse, TEntry, TExtractorsMap, true>(ev);
+
+ if (!record.GetLastBatch()) {
+ Y_VERIFY(record.HasNext());
+ Request.MutableFrom()->CopyFrom(record.GetNext());
+ Request.SetInclusiveFrom(true);
+ }
+
+ BatchRequestInFlight = false;
+ }
+
+ void PassAway() override {
+ Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(0));
+ TBase::PassAway();
+ }
+
+private:
+ NKikimrSysView::TEvGetQueryMetricsRequest Request;
+};
+
THolder<IActor> CreateQueryMetricsScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId,
- const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns)
-{
+ const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns)
+{
return MakeHolder<TQueryMetricsScan>(ownerId, scanId, tableId, tableRange, columns);
-}
-
-} // NSysView
-} // NKikimr
+}
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/query_stats/query_metrics.h b/ydb/core/sys_view/query_stats/query_metrics.h
index e3740811265..7e3997ab0f9 100644
--- a/ydb/core/sys_view/query_stats/query_metrics.h
+++ b/ydb/core/sys_view/query_stats/query_metrics.h
@@ -1,13 +1,13 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/kqp/runtime/kqp_compute.h>
-
-namespace NKikimr {
-namespace NSysView {
-
+
+namespace NKikimr {
+namespace NSysView {
+
THolder<IActor> CreateQueryMetricsScan(const TActorId& ownerId, ui32 scanId,
- const TTableId& tableId, const TTableRange& tableRange,
- const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns);
-
-} // NSysView
-} // NKikimr
+ const TTableId& tableId, const TTableRange& tableRange,
+ const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns);
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/query_stats/query_stats.cpp b/ydb/core/sys_view/query_stats/query_stats.cpp
index 918c9b781e6..b608b7545d7 100644
--- a/ydb/core/sys_view/query_stats/query_stats.cpp
+++ b/ydb/core/sys_view/query_stats/query_stats.cpp
@@ -1,5 +1,5 @@
-#include "query_stats.h"
-
+#include "query_stats.h"
+
#include <ydb/core/base/appdata.h>
#include <ydb/core/sys_view/common/common.h>
#include <ydb/core/sys_view/common/events.h>
@@ -8,548 +8,548 @@
#include <ydb/core/sys_view/common/scan_actor_base_impl.h>
#include <ydb/core/sys_view/service/query_history.h>
#include <ydb/core/sys_view/service/sysview_service.h>
-
+
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h>
#include <library/cpp/actors/core/interconnect.h>
#include <library/cpp/actors/interconnect/interconnect.h>
#include <library/cpp/actors/core/hfunc.h>
#include <library/cpp/actors/core/log.h>
-
-namespace NKikimr {
-namespace NSysView {
-
-using namespace NActors;
-
-TQueryStatsBucketRange::TQueryStatsBucketRange(const TSerializedTableRange& range, const TDuration& bucketSize) {
- const ui64 bucketSizeUs = bucketSize.MicroSeconds();
-
- const auto& cellsFrom = range.From.GetCells();
- const auto& cellsTo = range.To.GetCells();
-
- if (cellsFrom.size() > 0 && !cellsFrom[0].IsNull()) {
- auto fromUs = cellsFrom[0].AsValue<ui64>();
- FromBucket = fromUs / bucketSizeUs;
- bool exactBound = (FromBucket * bucketSizeUs == fromUs);
-
- TMaybe<ui32> rank;
- if (cellsFrom.size() > 1 && !cellsFrom[1].IsNull()) {
- rank = cellsFrom[1].AsValue<ui32>();
- if (!range.FromInclusive && *rank < std::numeric_limits<ui32>::max()) {
- ++*rank;
- }
- }
-
- if (FromBucket) {
- if (exactBound && (range.FromInclusive || rank)) {
- --FromBucket;
- if (rank) {
- FromRank = rank;
- }
- }
- }
- }
-
- if (cellsTo.size() > 0 && !cellsTo[0].IsNull()) {
- auto toUs = cellsTo[0].template AsValue<ui64>();
- ToBucket = toUs / bucketSizeUs;
- bool exactBound = (ToBucket * bucketSizeUs == toUs);
-
- TMaybe<ui32> rank;
- if (cellsTo.size() > 1 && !cellsTo[1].IsNull()) {
- rank = cellsTo[1].template AsValue<ui32>();
- if (!range.ToInclusive && *rank) {
- --*rank;
- }
- }
-
- if (ToBucket) {
- --ToBucket;
- if (exactBound) {
- if (rank) {
- ToRank = rank;
- } else if (!range.ToInclusive) {
- if (!ToBucket) {
- IsEmpty = true;
- } else {
- --ToBucket;
- }
- }
- }
- } else {
- IsEmpty = true;
- }
- }
-
- if (FromBucket > ToBucket) {
- IsEmpty = true;
- }
-}
-
-template <typename TGreater>
-class TQueryStatsScan : public TScanActorBase<TQueryStatsScan<TGreater>> {
-public:
- using TBase = TScanActorBase<TQueryStatsScan<TGreater>>;
-
+
+namespace NKikimr {
+namespace NSysView {
+
+using namespace NActors;
+
+TQueryStatsBucketRange::TQueryStatsBucketRange(const TSerializedTableRange& range, const TDuration& bucketSize) {
+ const ui64 bucketSizeUs = bucketSize.MicroSeconds();
+
+ const auto& cellsFrom = range.From.GetCells();
+ const auto& cellsTo = range.To.GetCells();
+
+ if (cellsFrom.size() > 0 && !cellsFrom[0].IsNull()) {
+ auto fromUs = cellsFrom[0].AsValue<ui64>();
+ FromBucket = fromUs / bucketSizeUs;
+ bool exactBound = (FromBucket * bucketSizeUs == fromUs);
+
+ TMaybe<ui32> rank;
+ if (cellsFrom.size() > 1 && !cellsFrom[1].IsNull()) {
+ rank = cellsFrom[1].AsValue<ui32>();
+ if (!range.FromInclusive && *rank < std::numeric_limits<ui32>::max()) {
+ ++*rank;
+ }
+ }
+
+ if (FromBucket) {
+ if (exactBound && (range.FromInclusive || rank)) {
+ --FromBucket;
+ if (rank) {
+ FromRank = rank;
+ }
+ }
+ }
+ }
+
+ if (cellsTo.size() > 0 && !cellsTo[0].IsNull()) {
+ auto toUs = cellsTo[0].template AsValue<ui64>();
+ ToBucket = toUs / bucketSizeUs;
+ bool exactBound = (ToBucket * bucketSizeUs == toUs);
+
+ TMaybe<ui32> rank;
+ if (cellsTo.size() > 1 && !cellsTo[1].IsNull()) {
+ rank = cellsTo[1].template AsValue<ui32>();
+ if (!range.ToInclusive && *rank) {
+ --*rank;
+ }
+ }
+
+ if (ToBucket) {
+ --ToBucket;
+ if (exactBound) {
+ if (rank) {
+ ToRank = rank;
+ } else if (!range.ToInclusive) {
+ if (!ToBucket) {
+ IsEmpty = true;
+ } else {
+ --ToBucket;
+ }
+ }
+ }
+ } else {
+ IsEmpty = true;
+ }
+ }
+
+ if (FromBucket > ToBucket) {
+ IsEmpty = true;
+ }
+}
+
+template <typename TGreater>
+class TQueryStatsScan : public TScanActorBase<TQueryStatsScan<TGreater>> {
+public:
+ using TBase = TScanActorBase<TQueryStatsScan<TGreater>>;
+
static constexpr auto ActorActivityType() {
- return NKikimrServices::TActivity::KQP_SYSTEM_VIEW_SCAN;
- }
-
+ return NKikimrServices::TActivity::KQP_SYSTEM_VIEW_SCAN;
+ }
+
TQueryStatsScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId,
- const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns,
- NKikimrSysView::EStatsType statsType,
- ui64 bucketCount, const TDuration& bucketSize)
- : TBase(ownerId, scanId, tableId, tableRange, columns)
- , StatsType(statsType)
- , BucketRange(this->TableRange, bucketSize)
- {
+ const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns,
+ NKikimrSysView::EStatsType statsType,
+ ui64 bucketCount, const TDuration& bucketSize)
+ : TBase(ownerId, scanId, tableId, tableRange, columns)
+ , StatsType(statsType)
+ , BucketRange(this->TableRange, bucketSize)
+ {
auto now = TAppData::TimeProvider->Now();
- History = MakeHolder<TScanQueryHistory<TGreater>>(bucketCount, bucketSize, now);
-
- ConvertKeyRange<NKikimrSysView::TEvGetQueryMetricsRequest, ui64, ui32>(Request, this->TableRange);
- Request.SetType(statsType);
- }
-
- STFUNC(StateScan) {
- switch (ev->GetTypeRewrite()) {
- hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, Handle);
- hFunc(TEvSysView::TEvGetQueryStatsResult, Handle);
- hFunc(TEvents::TEvUndelivered, Undelivered);
- hFunc(TEvInterconnect::TEvNodeConnected, Connected);
- hFunc(TEvInterconnect::TEvNodeDisconnected, Disconnected);
- hFunc(TEvPrivate::TEvRetryNode, RetryNode);
- hFunc(TEvSysView::TEvGetQueryStatsResponse, Handle);
- hFunc(TEvPipeCache::TEvDeliveryProblem, Handle);
+ History = MakeHolder<TScanQueryHistory<TGreater>>(bucketCount, bucketSize, now);
+
+ ConvertKeyRange<NKikimrSysView::TEvGetQueryMetricsRequest, ui64, ui32>(Request, this->TableRange);
+ Request.SetType(statsType);
+ }
+
+ STFUNC(StateScan) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, Handle);
+ hFunc(TEvSysView::TEvGetQueryStatsResult, Handle);
+ hFunc(TEvents::TEvUndelivered, Undelivered);
+ hFunc(TEvInterconnect::TEvNodeConnected, Connected);
+ hFunc(TEvInterconnect::TEvNodeDisconnected, Disconnected);
+ hFunc(TEvPrivate::TEvRetryNode, RetryNode);
+ hFunc(TEvSysView::TEvGetQueryStatsResponse, Handle);
+ hFunc(TEvPipeCache::TEvDeliveryProblem, Handle);
hFunc(NKqp::TEvKqp::TEvAbortExecution, TBase::HandleAbortExecution);
- cFunc(TEvents::TEvWakeup::EventType, TBase::HandleTimeout);
- cFunc(TEvents::TEvPoison::EventType, PassAway);
- default:
- LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS,
- "NSysView::TQueryStatsScan: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite());
- }
- }
-
-private:
- struct TEvPrivate {
- enum EEv {
- EvRetryNode = EventSpaceBegin(TEvents::ES_PRIVATE),
-
- EvEnd
- };
-
- struct TEvRetryNode : public TEventLocal<TEvRetryNode, EvRetryNode> {
- ui32 NodeId;
-
- explicit TEvRetryNode(ui32 nodeId)
- : NodeId(nodeId)
- {}
- };
- };
-
- void ProceedToScan() override {
- this->Become(&TQueryStatsScan<TGreater>::StateScan);
- if (this->AckReceived) {
- StartScan();
- }
- }
-
- void Handle(NKqp::TEvKqpCompute::TEvScanDataAck::TPtr&) {
- StartScan();
- }
-
- void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr&) {
- this->ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "Delivery problem in query stats scan");
- }
-
- void RequestBatch() {
- if (this->BatchRequestInFlight) {
- return;
- }
-
- this->UseProcessor = true;
-
- auto request = MakeHolder<TEvSysView::TEvGetQueryMetricsRequest>();
- request->Record.CopyFrom(Request);
-
- this->Send(MakePipePeNodeCacheID(false),
- new TEvPipeCache::TEvForward(request.Release(), this->SysViewProcessorId, true),
- IEventHandle::FlagTrackDelivery);
-
- this->BatchRequestInFlight = true;
- }
-
- void StartScan() {
+ cFunc(TEvents::TEvWakeup::EventType, TBase::HandleTimeout);
+ cFunc(TEvents::TEvPoison::EventType, PassAway);
+ default:
+ LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS,
+ "NSysView::TQueryStatsScan: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite());
+ }
+ }
+
+private:
+ struct TEvPrivate {
+ enum EEv {
+ EvRetryNode = EventSpaceBegin(TEvents::ES_PRIVATE),
+
+ EvEnd
+ };
+
+ struct TEvRetryNode : public TEventLocal<TEvRetryNode, EvRetryNode> {
+ ui32 NodeId;
+
+ explicit TEvRetryNode(ui32 nodeId)
+ : NodeId(nodeId)
+ {}
+ };
+ };
+
+ void ProceedToScan() override {
+ this->Become(&TQueryStatsScan<TGreater>::StateScan);
+ if (this->AckReceived) {
+ StartScan();
+ }
+ }
+
+ void Handle(NKqp::TEvKqpCompute::TEvScanDataAck::TPtr&) {
+ StartScan();
+ }
+
+ void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr&) {
+ this->ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "Delivery problem in query stats scan");
+ }
+
+ void RequestBatch() {
+ if (this->BatchRequestInFlight) {
+ return;
+ }
+
+ this->UseProcessor = true;
+
+ auto request = MakeHolder<TEvSysView::TEvGetQueryMetricsRequest>();
+ request->Record.CopyFrom(Request);
+
+ this->Send(MakePipePeNodeCacheID(false),
+ new TEvPipeCache::TEvForward(request.Release(), this->SysViewProcessorId, true),
+ IEventHandle::FlagTrackDelivery);
+
+ this->BatchRequestInFlight = true;
+ }
+
+ void StartScan() {
if (AppData()->FeatureFlags.GetEnablePersistentQueryStats() && this->SysViewProcessorId) {
- RequestBatch();
- } else {
- StartOldScan();
- }
- }
-
- void StartOldScan() {
- if (BucketRange.IsEmpty || this->TenantNodes.empty()) {
- this->ReplyEmptyAndDie();
- return;
- }
-
- NodesToRequest.reserve(this->TenantNodes.size());
- for (const auto& nodeId : this->TenantNodes) {
- Nodes[nodeId] = TRetryState{false, 0, StartRetryInterval};
- NodesToRequest.push_back(nodeId);
- }
-
- SendRequests();
- }
-
- void ScheduleRetry(ui32 nodeId) {
- if (NodesReplied.find(nodeId) != NodesReplied.end()) {
- return;
- }
-
- auto& retryState = Nodes[nodeId];
- if (++retryState.Attempt == MaxRetryAttempts) {
- NodesReplied.insert(nodeId); // TODO: warnings (answer is partial)
- } else {
- this->Schedule(retryState.Interval, new typename TEvPrivate::TEvRetryNode(nodeId));
- retryState.Interval = TDuration::MilliSeconds((ui64)(retryState.Interval.MilliSeconds() * RetryMultiplier));
- }
-
- SendRequests();
- }
-
- void Handle(TEvSysView::TEvGetQueryStatsResult::TPtr& ev) {
- ui32 nodeId = ev.Get()->Cookie;
-
- History->Merge(std::move(ev->Get()->Record));
- NodesReplied.insert(nodeId);
-
- auto& retryState = Nodes[nodeId];
- if (retryState.InFly) {
- --RequestsInFly;
- retryState.InFly = false;
- }
-
- SendRequests();
- }
-
- void RetryNode(typename TEvPrivate::TEvRetryNode::TPtr& ev) {
- auto nodeId = ev->Get()->NodeId;
- NodesToRequest.push_back(nodeId);
- SendRequests();
- }
-
- void Undelivered(TEvents::TEvUndelivered::TPtr& ev) {
- ui32 nodeId = ev.Get()->Cookie;
- auto& retryState = Nodes[nodeId];
-
- if (retryState.InFly) {
- retryState.InFly = false;
- --RequestsInFly;
- ScheduleRetry(nodeId);
- }
- }
-
- void Connected(TEvInterconnect::TEvNodeConnected::TPtr&) {
- }
-
- void Disconnected(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) {
- ui32 nodeId = ev->Get()->NodeId;
- auto& retryState = Nodes[nodeId];
-
- if (retryState.InFly) {
- --RequestsInFly;
- retryState.InFly = false;
- ScheduleRetry(nodeId);
- }
- }
-
- void SendRequests() {
- while (!NodesToRequest.empty() && RequestsInFly < MaxRequestsInFly) {
- auto nodeId = NodesToRequest.back();
- NodesToRequest.pop_back();
-
- auto sysViewServiceID = MakeSysViewServiceID(nodeId);
-
- auto request = MakeHolder<TEvSysView::TEvGetQueryStats>();
- request->Record.SetStatsType(StatsType);
- request->Record.SetStartBucket(std::max(History->GetStartBucket(), BucketRange.FromBucket));
- request->Record.SetTenantName(this->TenantName);
-
- this->Send(sysViewServiceID, std::move(request),
- IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession, nodeId);
-
- Nodes[nodeId].InFly = true;
- ++RequestsInFly;
- }
-
- if (NodesReplied.size() == Nodes.size()) {
- ReplyAndDie();
- }
- }
-
- using TEntry = NKikimrSysView::TQueryStatsEntry;
- using TExtractor = std::function<TCell(const TEntry&)>;
- using TSchema = Schema::QueryStats;
-
- struct TExtractorsMap : public THashMap<NTable::TTag, TExtractor> {
- TExtractorsMap() {
- insert({TSchema::IntervalEnd::ColumnId, [] (const TEntry& entry) {
- return TCell::Make<ui64>(entry.GetKey().GetIntervalEndUs());
- }});
- insert({TSchema::Rank::ColumnId, [] (const TEntry& entry) {
- return TCell::Make<ui32>(entry.GetKey().GetRank());
- }});
- insert({TSchema::QueryText::ColumnId, [] (const TEntry& entry) {
- const auto& text = entry.GetStats().GetQueryText();
- return TCell(text.data(), text.size());
- }});
- insert({TSchema::Duration::ColumnId, [] (const TEntry& entry) {
- return TCell::Make<i64>(entry.GetStats().GetDurationMs() * 1000);
- }});
- insert({TSchema::EndTime::ColumnId, [] (const TEntry& entry) {
- return TCell::Make<ui64>(entry.GetStats().GetEndTimeMs() * 1000);
- }});
- insert({TSchema::ReadRows::ColumnId, [] (const TEntry& entry) {
- return TCell::Make<ui64>(entry.GetStats().GetStats().GetReadRows());
- }});
- insert({TSchema::ReadBytes::ColumnId, [] (const TEntry& entry) {
- return TCell::Make<ui64>(entry.GetStats().GetStats().GetReadBytes());
- }});
- insert({TSchema::UpdateRows::ColumnId, [] (const TEntry& entry) {
- return TCell::Make<ui64>(entry.GetStats().GetStats().GetUpdateRows());
- }});
- insert({TSchema::UpdateBytes::ColumnId, [] (const TEntry& entry) {
- return TCell::Make<ui64>(entry.GetStats().GetStats().GetUpdateBytes());
- }});
- insert({TSchema::DeleteRows::ColumnId, [] (const TEntry& entry) {
- return TCell::Make<ui64>(entry.GetStats().GetStats().GetDeleteRows());
- }});
- insert({TSchema::DeleteBytes::ColumnId, [] (const TEntry&) {
- return TCell::Make<ui64>(0); // deprecated
- }});
- insert({TSchema::Partitions::ColumnId, [] (const TEntry& entry) {
- return TCell::Make<ui64>(entry.GetStats().GetStats().GetPartitionCount());
- }});
- insert({TSchema::UserSID::ColumnId, [] (const TEntry& entry) {
- const auto& stats = entry.GetStats();
- if (!stats.HasUserSID()) {
- return TCell();
- }
- const auto& sid = stats.GetUserSID();
- return TCell(sid.data(), sid.size());
- }});
- insert({TSchema::ParametersSize::ColumnId, [] (const TEntry& entry) {
- return TCell::Make<ui64>(entry.GetStats().GetParametersSize());
- }});
- insert({TSchema::CompileDuration::ColumnId, [] (const TEntry& entry) {
- const auto& stats = entry.GetStats();
- return stats.HasCompileDurationMs() ? TCell::Make<ui64>(stats.GetCompileDurationMs() * 1000) : TCell();
- }});
- insert({TSchema::FromQueryCache::ColumnId, [] (const TEntry& entry) {
- const auto& stats = entry.GetStats();
- return stats.HasFromQueryCache() ? TCell::Make<bool>(stats.GetFromQueryCache()) : TCell();
- }});
- insert({TSchema::CPUTime::ColumnId, [] (const TEntry& entry) {
- return TCell::Make<ui64>(entry.GetStats().GetTotalCpuTimeUs());
- }});
- insert({TSchema::ShardCount::ColumnId, [] (const TEntry& entry) {
- return TCell::Make<ui64>(entry.GetStats().GetShardsCpuTimeUs().GetCnt());
- }});
- insert({TSchema::SumShardCPUTime::ColumnId, [] (const TEntry& entry) {
- return TCell::Make<ui64>(entry.GetStats().GetShardsCpuTimeUs().GetSum());
- }});
- insert({TSchema::MinShardCPUTime::ColumnId, [] (const TEntry& entry) {
- return TCell::Make<ui64>(entry.GetStats().GetShardsCpuTimeUs().GetMin());
- }});
- insert({TSchema::MaxShardCPUTime::ColumnId, [] (const TEntry& entry) {
- return TCell::Make<ui64>(entry.GetStats().GetShardsCpuTimeUs().GetMax());
- }});
- insert({TSchema::ComputeNodesCount::ColumnId, [] (const TEntry& entry) {
- return TCell::Make<ui64>(entry.GetStats().GetComputeCpuTimeUs().GetCnt());
- }});
- insert({TSchema::SumComputeCPUTime::ColumnId, [] (const TEntry& entry) {
- return TCell::Make<ui64>(entry.GetStats().GetComputeCpuTimeUs().GetSum());
- }});
- insert({TSchema::MinComputeCPUTime::ColumnId, [] (const TEntry& entry) {
- return TCell::Make<ui64>(entry.GetStats().GetComputeCpuTimeUs().GetMin());
- }});
- insert({TSchema::MaxComputeCPUTime::ColumnId, [] (const TEntry& entry) {
- return TCell::Make<ui64>(entry.GetStats().GetComputeCpuTimeUs().GetMax());
- }});
- insert({TSchema::CompileCPUTime::ColumnId, [] (const TEntry& entry) {
- const auto& stats = entry.GetStats();
- return stats.HasCompileCpuTimeUs() ? TCell::Make<ui64>(stats.GetCompileCpuTimeUs()) : TCell();
- }});
- insert({TSchema::ProcessCPUTime::ColumnId, [] (const TEntry& entry) {
- return TCell::Make<ui64>(entry.GetStats().GetProcessCpuTimeUs());
- }});
- insert({TSchema::TypeCol::ColumnId, [] (const TEntry& entry) {
- const auto& type = entry.GetStats().GetType();
- return TCell(type.data(), type.size());
- }});
- insert({TSchema::RequestUnits::ColumnId, [] (const TEntry& entry) {
- return TCell::Make<ui64>(entry.GetStats().GetRequestUnits());
- }});
- }
- };
-
- void Handle(TEvSysView::TEvGetQueryStatsResponse::TPtr& ev) {
- const auto& record = ev->Get()->Record;
- if (record.HasOverloaded() && record.GetOverloaded()) {
- this->ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "SysViewProcessor is overloaded");
- return;
- }
-
- this->template ReplyBatch<TEvSysView::TEvGetQueryStatsResponse, TEntry, TExtractorsMap, true>(ev);
-
- if (!record.GetLastBatch()) {
- Y_VERIFY(record.HasNext());
- Request.MutableFrom()->CopyFrom(record.GetNext());
- Request.SetInclusiveFrom(true);
- }
-
- this->BatchRequestInFlight = false;
- }
-
- void ReplyAndDie() {
- static TExtractorsMap extractors;
-
- auto fromBucket = std::max(History->GetStartBucket(), BucketRange.FromBucket);
- auto toBucket = std::min(History->GetLastBucket(), BucketRange.ToBucket);
-
- if (fromBucket > toBucket) {
- this->ReplyEmptyAndDie();
- }
-
- auto batch = MakeHolder<NKqp::TEvKqpCompute::TEvScanData>(this->ScanId);
- batch->Finished = true;
-
- TEntry entry;
- auto& key = *entry.MutableKey();
- key.SetIntervalEndUs((fromBucket + 1) * History->GetBucketSizeMs() * 1000);
-
- for (ui64 bucket = fromBucket; bucket <= toBucket; ++bucket) {
- const auto& top = History->GetBucketTop(bucket);
-
- ui32 fromRank = 1;
- if (BucketRange.FromRank && bucket == BucketRange.FromBucket) {
- fromRank = std::max(fromRank, *BucketRange.FromRank);
- }
-
- ui32 toRank = top.StatsSize();
- if (BucketRange.ToRank && bucket == BucketRange.ToBucket) {
- toRank = std::min(toRank, *BucketRange.ToRank);
- }
-
- TVector<TCell> cells;
- for (ui32 r = fromRank; r <= toRank; ++r) {
- key.SetRank(r);
- entry.MutableStats()->CopyFrom(top.GetStats(r - 1));
-
- for (auto column : this->Columns) {
- auto extractor = extractors.find(column.Tag);
- if (extractor == extractors.end()) {
- cells.push_back(TCell());
- } else {
- cells.push_back(extractor->second(entry));
- }
- }
-
- TArrayRef<const TCell> ref(cells);
+ RequestBatch();
+ } else {
+ StartOldScan();
+ }
+ }
+
+ void StartOldScan() {
+ if (BucketRange.IsEmpty || this->TenantNodes.empty()) {
+ this->ReplyEmptyAndDie();
+ return;
+ }
+
+ NodesToRequest.reserve(this->TenantNodes.size());
+ for (const auto& nodeId : this->TenantNodes) {
+ Nodes[nodeId] = TRetryState{false, 0, StartRetryInterval};
+ NodesToRequest.push_back(nodeId);
+ }
+
+ SendRequests();
+ }
+
+ void ScheduleRetry(ui32 nodeId) {
+ if (NodesReplied.find(nodeId) != NodesReplied.end()) {
+ return;
+ }
+
+ auto& retryState = Nodes[nodeId];
+ if (++retryState.Attempt == MaxRetryAttempts) {
+ NodesReplied.insert(nodeId); // TODO: warnings (answer is partial)
+ } else {
+ this->Schedule(retryState.Interval, new typename TEvPrivate::TEvRetryNode(nodeId));
+ retryState.Interval = TDuration::MilliSeconds((ui64)(retryState.Interval.MilliSeconds() * RetryMultiplier));
+ }
+
+ SendRequests();
+ }
+
+ void Handle(TEvSysView::TEvGetQueryStatsResult::TPtr& ev) {
+ ui32 nodeId = ev.Get()->Cookie;
+
+ History->Merge(std::move(ev->Get()->Record));
+ NodesReplied.insert(nodeId);
+
+ auto& retryState = Nodes[nodeId];
+ if (retryState.InFly) {
+ --RequestsInFly;
+ retryState.InFly = false;
+ }
+
+ SendRequests();
+ }
+
+ void RetryNode(typename TEvPrivate::TEvRetryNode::TPtr& ev) {
+ auto nodeId = ev->Get()->NodeId;
+ NodesToRequest.push_back(nodeId);
+ SendRequests();
+ }
+
+ void Undelivered(TEvents::TEvUndelivered::TPtr& ev) {
+ ui32 nodeId = ev.Get()->Cookie;
+ auto& retryState = Nodes[nodeId];
+
+ if (retryState.InFly) {
+ retryState.InFly = false;
+ --RequestsInFly;
+ ScheduleRetry(nodeId);
+ }
+ }
+
+ void Connected(TEvInterconnect::TEvNodeConnected::TPtr&) {
+ }
+
+ void Disconnected(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) {
+ ui32 nodeId = ev->Get()->NodeId;
+ auto& retryState = Nodes[nodeId];
+
+ if (retryState.InFly) {
+ --RequestsInFly;
+ retryState.InFly = false;
+ ScheduleRetry(nodeId);
+ }
+ }
+
+ void SendRequests() {
+ while (!NodesToRequest.empty() && RequestsInFly < MaxRequestsInFly) {
+ auto nodeId = NodesToRequest.back();
+ NodesToRequest.pop_back();
+
+ auto sysViewServiceID = MakeSysViewServiceID(nodeId);
+
+ auto request = MakeHolder<TEvSysView::TEvGetQueryStats>();
+ request->Record.SetStatsType(StatsType);
+ request->Record.SetStartBucket(std::max(History->GetStartBucket(), BucketRange.FromBucket));
+ request->Record.SetTenantName(this->TenantName);
+
+ this->Send(sysViewServiceID, std::move(request),
+ IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession, nodeId);
+
+ Nodes[nodeId].InFly = true;
+ ++RequestsInFly;
+ }
+
+ if (NodesReplied.size() == Nodes.size()) {
+ ReplyAndDie();
+ }
+ }
+
+ using TEntry = NKikimrSysView::TQueryStatsEntry;
+ using TExtractor = std::function<TCell(const TEntry&)>;
+ using TSchema = Schema::QueryStats;
+
+ struct TExtractorsMap : public THashMap<NTable::TTag, TExtractor> {
+ TExtractorsMap() {
+ insert({TSchema::IntervalEnd::ColumnId, [] (const TEntry& entry) {
+ return TCell::Make<ui64>(entry.GetKey().GetIntervalEndUs());
+ }});
+ insert({TSchema::Rank::ColumnId, [] (const TEntry& entry) {
+ return TCell::Make<ui32>(entry.GetKey().GetRank());
+ }});
+ insert({TSchema::QueryText::ColumnId, [] (const TEntry& entry) {
+ const auto& text = entry.GetStats().GetQueryText();
+ return TCell(text.data(), text.size());
+ }});
+ insert({TSchema::Duration::ColumnId, [] (const TEntry& entry) {
+ return TCell::Make<i64>(entry.GetStats().GetDurationMs() * 1000);
+ }});
+ insert({TSchema::EndTime::ColumnId, [] (const TEntry& entry) {
+ return TCell::Make<ui64>(entry.GetStats().GetEndTimeMs() * 1000);
+ }});
+ insert({TSchema::ReadRows::ColumnId, [] (const TEntry& entry) {
+ return TCell::Make<ui64>(entry.GetStats().GetStats().GetReadRows());
+ }});
+ insert({TSchema::ReadBytes::ColumnId, [] (const TEntry& entry) {
+ return TCell::Make<ui64>(entry.GetStats().GetStats().GetReadBytes());
+ }});
+ insert({TSchema::UpdateRows::ColumnId, [] (const TEntry& entry) {
+ return TCell::Make<ui64>(entry.GetStats().GetStats().GetUpdateRows());
+ }});
+ insert({TSchema::UpdateBytes::ColumnId, [] (const TEntry& entry) {
+ return TCell::Make<ui64>(entry.GetStats().GetStats().GetUpdateBytes());
+ }});
+ insert({TSchema::DeleteRows::ColumnId, [] (const TEntry& entry) {
+ return TCell::Make<ui64>(entry.GetStats().GetStats().GetDeleteRows());
+ }});
+ insert({TSchema::DeleteBytes::ColumnId, [] (const TEntry&) {
+ return TCell::Make<ui64>(0); // deprecated
+ }});
+ insert({TSchema::Partitions::ColumnId, [] (const TEntry& entry) {
+ return TCell::Make<ui64>(entry.GetStats().GetStats().GetPartitionCount());
+ }});
+ insert({TSchema::UserSID::ColumnId, [] (const TEntry& entry) {
+ const auto& stats = entry.GetStats();
+ if (!stats.HasUserSID()) {
+ return TCell();
+ }
+ const auto& sid = stats.GetUserSID();
+ return TCell(sid.data(), sid.size());
+ }});
+ insert({TSchema::ParametersSize::ColumnId, [] (const TEntry& entry) {
+ return TCell::Make<ui64>(entry.GetStats().GetParametersSize());
+ }});
+ insert({TSchema::CompileDuration::ColumnId, [] (const TEntry& entry) {
+ const auto& stats = entry.GetStats();
+ return stats.HasCompileDurationMs() ? TCell::Make<ui64>(stats.GetCompileDurationMs() * 1000) : TCell();
+ }});
+ insert({TSchema::FromQueryCache::ColumnId, [] (const TEntry& entry) {
+ const auto& stats = entry.GetStats();
+ return stats.HasFromQueryCache() ? TCell::Make<bool>(stats.GetFromQueryCache()) : TCell();
+ }});
+ insert({TSchema::CPUTime::ColumnId, [] (const TEntry& entry) {
+ return TCell::Make<ui64>(entry.GetStats().GetTotalCpuTimeUs());
+ }});
+ insert({TSchema::ShardCount::ColumnId, [] (const TEntry& entry) {
+ return TCell::Make<ui64>(entry.GetStats().GetShardsCpuTimeUs().GetCnt());
+ }});
+ insert({TSchema::SumShardCPUTime::ColumnId, [] (const TEntry& entry) {
+ return TCell::Make<ui64>(entry.GetStats().GetShardsCpuTimeUs().GetSum());
+ }});
+ insert({TSchema::MinShardCPUTime::ColumnId, [] (const TEntry& entry) {
+ return TCell::Make<ui64>(entry.GetStats().GetShardsCpuTimeUs().GetMin());
+ }});
+ insert({TSchema::MaxShardCPUTime::ColumnId, [] (const TEntry& entry) {
+ return TCell::Make<ui64>(entry.GetStats().GetShardsCpuTimeUs().GetMax());
+ }});
+ insert({TSchema::ComputeNodesCount::ColumnId, [] (const TEntry& entry) {
+ return TCell::Make<ui64>(entry.GetStats().GetComputeCpuTimeUs().GetCnt());
+ }});
+ insert({TSchema::SumComputeCPUTime::ColumnId, [] (const TEntry& entry) {
+ return TCell::Make<ui64>(entry.GetStats().GetComputeCpuTimeUs().GetSum());
+ }});
+ insert({TSchema::MinComputeCPUTime::ColumnId, [] (const TEntry& entry) {
+ return TCell::Make<ui64>(entry.GetStats().GetComputeCpuTimeUs().GetMin());
+ }});
+ insert({TSchema::MaxComputeCPUTime::ColumnId, [] (const TEntry& entry) {
+ return TCell::Make<ui64>(entry.GetStats().GetComputeCpuTimeUs().GetMax());
+ }});
+ insert({TSchema::CompileCPUTime::ColumnId, [] (const TEntry& entry) {
+ const auto& stats = entry.GetStats();
+ return stats.HasCompileCpuTimeUs() ? TCell::Make<ui64>(stats.GetCompileCpuTimeUs()) : TCell();
+ }});
+ insert({TSchema::ProcessCPUTime::ColumnId, [] (const TEntry& entry) {
+ return TCell::Make<ui64>(entry.GetStats().GetProcessCpuTimeUs());
+ }});
+ insert({TSchema::TypeCol::ColumnId, [] (const TEntry& entry) {
+ const auto& type = entry.GetStats().GetType();
+ return TCell(type.data(), type.size());
+ }});
+ insert({TSchema::RequestUnits::ColumnId, [] (const TEntry& entry) {
+ return TCell::Make<ui64>(entry.GetStats().GetRequestUnits());
+ }});
+ }
+ };
+
+ void Handle(TEvSysView::TEvGetQueryStatsResponse::TPtr& ev) {
+ const auto& record = ev->Get()->Record;
+ if (record.HasOverloaded() && record.GetOverloaded()) {
+ this->ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "SysViewProcessor is overloaded");
+ return;
+ }
+
+ this->template ReplyBatch<TEvSysView::TEvGetQueryStatsResponse, TEntry, TExtractorsMap, true>(ev);
+
+ if (!record.GetLastBatch()) {
+ Y_VERIFY(record.HasNext());
+ Request.MutableFrom()->CopyFrom(record.GetNext());
+ Request.SetInclusiveFrom(true);
+ }
+
+ this->BatchRequestInFlight = false;
+ }
+
+ void ReplyAndDie() {
+ static TExtractorsMap extractors;
+
+ auto fromBucket = std::max(History->GetStartBucket(), BucketRange.FromBucket);
+ auto toBucket = std::min(History->GetLastBucket(), BucketRange.ToBucket);
+
+ if (fromBucket > toBucket) {
+ this->ReplyEmptyAndDie();
+ }
+
+ auto batch = MakeHolder<NKqp::TEvKqpCompute::TEvScanData>(this->ScanId);
+ batch->Finished = true;
+
+ TEntry entry;
+ auto& key = *entry.MutableKey();
+ key.SetIntervalEndUs((fromBucket + 1) * History->GetBucketSizeMs() * 1000);
+
+ for (ui64 bucket = fromBucket; bucket <= toBucket; ++bucket) {
+ const auto& top = History->GetBucketTop(bucket);
+
+ ui32 fromRank = 1;
+ if (BucketRange.FromRank && bucket == BucketRange.FromBucket) {
+ fromRank = std::max(fromRank, *BucketRange.FromRank);
+ }
+
+ ui32 toRank = top.StatsSize();
+ if (BucketRange.ToRank && bucket == BucketRange.ToBucket) {
+ toRank = std::min(toRank, *BucketRange.ToRank);
+ }
+
+ TVector<TCell> cells;
+ for (ui32 r = fromRank; r <= toRank; ++r) {
+ key.SetRank(r);
+ entry.MutableStats()->CopyFrom(top.GetStats(r - 1));
+
+ for (auto column : this->Columns) {
+ auto extractor = extractors.find(column.Tag);
+ if (extractor == extractors.end()) {
+ cells.push_back(TCell());
+ } else {
+ cells.push_back(extractor->second(entry));
+ }
+ }
+
+ TArrayRef<const TCell> ref(cells);
batch->Rows.emplace_back(TOwnedCellVec::Make(ref));
- cells.clear();
- }
-
- key.SetIntervalEndUs(key.GetIntervalEndUs() + History->GetBucketSizeMs() * 1000);
- }
-
- TBase::SendBatch(std::move(batch));
- }
-
- void PassAway() override {
- if (UseProcessor) {
- this->Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(0));
- } else {
- for (auto& [nodeId, _] : Nodes) {
- this->Send(TActorContext::InterconnectProxy(nodeId), new TEvents::TEvUnsubscribe);
- }
- }
- TBase::PassAway();
- }
-
-private:
- static constexpr size_t MaxRequestsInFly = 10;
-
- static constexpr size_t MaxRetryAttempts = 5;
- static constexpr TDuration StartRetryInterval = TDuration::MilliSeconds(500);
- static constexpr double RetryMultiplier = 2.0;
-
- const NKikimrSysView::EStatsType StatsType;
-
- TQueryStatsBucketRange BucketRange;
-
- struct TRetryState {
- bool InFly = false;
- ui32 Attempt = 0;
- TDuration Interval;
- };
-
- THashMap<ui32, TRetryState> Nodes;
- THashSet<ui32> NodesReplied;
-
- TVector<ui32> NodesToRequest;
- size_t RequestsInFly = 0;
-
- THolder<TScanQueryHistory<TGreater>> History;
-
- bool UseProcessor = false;
- NKikimrSysView::TEvGetQueryMetricsRequest Request;
-};
-
+ cells.clear();
+ }
+
+ key.SetIntervalEndUs(key.GetIntervalEndUs() + History->GetBucketSizeMs() * 1000);
+ }
+
+ TBase::SendBatch(std::move(batch));
+ }
+
+ void PassAway() override {
+ if (UseProcessor) {
+ this->Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(0));
+ } else {
+ for (auto& [nodeId, _] : Nodes) {
+ this->Send(TActorContext::InterconnectProxy(nodeId), new TEvents::TEvUnsubscribe);
+ }
+ }
+ TBase::PassAway();
+ }
+
+private:
+ static constexpr size_t MaxRequestsInFly = 10;
+
+ static constexpr size_t MaxRetryAttempts = 5;
+ static constexpr TDuration StartRetryInterval = TDuration::MilliSeconds(500);
+ static constexpr double RetryMultiplier = 2.0;
+
+ const NKikimrSysView::EStatsType StatsType;
+
+ TQueryStatsBucketRange BucketRange;
+
+ struct TRetryState {
+ bool InFly = false;
+ ui32 Attempt = 0;
+ TDuration Interval;
+ };
+
+ THashMap<ui32, TRetryState> Nodes;
+ THashSet<ui32> NodesReplied;
+
+ TVector<ui32> NodesToRequest;
+ size_t RequestsInFly = 0;
+
+ THolder<TScanQueryHistory<TGreater>> History;
+
+ bool UseProcessor = false;
+ NKikimrSysView::TEvGetQueryMetricsRequest Request;
+};
+
THolder<IActor> CreateQueryStatsScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId, const TStringBuf viewName,
- const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns)
-{
- if (viewName == TopQueriesByDuration1MinuteName) {
+ const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns)
+{
+ if (viewName == TopQueriesByDuration1MinuteName) {
return MakeHolder<TQueryStatsScan<TDurationGreater>>(ownerId, scanId, tableId, tableRange, columns,
- NKikimrSysView::TOP_DURATION_ONE_MINUTE,
- ONE_MINUTE_BUCKET_COUNT, ONE_MINUTE_BUCKET_SIZE);
-
- } else if (viewName == TopQueriesByDuration1HourName) {
+ NKikimrSysView::TOP_DURATION_ONE_MINUTE,
+ ONE_MINUTE_BUCKET_COUNT, ONE_MINUTE_BUCKET_SIZE);
+
+ } else if (viewName == TopQueriesByDuration1HourName) {
return MakeHolder<TQueryStatsScan<TDurationGreater>>(ownerId, scanId, tableId, tableRange, columns,
- NKikimrSysView::TOP_DURATION_ONE_HOUR,
- ONE_HOUR_BUCKET_COUNT, ONE_HOUR_BUCKET_SIZE);
-
- } else if (viewName == TopQueriesByReadBytes1MinuteName) {
+ NKikimrSysView::TOP_DURATION_ONE_HOUR,
+ ONE_HOUR_BUCKET_COUNT, ONE_HOUR_BUCKET_SIZE);
+
+ } else if (viewName == TopQueriesByReadBytes1MinuteName) {
return MakeHolder<TQueryStatsScan<TReadBytesGreater>>(ownerId, scanId, tableId, tableRange, columns,
- NKikimrSysView::TOP_READ_BYTES_ONE_MINUTE,
- ONE_MINUTE_BUCKET_COUNT, ONE_MINUTE_BUCKET_SIZE);
-
- } else if (viewName == TopQueriesByReadBytes1HourName) {
+ NKikimrSysView::TOP_READ_BYTES_ONE_MINUTE,
+ ONE_MINUTE_BUCKET_COUNT, ONE_MINUTE_BUCKET_SIZE);
+
+ } else if (viewName == TopQueriesByReadBytes1HourName) {
return MakeHolder<TQueryStatsScan<TReadBytesGreater>>(ownerId, scanId, tableId, tableRange, columns,
- NKikimrSysView::TOP_READ_BYTES_ONE_HOUR,
- ONE_HOUR_BUCKET_COUNT, ONE_HOUR_BUCKET_SIZE);
-
- } else if (viewName == TopQueriesByCpuTime1MinuteName) {
+ NKikimrSysView::TOP_READ_BYTES_ONE_HOUR,
+ ONE_HOUR_BUCKET_COUNT, ONE_HOUR_BUCKET_SIZE);
+
+ } else if (viewName == TopQueriesByCpuTime1MinuteName) {
return MakeHolder<TQueryStatsScan<TCpuTimeGreater>>(ownerId, scanId, tableId, tableRange, columns,
- NKikimrSysView::TOP_CPU_TIME_ONE_MINUTE,
- ONE_MINUTE_BUCKET_COUNT, ONE_MINUTE_BUCKET_SIZE);
-
- } else if (viewName == TopQueriesByCpuTime1HourName) {
+ NKikimrSysView::TOP_CPU_TIME_ONE_MINUTE,
+ ONE_MINUTE_BUCKET_COUNT, ONE_MINUTE_BUCKET_SIZE);
+
+ } else if (viewName == TopQueriesByCpuTime1HourName) {
return MakeHolder<TQueryStatsScan<TCpuTimeGreater>>(ownerId, scanId, tableId, tableRange, columns,
- NKikimrSysView::TOP_CPU_TIME_ONE_HOUR,
- ONE_HOUR_BUCKET_COUNT, ONE_HOUR_BUCKET_SIZE);
- } else if (viewName == TopQueriesByRequestUnits1MinuteName) {
- return MakeHolder<TQueryStatsScan<TRequestUnitsGreater>>(ownerId, scanId, tableId, tableRange, columns,
- NKikimrSysView::TOP_REQUEST_UNITS_ONE_MINUTE,
- ONE_MINUTE_BUCKET_COUNT, ONE_MINUTE_BUCKET_SIZE);
-
- } else if (viewName == TopQueriesByRequestUnits1HourName) {
- return MakeHolder<TQueryStatsScan<TRequestUnitsGreater>>(ownerId, scanId, tableId, tableRange, columns,
- NKikimrSysView::TOP_REQUEST_UNITS_ONE_HOUR,
- ONE_HOUR_BUCKET_COUNT, ONE_HOUR_BUCKET_SIZE);
- }
- return {};
-}
-
-} // NSysView
-} // NKikimr
+ NKikimrSysView::TOP_CPU_TIME_ONE_HOUR,
+ ONE_HOUR_BUCKET_COUNT, ONE_HOUR_BUCKET_SIZE);
+ } else if (viewName == TopQueriesByRequestUnits1MinuteName) {
+ return MakeHolder<TQueryStatsScan<TRequestUnitsGreater>>(ownerId, scanId, tableId, tableRange, columns,
+ NKikimrSysView::TOP_REQUEST_UNITS_ONE_MINUTE,
+ ONE_MINUTE_BUCKET_COUNT, ONE_MINUTE_BUCKET_SIZE);
+
+ } else if (viewName == TopQueriesByRequestUnits1HourName) {
+ return MakeHolder<TQueryStatsScan<TRequestUnitsGreater>>(ownerId, scanId, tableId, tableRange, columns,
+ NKikimrSysView::TOP_REQUEST_UNITS_ONE_HOUR,
+ ONE_HOUR_BUCKET_COUNT, ONE_HOUR_BUCKET_SIZE);
+ }
+ return {};
+}
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/query_stats/query_stats.h b/ydb/core/sys_view/query_stats/query_stats.h
index b9fe4b7ecc6..47786c7bf5a 100644
--- a/ydb/core/sys_view/query_stats/query_stats.h
+++ b/ydb/core/sys_view/query_stats/query_stats.h
@@ -1,25 +1,25 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/kqp/runtime/kqp_compute.h>
-
-namespace NKikimr {
-namespace NSysView {
-
-struct TQueryStatsBucketRange {
- ui64 FromBucket = 0;
- TMaybe<ui32> FromRank;
-
- ui64 ToBucket = std::numeric_limits<ui64>::max();
- TMaybe<ui32> ToRank;
-
- bool IsEmpty = false;
-
- explicit TQueryStatsBucketRange(const TSerializedTableRange& range, const TDuration& bucketSize);
-};
-
+
+namespace NKikimr {
+namespace NSysView {
+
+struct TQueryStatsBucketRange {
+ ui64 FromBucket = 0;
+ TMaybe<ui32> FromRank;
+
+ ui64 ToBucket = std::numeric_limits<ui64>::max();
+ TMaybe<ui32> ToRank;
+
+ bool IsEmpty = false;
+
+ explicit TQueryStatsBucketRange(const TSerializedTableRange& range, const TDuration& bucketSize);
+};
+
THolder<IActor> CreateQueryStatsScan(const TActorId& ownerId, ui32 scanId,
- const TTableId& tableId, const TStringBuf viewName,
- const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns);
-
-} // NSysView
-} // NKikimr
+ const TTableId& tableId, const TStringBuf viewName,
+ const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns);
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/query_stats/query_stats_ut.cpp b/ydb/core/sys_view/query_stats/query_stats_ut.cpp
index 4b8c5141cb9..f8859a7ddc6 100644
--- a/ydb/core/sys_view/query_stats/query_stats_ut.cpp
+++ b/ydb/core/sys_view/query_stats/query_stats_ut.cpp
@@ -1,101 +1,101 @@
-#include "query_stats.h"
-
+#include "query_stats.h"
+
#include <library/cpp/testing/unittest/registar.h>
-
-namespace NKikimr {
-namespace NSysView {
-
-Y_UNIT_TEST_SUITE(QueryStats) {
- Y_UNIT_TEST(Ranges) {
- auto test = [] (TMaybe<ui64> fromInstant, TMaybe<ui32> fromRank, bool inclusiveFrom,
- TMaybe<ui64> toInstant, TMaybe<ui32> toRank, bool inclusiveTo,
- ui64 fromBucketExpected, TMaybe<ui32> fromRankExpected,
- ui64 toBucketExpected, TMaybe<ui32> toRankExpected,
- bool isEmptyExpected)
- {
- TVector<TCell> from;
- if (fromInstant) {
- from.push_back(TCell::Make<ui64>(*fromInstant));
- if (fromRank) {
- from.push_back(TCell::Make<ui32>(*fromRank));
- }
- }
- TVector<TCell> to;
- if (toInstant) {
- to.push_back(TCell::Make<ui64>(*toInstant));
- if (toRank) {
- to.push_back(TCell::Make<ui32>(*toRank));
- }
- }
- TSerializedTableRange tableRange(from, inclusiveFrom, to, inclusiveTo);
-
- TQueryStatsBucketRange range(tableRange, TDuration::MicroSeconds(10));
-
- UNIT_ASSERT_VALUES_EQUAL(range.FromBucket, fromBucketExpected);
- UNIT_ASSERT_VALUES_EQUAL(range.FromRank, fromRankExpected);
- UNIT_ASSERT_VALUES_EQUAL(range.ToBucket, toBucketExpected);
- UNIT_ASSERT_VALUES_EQUAL(range.ToRank, toRankExpected);
- UNIT_ASSERT_VALUES_EQUAL(range.IsEmpty, isEmptyExpected);
- };
-
- auto maxUi64 = std::numeric_limits<ui64>::max();
-
- test({}, {}, false, {}, {}, false,
- 0, {}, maxUi64, {}, false);
- test({}, {}, true, {}, {}, true,
- 0, {}, maxUi64, {}, false);
-
- test(0, {}, false, 0, {}, false,
- 0, {}, 0, {}, true);
- test(0, {}, true, 0, {}, true,
- 0, {}, 0, {}, true);
-
- test(5, {}, false, 5, {}, false,
- 0, {}, 0, {}, true);
- test(5, {}, true, 5, {}, true,
- 0, {}, 0, {}, true);
-
- test(10, {}, false, 10, {}, false,
- 1, {}, 0, {}, true);
- test(10, {}, true, 10, {}, true,
- 0, {}, 0, {}, false);
-
- test(15, {}, false, 15, {}, false,
- 1, {}, 0, {}, true);
- test(15, {}, true, 15, {}, true,
- 1, {}, 0, {}, true);
-
- test(20, {}, false, 20, {}, false,
- 2, {}, 0, {}, true);
- test(20, {}, true, 20, {}, true,
- 1, {}, 1, {}, false);
-
- test(0, 3, false, 0, 7, false,
- 0, {}, 0, {}, true);
- test(0, 3, true, 0, 3, true,
- 0, {}, 0, {}, true);
-
- test(5, 3, false, 5, 7, false,
- 0, {}, 0, {}, true);
- test(5, 3, true, 5, 7, true,
- 0, {}, 0, {}, true);
-
- test(10, 3, false, 10, 7, false,
- 0, 4, 0, 6, false);
- test(10, 3, true, 10, 7, true,
- 0, 3, 0, 7, false);
-
- test(15, 3, false, 15, 7, false,
- 1, {}, 0, {}, true);
- test(15, 3, true, 15, 7, true,
- 1, {}, 0, {}, true);
-
- test(20, 3, false, 20, 7, false,
- 1, 4, 1, 6, false);
- test(20, 3, true, 20, 7, true,
- 1, 3, 1, 7, false);
- }
-}
-
-} // NSysView
-} // NKikimr
+
+namespace NKikimr {
+namespace NSysView {
+
+Y_UNIT_TEST_SUITE(QueryStats) {
+ Y_UNIT_TEST(Ranges) {
+ auto test = [] (TMaybe<ui64> fromInstant, TMaybe<ui32> fromRank, bool inclusiveFrom,
+ TMaybe<ui64> toInstant, TMaybe<ui32> toRank, bool inclusiveTo,
+ ui64 fromBucketExpected, TMaybe<ui32> fromRankExpected,
+ ui64 toBucketExpected, TMaybe<ui32> toRankExpected,
+ bool isEmptyExpected)
+ {
+ TVector<TCell> from;
+ if (fromInstant) {
+ from.push_back(TCell::Make<ui64>(*fromInstant));
+ if (fromRank) {
+ from.push_back(TCell::Make<ui32>(*fromRank));
+ }
+ }
+ TVector<TCell> to;
+ if (toInstant) {
+ to.push_back(TCell::Make<ui64>(*toInstant));
+ if (toRank) {
+ to.push_back(TCell::Make<ui32>(*toRank));
+ }
+ }
+ TSerializedTableRange tableRange(from, inclusiveFrom, to, inclusiveTo);
+
+ TQueryStatsBucketRange range(tableRange, TDuration::MicroSeconds(10));
+
+ UNIT_ASSERT_VALUES_EQUAL(range.FromBucket, fromBucketExpected);
+ UNIT_ASSERT_VALUES_EQUAL(range.FromRank, fromRankExpected);
+ UNIT_ASSERT_VALUES_EQUAL(range.ToBucket, toBucketExpected);
+ UNIT_ASSERT_VALUES_EQUAL(range.ToRank, toRankExpected);
+ UNIT_ASSERT_VALUES_EQUAL(range.IsEmpty, isEmptyExpected);
+ };
+
+ auto maxUi64 = std::numeric_limits<ui64>::max();
+
+ test({}, {}, false, {}, {}, false,
+ 0, {}, maxUi64, {}, false);
+ test({}, {}, true, {}, {}, true,
+ 0, {}, maxUi64, {}, false);
+
+ test(0, {}, false, 0, {}, false,
+ 0, {}, 0, {}, true);
+ test(0, {}, true, 0, {}, true,
+ 0, {}, 0, {}, true);
+
+ test(5, {}, false, 5, {}, false,
+ 0, {}, 0, {}, true);
+ test(5, {}, true, 5, {}, true,
+ 0, {}, 0, {}, true);
+
+ test(10, {}, false, 10, {}, false,
+ 1, {}, 0, {}, true);
+ test(10, {}, true, 10, {}, true,
+ 0, {}, 0, {}, false);
+
+ test(15, {}, false, 15, {}, false,
+ 1, {}, 0, {}, true);
+ test(15, {}, true, 15, {}, true,
+ 1, {}, 0, {}, true);
+
+ test(20, {}, false, 20, {}, false,
+ 2, {}, 0, {}, true);
+ test(20, {}, true, 20, {}, true,
+ 1, {}, 1, {}, false);
+
+ test(0, 3, false, 0, 7, false,
+ 0, {}, 0, {}, true);
+ test(0, 3, true, 0, 3, true,
+ 0, {}, 0, {}, true);
+
+ test(5, 3, false, 5, 7, false,
+ 0, {}, 0, {}, true);
+ test(5, 3, true, 5, 7, true,
+ 0, {}, 0, {}, true);
+
+ test(10, 3, false, 10, 7, false,
+ 0, 4, 0, 6, false);
+ test(10, 3, true, 10, 7, true,
+ 0, 3, 0, 7, false);
+
+ test(15, 3, false, 15, 7, false,
+ 1, {}, 0, {}, true);
+ test(15, 3, true, 15, 7, true,
+ 1, {}, 0, {}, true);
+
+ test(20, 3, false, 20, 7, false,
+ 1, 4, 1, 6, false);
+ test(20, 3, true, 20, 7, true,
+ 1, 3, 1, 7, false);
+ }
+}
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/query_stats/ut/ya.make b/ydb/core/sys_view/query_stats/ut/ya.make
index 89561102ce4..0674a86a62c 100644
--- a/ydb/core/sys_view/query_stats/ut/ya.make
+++ b/ydb/core/sys_view/query_stats/ut/ya.make
@@ -1,25 +1,25 @@
UNITTEST_FOR(ydb/core/sys_view/query_stats)
-
-OWNER(
- monster
- g:kikimr
-)
-
-FORK_SUBTESTS()
-
-SIZE(MEDIUM)
-
-TIMEOUT(600)
-
-PEERDIR(
+
+OWNER(
+ monster
+ g:kikimr
+)
+
+FORK_SUBTESTS()
+
+SIZE(MEDIUM)
+
+TIMEOUT(600)
+
+PEERDIR(
library/cpp/testing/unittest
ydb/core/testlib
-)
-
+)
+
YQL_LAST_ABI_VERSION()
-SRCS(
- query_stats_ut.cpp
-)
-
-END()
+SRCS(
+ query_stats_ut.cpp
+)
+
+END()
diff --git a/ydb/core/sys_view/query_stats/ya.make b/ydb/core/sys_view/query_stats/ya.make
index 8a572dbeae9..7683ef20889 100644
--- a/ydb/core/sys_view/query_stats/ya.make
+++ b/ydb/core/sys_view/query_stats/ya.make
@@ -1,25 +1,25 @@
-LIBRARY()
-
-OWNER(
- monster
- g:kikimr
-)
-
-SRCS(
- query_metrics.h
- query_metrics.cpp
- query_stats.h
- query_stats.cpp
-)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(
+ monster
+ g:kikimr
+)
+
+SRCS(
+ query_metrics.h
+ query_metrics.cpp
+ query_stats.h
+ query_stats.cpp
+)
+
+PEERDIR(
library/cpp/actors/core
ydb/core/base
ydb/core/kqp/runtime
ydb/core/sys_view/common
ydb/core/sys_view/service
-)
-
+)
+
YQL_LAST_ABI_VERSION()
END()
diff --git a/ydb/core/sys_view/scan.cpp b/ydb/core/sys_view/scan.cpp
index f337cb6ba77..97a169b675d 100644
--- a/ydb/core/sys_view/scan.cpp
+++ b/ydb/core/sys_view/scan.cpp
@@ -1,5 +1,5 @@
-#include "scan.h"
-
+#include "scan.h"
+
#include <ydb/core/sys_view/common/schema.h>
#include <ydb/core/sys_view/partition_stats/partition_stats.h>
#include <ydb/core/sys_view/nodes/nodes.h>
@@ -11,63 +11,63 @@
#include <ydb/core/sys_view/storage/storage_pools.h>
#include <ydb/core/sys_view/storage/storage_stats.h>
#include <ydb/core/sys_view/tablets/tablets.h>
-
-namespace NKikimr {
-namespace NSysView {
-
+
+namespace NKikimr {
+namespace NSysView {
+
THolder<IActor> CreateSystemViewScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId,
- const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns)
-{
- if (tableId.SysViewInfo == PartitionStatsName) {
- return CreatePartitionStatsScan(ownerId, scanId, tableId, tableRange, columns);
- }
+ const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns)
+{
+ if (tableId.SysViewInfo == PartitionStatsName) {
+ return CreatePartitionStatsScan(ownerId, scanId, tableId, tableRange, columns);
+ }
- if (tableId.SysViewInfo == NodesName) {
- return CreateNodesScan(ownerId, scanId, tableId, tableRange, columns);
- }
+ if (tableId.SysViewInfo == NodesName) {
+ return CreateNodesScan(ownerId, scanId, tableId, tableRange, columns);
+ }
- if (tableId.SysViewInfo == TopQueriesByDuration1MinuteName ||
- tableId.SysViewInfo == TopQueriesByDuration1HourName ||
- tableId.SysViewInfo == TopQueriesByReadBytes1MinuteName ||
- tableId.SysViewInfo == TopQueriesByReadBytes1HourName ||
- tableId.SysViewInfo == TopQueriesByCpuTime1MinuteName ||
- tableId.SysViewInfo == TopQueriesByCpuTime1HourName ||
- tableId.SysViewInfo == TopQueriesByRequestUnits1MinuteName ||
- tableId.SysViewInfo == TopQueriesByRequestUnits1HourName)
- {
- return CreateQueryStatsScan(ownerId, scanId, tableId, tableId.SysViewInfo, tableRange, columns);
- }
+ if (tableId.SysViewInfo == TopQueriesByDuration1MinuteName ||
+ tableId.SysViewInfo == TopQueriesByDuration1HourName ||
+ tableId.SysViewInfo == TopQueriesByReadBytes1MinuteName ||
+ tableId.SysViewInfo == TopQueriesByReadBytes1HourName ||
+ tableId.SysViewInfo == TopQueriesByCpuTime1MinuteName ||
+ tableId.SysViewInfo == TopQueriesByCpuTime1HourName ||
+ tableId.SysViewInfo == TopQueriesByRequestUnits1MinuteName ||
+ tableId.SysViewInfo == TopQueriesByRequestUnits1HourName)
+ {
+ return CreateQueryStatsScan(ownerId, scanId, tableId, tableId.SysViewInfo, tableRange, columns);
+ }
+
+ if (tableId.SysViewInfo == PDisksName) {
+ return CreatePDisksScan(ownerId, scanId, tableId, tableRange, columns);
+ }
- if (tableId.SysViewInfo == PDisksName) {
- return CreatePDisksScan(ownerId, scanId, tableId, tableRange, columns);
- }
+ if (tableId.SysViewInfo == VSlotsName) {
+ return CreateVSlotsScan(ownerId, scanId, tableId, tableRange, columns);
+ }
- if (tableId.SysViewInfo == VSlotsName) {
- return CreateVSlotsScan(ownerId, scanId, tableId, tableRange, columns);
- }
-
- if (tableId.SysViewInfo == GroupsName) {
- return CreateGroupsScan(ownerId, scanId, tableId, tableRange, columns);
- }
-
- if (tableId.SysViewInfo == StoragePoolsName) {
- return CreateStoragePoolsScan(ownerId, scanId, tableId, tableRange, columns);
- }
+ if (tableId.SysViewInfo == GroupsName) {
+ return CreateGroupsScan(ownerId, scanId, tableId, tableRange, columns);
+ }
+ if (tableId.SysViewInfo == StoragePoolsName) {
+ return CreateStoragePoolsScan(ownerId, scanId, tableId, tableRange, columns);
+ }
+
if (tableId.SysViewInfo == StorageStatsName) {
return CreateStorageStatsScan(ownerId, scanId, tableId, tableRange, columns);
}
- if (tableId.SysViewInfo == TabletsName) {
- return CreateTabletsScan(ownerId, scanId, tableId, tableRange, columns);
- }
-
- if (tableId.SysViewInfo == QueryMetricsName) {
- return CreateQueryMetricsScan(ownerId, scanId, tableId, tableRange, columns);
- }
-
- return nullptr;
-}
-
-} // NSysView
-} // NKikimr
+ if (tableId.SysViewInfo == TabletsName) {
+ return CreateTabletsScan(ownerId, scanId, tableId, tableRange, columns);
+ }
+
+ if (tableId.SysViewInfo == QueryMetricsName) {
+ return CreateQueryMetricsScan(ownerId, scanId, tableId, tableRange, columns);
+ }
+
+ return nullptr;
+}
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/scan.h b/ydb/core/sys_view/scan.h
index bfac578bfae..19034c317dc 100644
--- a/ydb/core/sys_view/scan.h
+++ b/ydb/core/sys_view/scan.h
@@ -1,12 +1,12 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/kqp/runtime/kqp_compute.h>
-
-namespace NKikimr {
-namespace NSysView {
-
+
+namespace NKikimr {
+namespace NSysView {
+
THolder<IActor> CreateSystemViewScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId,
- const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns);
-
-} // NSysView
-} // NKikimr
+ const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns);
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/service/db_counters.cpp b/ydb/core/sys_view/service/db_counters.cpp
index cc3a16d16ac..3395137237b 100644
--- a/ydb/core/sys_view/service/db_counters.cpp
+++ b/ydb/core/sys_view/service/db_counters.cpp
@@ -1,117 +1,117 @@
-#include "db_counters.h"
-
+#include "db_counters.h"
+
#include <ydb/core/base/path.h>
#include <ydb/core/sys_view/common/events.h>
#include <ydb/core/tx/scheme_cache/scheme_cache.h>
-
-#include <library/cpp/actors/core/log.h>
-
-namespace NKikimr {
-namespace NSysView {
-
-class TDbWatcherActor
- : public TActor<TDbWatcherActor>
-{
- using TNavigate = NSchemeCache::TSchemeCacheNavigate;
-
- TIntrusivePtr<TDbWatcherCallback> Callback;
-
- THashMap<TString, TPathId> DatabaseToPathId;
- THashMap<TPathId, TString> PathIdToDatabase;
-
-public:
- static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
- return NKikimrServices::TActivity::DB_WATCHER_ACTOR;
- }
-
- explicit TDbWatcherActor(TIntrusivePtr<TDbWatcherCallback> callback)
- : TActor(&TThis::StateFunc)
- , Callback(callback)
- {
- }
-
-private:
- void Handle(NSysView::TEvSysView::TEvWatchDatabase::TPtr& ev) {
- auto database = ev->Get()->Database;
- auto pathId = ev->Get()->PathId;
-
- if (!database) {
- if (!pathId || PathIdToDatabase.FindPtr(pathId)) {
- return;
- }
- PathIdToDatabase.emplace(pathId, "");
-
- auto key = pathId.Hash();
- Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvWatchPathId(pathId, key));
- return;
- }
-
- if (DatabaseToPathId.FindPtr(database)) {
- return;
- }
- DatabaseToPathId.emplace(database, TPathId());
-
- auto request = MakeHolder<TNavigate>();
- request->ResultSet.push_back({});
-
- auto& entry = request->ResultSet.back();
- entry.Path = SplitPath(database);
- entry.Operation = TNavigate::EOp::OpPath;
- entry.RequestType = TNavigate::TEntry::ERequestType::ByPath;
- entry.RedirectRequired = false;
-
- Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release()));
- }
-
- void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) {
- auto* request = ev->Get()->Request.Get();
- if (request->ResultSet.size() != 1) {
- return;
- }
- auto& entry = request->ResultSet.back();
- auto database = CanonizePath(entry.Path);
- if (entry.Status != TNavigate::EStatus::Ok) {
- DatabaseToPathId.erase(database);
- Callback->OnDatabaseRemoved(database, TPathId());
- return;
- }
- auto pathId = entry.DomainInfo->DomainKey;
- DatabaseToPathId[database] = pathId;
- PathIdToDatabase[pathId] = database;
-
- Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvWatchPathId(pathId));
- }
-
- void Handle(TEvTxProxySchemeCache::TEvWatchNotifyDeleted::TPtr& ev) {
- auto pathId = ev->Get()->PathId;
-
- if (auto* db = PathIdToDatabase.FindPtr(pathId)) {
- auto database = *db;
- if (database) {
- DatabaseToPathId.erase(database);
- }
- PathIdToDatabase.erase(pathId);
-
- Callback->OnDatabaseRemoved(database, pathId);
-
- auto key = pathId.Hash();
- Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvWatchRemove(key));
- }
- }
-
- STRICT_STFUNC(StateFunc, {
- hFunc(NSysView::TEvSysView::TEvWatchDatabase, Handle)
- hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle);
- IgnoreFunc(TEvTxProxySchemeCache::TEvWatchNotifyUpdated);
- IgnoreFunc(TEvTxProxySchemeCache::TEvWatchNotifyUnavailable);
- hFunc(TEvTxProxySchemeCache::TEvWatchNotifyDeleted, Handle);
- })
-};
-
-IActor* CreateDbWatcherActor(TIntrusivePtr<TDbWatcherCallback> callback) {
- return new TDbWatcherActor(callback);
-}
-
-}
-}
-
+
+#include <library/cpp/actors/core/log.h>
+
+namespace NKikimr {
+namespace NSysView {
+
+class TDbWatcherActor
+ : public TActor<TDbWatcherActor>
+{
+ using TNavigate = NSchemeCache::TSchemeCacheNavigate;
+
+ TIntrusivePtr<TDbWatcherCallback> Callback;
+
+ THashMap<TString, TPathId> DatabaseToPathId;
+ THashMap<TPathId, TString> PathIdToDatabase;
+
+public:
+ static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
+ return NKikimrServices::TActivity::DB_WATCHER_ACTOR;
+ }
+
+ explicit TDbWatcherActor(TIntrusivePtr<TDbWatcherCallback> callback)
+ : TActor(&TThis::StateFunc)
+ , Callback(callback)
+ {
+ }
+
+private:
+ void Handle(NSysView::TEvSysView::TEvWatchDatabase::TPtr& ev) {
+ auto database = ev->Get()->Database;
+ auto pathId = ev->Get()->PathId;
+
+ if (!database) {
+ if (!pathId || PathIdToDatabase.FindPtr(pathId)) {
+ return;
+ }
+ PathIdToDatabase.emplace(pathId, "");
+
+ auto key = pathId.Hash();
+ Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvWatchPathId(pathId, key));
+ return;
+ }
+
+ if (DatabaseToPathId.FindPtr(database)) {
+ return;
+ }
+ DatabaseToPathId.emplace(database, TPathId());
+
+ auto request = MakeHolder<TNavigate>();
+ request->ResultSet.push_back({});
+
+ auto& entry = request->ResultSet.back();
+ entry.Path = SplitPath(database);
+ entry.Operation = TNavigate::EOp::OpPath;
+ entry.RequestType = TNavigate::TEntry::ERequestType::ByPath;
+ entry.RedirectRequired = false;
+
+ Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release()));
+ }
+
+ void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) {
+ auto* request = ev->Get()->Request.Get();
+ if (request->ResultSet.size() != 1) {
+ return;
+ }
+ auto& entry = request->ResultSet.back();
+ auto database = CanonizePath(entry.Path);
+ if (entry.Status != TNavigate::EStatus::Ok) {
+ DatabaseToPathId.erase(database);
+ Callback->OnDatabaseRemoved(database, TPathId());
+ return;
+ }
+ auto pathId = entry.DomainInfo->DomainKey;
+ DatabaseToPathId[database] = pathId;
+ PathIdToDatabase[pathId] = database;
+
+ Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvWatchPathId(pathId));
+ }
+
+ void Handle(TEvTxProxySchemeCache::TEvWatchNotifyDeleted::TPtr& ev) {
+ auto pathId = ev->Get()->PathId;
+
+ if (auto* db = PathIdToDatabase.FindPtr(pathId)) {
+ auto database = *db;
+ if (database) {
+ DatabaseToPathId.erase(database);
+ }
+ PathIdToDatabase.erase(pathId);
+
+ Callback->OnDatabaseRemoved(database, pathId);
+
+ auto key = pathId.Hash();
+ Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvWatchRemove(key));
+ }
+ }
+
+ STRICT_STFUNC(StateFunc, {
+ hFunc(NSysView::TEvSysView::TEvWatchDatabase, Handle)
+ hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle);
+ IgnoreFunc(TEvTxProxySchemeCache::TEvWatchNotifyUpdated);
+ IgnoreFunc(TEvTxProxySchemeCache::TEvWatchNotifyUnavailable);
+ hFunc(TEvTxProxySchemeCache::TEvWatchNotifyDeleted, Handle);
+ })
+};
+
+IActor* CreateDbWatcherActor(TIntrusivePtr<TDbWatcherCallback> callback) {
+ return new TDbWatcherActor(callback);
+}
+
+}
+}
+
diff --git a/ydb/core/sys_view/service/db_counters.h b/ydb/core/sys_view/service/db_counters.h
index 92e15e51e2f..9943fa01e75 100644
--- a/ydb/core/sys_view/service/db_counters.h
+++ b/ydb/core/sys_view/service/db_counters.h
@@ -1,15 +1,15 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/scheme/scheme_tabledefs.h>
-
-namespace NKikimr {
-namespace NSysView {
-
-struct TDbWatcherCallback : public TThrRefBase {
- virtual void OnDatabaseRemoved(const TString& database, TPathId pathId) = 0;
-};
-
-IActor* CreateDbWatcherActor(TIntrusivePtr<TDbWatcherCallback> callback);
-
-}
-}
+
+namespace NKikimr {
+namespace NSysView {
+
+struct TDbWatcherCallback : public TThrRefBase {
+ virtual void OnDatabaseRemoved(const TString& database, TPathId pathId) = 0;
+};
+
+IActor* CreateDbWatcherActor(TIntrusivePtr<TDbWatcherCallback> callback);
+
+}
+}
diff --git a/ydb/core/sys_view/service/ext_counters.cpp b/ydb/core/sys_view/service/ext_counters.cpp
index bebbd047862..ae97e7b179e 100644
--- a/ydb/core/sys_view/service/ext_counters.cpp
+++ b/ydb/core/sys_view/service/ext_counters.cpp
@@ -1,109 +1,109 @@
-#include "ext_counters.h"
-
+#include "ext_counters.h"
+
#include <ydb/core/base/appdata.h>
#include <ydb/core/base/counters.h>
-
-namespace NKikimr {
-namespace NSysView {
-
-class TExtCountersUpdaterActor
- : public TActorBootstrapped<TExtCountersUpdaterActor>
-{
- using TCounterPtr = NMonitoring::TDynamicCounters::TCounterPtr;
-
- const TExtCountersConfig Config;
-
- TCounterPtr MemoryUsedBytes;
- TCounterPtr MemoryLimitBytes;
- TVector<TCounterPtr> CpuUsedCorePercents;
- TVector<TCounterPtr> CpuLimitCorePercents;
-
- TCounterPtr AnonRssSize;
- TCounterPtr CGroupMemLimit;
- TVector<TCounterPtr> PoolElapsedMicrosec;
-
-public:
- static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
- return NKikimrServices::TActivity::EXT_COUNTERS_UPDATER_ACTOR;
- }
-
- explicit TExtCountersUpdaterActor(TExtCountersConfig&& config)
- : Config(std::move(config))
- {}
-
- void Bootstrap() {
- auto ydbGroup = GetServiceCounters(AppData()->Counters, "ydb");
-
- MemoryUsedBytes = ydbGroup->GetNamedCounter("name",
- "resources.memory.used_bytes", false);
- MemoryLimitBytes = ydbGroup->GetNamedCounter("name",
- "resources.memory.limit_bytes", false);
-
- auto poolCount = Config.Pools.size();
- CpuUsedCorePercents.resize(poolCount);
- CpuLimitCorePercents.resize(poolCount);
-
- for (size_t i = 0; i < poolCount; ++i) {
- auto name = to_lower(Config.Pools[i].Name);
- CpuUsedCorePercents[i] = ydbGroup->GetSubgroup("pool", name)->GetNamedCounter("name",
- "resources.cpu.used_core_percents", true);
- CpuLimitCorePercents[i] = ydbGroup->GetSubgroup("pool", name)->GetNamedCounter("name",
- "resources.cpu.limit_core_percents", false);
- }
-
- Schedule(TDuration::Seconds(1), new TEvents::TEvWakeup);
- Become(&TThis::StateWork);
- }
-
-private:
- void Initialize() {
- if (!AnonRssSize) {
- auto utilsGroup = GetServiceCounters(AppData()->Counters, "utils");
-
- AnonRssSize = utilsGroup->FindCounter("Process/AnonRssSize");
- CGroupMemLimit = utilsGroup->FindCounter("Process/CGroupMemLimit");
-
- PoolElapsedMicrosec.resize(Config.Pools.size());
- for (size_t i = 0; i < Config.Pools.size(); ++i) {
- auto poolGroup = utilsGroup->FindSubgroup("execpool", Config.Pools[i].Name);
- if (poolGroup) {
- PoolElapsedMicrosec[i] = poolGroup->FindCounter("ElapsedMicrosec");
- }
- }
- }
- }
-
- void Transform() {
- Initialize();
-
- if (AnonRssSize) {
- MemoryUsedBytes->Set(AnonRssSize->Val());
- }
- if (CGroupMemLimit) {
- MemoryLimitBytes->Set(CGroupMemLimit->Val());
- }
- for (size_t i = 0; i < Config.Pools.size(); ++i) {
- if (PoolElapsedMicrosec[i]) {
- CpuUsedCorePercents[i]->Set(PoolElapsedMicrosec[i]->Val() / 10000.);
- CpuLimitCorePercents[i]->Set(Config.Pools[i].ThreadCount * 100);
- }
- }
- }
-
- void HandleWakeup() {
- Schedule(TDuration::Seconds(1), new TEvents::TEvWakeup);
- Transform();
- }
-
- STRICT_STFUNC(StateWork, {
- cFunc(TEvents::TSystem::Wakeup, HandleWakeup);
- })
-};
-
-IActor* CreateExtCountersUpdater(TExtCountersConfig&& config) {
- return new TExtCountersUpdaterActor(std::move(config));
-}
-
-}
-}
-
+
+namespace NKikimr {
+namespace NSysView {
+
+class TExtCountersUpdaterActor
+ : public TActorBootstrapped<TExtCountersUpdaterActor>
+{
+ using TCounterPtr = NMonitoring::TDynamicCounters::TCounterPtr;
+
+ const TExtCountersConfig Config;
+
+ TCounterPtr MemoryUsedBytes;
+ TCounterPtr MemoryLimitBytes;
+ TVector<TCounterPtr> CpuUsedCorePercents;
+ TVector<TCounterPtr> CpuLimitCorePercents;
+
+ TCounterPtr AnonRssSize;
+ TCounterPtr CGroupMemLimit;
+ TVector<TCounterPtr> PoolElapsedMicrosec;
+
+public:
+ static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
+ return NKikimrServices::TActivity::EXT_COUNTERS_UPDATER_ACTOR;
+ }
+
+ explicit TExtCountersUpdaterActor(TExtCountersConfig&& config)
+ : Config(std::move(config))
+ {}
+
+ void Bootstrap() {
+ auto ydbGroup = GetServiceCounters(AppData()->Counters, "ydb");
+
+ MemoryUsedBytes = ydbGroup->GetNamedCounter("name",
+ "resources.memory.used_bytes", false);
+ MemoryLimitBytes = ydbGroup->GetNamedCounter("name",
+ "resources.memory.limit_bytes", false);
+
+ auto poolCount = Config.Pools.size();
+ CpuUsedCorePercents.resize(poolCount);
+ CpuLimitCorePercents.resize(poolCount);
+
+ for (size_t i = 0; i < poolCount; ++i) {
+ auto name = to_lower(Config.Pools[i].Name);
+ CpuUsedCorePercents[i] = ydbGroup->GetSubgroup("pool", name)->GetNamedCounter("name",
+ "resources.cpu.used_core_percents", true);
+ CpuLimitCorePercents[i] = ydbGroup->GetSubgroup("pool", name)->GetNamedCounter("name",
+ "resources.cpu.limit_core_percents", false);
+ }
+
+ Schedule(TDuration::Seconds(1), new TEvents::TEvWakeup);
+ Become(&TThis::StateWork);
+ }
+
+private:
+ void Initialize() {
+ if (!AnonRssSize) {
+ auto utilsGroup = GetServiceCounters(AppData()->Counters, "utils");
+
+ AnonRssSize = utilsGroup->FindCounter("Process/AnonRssSize");
+ CGroupMemLimit = utilsGroup->FindCounter("Process/CGroupMemLimit");
+
+ PoolElapsedMicrosec.resize(Config.Pools.size());
+ for (size_t i = 0; i < Config.Pools.size(); ++i) {
+ auto poolGroup = utilsGroup->FindSubgroup("execpool", Config.Pools[i].Name);
+ if (poolGroup) {
+ PoolElapsedMicrosec[i] = poolGroup->FindCounter("ElapsedMicrosec");
+ }
+ }
+ }
+ }
+
+ void Transform() {
+ Initialize();
+
+ if (AnonRssSize) {
+ MemoryUsedBytes->Set(AnonRssSize->Val());
+ }
+ if (CGroupMemLimit) {
+ MemoryLimitBytes->Set(CGroupMemLimit->Val());
+ }
+ for (size_t i = 0; i < Config.Pools.size(); ++i) {
+ if (PoolElapsedMicrosec[i]) {
+ CpuUsedCorePercents[i]->Set(PoolElapsedMicrosec[i]->Val() / 10000.);
+ CpuLimitCorePercents[i]->Set(Config.Pools[i].ThreadCount * 100);
+ }
+ }
+ }
+
+ void HandleWakeup() {
+ Schedule(TDuration::Seconds(1), new TEvents::TEvWakeup);
+ Transform();
+ }
+
+ STRICT_STFUNC(StateWork, {
+ cFunc(TEvents::TSystem::Wakeup, HandleWakeup);
+ })
+};
+
+IActor* CreateExtCountersUpdater(TExtCountersConfig&& config) {
+ return new TExtCountersUpdaterActor(std::move(config));
+}
+
+}
+}
+
diff --git a/ydb/core/sys_view/service/ext_counters.h b/ydb/core/sys_view/service/ext_counters.h
index 0097591227f..84f91dc9cd1 100644
--- a/ydb/core/sys_view/service/ext_counters.h
+++ b/ydb/core/sys_view/service/ext_counters.h
@@ -1,19 +1,19 @@
-#pragma once
-
-#include <library/cpp/actors/core/actor.h>
-
-namespace NKikimr {
-namespace NSysView {
-
-struct TExtCountersConfig {
- struct TPool {
- TString Name;
- ui32 ThreadCount = 0;
- };
- TVector<TPool> Pools;
-};
-
-NActors::IActor* CreateExtCountersUpdater(TExtCountersConfig&& config);
-
-}
-}
+#pragma once
+
+#include <library/cpp/actors/core/actor.h>
+
+namespace NKikimr {
+namespace NSysView {
+
+struct TExtCountersConfig {
+ struct TPool {
+ TString Name;
+ ui32 ThreadCount = 0;
+ };
+ TVector<TPool> Pools;
+};
+
+NActors::IActor* CreateExtCountersUpdater(TExtCountersConfig&& config);
+
+}
+}
diff --git a/ydb/core/sys_view/service/query_history.h b/ydb/core/sys_view/service/query_history.h
index 08dd23fc734..6d92d789a10 100644
--- a/ydb/core/sys_view/service/query_history.h
+++ b/ydb/core/sys_view/service/query_history.h
@@ -1,511 +1,511 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/protos/sys_view.pb.h>
-
-#include <util/generic/vector.h>
-#include <util/datetime/base.h>
-
-namespace NKikimr {
-namespace NSysView {
-
-// TODO: configs
-constexpr TDuration ONE_MINUTE_BUCKET_SIZE = TDuration::Minutes(1);
-constexpr ui64 ONE_MINUTE_BUCKET_COUNT = 6 * 60; // 6 hours
-
-constexpr TDuration ONE_HOUR_BUCKET_SIZE = TDuration::Hours(1);
-constexpr ui64 ONE_HOUR_BUCKET_COUNT = 24 * 7 * 2; // 2 weeks
-
-constexpr size_t TOP_QUERIES_COUNT = 5;
-
-using TNodeId = ui32;
-using TQueryHash = ui64;
-
-using TQueryStatsPtr = std::shared_ptr<NKikimrSysView::TQueryStats>;
-
-using TQueryTextsByHash = std::unordered_map<TQueryHash, TString>;
-
-struct TReadBytesGreater {
- using TValue = ui64;
-
- static TValue GetValue(const TQueryStatsPtr stats) {
- return stats->GetStats().GetReadBytes();
- }
- bool operator() (const NKikimrSysView::TQueryStats* a, const NKikimrSysView::TQueryStats* b) const {
- return a->GetStats().GetReadBytes() == b->GetStats().GetReadBytes()
- ? a->GetNodeId() > b->GetNodeId()
- : a->GetStats().GetReadBytes() > b->GetStats().GetReadBytes();
- }
- bool operator() (const TQueryStatsPtr a, const TQueryStatsPtr b) const {
- return operator()(a.get(), b.get());
- }
-};
-
-struct TDurationGreater {
- using TValue = ui64;
-
- static TValue GetValue(const TQueryStatsPtr stats) {
- return stats->GetDurationMs();
- }
- bool operator() (const NKikimrSysView::TQueryStats* a, const NKikimrSysView::TQueryStats* b) const {
- return a->GetDurationMs() == b->GetDurationMs()
- ? a->GetNodeId() > b->GetNodeId()
- : a->GetDurationMs() > b->GetDurationMs();
- }
- bool operator() (const TQueryStatsPtr a, const TQueryStatsPtr b) const {
- return operator()(a.get(), b.get());
- }
-};
-
-struct TCpuTimeGreater {
- using TValue = ui64;
-
- static TValue GetValue(const TQueryStatsPtr stats) {
- return stats->GetTotalCpuTimeUs();
- }
- bool operator() (const NKikimrSysView::TQueryStats* a, const NKikimrSysView::TQueryStats* b) const {
- return a->GetTotalCpuTimeUs() == b->GetTotalCpuTimeUs()
- ? a->GetNodeId() > b->GetNodeId()
- : a->GetTotalCpuTimeUs() > b->GetTotalCpuTimeUs();
- }
- bool operator() (const TQueryStatsPtr a, const TQueryStatsPtr b) const {
- return operator()(a.get(), b.get());
- }
-};
-
-struct TRequestUnitsGreater {
- using TValue = ui64;
-
- static TValue GetValue(const TQueryStatsPtr stats) {
- return stats->GetRequestUnits();
- }
- bool operator() (const NKikimrSysView::TQueryStats* a, const NKikimrSysView::TQueryStats* b) const {
- return a->GetRequestUnits() == b->GetRequestUnits()
- ? a->GetNodeId() > b->GetNodeId()
- : a->GetRequestUnits() > b->GetRequestUnits();
- }
- bool operator() (const TQueryStatsPtr a, const TQueryStatsPtr b) const {
- return operator()(a.get(), b.get());
- }
-};
-
-inline void CopyQueryStatsNoText(NKikimrSysView::TQueryStats* to, const NKikimrSysView::TQueryStats& from) {
- to->SetQueryTextHash(from.GetQueryTextHash());
- to->SetDurationMs(from.GetDurationMs());
- to->SetEndTimeMs(from.GetEndTimeMs());
- to->MutableStats()->CopyFrom(from.GetStats());
- if (from.HasUserSID()) {
- to->SetUserSID(from.GetUserSID());
- }
- to->SetParametersSize(from.GetParametersSize());
- if (from.HasCompileDurationMs()) {
- to->SetCompileDurationMs(from.GetCompileDurationMs());
- }
- if (from.HasFromQueryCache()) {
- to->SetFromQueryCache(from.GetFromQueryCache());
- }
- to->SetNodeId(from.GetNodeId());
- to->MutableShardsCpuTimeUs()->CopyFrom(from.GetShardsCpuTimeUs());
- to->MutableComputeCpuTimeUs()->CopyFrom(from.GetComputeCpuTimeUs());
- if (from.HasCompileCpuTimeUs()) {
- to->SetCompileCpuTimeUs(from.GetCompileCpuTimeUs());
- }
- to->SetProcessCpuTimeUs(from.GetProcessCpuTimeUs());
- to->SetTotalCpuTimeUs(from.GetTotalCpuTimeUs());
- to->SetType(from.GetType());
- to->SetRequestUnits(from.GetRequestUnits());
-}
-
-template <typename TGreater>
-struct TQueryStatsBucket {
- TVector<TQueryStatsPtr> Top;
-
- void Add(TQueryStatsPtr stats) {
- auto it = std::lower_bound(Top.begin(), Top.end(), stats, TGreater());
- if (Top.size() == TOP_QUERIES_COUNT && it == Top.end()) {
- return;
- }
- Top.insert(it, stats);
- if (Top.size() > TOP_QUERIES_COUNT) {
- Top.resize(TOP_QUERIES_COUNT);
- }
- }
-
- void Clear() {
- Top.clear();
- }
-
- void Fill(NKikimrSysView::TTopQueryStats* bucket, TQueryTextsByHash& texts) const {
- for (auto& queryStats : Top) {
- CopyQueryStatsNoText(bucket->AddStats(), *queryStats);
- texts.try_emplace(queryStats->GetQueryTextHash(), queryStats->GetQueryText());
- }
- }
-};
-
-template <typename TGreater>
-struct TQueryStatsDedupBucket {
- using TValue = typename TGreater::TValue;
-
- std::unordered_map<TQueryHash, TQueryStatsPtr> Stats; // query hash -> stats
- std::multimap<TValue, TQueryHash> Top; // top by characteristic -> query hash
-
- void Add(TQueryStatsPtr stats) {
- if (!stats->HasQueryTextHash()) {
- return;
- }
-
- auto queryHash = stats->GetQueryTextHash();
- auto hashIt = Stats.find(queryHash);
-
- if (hashIt != Stats.end()) {
- if (!TGreater()(stats, hashIt->second)) {
- return;
- }
-
- TValue old = TGreater::GetValue(hashIt->second);
- const auto range = Top.equal_range(old);
- auto it = range.first;
- for (; it != range.second; ++it) {
- if (it->second == queryHash) {
- break;
- }
- }
-
- Y_VERIFY(it != range.second);
- Top.erase(it);
-
- TValue current = TGreater::GetValue(stats);
- Top.emplace(std::make_pair(current, queryHash));
-
- hashIt->second = stats;
-
- } else {
- if (Top.size() == TOP_QUERIES_COUNT) {
- auto it = Top.begin();
- TValue value = TGreater::GetValue(stats);
- if (value <= it->first) {
- return;
- }
- Stats.erase(it->second);
- Top.erase(it);
- }
-
- TValue current = TGreater::GetValue(stats);
- Top.emplace(std::make_pair(current, queryHash));
- Stats.emplace(std::make_pair(queryHash, stats));
- }
- }
-
- void Clear() {
- Stats.clear();
- Top.clear();
- }
-
- void Swap(TQueryStatsDedupBucket& other) {
- Stats.swap(other.Stats);
- Top.swap(other.Top);
- }
-
- void Fill(NKikimrSysView::TTopQueryStats* bucket, TQueryTextsByHash& texts) const {
- for (auto it = Top.rbegin(); it != Top.rend(); ++it) {
- auto hashIt = Stats.find(it->second);
- Y_VERIFY(hashIt != Stats.end());
-
- auto* dstStats = bucket->AddStats();
- auto& srcStats = *hashIt->second;
- CopyQueryStatsNoText(dstStats, srcStats);
- texts.try_emplace(srcStats.GetQueryTextHash(), srcStats.GetQueryText());
- }
- }
-
- void FillSummary(NKikimrSysView::TEvIntervalQuerySummary::TQuerySet& queries) const {
- for (auto it = Top.rbegin(); it != Top.rend(); ++it) {
- queries.AddValues(it->first);
- queries.AddHashes(it->second);
- }
- }
-
- void FillStats(const NProtoBuf::RepeatedField<ui64>& hashes,
- NProtoBuf::RepeatedPtrField<NKikimrSysView::TQueryStats>& stats) const
- {
- for (auto queryHash : hashes) {
- auto it = Stats.find(queryHash);
- if (it == Stats.end()) {
- continue;
- }
- stats.Add()->CopyFrom(*it->second);
- }
- }
-};
-
-template <typename TGreater>
-struct TAggrQueryStatsBucket
-{
- NKikimrSysView::TTopQueryStats Top;
-
- void Merge(NKikimrSysView::TTopQueryStats&& src, const TQueryTextsByHash& texts) {
- auto& dst = Top;
- NKikimrSysView::TTopQueryStats result;
-
- size_t dstIndex = 0;
- size_t srcIndex = 0;
-
- for (size_t i = 0; i < TOP_QUERIES_COUNT; ++i) {
- auto copySrc = [&result, &texts] (NKikimrSysView::TQueryStats* srcPtr) {
- auto* stats = result.AddStats();
- stats->Swap(srcPtr);
- if (auto it = texts.find(stats->GetQueryTextHash()); it != texts.end()) {
- stats->SetQueryText(it->second);
- }
- };
-
- if (dstIndex == dst.StatsSize()) {
- if (srcIndex == src.StatsSize()) {
- break;
- }
- copySrc(src.MutableStats(srcIndex++));
- } else {
- if (srcIndex == src.StatsSize()) {
- result.AddStats()->Swap(dst.MutableStats(dstIndex++));
- } else {
- auto* dstPtr = dst.MutableStats(dstIndex);
- auto* srcPtr = src.MutableStats(srcIndex);
- if (TGreater()(dstPtr, srcPtr)) {
- result.AddStats()->Swap(dstPtr);
- ++dstIndex;
- } else {
- copySrc(srcPtr);
- ++srcIndex;
- }
- }
- }
- }
-
- Top.Swap(&result);
- }
-
- void Clear() {
- Top.Clear();
- }
-};
-
-template <typename TGreater>
-struct TAggrQueryStatsDedupBucket
-{
- NKikimrSysView::TTopQueryStats Top;
-
- void Merge(NKikimrSysView::TTopQueryStats&& src, const TQueryTextsByHash& texts) {
- auto& dst = Top;
- NKikimrSysView::TTopQueryStats result;
-
- size_t dstIndex = 0;
- size_t srcIndex = 0;
- std::unordered_set<TQueryHash> seenHashes;
-
- while (result.StatsSize() < TOP_QUERIES_COUNT) {
- auto copySrc = [&result, &texts] (NKikimrSysView::TQueryStats* srcPtr) {
- auto* stats = result.AddStats();
- stats->Swap(srcPtr);
- if (auto it = texts.find(stats->GetQueryTextHash()); it != texts.end()) {
- stats->SetQueryText(it->second);
- }
- };
-
- if (dstIndex == dst.StatsSize()) {
- if (srcIndex == src.StatsSize()) {
- break;
- }
- auto* srcPtr = src.MutableStats(srcIndex++);
- auto srcHash = srcPtr->GetQueryTextHash();
- if (seenHashes.find(srcHash) != seenHashes.end()) {
- continue;
- }
- copySrc(srcPtr);
- seenHashes.insert(srcHash);
- } else {
- auto* dstPtr = dst.MutableStats(dstIndex);
- auto dstHash = dstPtr->GetQueryTextHash();
- if (seenHashes.find(dstHash) != seenHashes.end()) {
- ++dstIndex;
- continue;
- }
- if (srcIndex == src.StatsSize()) {
- result.AddStats()->Swap(dstPtr);
- seenHashes.insert(dstHash);
- ++dstIndex;
- continue;
- }
- auto* srcPtr = src.MutableStats(srcIndex);
- auto srcHash = srcPtr->GetQueryTextHash();
- if (seenHashes.find(srcHash) != seenHashes.end()) {
- ++srcIndex;
- continue;
- }
- if (TGreater()(dstPtr, srcPtr)) {
- result.AddStats()->Swap(dstPtr);
- seenHashes.insert(dstHash);
- ++dstIndex;
- } else {
- copySrc(srcPtr);
- seenHashes.insert(srcHash);
- ++srcIndex;
- }
- }
- }
-
- Top.Swap(&result);
- }
-
- void Clear() {
- Top.Clear();
- }
-};
-
-template <typename TBucket>
-class TQueryHistoryBase
-{
-public:
- TQueryHistoryBase(ui64 bucketCount, const TDuration& bucketSize, const TInstant& now)
- : BucketCount(bucketCount)
- , BucketSizeMs(bucketSize.MilliSeconds())
- {
- Y_VERIFY(bucketCount > 0);
- Y_VERIFY(BucketSizeMs > 0);
-
- Buckets.resize(bucketCount);
- LastBucket = now.MilliSeconds() / BucketSizeMs;
- }
-
- void Clear() {
- for (auto& bucket : Buckets) {
- bucket.Clear();
- }
- }
-
- ui64 GetBucketCount() const {
- return BucketCount;
- }
-
- ui64 GetBucketSizeMs() const {
- return BucketSizeMs;
- }
-
- ui64 GetStartBucket() const {
- return LastBucket > BucketCount - 1 ? LastBucket - BucketCount + 1 : 0;
- }
-
- ui64 GetLastBucket() const {
- return LastBucket;
- }
-
- void RefreshBuckets(ui64 bucket) {
- if (bucket > LastBucket) {
- auto toClear = std::min(bucket, LastBucket + BucketCount);
- for (auto b = LastBucket + 1; b <= toClear; ++b) {
- Buckets[b % BucketCount].Clear();
- }
- LastBucket = bucket;
- }
- }
-
-protected:
- const ui64 BucketCount;
- const ui64 BucketSizeMs;
-
- TVector<TBucket> Buckets;
- ui64 LastBucket = 0;
-};
-
-template <typename TGreater>
-class TServiceQueryHistory : public TQueryHistoryBase<TQueryStatsDedupBucket<TGreater>>
-{
- using TBase = TQueryHistoryBase<TQueryStatsDedupBucket<TGreater>>;
-
-public:
- TServiceQueryHistory(ui64 bucketCount, const TDuration& bucketSize, const TInstant& now)
- : TBase(bucketCount, bucketSize, now)
- {}
-
- void Add(TQueryStatsPtr stats) {
- ui64 timestampMs = stats->GetEndTimeMs();
- auto bucket = timestampMs / this->BucketSizeMs;
- if (bucket < this->GetStartBucket()) {
- return; // too old
- }
- this->RefreshBuckets(bucket);
- this->Buckets[bucket % this->BucketCount].Add(stats);
- }
-
- void ToProto(ui64 startBucket, NKikimrSysView::TEvGetQueryStatsResult& message) const {
- ui64 fromBucket = std::max(startBucket, this->GetStartBucket());
-
- TQueryTextsByHash texts;
-
- for (ui64 b = fromBucket; b <= this->LastBucket; ++b) {
- const auto& bucket = this->Buckets[b % this->BucketCount];
- bucket.Fill(message.AddBuckets(), texts);
- }
-
- for (const auto& [hash, text] : texts) {
- auto* hashText = message.AddQueryTexts();
- hashText->SetHash(hash);
- hashText->SetText(text);
- }
-
- message.SetLastBucket(this->LastBucket);
- message.SetBucketSizeMs(this->BucketSizeMs);
- }
-};
-
-template <typename TGreater>
-class TScanQueryHistory : public TQueryHistoryBase<TAggrQueryStatsDedupBucket<TGreater>>
-{
- using TBase = TQueryHistoryBase<TAggrQueryStatsDedupBucket<TGreater>>;
-
-public:
- TScanQueryHistory(ui64 bucketCount, const TDuration& bucketSize, const TInstant& now)
- : TBase(bucketCount, bucketSize, now)
- {}
-
- void Merge(NKikimrSysView::TEvGetQueryStatsResult&& message) {
- if (!message.HasLastBucket() || !message.HasBucketSizeMs()) {
- return;
- }
- if (message.GetBucketSizeMs() != this->BucketSizeMs) {
- return;
- }
-
- TQueryTextsByHash texts;
- for (const auto& hashedText : message.GetQueryTexts()) {
- texts.emplace(std::make_pair(hashedText.GetHash(), hashedText.GetText()));
- }
-
- ui64 lastBucket = message.GetLastBucket();
- this->RefreshBuckets(lastBucket);
-
- Y_VERIFY(lastBucket + 1 >= message.BucketsSize());
-
- ui64 startBucket = lastBucket + 1 - message.BucketsSize();
- ui64 thisStartBucket = this->GetStartBucket();
-
- ui64 shift = thisStartBucket > startBucket ? thisStartBucket - startBucket : 0;
- for (ui64 b = shift; b < message.BucketsSize(); ++b) {
- auto bucket = b + startBucket;
- this->Buckets[bucket % this->BucketCount].Merge(
- std::move(*message.MutableBuckets(b)), texts);
- }
- }
-
- ui64 GetStartBucketIntervalEndMs() const {
- return (this->GetStartBucket() + 1) * this->BucketSizeMs;
- }
-
- const NKikimrSysView::TTopQueryStats& GetBucketTop(ui64 bucket) const {
- Y_VERIFY(bucket >= this->GetStartBucket());
- Y_VERIFY(bucket <= this->GetLastBucket());
- return this->Buckets[bucket % this->BucketCount].Top;
- }
-};
-
-} // NSysView
-} // NKikimr
+
+#include <util/generic/vector.h>
+#include <util/datetime/base.h>
+
+namespace NKikimr {
+namespace NSysView {
+
+// TODO: configs
+constexpr TDuration ONE_MINUTE_BUCKET_SIZE = TDuration::Minutes(1);
+constexpr ui64 ONE_MINUTE_BUCKET_COUNT = 6 * 60; // 6 hours
+
+constexpr TDuration ONE_HOUR_BUCKET_SIZE = TDuration::Hours(1);
+constexpr ui64 ONE_HOUR_BUCKET_COUNT = 24 * 7 * 2; // 2 weeks
+
+constexpr size_t TOP_QUERIES_COUNT = 5;
+
+using TNodeId = ui32;
+using TQueryHash = ui64;
+
+using TQueryStatsPtr = std::shared_ptr<NKikimrSysView::TQueryStats>;
+
+using TQueryTextsByHash = std::unordered_map<TQueryHash, TString>;
+
+struct TReadBytesGreater {
+ using TValue = ui64;
+
+ static TValue GetValue(const TQueryStatsPtr stats) {
+ return stats->GetStats().GetReadBytes();
+ }
+ bool operator() (const NKikimrSysView::TQueryStats* a, const NKikimrSysView::TQueryStats* b) const {
+ return a->GetStats().GetReadBytes() == b->GetStats().GetReadBytes()
+ ? a->GetNodeId() > b->GetNodeId()
+ : a->GetStats().GetReadBytes() > b->GetStats().GetReadBytes();
+ }
+ bool operator() (const TQueryStatsPtr a, const TQueryStatsPtr b) const {
+ return operator()(a.get(), b.get());
+ }
+};
+
+struct TDurationGreater {
+ using TValue = ui64;
+
+ static TValue GetValue(const TQueryStatsPtr stats) {
+ return stats->GetDurationMs();
+ }
+ bool operator() (const NKikimrSysView::TQueryStats* a, const NKikimrSysView::TQueryStats* b) const {
+ return a->GetDurationMs() == b->GetDurationMs()
+ ? a->GetNodeId() > b->GetNodeId()
+ : a->GetDurationMs() > b->GetDurationMs();
+ }
+ bool operator() (const TQueryStatsPtr a, const TQueryStatsPtr b) const {
+ return operator()(a.get(), b.get());
+ }
+};
+
+struct TCpuTimeGreater {
+ using TValue = ui64;
+
+ static TValue GetValue(const TQueryStatsPtr stats) {
+ return stats->GetTotalCpuTimeUs();
+ }
+ bool operator() (const NKikimrSysView::TQueryStats* a, const NKikimrSysView::TQueryStats* b) const {
+ return a->GetTotalCpuTimeUs() == b->GetTotalCpuTimeUs()
+ ? a->GetNodeId() > b->GetNodeId()
+ : a->GetTotalCpuTimeUs() > b->GetTotalCpuTimeUs();
+ }
+ bool operator() (const TQueryStatsPtr a, const TQueryStatsPtr b) const {
+ return operator()(a.get(), b.get());
+ }
+};
+
+struct TRequestUnitsGreater {
+ using TValue = ui64;
+
+ static TValue GetValue(const TQueryStatsPtr stats) {
+ return stats->GetRequestUnits();
+ }
+ bool operator() (const NKikimrSysView::TQueryStats* a, const NKikimrSysView::TQueryStats* b) const {
+ return a->GetRequestUnits() == b->GetRequestUnits()
+ ? a->GetNodeId() > b->GetNodeId()
+ : a->GetRequestUnits() > b->GetRequestUnits();
+ }
+ bool operator() (const TQueryStatsPtr a, const TQueryStatsPtr b) const {
+ return operator()(a.get(), b.get());
+ }
+};
+
+inline void CopyQueryStatsNoText(NKikimrSysView::TQueryStats* to, const NKikimrSysView::TQueryStats& from) {
+ to->SetQueryTextHash(from.GetQueryTextHash());
+ to->SetDurationMs(from.GetDurationMs());
+ to->SetEndTimeMs(from.GetEndTimeMs());
+ to->MutableStats()->CopyFrom(from.GetStats());
+ if (from.HasUserSID()) {
+ to->SetUserSID(from.GetUserSID());
+ }
+ to->SetParametersSize(from.GetParametersSize());
+ if (from.HasCompileDurationMs()) {
+ to->SetCompileDurationMs(from.GetCompileDurationMs());
+ }
+ if (from.HasFromQueryCache()) {
+ to->SetFromQueryCache(from.GetFromQueryCache());
+ }
+ to->SetNodeId(from.GetNodeId());
+ to->MutableShardsCpuTimeUs()->CopyFrom(from.GetShardsCpuTimeUs());
+ to->MutableComputeCpuTimeUs()->CopyFrom(from.GetComputeCpuTimeUs());
+ if (from.HasCompileCpuTimeUs()) {
+ to->SetCompileCpuTimeUs(from.GetCompileCpuTimeUs());
+ }
+ to->SetProcessCpuTimeUs(from.GetProcessCpuTimeUs());
+ to->SetTotalCpuTimeUs(from.GetTotalCpuTimeUs());
+ to->SetType(from.GetType());
+ to->SetRequestUnits(from.GetRequestUnits());
+}
+
+template <typename TGreater>
+struct TQueryStatsBucket {
+ TVector<TQueryStatsPtr> Top;
+
+ void Add(TQueryStatsPtr stats) {
+ auto it = std::lower_bound(Top.begin(), Top.end(), stats, TGreater());
+ if (Top.size() == TOP_QUERIES_COUNT && it == Top.end()) {
+ return;
+ }
+ Top.insert(it, stats);
+ if (Top.size() > TOP_QUERIES_COUNT) {
+ Top.resize(TOP_QUERIES_COUNT);
+ }
+ }
+
+ void Clear() {
+ Top.clear();
+ }
+
+ void Fill(NKikimrSysView::TTopQueryStats* bucket, TQueryTextsByHash& texts) const {
+ for (auto& queryStats : Top) {
+ CopyQueryStatsNoText(bucket->AddStats(), *queryStats);
+ texts.try_emplace(queryStats->GetQueryTextHash(), queryStats->GetQueryText());
+ }
+ }
+};
+
+template <typename TGreater>
+struct TQueryStatsDedupBucket {
+ using TValue = typename TGreater::TValue;
+
+ std::unordered_map<TQueryHash, TQueryStatsPtr> Stats; // query hash -> stats
+ std::multimap<TValue, TQueryHash> Top; // top by characteristic -> query hash
+
+ void Add(TQueryStatsPtr stats) {
+ if (!stats->HasQueryTextHash()) {
+ return;
+ }
+
+ auto queryHash = stats->GetQueryTextHash();
+ auto hashIt = Stats.find(queryHash);
+
+ if (hashIt != Stats.end()) {
+ if (!TGreater()(stats, hashIt->second)) {
+ return;
+ }
+
+ TValue old = TGreater::GetValue(hashIt->second);
+ const auto range = Top.equal_range(old);
+ auto it = range.first;
+ for (; it != range.second; ++it) {
+ if (it->second == queryHash) {
+ break;
+ }
+ }
+
+ Y_VERIFY(it != range.second);
+ Top.erase(it);
+
+ TValue current = TGreater::GetValue(stats);
+ Top.emplace(std::make_pair(current, queryHash));
+
+ hashIt->second = stats;
+
+ } else {
+ if (Top.size() == TOP_QUERIES_COUNT) {
+ auto it = Top.begin();
+ TValue value = TGreater::GetValue(stats);
+ if (value <= it->first) {
+ return;
+ }
+ Stats.erase(it->second);
+ Top.erase(it);
+ }
+
+ TValue current = TGreater::GetValue(stats);
+ Top.emplace(std::make_pair(current, queryHash));
+ Stats.emplace(std::make_pair(queryHash, stats));
+ }
+ }
+
+ void Clear() {
+ Stats.clear();
+ Top.clear();
+ }
+
+ void Swap(TQueryStatsDedupBucket& other) {
+ Stats.swap(other.Stats);
+ Top.swap(other.Top);
+ }
+
+ void Fill(NKikimrSysView::TTopQueryStats* bucket, TQueryTextsByHash& texts) const {
+ for (auto it = Top.rbegin(); it != Top.rend(); ++it) {
+ auto hashIt = Stats.find(it->second);
+ Y_VERIFY(hashIt != Stats.end());
+
+ auto* dstStats = bucket->AddStats();
+ auto& srcStats = *hashIt->second;
+ CopyQueryStatsNoText(dstStats, srcStats);
+ texts.try_emplace(srcStats.GetQueryTextHash(), srcStats.GetQueryText());
+ }
+ }
+
+ void FillSummary(NKikimrSysView::TEvIntervalQuerySummary::TQuerySet& queries) const {
+ for (auto it = Top.rbegin(); it != Top.rend(); ++it) {
+ queries.AddValues(it->first);
+ queries.AddHashes(it->second);
+ }
+ }
+
+ void FillStats(const NProtoBuf::RepeatedField<ui64>& hashes,
+ NProtoBuf::RepeatedPtrField<NKikimrSysView::TQueryStats>& stats) const
+ {
+ for (auto queryHash : hashes) {
+ auto it = Stats.find(queryHash);
+ if (it == Stats.end()) {
+ continue;
+ }
+ stats.Add()->CopyFrom(*it->second);
+ }
+ }
+};
+
+template <typename TGreater>
+struct TAggrQueryStatsBucket
+{
+ NKikimrSysView::TTopQueryStats Top;
+
+ void Merge(NKikimrSysView::TTopQueryStats&& src, const TQueryTextsByHash& texts) {
+ auto& dst = Top;
+ NKikimrSysView::TTopQueryStats result;
+
+ size_t dstIndex = 0;
+ size_t srcIndex = 0;
+
+ for (size_t i = 0; i < TOP_QUERIES_COUNT; ++i) {
+ auto copySrc = [&result, &texts] (NKikimrSysView::TQueryStats* srcPtr) {
+ auto* stats = result.AddStats();
+ stats->Swap(srcPtr);
+ if (auto it = texts.find(stats->GetQueryTextHash()); it != texts.end()) {
+ stats->SetQueryText(it->second);
+ }
+ };
+
+ if (dstIndex == dst.StatsSize()) {
+ if (srcIndex == src.StatsSize()) {
+ break;
+ }
+ copySrc(src.MutableStats(srcIndex++));
+ } else {
+ if (srcIndex == src.StatsSize()) {
+ result.AddStats()->Swap(dst.MutableStats(dstIndex++));
+ } else {
+ auto* dstPtr = dst.MutableStats(dstIndex);
+ auto* srcPtr = src.MutableStats(srcIndex);
+ if (TGreater()(dstPtr, srcPtr)) {
+ result.AddStats()->Swap(dstPtr);
+ ++dstIndex;
+ } else {
+ copySrc(srcPtr);
+ ++srcIndex;
+ }
+ }
+ }
+ }
+
+ Top.Swap(&result);
+ }
+
+ void Clear() {
+ Top.Clear();
+ }
+};
+
+template <typename TGreater>
+struct TAggrQueryStatsDedupBucket
+{
+ NKikimrSysView::TTopQueryStats Top;
+
+ void Merge(NKikimrSysView::TTopQueryStats&& src, const TQueryTextsByHash& texts) {
+ auto& dst = Top;
+ NKikimrSysView::TTopQueryStats result;
+
+ size_t dstIndex = 0;
+ size_t srcIndex = 0;
+ std::unordered_set<TQueryHash> seenHashes;
+
+ while (result.StatsSize() < TOP_QUERIES_COUNT) {
+ auto copySrc = [&result, &texts] (NKikimrSysView::TQueryStats* srcPtr) {
+ auto* stats = result.AddStats();
+ stats->Swap(srcPtr);
+ if (auto it = texts.find(stats->GetQueryTextHash()); it != texts.end()) {
+ stats->SetQueryText(it->second);
+ }
+ };
+
+ if (dstIndex == dst.StatsSize()) {
+ if (srcIndex == src.StatsSize()) {
+ break;
+ }
+ auto* srcPtr = src.MutableStats(srcIndex++);
+ auto srcHash = srcPtr->GetQueryTextHash();
+ if (seenHashes.find(srcHash) != seenHashes.end()) {
+ continue;
+ }
+ copySrc(srcPtr);
+ seenHashes.insert(srcHash);
+ } else {
+ auto* dstPtr = dst.MutableStats(dstIndex);
+ auto dstHash = dstPtr->GetQueryTextHash();
+ if (seenHashes.find(dstHash) != seenHashes.end()) {
+ ++dstIndex;
+ continue;
+ }
+ if (srcIndex == src.StatsSize()) {
+ result.AddStats()->Swap(dstPtr);
+ seenHashes.insert(dstHash);
+ ++dstIndex;
+ continue;
+ }
+ auto* srcPtr = src.MutableStats(srcIndex);
+ auto srcHash = srcPtr->GetQueryTextHash();
+ if (seenHashes.find(srcHash) != seenHashes.end()) {
+ ++srcIndex;
+ continue;
+ }
+ if (TGreater()(dstPtr, srcPtr)) {
+ result.AddStats()->Swap(dstPtr);
+ seenHashes.insert(dstHash);
+ ++dstIndex;
+ } else {
+ copySrc(srcPtr);
+ seenHashes.insert(srcHash);
+ ++srcIndex;
+ }
+ }
+ }
+
+ Top.Swap(&result);
+ }
+
+ void Clear() {
+ Top.Clear();
+ }
+};
+
+template <typename TBucket>
+class TQueryHistoryBase
+{
+public:
+ TQueryHistoryBase(ui64 bucketCount, const TDuration& bucketSize, const TInstant& now)
+ : BucketCount(bucketCount)
+ , BucketSizeMs(bucketSize.MilliSeconds())
+ {
+ Y_VERIFY(bucketCount > 0);
+ Y_VERIFY(BucketSizeMs > 0);
+
+ Buckets.resize(bucketCount);
+ LastBucket = now.MilliSeconds() / BucketSizeMs;
+ }
+
+ void Clear() {
+ for (auto& bucket : Buckets) {
+ bucket.Clear();
+ }
+ }
+
+ ui64 GetBucketCount() const {
+ return BucketCount;
+ }
+
+ ui64 GetBucketSizeMs() const {
+ return BucketSizeMs;
+ }
+
+ ui64 GetStartBucket() const {
+ return LastBucket > BucketCount - 1 ? LastBucket - BucketCount + 1 : 0;
+ }
+
+ ui64 GetLastBucket() const {
+ return LastBucket;
+ }
+
+ void RefreshBuckets(ui64 bucket) {
+ if (bucket > LastBucket) {
+ auto toClear = std::min(bucket, LastBucket + BucketCount);
+ for (auto b = LastBucket + 1; b <= toClear; ++b) {
+ Buckets[b % BucketCount].Clear();
+ }
+ LastBucket = bucket;
+ }
+ }
+
+protected:
+ const ui64 BucketCount;
+ const ui64 BucketSizeMs;
+
+ TVector<TBucket> Buckets;
+ ui64 LastBucket = 0;
+};
+
+template <typename TGreater>
+class TServiceQueryHistory : public TQueryHistoryBase<TQueryStatsDedupBucket<TGreater>>
+{
+ using TBase = TQueryHistoryBase<TQueryStatsDedupBucket<TGreater>>;
+
+public:
+ TServiceQueryHistory(ui64 bucketCount, const TDuration& bucketSize, const TInstant& now)
+ : TBase(bucketCount, bucketSize, now)
+ {}
+
+ void Add(TQueryStatsPtr stats) {
+ ui64 timestampMs = stats->GetEndTimeMs();
+ auto bucket = timestampMs / this->BucketSizeMs;
+ if (bucket < this->GetStartBucket()) {
+ return; // too old
+ }
+ this->RefreshBuckets(bucket);
+ this->Buckets[bucket % this->BucketCount].Add(stats);
+ }
+
+ void ToProto(ui64 startBucket, NKikimrSysView::TEvGetQueryStatsResult& message) const {
+ ui64 fromBucket = std::max(startBucket, this->GetStartBucket());
+
+ TQueryTextsByHash texts;
+
+ for (ui64 b = fromBucket; b <= this->LastBucket; ++b) {
+ const auto& bucket = this->Buckets[b % this->BucketCount];
+ bucket.Fill(message.AddBuckets(), texts);
+ }
+
+ for (const auto& [hash, text] : texts) {
+ auto* hashText = message.AddQueryTexts();
+ hashText->SetHash(hash);
+ hashText->SetText(text);
+ }
+
+ message.SetLastBucket(this->LastBucket);
+ message.SetBucketSizeMs(this->BucketSizeMs);
+ }
+};
+
+template <typename TGreater>
+class TScanQueryHistory : public TQueryHistoryBase<TAggrQueryStatsDedupBucket<TGreater>>
+{
+ using TBase = TQueryHistoryBase<TAggrQueryStatsDedupBucket<TGreater>>;
+
+public:
+ TScanQueryHistory(ui64 bucketCount, const TDuration& bucketSize, const TInstant& now)
+ : TBase(bucketCount, bucketSize, now)
+ {}
+
+ void Merge(NKikimrSysView::TEvGetQueryStatsResult&& message) {
+ if (!message.HasLastBucket() || !message.HasBucketSizeMs()) {
+ return;
+ }
+ if (message.GetBucketSizeMs() != this->BucketSizeMs) {
+ return;
+ }
+
+ TQueryTextsByHash texts;
+ for (const auto& hashedText : message.GetQueryTexts()) {
+ texts.emplace(std::make_pair(hashedText.GetHash(), hashedText.GetText()));
+ }
+
+ ui64 lastBucket = message.GetLastBucket();
+ this->RefreshBuckets(lastBucket);
+
+ Y_VERIFY(lastBucket + 1 >= message.BucketsSize());
+
+ ui64 startBucket = lastBucket + 1 - message.BucketsSize();
+ ui64 thisStartBucket = this->GetStartBucket();
+
+ ui64 shift = thisStartBucket > startBucket ? thisStartBucket - startBucket : 0;
+ for (ui64 b = shift; b < message.BucketsSize(); ++b) {
+ auto bucket = b + startBucket;
+ this->Buckets[bucket % this->BucketCount].Merge(
+ std::move(*message.MutableBuckets(b)), texts);
+ }
+ }
+
+ ui64 GetStartBucketIntervalEndMs() const {
+ return (this->GetStartBucket() + 1) * this->BucketSizeMs;
+ }
+
+ const NKikimrSysView::TTopQueryStats& GetBucketTop(ui64 bucket) const {
+ Y_VERIFY(bucket >= this->GetStartBucket());
+ Y_VERIFY(bucket <= this->GetLastBucket());
+ return this->Buckets[bucket % this->BucketCount].Top;
+ }
+};
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/service/query_history_ut.cpp b/ydb/core/sys_view/service/query_history_ut.cpp
index 9c64e9a4f33..edbbd9007b3 100644
--- a/ydb/core/sys_view/service/query_history_ut.cpp
+++ b/ydb/core/sys_view/service/query_history_ut.cpp
@@ -1,476 +1,476 @@
-#include "query_history.h"
-
+#include "query_history.h"
+
#include <library/cpp/testing/unittest/registar.h>
-
-#include <util/generic/xrange.h>
-#include <util/digest/murmur.h>
-#include <util/random/random.h>
-
-namespace NKikimr {
-namespace NSysView {
-
-namespace {
-
-auto MakeStatsPtr() {
- return std::make_shared<NKikimrSysView::TQueryStats>();
-}
-
-ui64 QueryHash(const TString& queryText) {
- return MurmurHash<ui64>(queryText.data(), queryText.size());
-}
-
-void FillStatsWithNodeId(TQueryTextsByHash& texts, NKikimrSysView::TQueryStats* message, ui32 nodeId,
- const TString& text, ui64 duration)
-{
- message->SetNodeId(nodeId);
-
- auto hash = MurmurHash<ui64>(text.data(), text.size());
- texts[hash] = text;
- message->SetQueryText(text);
- message->SetQueryTextHash(hash);
-
- message->SetDurationMs(duration);
-}
-
-void FillStats(TQueryTextsByHash& texts, NKikimrSysView::TQueryStats* message,
- const TString& text, ui64 duration, ui64 endTime = 0, ui64 readBytes = 0)
-{
- message->SetNodeId(1);
-
- auto hash = MurmurHash<ui64>(text.data(), text.size());
- texts[hash] = text;
- message->SetQueryText(text);
- message->SetQueryTextHash(hash);
-
- message->SetDurationMs(duration);
- message->SetEndTimeMs(endTime);
-
- auto* stats = message->MutableStats();
- stats->SetReadBytes(readBytes);
-}
-
-void FillStats(TQueryTextsByHash& texts, TQueryStatsPtr messagePtr,
- const TString& text, ui64 duration, ui64 endTime = 0, ui64 readBytes = 0)
-{
- FillStats(texts, messagePtr.get(), text, duration, endTime, readBytes);
-}
-
-template <typename TBucket>
-void AddStatsToBucket(TQueryTextsByHash& texts, TBucket& bucket,
- const TString& text, ui64 duration, ui64 endTime = 0, ui64 readBytes = 0)
-{
- auto stats = MakeStatsPtr();
- FillStats(texts, stats.get(), text, duration, endTime, readBytes);
- bucket.Add(stats);
-}
-
-using TResult = NKikimrSysView::TEvGetQueryStatsResult;
-using THistory = TScanQueryHistory<TDurationGreater>;
-
-void FillStatsResult(TQueryTextsByHash& texts, TResult* message, ui64 bucketSizeMs, const TString& text,
- ui64 lastBucket, const std::initializer_list<ui64>& bucketsStatsCount)
-{
- message->SetLastBucket(lastBucket);
- message->SetBucketSizeMs(bucketSizeMs);
-
- auto bucketIndex = lastBucket - bucketsStatsCount.size() + 1;
-
- for (auto count : bucketsStatsCount) {
- auto* bucket = message->AddBuckets();
- for (size_t i : xrange(count)) {
- auto* stats = bucket->AddStats();
- FillStats(texts, stats, text + ToString(i), i, bucketIndex * bucketSizeMs);
- }
- ++bucketIndex;
- }
-}
-
-void ValidateResultBuckets(const THistory& history,
- ui64 lastBucket, const std::initializer_list<ui64>& bucketsStatsCount)
-{
- UNIT_ASSERT_EQUAL(history.GetLastBucket(), lastBucket);
- UNIT_ASSERT_EQUAL(history.GetBucketCount(), bucketsStatsCount.size());
-
- size_t i = history.GetStartBucket();
- for (auto count : bucketsStatsCount) {
- UNIT_ASSERT_EQUAL(history.GetBucketTop(i).StatsSize(), count);
- ++i;
- }
-}
-
-void TestMerge(
- ui64 lastBucket1, const std::initializer_list<ui64>& bucketsStatsCount1,
- ui64 lastBucket2, const std::initializer_list<ui64>& bucketsStatsCount2,
- ui64 lastBucketHistory, ui64 historySize,
- ui64 lastBucketResult, const std::initializer_list<ui64>& bucketsStatsCountResult)
-{
- constexpr ui64 bucketSizeMs = 10;
-
- TQueryTextsByHash texts;
-
- TResult r1;
- FillStatsResult(texts, &r1, bucketSizeMs, "a", lastBucket1, bucketsStatsCount1);
-
- TResult r2;
- FillStatsResult(texts, &r2, bucketSizeMs, "b", lastBucket2, bucketsStatsCount2);
-
- THistory history(historySize, TDuration::MilliSeconds(bucketSizeMs),
- TInstant::MilliSeconds(lastBucketHistory * bucketSizeMs));
-
- history.Merge(std::move(r1));
- history.Merge(std::move(r2));
-
- ValidateResultBuckets(history, lastBucketResult, bucketsStatsCountResult);
-}
-
-} // namespace
-
-Y_UNIT_TEST_SUITE(SysViewQueryHistory) {
-
- Y_UNIT_TEST(TopDurationAdd) {
- TQueryStatsBucket<TDurationGreater> bucket;
-
- TQueryTextsByHash texts;
-
- AddStatsToBucket(texts, bucket, "a", 10);
- AddStatsToBucket(texts, bucket, "b", 20);
- AddStatsToBucket(texts, bucket, "c", 15);
-
- NKikimrSysView::TTopQueryStats proto;
- bucket.Fill(&proto, texts);
-
- UNIT_ASSERT_EQUAL(proto.StatsSize(), 3);
-
- UNIT_ASSERT_EQUAL(proto.GetStats(0).GetDurationMs(), 20);
- UNIT_ASSERT_EQUAL(proto.GetStats(1).GetDurationMs(), 15);
- UNIT_ASSERT_EQUAL(proto.GetStats(2).GetDurationMs(), 10);
- }
-
- Y_UNIT_TEST(TopReadBytesAdd) {
- TQueryStatsBucket<TReadBytesGreater> bucket;
-
- TQueryTextsByHash texts;
-
- AddStatsToBucket(texts, bucket, "a", 10, 0, 7);
- AddStatsToBucket(texts, bucket, "b", 20, 0, 20);
- AddStatsToBucket(texts, bucket, "c", 15, 0, 100);
-
- NKikimrSysView::TTopQueryStats proto;
- bucket.Fill(&proto, texts);
-
- UNIT_ASSERT_EQUAL(proto.StatsSize(), 3);
-
- UNIT_ASSERT_EQUAL(proto.GetStats(0).GetStats().GetReadBytes(), 100);
- UNIT_ASSERT_EQUAL(proto.GetStats(1).GetStats().GetReadBytes(), 20);
- UNIT_ASSERT_EQUAL(proto.GetStats(2).GetStats().GetReadBytes(), 7);
- }
-
- Y_UNIT_TEST(AddDedup) {
- TQueryStatsDedupBucket<TDurationGreater> bucket;
-
- TQueryTextsByHash texts;
-
- AddStatsToBucket(texts, bucket, "a", 10);
- AddStatsToBucket(texts, bucket, "b", 20);
- AddStatsToBucket(texts, bucket, "c", 15);
- AddStatsToBucket(texts, bucket, "b", 5);
- AddStatsToBucket(texts, bucket, "c", 25);
-
- NKikimrSysView::TTopQueryStats proto;
- bucket.Fill(&proto, texts);
-
- UNIT_ASSERT_EQUAL(proto.StatsSize(), 3);
-
- UNIT_ASSERT_EQUAL(proto.GetStats(0).GetDurationMs(), 25);
- UNIT_ASSERT_EQUAL(proto.GetStats(0).GetQueryTextHash(), QueryHash("c"));
- UNIT_ASSERT_EQUAL(proto.GetStats(1).GetDurationMs(), 20);
- UNIT_ASSERT_EQUAL(proto.GetStats(1).GetQueryTextHash(), QueryHash("b"));
- UNIT_ASSERT_EQUAL(proto.GetStats(2).GetDurationMs(), 10);
- UNIT_ASSERT_EQUAL(proto.GetStats(2).GetQueryTextHash(), QueryHash("a"));
- }
-
- Y_UNIT_TEST(AddDedup2) {
- TQueryStatsDedupBucket<TDurationGreater> bucket;
-
- TQueryTextsByHash texts;
-
- for (size_t i : xrange(TOP_QUERIES_COUNT)) {
- AddStatsToBucket(texts, bucket, ToString(i), (i + 1) * 10);
- }
- AddStatsToBucket(texts, bucket, "a", 5);
- AddStatsToBucket(texts, bucket, "b", 15);
-
- NKikimrSysView::TTopQueryStats proto;
- bucket.Fill(&proto, texts);
-
- UNIT_ASSERT_EQUAL(proto.StatsSize(), TOP_QUERIES_COUNT);
- auto last = TOP_QUERIES_COUNT - 1;
-
- UNIT_ASSERT_EQUAL(proto.GetStats(last).GetDurationMs(), 15);
- UNIT_ASSERT_EQUAL(proto.GetStats(last).GetQueryTextHash(), QueryHash("b"));
- }
-
- Y_UNIT_TEST(AddDedupRandom) {
- TQueryStatsDedupBucket<TDurationGreater> bucket;
-
- TQueryTextsByHash texts;
-
- for (size_t i : xrange(10000)) {
- Y_UNUSED(i);
- AddStatsToBucket(texts, bucket, ToString(RandomNumber<ui32>(1000)), RandomNumber<ui64>(5000));
- }
-
- UNIT_ASSERT_EQUAL(bucket.Top.size(), TOP_QUERIES_COUNT);
- UNIT_ASSERT_EQUAL(bucket.Stats.size(), TOP_QUERIES_COUNT);
-
- for (auto& top : bucket.Top) {
- auto it = bucket.Stats.find(top.second);
- UNIT_ASSERT(it != bucket.Stats.end());
- UNIT_ASSERT_EQUAL(it->second->GetDurationMs(), top.first);
- }
- }
-
- Y_UNIT_TEST(AggrMerge) {
- TAggrQueryStatsBucket<TDurationGreater> bucket;
-
- TQueryTextsByHash texts;
-
- NKikimrSysView::TTopQueryStats s1;
- FillStats(texts, s1.AddStats(), "a", 30);
- FillStats(texts, s1.AddStats(), "b", 20);
- FillStats(texts, s1.AddStats(), "c", 10);
-
- NKikimrSysView::TTopQueryStats s2;
- FillStats(texts, s2.AddStats(), "x", 35);
- FillStats(texts, s2.AddStats(), "y", 25);
- FillStats(texts, s2.AddStats(), "z", 22);
-
- bucket.Merge(std::move(s1), texts);
- bucket.Merge(std::move(s2), texts);
-
- UNIT_ASSERT_EQUAL(bucket.Top.StatsSize(), 5);
-
- UNIT_ASSERT_EQUAL(bucket.Top.GetStats(0).GetDurationMs(), 35);
- UNIT_ASSERT_EQUAL(bucket.Top.GetStats(1).GetDurationMs(), 30);
- UNIT_ASSERT_EQUAL(bucket.Top.GetStats(2).GetDurationMs(), 25);
- UNIT_ASSERT_EQUAL(bucket.Top.GetStats(3).GetDurationMs(), 22);
- UNIT_ASSERT_EQUAL(bucket.Top.GetStats(4).GetDurationMs(), 20);
- }
-
- Y_UNIT_TEST(AggrMergeDedup) {
- TAggrQueryStatsDedupBucket<TDurationGreater> bucket;
-
- TQueryTextsByHash texts;
-
- NKikimrSysView::TTopQueryStats s1;
- FillStats(texts, s1.AddStats(), "a", 30);
- FillStats(texts, s1.AddStats(), "b", 20);
- FillStats(texts, s1.AddStats(), "c", 10);
- FillStats(texts, s1.AddStats(), "d", 5);
-
- NKikimrSysView::TTopQueryStats s2;
- FillStats(texts, s2.AddStats(), "c", 35);
- FillStats(texts, s2.AddStats(), "a", 25);
- FillStats(texts, s2.AddStats(), "b", 25);
- FillStats(texts, s2.AddStats(), "e", 15);
-
- bucket.Merge(std::move(s1), texts);
- bucket.Merge(std::move(s2), texts);
-
- UNIT_ASSERT_EQUAL(bucket.Top.StatsSize(), 5);
-
- UNIT_ASSERT_EQUAL(bucket.Top.GetStats(0).GetDurationMs(), 35);
- UNIT_ASSERT_EQUAL(bucket.Top.GetStats(0).GetQueryText(), "c");
-
- UNIT_ASSERT_EQUAL(bucket.Top.GetStats(1).GetDurationMs(), 30);
- UNIT_ASSERT_EQUAL(bucket.Top.GetStats(1).GetQueryText(), "a");
-
- UNIT_ASSERT_EQUAL(bucket.Top.GetStats(2).GetDurationMs(), 25);
- UNIT_ASSERT_EQUAL(bucket.Top.GetStats(2).GetQueryText(), "b");
-
- UNIT_ASSERT_EQUAL(bucket.Top.GetStats(3).GetDurationMs(), 15);
- UNIT_ASSERT_EQUAL(bucket.Top.GetStats(3).GetQueryText(), "e");
-
- UNIT_ASSERT_EQUAL(bucket.Top.GetStats(4).GetDurationMs(), 5);
- UNIT_ASSERT_EQUAL(bucket.Top.GetStats(4).GetQueryText(), "d");
- }
-
- Y_UNIT_TEST(StableMerge) {
- TAggrQueryStatsDedupBucket<TDurationGreater> bucket;
-
- TQueryTextsByHash texts;
-
- NKikimrSysView::TTopQueryStats s1;
- FillStatsWithNodeId(texts, s1.AddStats(), 1, "a", 10);
- FillStatsWithNodeId(texts, s1.AddStats(), 1, "b", 10);
-
- NKikimrSysView::TTopQueryStats s2;
- FillStatsWithNodeId(texts, s2.AddStats(), 2, "x", 10);
- FillStatsWithNodeId(texts, s2.AddStats(), 2, "y", 10);
-
- bucket.Merge(std::move(s1), texts);
-
- bucket.Merge(std::move(s2), texts);
-
- UNIT_ASSERT_EQUAL(bucket.Top.StatsSize(), 4);
-
- UNIT_ASSERT_EQUAL(bucket.Top.GetStats(0).GetQueryText(), "x");
- UNIT_ASSERT_EQUAL(bucket.Top.GetStats(1).GetQueryText(), "y");
- UNIT_ASSERT_EQUAL(bucket.Top.GetStats(2).GetQueryText(), "a");
- UNIT_ASSERT_EQUAL(bucket.Top.GetStats(3).GetQueryText(), "b");
- }
-
- Y_UNIT_TEST(StableMerge2) {
- TAggrQueryStatsDedupBucket<TDurationGreater> bucket;
-
- TQueryTextsByHash texts;
-
- NKikimrSysView::TTopQueryStats s1;
- FillStatsWithNodeId(texts, s1.AddStats(), 1, "a", 10);
- FillStatsWithNodeId(texts, s1.AddStats(), 1, "b", 10);
-
- NKikimrSysView::TTopQueryStats s2;
- FillStatsWithNodeId(texts, s2.AddStats(), 2, "x", 10);
- FillStatsWithNodeId(texts, s2.AddStats(), 2, "y", 10);
-
- bucket.Merge(std::move(s2), texts);
-
- bucket.Merge(std::move(s1), texts);
-
- UNIT_ASSERT_EQUAL(bucket.Top.StatsSize(), 4);
-
- UNIT_ASSERT_EQUAL(bucket.Top.GetStats(0).GetQueryText(), "x");
- UNIT_ASSERT_EQUAL(bucket.Top.GetStats(1).GetQueryText(), "y");
- UNIT_ASSERT_EQUAL(bucket.Top.GetStats(2).GetQueryText(), "a");
- UNIT_ASSERT_EQUAL(bucket.Top.GetStats(3).GetQueryText(), "b");
- }
-
- Y_UNIT_TEST(ServiceQueryHistoryAdd) {
- TServiceQueryHistory<TDurationGreater> history(5, TDuration::MilliSeconds(10), TInstant::MilliSeconds(75));
-
- TQueryTextsByHash texts;
-
- auto s1 = MakeStatsPtr();
- FillStats(texts, s1, "a", 1, 25);
- history.Add(s1);
-
- auto s2 = MakeStatsPtr();
- FillStats(texts, s2, "b", 5, 35);
- history.Add(s2);
-
- auto s3 = MakeStatsPtr();
- FillStats(texts, s3, "c", 1, 50);
- history.Add(s3);
-
- auto s4 = MakeStatsPtr();
- FillStats(texts, s4, "d", 10, 51);
- history.Add(s4);
-
- NKikimrSysView::TEvGetQueryStatsResult result;
- history.ToProto(3, result);
-
- UNIT_ASSERT_EQUAL(result.GetLastBucket(), 7);
- UNIT_ASSERT_EQUAL(result.GetBucketSizeMs(), 10);
-
- UNIT_ASSERT_EQUAL(result.BucketsSize(), 5);
-
- UNIT_ASSERT_EQUAL(result.GetBuckets(0).StatsSize(), 1);
- UNIT_ASSERT_EQUAL(result.GetBuckets(0).GetStats(0).GetEndTimeMs(), 35);
- UNIT_ASSERT_EQUAL(result.GetBuckets(0).GetStats(0).GetDurationMs(), 5);
-
- UNIT_ASSERT_EQUAL(result.GetBuckets(1).StatsSize(), 0);
-
- UNIT_ASSERT_EQUAL(result.GetBuckets(2).StatsSize(), 2);
- UNIT_ASSERT_EQUAL(result.GetBuckets(2).GetStats(0).GetEndTimeMs(), 51);
- UNIT_ASSERT_EQUAL(result.GetBuckets(2).GetStats(0).GetDurationMs(), 10);
- UNIT_ASSERT_EQUAL(result.GetBuckets(2).GetStats(1).GetEndTimeMs(), 50);
- UNIT_ASSERT_EQUAL(result.GetBuckets(2).GetStats(1).GetDurationMs(), 1);
-
- UNIT_ASSERT_EQUAL(result.GetBuckets(3).StatsSize(), 0);
- UNIT_ASSERT_EQUAL(result.GetBuckets(4).StatsSize(), 0);
-
- UNIT_ASSERT_EQUAL(result.QueryTextsSize(), 3);
-
- auto s5 = MakeStatsPtr();
- FillStats(texts, s5, "e", 5, 93);
- history.Add(s5);
-
- result.Clear();
- history.ToProto(5, result);
-
- UNIT_ASSERT_EQUAL(result.GetLastBucket(), 9);
-
- UNIT_ASSERT_EQUAL(result.BucketsSize(), 5);
-
- UNIT_ASSERT_EQUAL(result.GetBuckets(0).StatsSize(), 2);
- UNIT_ASSERT_EQUAL(result.GetBuckets(1).StatsSize(), 0);
- UNIT_ASSERT_EQUAL(result.GetBuckets(2).StatsSize(), 0);
- UNIT_ASSERT_EQUAL(result.GetBuckets(3).StatsSize(), 0);
- UNIT_ASSERT_EQUAL(result.GetBuckets(4).StatsSize(), 1);
-
- UNIT_ASSERT_EQUAL(result.GetBuckets(4).GetStats(0).GetQueryTextHash(), QueryHash("e"));
-
- UNIT_ASSERT_EQUAL(result.QueryTextsSize(), 3);
-
- result.Clear();
- history.ToProto(7, result);
-
- UNIT_ASSERT_EQUAL(result.GetLastBucket(), 9);
- UNIT_ASSERT_EQUAL(result.BucketsSize(), 3);
-
- UNIT_ASSERT_EQUAL(result.GetBuckets(0).StatsSize(), 0);
- UNIT_ASSERT_EQUAL(result.GetBuckets(1).StatsSize(), 0);
- UNIT_ASSERT_EQUAL(result.GetBuckets(2).StatsSize(), 1);
-
- UNIT_ASSERT_EQUAL(result.GetBuckets(2).GetStats(0).GetQueryTextHash(), QueryHash("e"));
-
- UNIT_ASSERT_EQUAL(result.QueryTextsSize(), 1);
- }
-
- Y_UNIT_TEST(ScanQueryHistoryMerge) {
- TestMerge(
- 100, {1, 2, 3},
- 100, {1, 2},
- 100, 3,
- 100, {1, 3, 5});
-
- TestMerge(
- 100, {1, 2, 3},
- 99, {1, 2},
- 100, 3,
- 100, {2, 4, 3});
-
- TestMerge(
- 100, {1, 2, 3},
- 98, {1, 2},
- 100, 3,
- 100, {3, 2, 3});
-
- TestMerge(
- 100, {1, 2, 3},
- 100, {1, 2},
- 90, 5,
- 100, {0, 0, 1, 3, 5});
-
- TestMerge(
- 100, {1, 2, 3, 4},
- 99, {1, 2},
- 101, 5,
- 101, {1, 3, 5, 4, 0});
-
- TestMerge(
- 100, {1, 2, 3},
- 102, {1, 2, 3, 4},
- 90, 5,
- 102, {1, 3, 5, 3, 4});
-
- TestMerge(
- 100, {1, 2, 3},
- 101, {2, 1, 2, 3},
- 90, 2,
- 101, {5, 3});
- }
-}
-
-} // NSysView
-} // NKikimr
+
+#include <util/generic/xrange.h>
+#include <util/digest/murmur.h>
+#include <util/random/random.h>
+
+namespace NKikimr {
+namespace NSysView {
+
+namespace {
+
+auto MakeStatsPtr() {
+ return std::make_shared<NKikimrSysView::TQueryStats>();
+}
+
+ui64 QueryHash(const TString& queryText) {
+ return MurmurHash<ui64>(queryText.data(), queryText.size());
+}
+
+void FillStatsWithNodeId(TQueryTextsByHash& texts, NKikimrSysView::TQueryStats* message, ui32 nodeId,
+ const TString& text, ui64 duration)
+{
+ message->SetNodeId(nodeId);
+
+ auto hash = MurmurHash<ui64>(text.data(), text.size());
+ texts[hash] = text;
+ message->SetQueryText(text);
+ message->SetQueryTextHash(hash);
+
+ message->SetDurationMs(duration);
+}
+
+void FillStats(TQueryTextsByHash& texts, NKikimrSysView::TQueryStats* message,
+ const TString& text, ui64 duration, ui64 endTime = 0, ui64 readBytes = 0)
+{
+ message->SetNodeId(1);
+
+ auto hash = MurmurHash<ui64>(text.data(), text.size());
+ texts[hash] = text;
+ message->SetQueryText(text);
+ message->SetQueryTextHash(hash);
+
+ message->SetDurationMs(duration);
+ message->SetEndTimeMs(endTime);
+
+ auto* stats = message->MutableStats();
+ stats->SetReadBytes(readBytes);
+}
+
+void FillStats(TQueryTextsByHash& texts, TQueryStatsPtr messagePtr,
+ const TString& text, ui64 duration, ui64 endTime = 0, ui64 readBytes = 0)
+{
+ FillStats(texts, messagePtr.get(), text, duration, endTime, readBytes);
+}
+
+template <typename TBucket>
+void AddStatsToBucket(TQueryTextsByHash& texts, TBucket& bucket,
+ const TString& text, ui64 duration, ui64 endTime = 0, ui64 readBytes = 0)
+{
+ auto stats = MakeStatsPtr();
+ FillStats(texts, stats.get(), text, duration, endTime, readBytes);
+ bucket.Add(stats);
+}
+
+using TResult = NKikimrSysView::TEvGetQueryStatsResult;
+using THistory = TScanQueryHistory<TDurationGreater>;
+
+void FillStatsResult(TQueryTextsByHash& texts, TResult* message, ui64 bucketSizeMs, const TString& text,
+ ui64 lastBucket, const std::initializer_list<ui64>& bucketsStatsCount)
+{
+ message->SetLastBucket(lastBucket);
+ message->SetBucketSizeMs(bucketSizeMs);
+
+ auto bucketIndex = lastBucket - bucketsStatsCount.size() + 1;
+
+ for (auto count : bucketsStatsCount) {
+ auto* bucket = message->AddBuckets();
+ for (size_t i : xrange(count)) {
+ auto* stats = bucket->AddStats();
+ FillStats(texts, stats, text + ToString(i), i, bucketIndex * bucketSizeMs);
+ }
+ ++bucketIndex;
+ }
+}
+
+void ValidateResultBuckets(const THistory& history,
+ ui64 lastBucket, const std::initializer_list<ui64>& bucketsStatsCount)
+{
+ UNIT_ASSERT_EQUAL(history.GetLastBucket(), lastBucket);
+ UNIT_ASSERT_EQUAL(history.GetBucketCount(), bucketsStatsCount.size());
+
+ size_t i = history.GetStartBucket();
+ for (auto count : bucketsStatsCount) {
+ UNIT_ASSERT_EQUAL(history.GetBucketTop(i).StatsSize(), count);
+ ++i;
+ }
+}
+
+void TestMerge(
+ ui64 lastBucket1, const std::initializer_list<ui64>& bucketsStatsCount1,
+ ui64 lastBucket2, const std::initializer_list<ui64>& bucketsStatsCount2,
+ ui64 lastBucketHistory, ui64 historySize,
+ ui64 lastBucketResult, const std::initializer_list<ui64>& bucketsStatsCountResult)
+{
+ constexpr ui64 bucketSizeMs = 10;
+
+ TQueryTextsByHash texts;
+
+ TResult r1;
+ FillStatsResult(texts, &r1, bucketSizeMs, "a", lastBucket1, bucketsStatsCount1);
+
+ TResult r2;
+ FillStatsResult(texts, &r2, bucketSizeMs, "b", lastBucket2, bucketsStatsCount2);
+
+ THistory history(historySize, TDuration::MilliSeconds(bucketSizeMs),
+ TInstant::MilliSeconds(lastBucketHistory * bucketSizeMs));
+
+ history.Merge(std::move(r1));
+ history.Merge(std::move(r2));
+
+ ValidateResultBuckets(history, lastBucketResult, bucketsStatsCountResult);
+}
+
+} // namespace
+
+Y_UNIT_TEST_SUITE(SysViewQueryHistory) {
+
+ Y_UNIT_TEST(TopDurationAdd) {
+ TQueryStatsBucket<TDurationGreater> bucket;
+
+ TQueryTextsByHash texts;
+
+ AddStatsToBucket(texts, bucket, "a", 10);
+ AddStatsToBucket(texts, bucket, "b", 20);
+ AddStatsToBucket(texts, bucket, "c", 15);
+
+ NKikimrSysView::TTopQueryStats proto;
+ bucket.Fill(&proto, texts);
+
+ UNIT_ASSERT_EQUAL(proto.StatsSize(), 3);
+
+ UNIT_ASSERT_EQUAL(proto.GetStats(0).GetDurationMs(), 20);
+ UNIT_ASSERT_EQUAL(proto.GetStats(1).GetDurationMs(), 15);
+ UNIT_ASSERT_EQUAL(proto.GetStats(2).GetDurationMs(), 10);
+ }
+
+ Y_UNIT_TEST(TopReadBytesAdd) {
+ TQueryStatsBucket<TReadBytesGreater> bucket;
+
+ TQueryTextsByHash texts;
+
+ AddStatsToBucket(texts, bucket, "a", 10, 0, 7);
+ AddStatsToBucket(texts, bucket, "b", 20, 0, 20);
+ AddStatsToBucket(texts, bucket, "c", 15, 0, 100);
+
+ NKikimrSysView::TTopQueryStats proto;
+ bucket.Fill(&proto, texts);
+
+ UNIT_ASSERT_EQUAL(proto.StatsSize(), 3);
+
+ UNIT_ASSERT_EQUAL(proto.GetStats(0).GetStats().GetReadBytes(), 100);
+ UNIT_ASSERT_EQUAL(proto.GetStats(1).GetStats().GetReadBytes(), 20);
+ UNIT_ASSERT_EQUAL(proto.GetStats(2).GetStats().GetReadBytes(), 7);
+ }
+
+ Y_UNIT_TEST(AddDedup) {
+ TQueryStatsDedupBucket<TDurationGreater> bucket;
+
+ TQueryTextsByHash texts;
+
+ AddStatsToBucket(texts, bucket, "a", 10);
+ AddStatsToBucket(texts, bucket, "b", 20);
+ AddStatsToBucket(texts, bucket, "c", 15);
+ AddStatsToBucket(texts, bucket, "b", 5);
+ AddStatsToBucket(texts, bucket, "c", 25);
+
+ NKikimrSysView::TTopQueryStats proto;
+ bucket.Fill(&proto, texts);
+
+ UNIT_ASSERT_EQUAL(proto.StatsSize(), 3);
+
+ UNIT_ASSERT_EQUAL(proto.GetStats(0).GetDurationMs(), 25);
+ UNIT_ASSERT_EQUAL(proto.GetStats(0).GetQueryTextHash(), QueryHash("c"));
+ UNIT_ASSERT_EQUAL(proto.GetStats(1).GetDurationMs(), 20);
+ UNIT_ASSERT_EQUAL(proto.GetStats(1).GetQueryTextHash(), QueryHash("b"));
+ UNIT_ASSERT_EQUAL(proto.GetStats(2).GetDurationMs(), 10);
+ UNIT_ASSERT_EQUAL(proto.GetStats(2).GetQueryTextHash(), QueryHash("a"));
+ }
+
+ Y_UNIT_TEST(AddDedup2) {
+ TQueryStatsDedupBucket<TDurationGreater> bucket;
+
+ TQueryTextsByHash texts;
+
+ for (size_t i : xrange(TOP_QUERIES_COUNT)) {
+ AddStatsToBucket(texts, bucket, ToString(i), (i + 1) * 10);
+ }
+ AddStatsToBucket(texts, bucket, "a", 5);
+ AddStatsToBucket(texts, bucket, "b", 15);
+
+ NKikimrSysView::TTopQueryStats proto;
+ bucket.Fill(&proto, texts);
+
+ UNIT_ASSERT_EQUAL(proto.StatsSize(), TOP_QUERIES_COUNT);
+ auto last = TOP_QUERIES_COUNT - 1;
+
+ UNIT_ASSERT_EQUAL(proto.GetStats(last).GetDurationMs(), 15);
+ UNIT_ASSERT_EQUAL(proto.GetStats(last).GetQueryTextHash(), QueryHash("b"));
+ }
+
+ Y_UNIT_TEST(AddDedupRandom) {
+ TQueryStatsDedupBucket<TDurationGreater> bucket;
+
+ TQueryTextsByHash texts;
+
+ for (size_t i : xrange(10000)) {
+ Y_UNUSED(i);
+ AddStatsToBucket(texts, bucket, ToString(RandomNumber<ui32>(1000)), RandomNumber<ui64>(5000));
+ }
+
+ UNIT_ASSERT_EQUAL(bucket.Top.size(), TOP_QUERIES_COUNT);
+ UNIT_ASSERT_EQUAL(bucket.Stats.size(), TOP_QUERIES_COUNT);
+
+ for (auto& top : bucket.Top) {
+ auto it = bucket.Stats.find(top.second);
+ UNIT_ASSERT(it != bucket.Stats.end());
+ UNIT_ASSERT_EQUAL(it->second->GetDurationMs(), top.first);
+ }
+ }
+
+ Y_UNIT_TEST(AggrMerge) {
+ TAggrQueryStatsBucket<TDurationGreater> bucket;
+
+ TQueryTextsByHash texts;
+
+ NKikimrSysView::TTopQueryStats s1;
+ FillStats(texts, s1.AddStats(), "a", 30);
+ FillStats(texts, s1.AddStats(), "b", 20);
+ FillStats(texts, s1.AddStats(), "c", 10);
+
+ NKikimrSysView::TTopQueryStats s2;
+ FillStats(texts, s2.AddStats(), "x", 35);
+ FillStats(texts, s2.AddStats(), "y", 25);
+ FillStats(texts, s2.AddStats(), "z", 22);
+
+ bucket.Merge(std::move(s1), texts);
+ bucket.Merge(std::move(s2), texts);
+
+ UNIT_ASSERT_EQUAL(bucket.Top.StatsSize(), 5);
+
+ UNIT_ASSERT_EQUAL(bucket.Top.GetStats(0).GetDurationMs(), 35);
+ UNIT_ASSERT_EQUAL(bucket.Top.GetStats(1).GetDurationMs(), 30);
+ UNIT_ASSERT_EQUAL(bucket.Top.GetStats(2).GetDurationMs(), 25);
+ UNIT_ASSERT_EQUAL(bucket.Top.GetStats(3).GetDurationMs(), 22);
+ UNIT_ASSERT_EQUAL(bucket.Top.GetStats(4).GetDurationMs(), 20);
+ }
+
+ Y_UNIT_TEST(AggrMergeDedup) {
+ TAggrQueryStatsDedupBucket<TDurationGreater> bucket;
+
+ TQueryTextsByHash texts;
+
+ NKikimrSysView::TTopQueryStats s1;
+ FillStats(texts, s1.AddStats(), "a", 30);
+ FillStats(texts, s1.AddStats(), "b", 20);
+ FillStats(texts, s1.AddStats(), "c", 10);
+ FillStats(texts, s1.AddStats(), "d", 5);
+
+ NKikimrSysView::TTopQueryStats s2;
+ FillStats(texts, s2.AddStats(), "c", 35);
+ FillStats(texts, s2.AddStats(), "a", 25);
+ FillStats(texts, s2.AddStats(), "b", 25);
+ FillStats(texts, s2.AddStats(), "e", 15);
+
+ bucket.Merge(std::move(s1), texts);
+ bucket.Merge(std::move(s2), texts);
+
+ UNIT_ASSERT_EQUAL(bucket.Top.StatsSize(), 5);
+
+ UNIT_ASSERT_EQUAL(bucket.Top.GetStats(0).GetDurationMs(), 35);
+ UNIT_ASSERT_EQUAL(bucket.Top.GetStats(0).GetQueryText(), "c");
+
+ UNIT_ASSERT_EQUAL(bucket.Top.GetStats(1).GetDurationMs(), 30);
+ UNIT_ASSERT_EQUAL(bucket.Top.GetStats(1).GetQueryText(), "a");
+
+ UNIT_ASSERT_EQUAL(bucket.Top.GetStats(2).GetDurationMs(), 25);
+ UNIT_ASSERT_EQUAL(bucket.Top.GetStats(2).GetQueryText(), "b");
+
+ UNIT_ASSERT_EQUAL(bucket.Top.GetStats(3).GetDurationMs(), 15);
+ UNIT_ASSERT_EQUAL(bucket.Top.GetStats(3).GetQueryText(), "e");
+
+ UNIT_ASSERT_EQUAL(bucket.Top.GetStats(4).GetDurationMs(), 5);
+ UNIT_ASSERT_EQUAL(bucket.Top.GetStats(4).GetQueryText(), "d");
+ }
+
+ Y_UNIT_TEST(StableMerge) {
+ TAggrQueryStatsDedupBucket<TDurationGreater> bucket;
+
+ TQueryTextsByHash texts;
+
+ NKikimrSysView::TTopQueryStats s1;
+ FillStatsWithNodeId(texts, s1.AddStats(), 1, "a", 10);
+ FillStatsWithNodeId(texts, s1.AddStats(), 1, "b", 10);
+
+ NKikimrSysView::TTopQueryStats s2;
+ FillStatsWithNodeId(texts, s2.AddStats(), 2, "x", 10);
+ FillStatsWithNodeId(texts, s2.AddStats(), 2, "y", 10);
+
+ bucket.Merge(std::move(s1), texts);
+
+ bucket.Merge(std::move(s2), texts);
+
+ UNIT_ASSERT_EQUAL(bucket.Top.StatsSize(), 4);
+
+ UNIT_ASSERT_EQUAL(bucket.Top.GetStats(0).GetQueryText(), "x");
+ UNIT_ASSERT_EQUAL(bucket.Top.GetStats(1).GetQueryText(), "y");
+ UNIT_ASSERT_EQUAL(bucket.Top.GetStats(2).GetQueryText(), "a");
+ UNIT_ASSERT_EQUAL(bucket.Top.GetStats(3).GetQueryText(), "b");
+ }
+
+ Y_UNIT_TEST(StableMerge2) {
+ TAggrQueryStatsDedupBucket<TDurationGreater> bucket;
+
+ TQueryTextsByHash texts;
+
+ NKikimrSysView::TTopQueryStats s1;
+ FillStatsWithNodeId(texts, s1.AddStats(), 1, "a", 10);
+ FillStatsWithNodeId(texts, s1.AddStats(), 1, "b", 10);
+
+ NKikimrSysView::TTopQueryStats s2;
+ FillStatsWithNodeId(texts, s2.AddStats(), 2, "x", 10);
+ FillStatsWithNodeId(texts, s2.AddStats(), 2, "y", 10);
+
+ bucket.Merge(std::move(s2), texts);
+
+ bucket.Merge(std::move(s1), texts);
+
+ UNIT_ASSERT_EQUAL(bucket.Top.StatsSize(), 4);
+
+ UNIT_ASSERT_EQUAL(bucket.Top.GetStats(0).GetQueryText(), "x");
+ UNIT_ASSERT_EQUAL(bucket.Top.GetStats(1).GetQueryText(), "y");
+ UNIT_ASSERT_EQUAL(bucket.Top.GetStats(2).GetQueryText(), "a");
+ UNIT_ASSERT_EQUAL(bucket.Top.GetStats(3).GetQueryText(), "b");
+ }
+
+ Y_UNIT_TEST(ServiceQueryHistoryAdd) {
+ TServiceQueryHistory<TDurationGreater> history(5, TDuration::MilliSeconds(10), TInstant::MilliSeconds(75));
+
+ TQueryTextsByHash texts;
+
+ auto s1 = MakeStatsPtr();
+ FillStats(texts, s1, "a", 1, 25);
+ history.Add(s1);
+
+ auto s2 = MakeStatsPtr();
+ FillStats(texts, s2, "b", 5, 35);
+ history.Add(s2);
+
+ auto s3 = MakeStatsPtr();
+ FillStats(texts, s3, "c", 1, 50);
+ history.Add(s3);
+
+ auto s4 = MakeStatsPtr();
+ FillStats(texts, s4, "d", 10, 51);
+ history.Add(s4);
+
+ NKikimrSysView::TEvGetQueryStatsResult result;
+ history.ToProto(3, result);
+
+ UNIT_ASSERT_EQUAL(result.GetLastBucket(), 7);
+ UNIT_ASSERT_EQUAL(result.GetBucketSizeMs(), 10);
+
+ UNIT_ASSERT_EQUAL(result.BucketsSize(), 5);
+
+ UNIT_ASSERT_EQUAL(result.GetBuckets(0).StatsSize(), 1);
+ UNIT_ASSERT_EQUAL(result.GetBuckets(0).GetStats(0).GetEndTimeMs(), 35);
+ UNIT_ASSERT_EQUAL(result.GetBuckets(0).GetStats(0).GetDurationMs(), 5);
+
+ UNIT_ASSERT_EQUAL(result.GetBuckets(1).StatsSize(), 0);
+
+ UNIT_ASSERT_EQUAL(result.GetBuckets(2).StatsSize(), 2);
+ UNIT_ASSERT_EQUAL(result.GetBuckets(2).GetStats(0).GetEndTimeMs(), 51);
+ UNIT_ASSERT_EQUAL(result.GetBuckets(2).GetStats(0).GetDurationMs(), 10);
+ UNIT_ASSERT_EQUAL(result.GetBuckets(2).GetStats(1).GetEndTimeMs(), 50);
+ UNIT_ASSERT_EQUAL(result.GetBuckets(2).GetStats(1).GetDurationMs(), 1);
+
+ UNIT_ASSERT_EQUAL(result.GetBuckets(3).StatsSize(), 0);
+ UNIT_ASSERT_EQUAL(result.GetBuckets(4).StatsSize(), 0);
+
+ UNIT_ASSERT_EQUAL(result.QueryTextsSize(), 3);
+
+ auto s5 = MakeStatsPtr();
+ FillStats(texts, s5, "e", 5, 93);
+ history.Add(s5);
+
+ result.Clear();
+ history.ToProto(5, result);
+
+ UNIT_ASSERT_EQUAL(result.GetLastBucket(), 9);
+
+ UNIT_ASSERT_EQUAL(result.BucketsSize(), 5);
+
+ UNIT_ASSERT_EQUAL(result.GetBuckets(0).StatsSize(), 2);
+ UNIT_ASSERT_EQUAL(result.GetBuckets(1).StatsSize(), 0);
+ UNIT_ASSERT_EQUAL(result.GetBuckets(2).StatsSize(), 0);
+ UNIT_ASSERT_EQUAL(result.GetBuckets(3).StatsSize(), 0);
+ UNIT_ASSERT_EQUAL(result.GetBuckets(4).StatsSize(), 1);
+
+ UNIT_ASSERT_EQUAL(result.GetBuckets(4).GetStats(0).GetQueryTextHash(), QueryHash("e"));
+
+ UNIT_ASSERT_EQUAL(result.QueryTextsSize(), 3);
+
+ result.Clear();
+ history.ToProto(7, result);
+
+ UNIT_ASSERT_EQUAL(result.GetLastBucket(), 9);
+ UNIT_ASSERT_EQUAL(result.BucketsSize(), 3);
+
+ UNIT_ASSERT_EQUAL(result.GetBuckets(0).StatsSize(), 0);
+ UNIT_ASSERT_EQUAL(result.GetBuckets(1).StatsSize(), 0);
+ UNIT_ASSERT_EQUAL(result.GetBuckets(2).StatsSize(), 1);
+
+ UNIT_ASSERT_EQUAL(result.GetBuckets(2).GetStats(0).GetQueryTextHash(), QueryHash("e"));
+
+ UNIT_ASSERT_EQUAL(result.QueryTextsSize(), 1);
+ }
+
+ Y_UNIT_TEST(ScanQueryHistoryMerge) {
+ TestMerge(
+ 100, {1, 2, 3},
+ 100, {1, 2},
+ 100, 3,
+ 100, {1, 3, 5});
+
+ TestMerge(
+ 100, {1, 2, 3},
+ 99, {1, 2},
+ 100, 3,
+ 100, {2, 4, 3});
+
+ TestMerge(
+ 100, {1, 2, 3},
+ 98, {1, 2},
+ 100, 3,
+ 100, {3, 2, 3});
+
+ TestMerge(
+ 100, {1, 2, 3},
+ 100, {1, 2},
+ 90, 5,
+ 100, {0, 0, 1, 3, 5});
+
+ TestMerge(
+ 100, {1, 2, 3, 4},
+ 99, {1, 2},
+ 101, 5,
+ 101, {1, 3, 5, 4, 0});
+
+ TestMerge(
+ 100, {1, 2, 3},
+ 102, {1, 2, 3, 4},
+ 90, 5,
+ 102, {1, 3, 5, 3, 4});
+
+ TestMerge(
+ 100, {1, 2, 3},
+ 101, {2, 1, 2, 3},
+ 90, 2,
+ 101, {5, 3});
+ }
+}
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/service/query_interval.cpp b/ydb/core/sys_view/service/query_interval.cpp
index daa4241b882..a19feff095e 100644
--- a/ydb/core/sys_view/service/query_interval.cpp
+++ b/ydb/core/sys_view/service/query_interval.cpp
@@ -1,155 +1,155 @@
-#include "query_interval.h"
-
-namespace NKikimr {
-namespace NSysView {
-
-void Initialize(NKikimrSysView::TQueryMetrics& metrics) {
- metrics.MutableDurationUs()->SetMin(Max<ui64>());
- metrics.MutableCpuTimeUs()->SetMin(Max<ui64>());
- metrics.MutableRequestUnits()->SetMin(Max<ui64>());
- metrics.MutableReadRows()->SetMin(Max<ui64>());
- metrics.MutableReadBytes()->SetMin(Max<ui64>());
- metrics.MutableUpdateRows()->SetMin(Max<ui64>());
- metrics.MutableUpdateBytes()->SetMin(Max<ui64>());
- metrics.MutableDeleteRows()->SetMin(Max<ui64>());
-}
-
-static void Aggregate(NKikimrSysView::TQueryMetrics::TMetrics& metrics, ui64 value) {
- metrics.SetSum(metrics.GetSum() + value);
- metrics.SetMin(std::min(metrics.GetMin(), value));
- metrics.SetMax(std::max(metrics.GetMax(), value));
-}
-
-static void Aggregate(NKikimrSysView::TQueryMetrics& metrics, TQueryStatsPtr stats) {
- auto& dataStats = stats->GetStats();
- metrics.SetCount(metrics.GetCount() + 1);
-
- Aggregate(*metrics.MutableDurationUs(), stats->GetDurationMs() * 1000);
- Aggregate(*metrics.MutableCpuTimeUs(), stats->GetTotalCpuTimeUs());
- Aggregate(*metrics.MutableRequestUnits(), stats->GetRequestUnits());
-
- Aggregate(*metrics.MutableReadRows(), dataStats.GetReadRows());
- Aggregate(*metrics.MutableReadBytes(), dataStats.GetReadBytes());
- Aggregate(*metrics.MutableUpdateRows(), dataStats.GetUpdateRows());
- Aggregate(*metrics.MutableUpdateBytes(), dataStats.GetUpdateBytes());
- Aggregate(*metrics.MutableDeleteRows(), dataStats.GetDeleteRows());
-}
-
-static void Aggregate(NKikimrSysView::TQueryMetrics::TMetrics& metrics,
- const NKikimrSysView::TQueryMetrics::TMetrics& from)
-{
- metrics.SetSum(metrics.GetSum() + from.GetSum());
- metrics.SetMin(std::min(metrics.GetMin(), from.GetMin()));
- metrics.SetMax(std::max(metrics.GetMax(), from.GetMax()));
-}
-
-void Aggregate(NKikimrSysView::TQueryMetrics& metrics,
- const NKikimrSysView::TQueryMetrics& from)
-{
- metrics.SetCount(metrics.GetCount() + from.GetCount());
-
- Aggregate(*metrics.MutableDurationUs(), from.GetDurationUs());
- Aggregate(*metrics.MutableCpuTimeUs(), from.GetCpuTimeUs());
- Aggregate(*metrics.MutableRequestUnits(), from.GetRequestUnits());
- Aggregate(*metrics.MutableReadRows(), from.GetReadRows());
- Aggregate(*metrics.MutableReadBytes(), from.GetReadBytes());
- Aggregate(*metrics.MutableUpdateRows(), from.GetUpdateRows());
- Aggregate(*metrics.MutableUpdateBytes(), from.GetUpdateBytes());
- Aggregate(*metrics.MutableDeleteRows(), from.GetDeleteRows());
-}
-
-bool TQueryInterval::Empty() const {
- return Metrics.empty();
-}
-
-void TQueryInterval::Clear() {
- Texts.clear();
- Metrics.clear();
- ByCpu.clear();
-}
-
-void TQueryInterval::Swap(TQueryInterval& other) {
- Texts.swap(other.Texts);
- Metrics.swap(other.Metrics);
- ByCpu.swap(other.ByCpu);
-}
-
-void TQueryInterval::Add(TQueryStatsPtr stats) {
- if (!stats->HasQueryTextHash()) {
- return;
- }
- auto queryHash = stats->GetQueryTextHash();
-
- if (auto metricsIt = Metrics.find(queryHash); metricsIt != Metrics.end()) {
- auto oldCpu = metricsIt->second.GetCpuTimeUs().GetSum();
- Aggregate(metricsIt->second, stats);
-
- auto range = ByCpu.equal_range(oldCpu);
- auto it = range.first;
- for (; it != range.second; ++it) {
- if (it->second == queryHash) {
- break;
- }
- }
- Y_VERIFY(it != range.second);
- ByCpu.erase(it);
-
- auto newCpu = metricsIt->second.GetCpuTimeUs().GetSum();
- ByCpu.emplace(newCpu, queryHash);
-
- } else {
- auto cpu = stats->GetTotalCpuTimeUs();
-
- if (ByCpu.size() == CountLimit) {
- auto it = ByCpu.begin();
- if (it->first >= cpu) {
- return;
- }
- Texts.erase(it->second);
- Metrics.erase(it->second);
- ByCpu.erase(it);
- }
-
- NKikimrSysView::TQueryMetrics metrics;
- Initialize(metrics);
- metrics.SetQueryTextHash(queryHash);
- Aggregate(metrics, stats);
-
- Texts.emplace(queryHash, stats->GetQueryText());
- Metrics.emplace(queryHash, std::move(metrics));
- ByCpu.emplace(cpu, queryHash);
- }
-}
-
-void TQueryInterval::FillSummary(NKikimrSysView::TEvIntervalQuerySummary::TQuerySet& queries) const {
- for (auto it = ByCpu.rbegin(); it != ByCpu.rend(); ++it) {
- queries.AddValues(it->first);
- queries.AddHashes(it->second);
- }
-}
-
-void TQueryInterval::FillMetrics(const NKikimrSysView::TEvGetIntervalMetricsRequest& request,
- NKikimrSysView::TEvGetIntervalMetricsResponse& response) const
-{
- for (auto queryHash : request.GetMetrics()) {
- auto metricsIt = Metrics.find(queryHash);
- if (metricsIt == Metrics.end()) {
- continue;
- }
- response.AddMetrics()->CopyFrom(metricsIt->second);
- }
-
- for (auto queryHash : request.GetQueryTextsToGet()) {
- auto textsIt = Texts.find(queryHash);
- if (textsIt == Texts.end()) {
- continue;
- }
- auto& queryText = *response.AddQueryTexts();
- queryText.SetHash(queryHash);
- queryText.SetText(textsIt->second);
- }
-}
-
-} // NSysView
-} // NKikimr
-
+#include "query_interval.h"
+
+namespace NKikimr {
+namespace NSysView {
+
+void Initialize(NKikimrSysView::TQueryMetrics& metrics) {
+ metrics.MutableDurationUs()->SetMin(Max<ui64>());
+ metrics.MutableCpuTimeUs()->SetMin(Max<ui64>());
+ metrics.MutableRequestUnits()->SetMin(Max<ui64>());
+ metrics.MutableReadRows()->SetMin(Max<ui64>());
+ metrics.MutableReadBytes()->SetMin(Max<ui64>());
+ metrics.MutableUpdateRows()->SetMin(Max<ui64>());
+ metrics.MutableUpdateBytes()->SetMin(Max<ui64>());
+ metrics.MutableDeleteRows()->SetMin(Max<ui64>());
+}
+
+static void Aggregate(NKikimrSysView::TQueryMetrics::TMetrics& metrics, ui64 value) {
+ metrics.SetSum(metrics.GetSum() + value);
+ metrics.SetMin(std::min(metrics.GetMin(), value));
+ metrics.SetMax(std::max(metrics.GetMax(), value));
+}
+
+static void Aggregate(NKikimrSysView::TQueryMetrics& metrics, TQueryStatsPtr stats) {
+ auto& dataStats = stats->GetStats();
+ metrics.SetCount(metrics.GetCount() + 1);
+
+ Aggregate(*metrics.MutableDurationUs(), stats->GetDurationMs() * 1000);
+ Aggregate(*metrics.MutableCpuTimeUs(), stats->GetTotalCpuTimeUs());
+ Aggregate(*metrics.MutableRequestUnits(), stats->GetRequestUnits());
+
+ Aggregate(*metrics.MutableReadRows(), dataStats.GetReadRows());
+ Aggregate(*metrics.MutableReadBytes(), dataStats.GetReadBytes());
+ Aggregate(*metrics.MutableUpdateRows(), dataStats.GetUpdateRows());
+ Aggregate(*metrics.MutableUpdateBytes(), dataStats.GetUpdateBytes());
+ Aggregate(*metrics.MutableDeleteRows(), dataStats.GetDeleteRows());
+}
+
+static void Aggregate(NKikimrSysView::TQueryMetrics::TMetrics& metrics,
+ const NKikimrSysView::TQueryMetrics::TMetrics& from)
+{
+ metrics.SetSum(metrics.GetSum() + from.GetSum());
+ metrics.SetMin(std::min(metrics.GetMin(), from.GetMin()));
+ metrics.SetMax(std::max(metrics.GetMax(), from.GetMax()));
+}
+
+void Aggregate(NKikimrSysView::TQueryMetrics& metrics,
+ const NKikimrSysView::TQueryMetrics& from)
+{
+ metrics.SetCount(metrics.GetCount() + from.GetCount());
+
+ Aggregate(*metrics.MutableDurationUs(), from.GetDurationUs());
+ Aggregate(*metrics.MutableCpuTimeUs(), from.GetCpuTimeUs());
+ Aggregate(*metrics.MutableRequestUnits(), from.GetRequestUnits());
+ Aggregate(*metrics.MutableReadRows(), from.GetReadRows());
+ Aggregate(*metrics.MutableReadBytes(), from.GetReadBytes());
+ Aggregate(*metrics.MutableUpdateRows(), from.GetUpdateRows());
+ Aggregate(*metrics.MutableUpdateBytes(), from.GetUpdateBytes());
+ Aggregate(*metrics.MutableDeleteRows(), from.GetDeleteRows());
+}
+
+bool TQueryInterval::Empty() const {
+ return Metrics.empty();
+}
+
+void TQueryInterval::Clear() {
+ Texts.clear();
+ Metrics.clear();
+ ByCpu.clear();
+}
+
+void TQueryInterval::Swap(TQueryInterval& other) {
+ Texts.swap(other.Texts);
+ Metrics.swap(other.Metrics);
+ ByCpu.swap(other.ByCpu);
+}
+
+void TQueryInterval::Add(TQueryStatsPtr stats) {
+ if (!stats->HasQueryTextHash()) {
+ return;
+ }
+ auto queryHash = stats->GetQueryTextHash();
+
+ if (auto metricsIt = Metrics.find(queryHash); metricsIt != Metrics.end()) {
+ auto oldCpu = metricsIt->second.GetCpuTimeUs().GetSum();
+ Aggregate(metricsIt->second, stats);
+
+ auto range = ByCpu.equal_range(oldCpu);
+ auto it = range.first;
+ for (; it != range.second; ++it) {
+ if (it->second == queryHash) {
+ break;
+ }
+ }
+ Y_VERIFY(it != range.second);
+ ByCpu.erase(it);
+
+ auto newCpu = metricsIt->second.GetCpuTimeUs().GetSum();
+ ByCpu.emplace(newCpu, queryHash);
+
+ } else {
+ auto cpu = stats->GetTotalCpuTimeUs();
+
+ if (ByCpu.size() == CountLimit) {
+ auto it = ByCpu.begin();
+ if (it->first >= cpu) {
+ return;
+ }
+ Texts.erase(it->second);
+ Metrics.erase(it->second);
+ ByCpu.erase(it);
+ }
+
+ NKikimrSysView::TQueryMetrics metrics;
+ Initialize(metrics);
+ metrics.SetQueryTextHash(queryHash);
+ Aggregate(metrics, stats);
+
+ Texts.emplace(queryHash, stats->GetQueryText());
+ Metrics.emplace(queryHash, std::move(metrics));
+ ByCpu.emplace(cpu, queryHash);
+ }
+}
+
+void TQueryInterval::FillSummary(NKikimrSysView::TEvIntervalQuerySummary::TQuerySet& queries) const {
+ for (auto it = ByCpu.rbegin(); it != ByCpu.rend(); ++it) {
+ queries.AddValues(it->first);
+ queries.AddHashes(it->second);
+ }
+}
+
+void TQueryInterval::FillMetrics(const NKikimrSysView::TEvGetIntervalMetricsRequest& request,
+ NKikimrSysView::TEvGetIntervalMetricsResponse& response) const
+{
+ for (auto queryHash : request.GetMetrics()) {
+ auto metricsIt = Metrics.find(queryHash);
+ if (metricsIt == Metrics.end()) {
+ continue;
+ }
+ response.AddMetrics()->CopyFrom(metricsIt->second);
+ }
+
+ for (auto queryHash : request.GetQueryTextsToGet()) {
+ auto textsIt = Texts.find(queryHash);
+ if (textsIt == Texts.end()) {
+ continue;
+ }
+ auto& queryText = *response.AddQueryTexts();
+ queryText.SetHash(queryHash);
+ queryText.SetText(textsIt->second);
+ }
+}
+
+} // NSysView
+} // NKikimr
+
diff --git a/ydb/core/sys_view/service/query_interval.h b/ydb/core/sys_view/service/query_interval.h
index 126bd788ff6..8da12b82b6a 100644
--- a/ydb/core/sys_view/service/query_interval.h
+++ b/ydb/core/sys_view/service/query_interval.h
@@ -1,34 +1,34 @@
-#pragma once
-
-#include "query_history.h"
-
-namespace NKikimr {
-namespace NSysView {
-
-void Initialize(NKikimrSysView::TQueryMetrics& metrics);
-
-void Aggregate(NKikimrSysView::TQueryMetrics& metrics, const NKikimrSysView::TQueryMetrics& from);
-
-class TQueryInterval {
- std::unordered_map<TQueryHash, TString> Texts; // hash -> text
- std::unordered_map<TQueryHash, NKikimrSysView::TQueryMetrics> Metrics; // hash -> metrics
- std::multimap<ui64, TQueryHash> ByCpu; // cpu sum -> hash
-
- static constexpr size_t CountLimit = 1024;
-
-public:
- bool Empty() const;
- void Clear();
- void Swap(TQueryInterval& other);
-
- void Add(TQueryStatsPtr stats);
-
- void FillSummary(NKikimrSysView::TEvIntervalQuerySummary::TQuerySet& queries) const;
-
- void FillMetrics(const NKikimrSysView::TEvGetIntervalMetricsRequest& request,
- NKikimrSysView::TEvGetIntervalMetricsResponse& response) const;
-};
-
-} // NSysView
-} // NKikimr
-
+#pragma once
+
+#include "query_history.h"
+
+namespace NKikimr {
+namespace NSysView {
+
+void Initialize(NKikimrSysView::TQueryMetrics& metrics);
+
+void Aggregate(NKikimrSysView::TQueryMetrics& metrics, const NKikimrSysView::TQueryMetrics& from);
+
+class TQueryInterval {
+ std::unordered_map<TQueryHash, TString> Texts; // hash -> text
+ std::unordered_map<TQueryHash, NKikimrSysView::TQueryMetrics> Metrics; // hash -> metrics
+ std::multimap<ui64, TQueryHash> ByCpu; // cpu sum -> hash
+
+ static constexpr size_t CountLimit = 1024;
+
+public:
+ bool Empty() const;
+ void Clear();
+ void Swap(TQueryInterval& other);
+
+ void Add(TQueryStatsPtr stats);
+
+ void FillSummary(NKikimrSysView::TEvIntervalQuerySummary::TQuerySet& queries) const;
+
+ void FillMetrics(const NKikimrSysView::TEvGetIntervalMetricsRequest& request,
+ NKikimrSysView::TEvGetIntervalMetricsResponse& response) const;
+};
+
+} // NSysView
+} // NKikimr
+
diff --git a/ydb/core/sys_view/service/sysview_service.cpp b/ydb/core/sys_view/service/sysview_service.cpp
index 1f569065999..b407cd39ba1 100644
--- a/ydb/core/sys_view/service/sysview_service.cpp
+++ b/ydb/core/sys_view/service/sysview_service.cpp
@@ -1,1149 +1,1149 @@
-#include "db_counters.h"
-#include "query_history.h"
-#include "query_interval.h"
-#include "sysview_service.h"
-
+#include "db_counters.h"
+#include "query_history.h"
+#include "query_interval.h"
+#include "sysview_service.h"
+
#include <ydb/core/sys_view/common/common.h>
#include <ydb/core/sys_view/common/events.h>
#include <ydb/core/base/appdata.h>
#include <ydb/core/base/tablet_pipecache.h>
#include <ydb/core/tx/scheme_cache/scheme_cache.h>
#include <ydb/core/mind/tenant_pool.h>
-
+
#include <library/cpp/actors/core/actor_bootstrapped.h>
#include <library/cpp/actors/core/hfunc.h>
#include <library/cpp/actors/core/log.h>
-
-using namespace NActors;
-
-namespace NKikimr {
-namespace NSysView {
-
-constexpr size_t QUERY_TEXT_LIMIT = 4096;
-
-void CollectQueryStats(const TActorContext& ctx, const NKqpProto::TKqpStatsQuery* queryStats,
- TDuration queryDuration, const TString& queryText,
- const TString& userSID, ui64 parametersSize, const TString& database,
- const NKikimrKqp::EQueryType type, ui64 requestUnits)
-{
+
+using namespace NActors;
+
+namespace NKikimr {
+namespace NSysView {
+
+constexpr size_t QUERY_TEXT_LIMIT = 4096;
+
+void CollectQueryStats(const TActorContext& ctx, const NKqpProto::TKqpStatsQuery* queryStats,
+ TDuration queryDuration, const TString& queryText,
+ const TString& userSID, ui64 parametersSize, const TString& database,
+ const NKikimrKqp::EQueryType type, ui64 requestUnits)
+{
if (!AppData()->FeatureFlags.GetEnableSystemViews()) {
- return;
- }
-
- auto collectEv = MakeHolder<NSysView::TEvSysView::TEvCollectQueryStats>();
- collectEv->Database = database;
-
- auto& stats = collectEv->QueryStats;
- auto& dataStats = *stats.MutableStats();
- auto& shardsCpuTime = *stats.MutableShardsCpuTimeUs();
- auto& computeCpuTime = *stats.MutableComputeCpuTimeUs();
-
- auto nodeId = ctx.SelfID.NodeId();
- stats.SetNodeId(nodeId);
-
- stats.SetEndTimeMs(TInstant::Now().MilliSeconds());
- stats.SetDurationMs(queryDuration.MilliSeconds());
-
- stats.SetQueryTextHash(MurmurHash<ui64>(queryText.data(), queryText.size()));
- if (queryText.size() > QUERY_TEXT_LIMIT) {
- auto limitedText = queryText.substr(0, QUERY_TEXT_LIMIT);
- stats.SetQueryText(limitedText);
- } else {
- stats.SetQueryText(queryText);
- }
-
- if (userSID) {
- stats.SetUserSID(userSID);
- }
- stats.SetParametersSize(parametersSize);
-
- TString strType;
- switch (type) {
- case NKikimrKqp::QUERY_TYPE_SQL_DML:
- case NKikimrKqp::QUERY_TYPE_PREPARED_DML:
- strType = "data";
- break;
- case NKikimrKqp::QUERY_TYPE_SQL_SCAN:
- strType = "scan";
- break;
- case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT:
+ return;
+ }
+
+ auto collectEv = MakeHolder<NSysView::TEvSysView::TEvCollectQueryStats>();
+ collectEv->Database = database;
+
+ auto& stats = collectEv->QueryStats;
+ auto& dataStats = *stats.MutableStats();
+ auto& shardsCpuTime = *stats.MutableShardsCpuTimeUs();
+ auto& computeCpuTime = *stats.MutableComputeCpuTimeUs();
+
+ auto nodeId = ctx.SelfID.NodeId();
+ stats.SetNodeId(nodeId);
+
+ stats.SetEndTimeMs(TInstant::Now().MilliSeconds());
+ stats.SetDurationMs(queryDuration.MilliSeconds());
+
+ stats.SetQueryTextHash(MurmurHash<ui64>(queryText.data(), queryText.size()));
+ if (queryText.size() > QUERY_TEXT_LIMIT) {
+ auto limitedText = queryText.substr(0, QUERY_TEXT_LIMIT);
+ stats.SetQueryText(limitedText);
+ } else {
+ stats.SetQueryText(queryText);
+ }
+
+ if (userSID) {
+ stats.SetUserSID(userSID);
+ }
+ stats.SetParametersSize(parametersSize);
+
+ TString strType;
+ switch (type) {
+ case NKikimrKqp::QUERY_TYPE_SQL_DML:
+ case NKikimrKqp::QUERY_TYPE_PREPARED_DML:
+ strType = "data";
+ break;
+ case NKikimrKqp::QUERY_TYPE_SQL_SCAN:
+ strType = "scan";
+ break;
+ case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT:
case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT_STREAMING:
- strType = "script";
- break;
- default:
- break;
- }
- stats.SetType(strType);
-
- if (!queryStats || queryStats->GetExecutions().empty()) {
- ctx.Send(NSysView::MakeSysViewServiceID(nodeId), std::move(collectEv));
- return;
- }
-
- shardsCpuTime.SetMin(Max<ui64>());
- computeCpuTime.SetMin(Max<ui64>());
-
+ strType = "script";
+ break;
+ default:
+ break;
+ }
+ stats.SetType(strType);
+
+ if (!queryStats || queryStats->GetExecutions().empty()) {
+ ctx.Send(NSysView::MakeSysViewServiceID(nodeId), std::move(collectEv));
+ return;
+ }
+
+ shardsCpuTime.SetMin(Max<ui64>());
+ computeCpuTime.SetMin(Max<ui64>());
+
auto aggregate = [] (NKikimrSysView::TStatsAggr& to, const NYql::NDqProto::TDqStatsAggr& from) {
- to.SetMin(std::min(to.GetMin(), from.GetMin()));
- to.SetMax(std::max(to.GetMax(), from.GetMax()));
- to.SetSum(to.GetSum() + from.GetSum());
- to.SetCnt(to.GetCnt() + from.GetCnt());
- };
-
+ to.SetMin(std::min(to.GetMin(), from.GetMin()));
+ to.SetMax(std::max(to.GetMax(), from.GetMax()));
+ to.SetSum(to.GetSum() + from.GetSum());
+ to.SetCnt(to.GetCnt() + from.GetCnt());
+ };
+
for (const NYql::NDqProto::TDqExecutionStats& exec : queryStats->GetExecutions()) {
NKqpProto::TKqpExecutionExtraStats execExtra;
if (exec.HasExtra()) {
bool ok = exec.GetExtra().UnpackTo(&execExtra);
Y_UNUSED(ok);
}
-
+
dataStats.SetPartitionCount(dataStats.GetPartitionCount() + execExtra.GetAffectedShards());
- for (auto& table : exec.GetTables()) {
- dataStats.SetReadRows(dataStats.GetReadRows() + table.GetReadRows());
- dataStats.SetReadBytes(dataStats.GetReadBytes() + table.GetReadBytes());
- dataStats.SetUpdateRows(dataStats.GetUpdateRows() + table.GetWriteRows());
- dataStats.SetUpdateBytes(dataStats.GetUpdateBytes() + table.GetWriteBytes());
- dataStats.SetDeleteRows(dataStats.GetDeleteRows() + table.GetEraseRows());
- }
-
+ for (auto& table : exec.GetTables()) {
+ dataStats.SetReadRows(dataStats.GetReadRows() + table.GetReadRows());
+ dataStats.SetReadBytes(dataStats.GetReadBytes() + table.GetReadBytes());
+ dataStats.SetUpdateRows(dataStats.GetUpdateRows() + table.GetWriteRows());
+ dataStats.SetUpdateBytes(dataStats.GetUpdateBytes() + table.GetWriteBytes());
+ dataStats.SetDeleteRows(dataStats.GetDeleteRows() + table.GetEraseRows());
+ }
+
aggregate(shardsCpuTime, execExtra.GetShardsCpuTimeUs());
aggregate(computeCpuTime, execExtra.GetComputeCpuTimeUs());
- }
-
- if (queryStats->HasCompilation()) {
- auto& compileStats = queryStats->GetCompilation();
- stats.SetFromQueryCache(compileStats.GetFromCache());
- stats.SetCompileDurationMs(compileStats.GetDurationUs() / 1'000);
- stats.SetCompileCpuTimeUs(compileStats.GetCpuTimeUs());
- }
-
- stats.SetProcessCpuTimeUs(queryStats->GetWorkerCpuTimeUs());
- stats.SetTotalCpuTimeUs(
- stats.GetProcessCpuTimeUs() +
- stats.GetCompileCpuTimeUs() +
- stats.GetShardsCpuTimeUs().GetSum() +
- stats.GetComputeCpuTimeUs().GetSum()
- );
-
- stats.SetRequestUnits(requestUnits);
-
- ctx.Send(NSysView::MakeSysViewServiceID(nodeId), std::move(collectEv));
-}
-
-static void CopyCounters(NKikimrSysView::TDbCounters* diff,
- const NKikimrSysView::TDbCounters& current)
-{
- auto simpleSize = current.SimpleSize();
- auto cumulativeSize = current.CumulativeSize();
- auto histogramSize = current.HistogramSize();
-
- diff->MutableSimple()->Reserve(simpleSize);
- diff->MutableCumulative()->Reserve(cumulativeSize);
- diff->MutableHistogram()->Reserve(histogramSize);
-
- for (size_t i = 0; i < simpleSize; ++i) {
- diff->AddSimple(current.GetSimple(i));
- }
-
- diff->SetCumulativeCount(cumulativeSize);
- for (size_t i = 0; i < cumulativeSize; ++i) {
- auto value = current.GetCumulative(i);
- if (!value) {
- continue;
- }
- diff->AddCumulative(i);
- diff->AddCumulative(value);
- }
-
- for (size_t i = 0; i < histogramSize; ++i) {
- const auto& currentH = current.GetHistogram(i);
- auto bucketCount = currentH.BucketsSize();
-
- auto* histogram = diff->AddHistogram();
- histogram->MutableBuckets()->Reserve(bucketCount);
- histogram->SetBucketsCount(bucketCount);
- for (size_t b = 0; b < bucketCount; ++b) {
- auto value = currentH.GetBuckets(b);
- if (!value) {
- continue;
- }
- histogram->AddBuckets(b);
- histogram->AddBuckets(value);
- }
- }
-}
-
-static void CalculateCountersDiff(NKikimrSysView::TDbCounters* diff,
- const NKikimrSysView::TDbCounters& current,
- NKikimrSysView::TDbCounters& prev)
-{
- auto simpleSize = current.SimpleSize();
- auto cumulativeSize = current.CumulativeSize();
- auto histogramSize = current.HistogramSize();
-
- if (prev.SimpleSize() != simpleSize) {
- SVLOG_CRIT("CalculateCountersDiff: simple count mismatch, prev "
- << prev.SimpleSize() << ", current " << simpleSize);
- prev.MutableSimple()->Resize(simpleSize, 0);
- }
- if (prev.CumulativeSize() != cumulativeSize) {
- SVLOG_CRIT("CalculateCountersDiff: cumulative count mismatch, prev "
- << prev.CumulativeSize() << ", current " << cumulativeSize);
- prev.MutableCumulative()->Resize(cumulativeSize, 0);
- }
- if (prev.HistogramSize() != histogramSize) {
- SVLOG_CRIT("CalculateCountersDiff: histogram count mismatch, prev "
- << prev.HistogramSize() << ", current " << histogramSize);
- if (prev.HistogramSize() < histogramSize) {
- auto missing = histogramSize - prev.HistogramSize();
- for (; missing > 0; --missing) {
- prev.AddHistogram();
- }
- }
- }
-
- diff->MutableSimple()->Reserve(simpleSize);
- diff->MutableCumulative()->Reserve(cumulativeSize);
- diff->MutableHistogram()->Reserve(histogramSize);
-
- for (size_t i = 0; i < simpleSize; ++i) {
- diff->AddSimple(current.GetSimple(i));
- }
-
- diff->SetCumulativeCount(cumulativeSize);
- for (size_t i = 0; i < cumulativeSize; ++i) {
- auto value = current.GetCumulative(i) - prev.GetCumulative(i);
- if (!value) {
- continue;
- }
- diff->AddCumulative(i);
- diff->AddCumulative(value);
- }
-
- for (size_t i = 0; i < histogramSize; ++i) {
- const auto& currentH = current.GetHistogram(i);
- auto& prevH = *prev.MutableHistogram(i);
- auto bucketCount = currentH.BucketsSize();
- if (prevH.BucketsSize() != bucketCount) {
- SVLOG_CRIT("CalculateCountersDiff: histogram buckets count mismatch, index " << i
- << ", prev " << prevH.BucketsSize() << ", current " << bucketCount);
- prevH.MutableBuckets()->Resize(bucketCount, 0);
- }
- auto* histogram = diff->AddHistogram();
- histogram->MutableBuckets()->Reserve(bucketCount);
- histogram->SetBucketsCount(bucketCount);
- for (size_t b = 0; b < bucketCount; ++b) {
- auto value = currentH.GetBuckets(b) - prevH.GetBuckets(b);
- if (!value) {
- continue;
- }
- histogram->AddBuckets(b);
- histogram->AddBuckets(value);
- }
- }
-}
-
-static void CalculateCountersDiff(NKikimrSysView::TDbServiceCounters* diff,
- const NKikimr::NSysView::TDbServiceCounters& current,
- NKikimr::NSysView::TDbServiceCounters& prev)
-{
- if (current.Proto().HasMain()) {
- if (prev.Proto().HasMain()) {
- CalculateCountersDiff(diff->MutableMain(),
- current.Proto().GetMain(), *prev.Proto().MutableMain());
- } else {
- CopyCounters(diff->MutableMain(), current.Proto().GetMain());
- }
- }
-
- for (const auto& currentT : current.Proto().GetTabletCounters()) {
- const auto type = currentT.GetType();
- auto* diffT = diff->AddTabletCounters();
- diffT->SetType(type);
-
- auto* prevT = prev.FindTabletCounters(type);
- if (prevT) {
- CalculateCountersDiff(diffT->MutableExecutorCounters(),
- currentT.GetExecutorCounters(), *prevT->MutableExecutorCounters());
- CalculateCountersDiff(diffT->MutableAppCounters(),
- currentT.GetAppCounters(), *prevT->MutableAppCounters());
- } else {
- CopyCounters(diffT->MutableExecutorCounters(), currentT.GetExecutorCounters());
- CopyCounters(diffT->MutableAppCounters(), currentT.GetAppCounters());
- }
- CopyCounters(diffT->MutableMaxExecutorCounters(), currentT.GetMaxExecutorCounters());
- CopyCounters(diffT->MutableMaxAppCounters(), currentT.GetMaxAppCounters());
- }
-
- for (const auto& currentR : current.Proto().GetGRpcCounters()) {
- const auto grpcService = currentR.GetGRpcService();
- const auto grpcRequest = currentR.GetGRpcRequest();
- auto* diffR = diff->AddGRpcCounters();
- diffR->SetGRpcService(grpcService);
- diffR->SetGRpcRequest(grpcRequest);
-
- auto* prevR = prev.FindGRpcCounters(grpcService, grpcRequest);
- if (prevR) {
- CalculateCountersDiff(diffR->MutableRequestCounters(),
- currentR.GetRequestCounters(), *prevR->MutableRequestCounters());
- } else {
- CopyCounters(diffR->MutableRequestCounters(), currentR.GetRequestCounters());
- }
- }
-}
-
-class TSysViewService : public TActorBootstrapped<TSysViewService> {
-public:
- using TBase = TActorBootstrapped<TSysViewService>;
-
- static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
- return NKikimrServices::TActivity::SYSTEM_VIEW_SERVICE;
- }
-
- explicit TSysViewService(
- TExtCountersConfig&& config,
- bool hasExternalCounters,
- EProcessorMode processorMode)
- : Config(std::move(config))
- , HasExternalCounters(hasExternalCounters)
- , TotalInterval(TDuration::Seconds(processorMode == EProcessorMode::FAST ? 6 : 60))
- , CollectInterval(TDuration::Seconds(processorMode == EProcessorMode::FAST ? 3 : 30))
- , SendInterval(TDuration::Seconds(processorMode == EProcessorMode::FAST ? 2 : 20))
- {}
-
- void Bootstrap(const TActorContext &ctx) {
- auto now = AppData()->TimeProvider->Now();
-
- TopByDuration1Minute = MakeHolder<TServiceQueryHistory<TDurationGreater>>(
- ONE_MINUTE_BUCKET_COUNT, ONE_MINUTE_BUCKET_SIZE, now);
- TopByDuration1Hour = MakeHolder<TServiceQueryHistory<TDurationGreater>>(
- ONE_HOUR_BUCKET_COUNT, ONE_HOUR_BUCKET_SIZE, now);
-
- TopByReadBytes1Minute = MakeHolder<TServiceQueryHistory<TReadBytesGreater>>(
- ONE_MINUTE_BUCKET_COUNT, ONE_MINUTE_BUCKET_SIZE, now);
- TopByReadBytes1Hour = MakeHolder<TServiceQueryHistory<TReadBytesGreater>>(
- ONE_HOUR_BUCKET_COUNT, ONE_HOUR_BUCKET_SIZE, now);
-
- TopByCpuTime1Minute = MakeHolder<TServiceQueryHistory<TCpuTimeGreater>>(
- ONE_MINUTE_BUCKET_COUNT, ONE_MINUTE_BUCKET_SIZE, now);
- TopByCpuTime1Hour = MakeHolder<TServiceQueryHistory<TCpuTimeGreater>>(
- ONE_HOUR_BUCKET_COUNT, ONE_HOUR_BUCKET_SIZE, now);
-
- TopByRequestUnits1Minute = MakeHolder<TServiceQueryHistory<TRequestUnitsGreater>>(
- ONE_MINUTE_BUCKET_COUNT, ONE_MINUTE_BUCKET_SIZE, now);
- TopByRequestUnits1Hour = MakeHolder<TServiceQueryHistory<TRequestUnitsGreater>>(
- ONE_HOUR_BUCKET_COUNT, ONE_HOUR_BUCKET_SIZE, now);
-
- ctx.Send(MakeTenantPoolRootID(), new TEvents::TEvSubscribe());
-
- ScanLimiter = MakeIntrusive<TScanLimiter>(ConcurrentScansLimit);
-
+ }
+
+ if (queryStats->HasCompilation()) {
+ auto& compileStats = queryStats->GetCompilation();
+ stats.SetFromQueryCache(compileStats.GetFromCache());
+ stats.SetCompileDurationMs(compileStats.GetDurationUs() / 1'000);
+ stats.SetCompileCpuTimeUs(compileStats.GetCpuTimeUs());
+ }
+
+ stats.SetProcessCpuTimeUs(queryStats->GetWorkerCpuTimeUs());
+ stats.SetTotalCpuTimeUs(
+ stats.GetProcessCpuTimeUs() +
+ stats.GetCompileCpuTimeUs() +
+ stats.GetShardsCpuTimeUs().GetSum() +
+ stats.GetComputeCpuTimeUs().GetSum()
+ );
+
+ stats.SetRequestUnits(requestUnits);
+
+ ctx.Send(NSysView::MakeSysViewServiceID(nodeId), std::move(collectEv));
+}
+
+static void CopyCounters(NKikimrSysView::TDbCounters* diff,
+ const NKikimrSysView::TDbCounters& current)
+{
+ auto simpleSize = current.SimpleSize();
+ auto cumulativeSize = current.CumulativeSize();
+ auto histogramSize = current.HistogramSize();
+
+ diff->MutableSimple()->Reserve(simpleSize);
+ diff->MutableCumulative()->Reserve(cumulativeSize);
+ diff->MutableHistogram()->Reserve(histogramSize);
+
+ for (size_t i = 0; i < simpleSize; ++i) {
+ diff->AddSimple(current.GetSimple(i));
+ }
+
+ diff->SetCumulativeCount(cumulativeSize);
+ for (size_t i = 0; i < cumulativeSize; ++i) {
+ auto value = current.GetCumulative(i);
+ if (!value) {
+ continue;
+ }
+ diff->AddCumulative(i);
+ diff->AddCumulative(value);
+ }
+
+ for (size_t i = 0; i < histogramSize; ++i) {
+ const auto& currentH = current.GetHistogram(i);
+ auto bucketCount = currentH.BucketsSize();
+
+ auto* histogram = diff->AddHistogram();
+ histogram->MutableBuckets()->Reserve(bucketCount);
+ histogram->SetBucketsCount(bucketCount);
+ for (size_t b = 0; b < bucketCount; ++b) {
+ auto value = currentH.GetBuckets(b);
+ if (!value) {
+ continue;
+ }
+ histogram->AddBuckets(b);
+ histogram->AddBuckets(value);
+ }
+ }
+}
+
+static void CalculateCountersDiff(NKikimrSysView::TDbCounters* diff,
+ const NKikimrSysView::TDbCounters& current,
+ NKikimrSysView::TDbCounters& prev)
+{
+ auto simpleSize = current.SimpleSize();
+ auto cumulativeSize = current.CumulativeSize();
+ auto histogramSize = current.HistogramSize();
+
+ if (prev.SimpleSize() != simpleSize) {
+ SVLOG_CRIT("CalculateCountersDiff: simple count mismatch, prev "
+ << prev.SimpleSize() << ", current " << simpleSize);
+ prev.MutableSimple()->Resize(simpleSize, 0);
+ }
+ if (prev.CumulativeSize() != cumulativeSize) {
+ SVLOG_CRIT("CalculateCountersDiff: cumulative count mismatch, prev "
+ << prev.CumulativeSize() << ", current " << cumulativeSize);
+ prev.MutableCumulative()->Resize(cumulativeSize, 0);
+ }
+ if (prev.HistogramSize() != histogramSize) {
+ SVLOG_CRIT("CalculateCountersDiff: histogram count mismatch, prev "
+ << prev.HistogramSize() << ", current " << histogramSize);
+ if (prev.HistogramSize() < histogramSize) {
+ auto missing = histogramSize - prev.HistogramSize();
+ for (; missing > 0; --missing) {
+ prev.AddHistogram();
+ }
+ }
+ }
+
+ diff->MutableSimple()->Reserve(simpleSize);
+ diff->MutableCumulative()->Reserve(cumulativeSize);
+ diff->MutableHistogram()->Reserve(histogramSize);
+
+ for (size_t i = 0; i < simpleSize; ++i) {
+ diff->AddSimple(current.GetSimple(i));
+ }
+
+ diff->SetCumulativeCount(cumulativeSize);
+ for (size_t i = 0; i < cumulativeSize; ++i) {
+ auto value = current.GetCumulative(i) - prev.GetCumulative(i);
+ if (!value) {
+ continue;
+ }
+ diff->AddCumulative(i);
+ diff->AddCumulative(value);
+ }
+
+ for (size_t i = 0; i < histogramSize; ++i) {
+ const auto& currentH = current.GetHistogram(i);
+ auto& prevH = *prev.MutableHistogram(i);
+ auto bucketCount = currentH.BucketsSize();
+ if (prevH.BucketsSize() != bucketCount) {
+ SVLOG_CRIT("CalculateCountersDiff: histogram buckets count mismatch, index " << i
+ << ", prev " << prevH.BucketsSize() << ", current " << bucketCount);
+ prevH.MutableBuckets()->Resize(bucketCount, 0);
+ }
+ auto* histogram = diff->AddHistogram();
+ histogram->MutableBuckets()->Reserve(bucketCount);
+ histogram->SetBucketsCount(bucketCount);
+ for (size_t b = 0; b < bucketCount; ++b) {
+ auto value = currentH.GetBuckets(b) - prevH.GetBuckets(b);
+ if (!value) {
+ continue;
+ }
+ histogram->AddBuckets(b);
+ histogram->AddBuckets(value);
+ }
+ }
+}
+
+static void CalculateCountersDiff(NKikimrSysView::TDbServiceCounters* diff,
+ const NKikimr::NSysView::TDbServiceCounters& current,
+ NKikimr::NSysView::TDbServiceCounters& prev)
+{
+ if (current.Proto().HasMain()) {
+ if (prev.Proto().HasMain()) {
+ CalculateCountersDiff(diff->MutableMain(),
+ current.Proto().GetMain(), *prev.Proto().MutableMain());
+ } else {
+ CopyCounters(diff->MutableMain(), current.Proto().GetMain());
+ }
+ }
+
+ for (const auto& currentT : current.Proto().GetTabletCounters()) {
+ const auto type = currentT.GetType();
+ auto* diffT = diff->AddTabletCounters();
+ diffT->SetType(type);
+
+ auto* prevT = prev.FindTabletCounters(type);
+ if (prevT) {
+ CalculateCountersDiff(diffT->MutableExecutorCounters(),
+ currentT.GetExecutorCounters(), *prevT->MutableExecutorCounters());
+ CalculateCountersDiff(diffT->MutableAppCounters(),
+ currentT.GetAppCounters(), *prevT->MutableAppCounters());
+ } else {
+ CopyCounters(diffT->MutableExecutorCounters(), currentT.GetExecutorCounters());
+ CopyCounters(diffT->MutableAppCounters(), currentT.GetAppCounters());
+ }
+ CopyCounters(diffT->MutableMaxExecutorCounters(), currentT.GetMaxExecutorCounters());
+ CopyCounters(diffT->MutableMaxAppCounters(), currentT.GetMaxAppCounters());
+ }
+
+ for (const auto& currentR : current.Proto().GetGRpcCounters()) {
+ const auto grpcService = currentR.GetGRpcService();
+ const auto grpcRequest = currentR.GetGRpcRequest();
+ auto* diffR = diff->AddGRpcCounters();
+ diffR->SetGRpcService(grpcService);
+ diffR->SetGRpcRequest(grpcRequest);
+
+ auto* prevR = prev.FindGRpcCounters(grpcService, grpcRequest);
+ if (prevR) {
+ CalculateCountersDiff(diffR->MutableRequestCounters(),
+ currentR.GetRequestCounters(), *prevR->MutableRequestCounters());
+ } else {
+ CopyCounters(diffR->MutableRequestCounters(), currentR.GetRequestCounters());
+ }
+ }
+}
+
+class TSysViewService : public TActorBootstrapped<TSysViewService> {
+public:
+ using TBase = TActorBootstrapped<TSysViewService>;
+
+ static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
+ return NKikimrServices::TActivity::SYSTEM_VIEW_SERVICE;
+ }
+
+ explicit TSysViewService(
+ TExtCountersConfig&& config,
+ bool hasExternalCounters,
+ EProcessorMode processorMode)
+ : Config(std::move(config))
+ , HasExternalCounters(hasExternalCounters)
+ , TotalInterval(TDuration::Seconds(processorMode == EProcessorMode::FAST ? 6 : 60))
+ , CollectInterval(TDuration::Seconds(processorMode == EProcessorMode::FAST ? 3 : 30))
+ , SendInterval(TDuration::Seconds(processorMode == EProcessorMode::FAST ? 2 : 20))
+ {}
+
+ void Bootstrap(const TActorContext &ctx) {
+ auto now = AppData()->TimeProvider->Now();
+
+ TopByDuration1Minute = MakeHolder<TServiceQueryHistory<TDurationGreater>>(
+ ONE_MINUTE_BUCKET_COUNT, ONE_MINUTE_BUCKET_SIZE, now);
+ TopByDuration1Hour = MakeHolder<TServiceQueryHistory<TDurationGreater>>(
+ ONE_HOUR_BUCKET_COUNT, ONE_HOUR_BUCKET_SIZE, now);
+
+ TopByReadBytes1Minute = MakeHolder<TServiceQueryHistory<TReadBytesGreater>>(
+ ONE_MINUTE_BUCKET_COUNT, ONE_MINUTE_BUCKET_SIZE, now);
+ TopByReadBytes1Hour = MakeHolder<TServiceQueryHistory<TReadBytesGreater>>(
+ ONE_HOUR_BUCKET_COUNT, ONE_HOUR_BUCKET_SIZE, now);
+
+ TopByCpuTime1Minute = MakeHolder<TServiceQueryHistory<TCpuTimeGreater>>(
+ ONE_MINUTE_BUCKET_COUNT, ONE_MINUTE_BUCKET_SIZE, now);
+ TopByCpuTime1Hour = MakeHolder<TServiceQueryHistory<TCpuTimeGreater>>(
+ ONE_HOUR_BUCKET_COUNT, ONE_HOUR_BUCKET_SIZE, now);
+
+ TopByRequestUnits1Minute = MakeHolder<TServiceQueryHistory<TRequestUnitsGreater>>(
+ ONE_MINUTE_BUCKET_COUNT, ONE_MINUTE_BUCKET_SIZE, now);
+ TopByRequestUnits1Hour = MakeHolder<TServiceQueryHistory<TRequestUnitsGreater>>(
+ ONE_HOUR_BUCKET_COUNT, ONE_HOUR_BUCKET_SIZE, now);
+
+ ctx.Send(MakeTenantPoolRootID(), new TEvents::TEvSubscribe());
+
+ ScanLimiter = MakeIntrusive<TScanLimiter>(ConcurrentScansLimit);
+
if (AppData()->FeatureFlags.GetEnablePersistentQueryStats()) {
- IntervalEnd = GetNextIntervalEnd();
- Schedule(IntervalEnd, new TEvPrivate::TEvProcessInterval(IntervalEnd));
- }
-
- if (AppData()->FeatureFlags.GetEnableDbCounters()) {
- auto intervalSize = ProcessCountersInterval.MicroSeconds();
- auto deadline = (Now().MicroSeconds() / intervalSize + 1) * intervalSize;
- deadline += RandomNumber<ui64>(intervalSize / 5);
- Schedule(TInstant::MicroSeconds(deadline), new TEvPrivate::TEvProcessCounters());
-
- auto callback = MakeIntrusive<TServiceDbWatcherCallback>(ctx.ActorSystem());
- DbWatcherActorId = ctx.Register(CreateDbWatcherActor(callback));
- }
-
- if (HasExternalCounters) {
- ctx.Register(CreateExtCountersUpdater(std::move(Config)));
- }
-
- Become(&TSysViewService::StateWork);
- }
-
- STFUNC(StateWork) {
- Y_UNUSED(ctx);
- switch(ev->GetTypeRewrite()) {
- hFunc(TEvTenantPool::TEvTenantPoolStatus, Handle);
- hFunc(TEvSysView::TEvCollectQueryStats, Handle);
- hFunc(TEvSysView::TEvGetQueryStats, Handle);
- hFunc(TEvSysView::TEvGetScanLimiter, Handle);
- hFunc(TEvPrivate::TEvProcessInterval, Handle);
- hFunc(TEvPrivate::TEvSendSummary, Handle);
- hFunc(TEvPrivate::TEvProcessCounters, Handle);
- hFunc(TEvPrivate::TEvRemoveDatabase, Handle);
- hFunc(TEvSysView::TEvRegisterDbCounters, Handle);
- hFunc(TEvSysView::TEvSendDbCountersResponse, Handle);
- hFunc(TEvSysView::TEvGetIntervalMetricsRequest, Handle);
- hFunc(TEvPipeCache::TEvDeliveryProblem, Handle);
- hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle);
- cFunc(TEvents::TEvPoison::EventType, PassAway);
- default:
- SVLOG_CRIT("NSysView::TSysViewService: unexpected event# "
- << ev->GetTypeRewrite());
- }
- }
-
-private:
- struct TEvPrivate {
- enum EEv {
- EvProcessInterval = EventSpaceBegin(TEvents::ES_PRIVATE),
- EvSendSummary,
- EvProcessCounters,
- EvRemoveDatabase,
- EvEnd
- };
-
- struct TEvProcessInterval : public TEventLocal<TEvProcessInterval, EvProcessInterval> {
- TInstant IntervalEnd;
-
- explicit TEvProcessInterval(TInstant intervalEnd)
- : IntervalEnd(intervalEnd)
- {}
- };
-
- struct TEvSendSummary : public TEventLocal<TEvSendSummary, EvSendSummary> {
- TInstant IntervalEnd;
- TString Database;
-
- TEvSendSummary(TInstant intervalEnd, const TString& database)
- : IntervalEnd(intervalEnd)
- , Database(database)
- {}
- };
-
- struct TEvProcessCounters : public TEventLocal<TEvProcessCounters, EvProcessCounters> {
- };
-
- struct TEvRemoveDatabase : public TEventLocal<TEvRemoveDatabase, EvRemoveDatabase> {
- TString Database;
- TPathId PathId;
-
- TEvRemoveDatabase(const TString& database, TPathId pathId)
- : Database(database)
- , PathId(pathId)
- {}
- };
- };
-
- class TServiceDbWatcherCallback : public TDbWatcherCallback {
- TActorSystem* ActorSystem = {};
-
- public:
- explicit TServiceDbWatcherCallback(TActorSystem* actorSystem)
- : ActorSystem(actorSystem)
- {}
-
- void OnDatabaseRemoved(const TString& database, TPathId pathId) override {
- auto evRemove = MakeHolder<TEvPrivate::TEvRemoveDatabase>(database, pathId);
- auto service = MakeSysViewServiceID(ActorSystem->NodeId);
- ActorSystem->Send(service, evRemove.Release());
- }
- };
-
- TInstant GetNextIntervalEnd() {
- auto intervalSize = TotalInterval.MicroSeconds();
- auto rounded = (Now().MicroSeconds() / intervalSize + 1) * intervalSize;
- return TInstant::MicroSeconds(rounded);
- }
-
- void SendSummary(TInstant intervalEnd, const TString& database) {
- auto processorId = GetProcessorId(database);
- if (!processorId) {
- return;
- }
-
- auto summary = MakeHolder<TEvSysView::TEvIntervalQuerySummary>();
- auto& record = summary->Record;
- record.SetDatabase(database);
- record.SetIntervalEndUs(intervalEnd.MicroSeconds());
- record.SetNodeId(SelfId().NodeId());
-
- auto& log = QueryLogs[database];
- log.Metrics.Report.FillSummary(*record.MutableMetrics());
- log.TopByDuration.Report.FillSummary(*record.MutableTopByDuration());
- log.TopByReadBytes.Report.FillSummary(*record.MutableTopByReadBytes());
- log.TopByCpuTime.Report.FillSummary(*record.MutableTopByCpuTime());
- log.TopByRequestUnits.Report.FillSummary(*record.MutableTopByRequestUnits());
-
- SVLOG_D("Send interval summary: "
- << "service id# " << SelfId()
- << ", processor id# " << processorId
- << ", database# " << database
- << ", interval end# " << intervalEnd
- << ", query count# " << record.GetMetrics().HashesSize());
-
- Send(MakePipePeNodeCacheID(false),
- new TEvPipeCache::TEvForward(summary.Release(), processorId, true),
- IEventHandle::FlagTrackDelivery);
- }
-
- void Rotate() {
- auto summaryEnd = IntervalEnd;
- auto delta = Now() - summaryEnd;
-
- IntervalEnd = GetNextIntervalEnd();
- Schedule(IntervalEnd, new TEvPrivate::TEvProcessInterval(IntervalEnd));
-
- Attempts.clear();
-
- SVLOG_D("Rotate logs: service id# " << SelfId()
- << ", query logs count# " << QueryLogs.size()
- << ", processor ids count# " << ProcessorIds.size()
- << ", processor id to database count# " << ProcessorIdToDatabase.size());
-
- if (QueryLogs.empty()) {
- return;
- }
-
- if (delta > CollectInterval) {
- // too late to send summaries, skip
- QueryLogs.clear();
- return;
- }
-
- for (auto it = QueryLogs.begin(); it != QueryLogs.end(); ) {
- it->second.Rotate();
- if (it->second.Metrics.Report.Empty()) {
- it = QueryLogs.erase(it);
- } else {
- ++it;
- }
- }
-
- if (delta >= SendInterval) {
- // send immediately
- for (const auto& [database, _] : QueryLogs) {
- SendSummary(summaryEnd, database);
- }
- return;
- }
-
- // randomize send time
- auto windowUs = (SendInterval - delta).MicroSeconds() / 2;
- auto now = Now();
- for (const auto& [database, _] : QueryLogs) {
- TInstant deadline = now + TDuration::MicroSeconds(windowUs + RandomNumber<ui64>(windowUs));
- Schedule(deadline, new TEvPrivate::TEvSendSummary(summaryEnd, database));
- Attempts[database] = std::make_pair(summaryEnd, 1);
- }
- }
-
- ui64 GetProcessorId(const TString& database) {
- auto it = ProcessorIds.find(database);
- if (it == ProcessorIds.end() || !it->second) {
- return 0;
- }
- auto processorId = *it->second;
- if (!processorId) {
- RequestProcessorId(database);
- return 0;
- }
- return processorId;
- }
-
- void RequestProcessorId(const TString& database) {
- using TNavigate = NSchemeCache::TSchemeCacheNavigate;
-
- auto request = MakeHolder<TNavigate>();
- request->ResultSet.push_back({});
-
- auto& entry = request->ResultSet.back();
- entry.Path = SplitPath(database);
- entry.Operation = TNavigate::EOp::OpPath;
- entry.RequestType = TNavigate::TEntry::ERequestType::ByPath;
- entry.RedirectRequired = false;
-
- Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release()));
- }
-
- void SendCounters(const TString& database) {
- auto processorId = GetProcessorId(database);
- if (!processorId) {
- return;
- }
-
- auto& dbCounters = DatabaseCounters[database];
-
- auto sendEv = MakeHolder<TEvSysView::TEvSendDbCountersRequest>();
- auto& record = sendEv->Record;
-
- if (dbCounters.IsConfirmed) {
- for (auto& [service, state] : dbCounters.States) {
- state.Counters->ToProto(state.Current);
- }
- ++dbCounters.Generation;
- dbCounters.IsConfirmed = false;
- dbCounters.IsRetrying = false;
- } else {
- dbCounters.IsRetrying = true;
- }
-
- record.SetGeneration(dbCounters.Generation);
- record.SetNodeId(SelfId().NodeId());
-
- for (auto& [service, state] : dbCounters.States) {
- auto* serviceCounters = record.AddServiceCounters();
- serviceCounters->SetService(service);
- auto* diff = serviceCounters->MutableCounters();
-
- CalculateCountersDiff(diff, state.Current, state.Confirmed);
- }
-
- SVLOG_D("Send counters: "
- << "service id# " << SelfId()
- << ", processor id# " << processorId
- << ", database# " << database
- << ", generation# " << record.GetGeneration()
- << ", node id# " << record.GetNodeId()
- << ", is retrying# " << dbCounters.IsRetrying);
-
- Send(MakePipePeNodeCacheID(false),
- new TEvPipeCache::TEvForward(sendEv.Release(), processorId, true),
- IEventHandle::FlagTrackDelivery);
- }
-
- void RequestDatabaseName(TPathId pathId) {
- using TNavigate = NSchemeCache::TSchemeCacheNavigate;
- auto request = MakeHolder<TNavigate>();
- request->ResultSet.push_back({});
-
- auto& entry = request->ResultSet.back();
- entry.TableId.PathId = pathId;
- entry.Operation = TNavigate::EOp::OpPath;
- entry.RequestType = TNavigate::TEntry::ERequestType::ByTableId;
- entry.RedirectRequired = false;
-
- Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release()));
- }
-
- void RegisterDbCounters(const TString& database, NKikimrSysView::EDbCountersService service,
- TIntrusivePtr<IDbCounters> counters)
- {
- auto [it, inserted] = DatabaseCounters.try_emplace(database, TDbCounters());
- if (inserted) {
- if (ProcessorIds.find(database) == ProcessorIds.end()) {
- RequestProcessorId(database);
- }
-
- if (DbWatcherActorId) {
- auto evWatch = MakeHolder<NSysView::TEvSysView::TEvWatchDatabase>(database);
- Send(DbWatcherActorId, evWatch.Release());
- }
- }
- it->second.States[service].Counters = counters;
- }
-
- void Handle(TEvPrivate::TEvSendSummary::TPtr& ev) {
- auto prevIntervalEnd = IntervalEnd - TotalInterval;
- auto intervalEnd = ev->Get()->IntervalEnd;
- if (intervalEnd != prevIntervalEnd) {
- return;
- }
- if (Now() > intervalEnd + CollectInterval) {
- return;
- }
- SendSummary(intervalEnd, ev->Get()->Database);
- }
-
- void Handle(TEvPrivate::TEvProcessInterval::TPtr& ev) {
- SVLOG_D("Handle TEvPrivate::TEvProcessInterval: "
- << "service id# " << SelfId()
- << ", interval end# " << IntervalEnd
- << ", event interval end# " << ev->Get()->IntervalEnd);
-
- if (IntervalEnd == ev->Get()->IntervalEnd) {
- Rotate();
- }
- }
-
- void Handle(TEvSysView::TEvGetIntervalMetricsRequest::TPtr& ev) {
- auto response = MakeHolder<TEvSysView::TEvGetIntervalMetricsResponse>();
-
+ IntervalEnd = GetNextIntervalEnd();
+ Schedule(IntervalEnd, new TEvPrivate::TEvProcessInterval(IntervalEnd));
+ }
+
+ if (AppData()->FeatureFlags.GetEnableDbCounters()) {
+ auto intervalSize = ProcessCountersInterval.MicroSeconds();
+ auto deadline = (Now().MicroSeconds() / intervalSize + 1) * intervalSize;
+ deadline += RandomNumber<ui64>(intervalSize / 5);
+ Schedule(TInstant::MicroSeconds(deadline), new TEvPrivate::TEvProcessCounters());
+
+ auto callback = MakeIntrusive<TServiceDbWatcherCallback>(ctx.ActorSystem());
+ DbWatcherActorId = ctx.Register(CreateDbWatcherActor(callback));
+ }
+
+ if (HasExternalCounters) {
+ ctx.Register(CreateExtCountersUpdater(std::move(Config)));
+ }
+
+ Become(&TSysViewService::StateWork);
+ }
+
+ STFUNC(StateWork) {
+ Y_UNUSED(ctx);
+ switch(ev->GetTypeRewrite()) {
+ hFunc(TEvTenantPool::TEvTenantPoolStatus, Handle);
+ hFunc(TEvSysView::TEvCollectQueryStats, Handle);
+ hFunc(TEvSysView::TEvGetQueryStats, Handle);
+ hFunc(TEvSysView::TEvGetScanLimiter, Handle);
+ hFunc(TEvPrivate::TEvProcessInterval, Handle);
+ hFunc(TEvPrivate::TEvSendSummary, Handle);
+ hFunc(TEvPrivate::TEvProcessCounters, Handle);
+ hFunc(TEvPrivate::TEvRemoveDatabase, Handle);
+ hFunc(TEvSysView::TEvRegisterDbCounters, Handle);
+ hFunc(TEvSysView::TEvSendDbCountersResponse, Handle);
+ hFunc(TEvSysView::TEvGetIntervalMetricsRequest, Handle);
+ hFunc(TEvPipeCache::TEvDeliveryProblem, Handle);
+ hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle);
+ cFunc(TEvents::TEvPoison::EventType, PassAway);
+ default:
+ SVLOG_CRIT("NSysView::TSysViewService: unexpected event# "
+ << ev->GetTypeRewrite());
+ }
+ }
+
+private:
+ struct TEvPrivate {
+ enum EEv {
+ EvProcessInterval = EventSpaceBegin(TEvents::ES_PRIVATE),
+ EvSendSummary,
+ EvProcessCounters,
+ EvRemoveDatabase,
+ EvEnd
+ };
+
+ struct TEvProcessInterval : public TEventLocal<TEvProcessInterval, EvProcessInterval> {
+ TInstant IntervalEnd;
+
+ explicit TEvProcessInterval(TInstant intervalEnd)
+ : IntervalEnd(intervalEnd)
+ {}
+ };
+
+ struct TEvSendSummary : public TEventLocal<TEvSendSummary, EvSendSummary> {
+ TInstant IntervalEnd;
+ TString Database;
+
+ TEvSendSummary(TInstant intervalEnd, const TString& database)
+ : IntervalEnd(intervalEnd)
+ , Database(database)
+ {}
+ };
+
+ struct TEvProcessCounters : public TEventLocal<TEvProcessCounters, EvProcessCounters> {
+ };
+
+ struct TEvRemoveDatabase : public TEventLocal<TEvRemoveDatabase, EvRemoveDatabase> {
+ TString Database;
+ TPathId PathId;
+
+ TEvRemoveDatabase(const TString& database, TPathId pathId)
+ : Database(database)
+ , PathId(pathId)
+ {}
+ };
+ };
+
+ class TServiceDbWatcherCallback : public TDbWatcherCallback {
+ TActorSystem* ActorSystem = {};
+
+ public:
+ explicit TServiceDbWatcherCallback(TActorSystem* actorSystem)
+ : ActorSystem(actorSystem)
+ {}
+
+ void OnDatabaseRemoved(const TString& database, TPathId pathId) override {
+ auto evRemove = MakeHolder<TEvPrivate::TEvRemoveDatabase>(database, pathId);
+ auto service = MakeSysViewServiceID(ActorSystem->NodeId);
+ ActorSystem->Send(service, evRemove.Release());
+ }
+ };
+
+ TInstant GetNextIntervalEnd() {
+ auto intervalSize = TotalInterval.MicroSeconds();
+ auto rounded = (Now().MicroSeconds() / intervalSize + 1) * intervalSize;
+ return TInstant::MicroSeconds(rounded);
+ }
+
+ void SendSummary(TInstant intervalEnd, const TString& database) {
+ auto processorId = GetProcessorId(database);
+ if (!processorId) {
+ return;
+ }
+
+ auto summary = MakeHolder<TEvSysView::TEvIntervalQuerySummary>();
+ auto& record = summary->Record;
+ record.SetDatabase(database);
+ record.SetIntervalEndUs(intervalEnd.MicroSeconds());
+ record.SetNodeId(SelfId().NodeId());
+
+ auto& log = QueryLogs[database];
+ log.Metrics.Report.FillSummary(*record.MutableMetrics());
+ log.TopByDuration.Report.FillSummary(*record.MutableTopByDuration());
+ log.TopByReadBytes.Report.FillSummary(*record.MutableTopByReadBytes());
+ log.TopByCpuTime.Report.FillSummary(*record.MutableTopByCpuTime());
+ log.TopByRequestUnits.Report.FillSummary(*record.MutableTopByRequestUnits());
+
+ SVLOG_D("Send interval summary: "
+ << "service id# " << SelfId()
+ << ", processor id# " << processorId
+ << ", database# " << database
+ << ", interval end# " << intervalEnd
+ << ", query count# " << record.GetMetrics().HashesSize());
+
+ Send(MakePipePeNodeCacheID(false),
+ new TEvPipeCache::TEvForward(summary.Release(), processorId, true),
+ IEventHandle::FlagTrackDelivery);
+ }
+
+ void Rotate() {
+ auto summaryEnd = IntervalEnd;
+ auto delta = Now() - summaryEnd;
+
+ IntervalEnd = GetNextIntervalEnd();
+ Schedule(IntervalEnd, new TEvPrivate::TEvProcessInterval(IntervalEnd));
+
+ Attempts.clear();
+
+ SVLOG_D("Rotate logs: service id# " << SelfId()
+ << ", query logs count# " << QueryLogs.size()
+ << ", processor ids count# " << ProcessorIds.size()
+ << ", processor id to database count# " << ProcessorIdToDatabase.size());
+
+ if (QueryLogs.empty()) {
+ return;
+ }
+
+ if (delta > CollectInterval) {
+ // too late to send summaries, skip
+ QueryLogs.clear();
+ return;
+ }
+
+ for (auto it = QueryLogs.begin(); it != QueryLogs.end(); ) {
+ it->second.Rotate();
+ if (it->second.Metrics.Report.Empty()) {
+ it = QueryLogs.erase(it);
+ } else {
+ ++it;
+ }
+ }
+
+ if (delta >= SendInterval) {
+ // send immediately
+ for (const auto& [database, _] : QueryLogs) {
+ SendSummary(summaryEnd, database);
+ }
+ return;
+ }
+
+ // randomize send time
+ auto windowUs = (SendInterval - delta).MicroSeconds() / 2;
+ auto now = Now();
+ for (const auto& [database, _] : QueryLogs) {
+ TInstant deadline = now + TDuration::MicroSeconds(windowUs + RandomNumber<ui64>(windowUs));
+ Schedule(deadline, new TEvPrivate::TEvSendSummary(summaryEnd, database));
+ Attempts[database] = std::make_pair(summaryEnd, 1);
+ }
+ }
+
+ ui64 GetProcessorId(const TString& database) {
+ auto it = ProcessorIds.find(database);
+ if (it == ProcessorIds.end() || !it->second) {
+ return 0;
+ }
+ auto processorId = *it->second;
+ if (!processorId) {
+ RequestProcessorId(database);
+ return 0;
+ }
+ return processorId;
+ }
+
+ void RequestProcessorId(const TString& database) {
+ using TNavigate = NSchemeCache::TSchemeCacheNavigate;
+
+ auto request = MakeHolder<TNavigate>();
+ request->ResultSet.push_back({});
+
+ auto& entry = request->ResultSet.back();
+ entry.Path = SplitPath(database);
+ entry.Operation = TNavigate::EOp::OpPath;
+ entry.RequestType = TNavigate::TEntry::ERequestType::ByPath;
+ entry.RedirectRequired = false;
+
+ Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release()));
+ }
+
+ void SendCounters(const TString& database) {
+ auto processorId = GetProcessorId(database);
+ if (!processorId) {
+ return;
+ }
+
+ auto& dbCounters = DatabaseCounters[database];
+
+ auto sendEv = MakeHolder<TEvSysView::TEvSendDbCountersRequest>();
+ auto& record = sendEv->Record;
+
+ if (dbCounters.IsConfirmed) {
+ for (auto& [service, state] : dbCounters.States) {
+ state.Counters->ToProto(state.Current);
+ }
+ ++dbCounters.Generation;
+ dbCounters.IsConfirmed = false;
+ dbCounters.IsRetrying = false;
+ } else {
+ dbCounters.IsRetrying = true;
+ }
+
+ record.SetGeneration(dbCounters.Generation);
+ record.SetNodeId(SelfId().NodeId());
+
+ for (auto& [service, state] : dbCounters.States) {
+ auto* serviceCounters = record.AddServiceCounters();
+ serviceCounters->SetService(service);
+ auto* diff = serviceCounters->MutableCounters();
+
+ CalculateCountersDiff(diff, state.Current, state.Confirmed);
+ }
+
+ SVLOG_D("Send counters: "
+ << "service id# " << SelfId()
+ << ", processor id# " << processorId
+ << ", database# " << database
+ << ", generation# " << record.GetGeneration()
+ << ", node id# " << record.GetNodeId()
+ << ", is retrying# " << dbCounters.IsRetrying);
+
+ Send(MakePipePeNodeCacheID(false),
+ new TEvPipeCache::TEvForward(sendEv.Release(), processorId, true),
+ IEventHandle::FlagTrackDelivery);
+ }
+
+ void RequestDatabaseName(TPathId pathId) {
+ using TNavigate = NSchemeCache::TSchemeCacheNavigate;
+ auto request = MakeHolder<TNavigate>();
+ request->ResultSet.push_back({});
+
+ auto& entry = request->ResultSet.back();
+ entry.TableId.PathId = pathId;
+ entry.Operation = TNavigate::EOp::OpPath;
+ entry.RequestType = TNavigate::TEntry::ERequestType::ByTableId;
+ entry.RedirectRequired = false;
+
+ Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release()));
+ }
+
+ void RegisterDbCounters(const TString& database, NKikimrSysView::EDbCountersService service,
+ TIntrusivePtr<IDbCounters> counters)
+ {
+ auto [it, inserted] = DatabaseCounters.try_emplace(database, TDbCounters());
+ if (inserted) {
+ if (ProcessorIds.find(database) == ProcessorIds.end()) {
+ RequestProcessorId(database);
+ }
+
+ if (DbWatcherActorId) {
+ auto evWatch = MakeHolder<NSysView::TEvSysView::TEvWatchDatabase>(database);
+ Send(DbWatcherActorId, evWatch.Release());
+ }
+ }
+ it->second.States[service].Counters = counters;
+ }
+
+ void Handle(TEvPrivate::TEvSendSummary::TPtr& ev) {
+ auto prevIntervalEnd = IntervalEnd - TotalInterval;
+ auto intervalEnd = ev->Get()->IntervalEnd;
+ if (intervalEnd != prevIntervalEnd) {
+ return;
+ }
+ if (Now() > intervalEnd + CollectInterval) {
+ return;
+ }
+ SendSummary(intervalEnd, ev->Get()->Database);
+ }
+
+ void Handle(TEvPrivate::TEvProcessInterval::TPtr& ev) {
+ SVLOG_D("Handle TEvPrivate::TEvProcessInterval: "
+ << "service id# " << SelfId()
+ << ", interval end# " << IntervalEnd
+ << ", event interval end# " << ev->Get()->IntervalEnd);
+
+ if (IntervalEnd == ev->Get()->IntervalEnd) {
+ Rotate();
+ }
+ }
+
+ void Handle(TEvSysView::TEvGetIntervalMetricsRequest::TPtr& ev) {
+ auto response = MakeHolder<TEvSysView::TEvGetIntervalMetricsResponse>();
+
if (!AppData()->FeatureFlags.GetEnablePersistentQueryStats()) {
- Send(ev->Sender, std::move(response), 0, ev->Cookie);
- return;
- }
-
- const auto& record = ev->Get()->Record;
- response->Record.SetIntervalEndUs(record.GetIntervalEndUs());
- const auto& database = record.GetDatabase();
-
- auto prevIntervalEnd = IntervalEnd - TotalInterval;
- if (record.GetIntervalEndUs() != prevIntervalEnd.MicroSeconds()) {
- SVLOG_W("Handle TEvSysView::TEvGetIntervalMetricsRequest, time mismatch: "
- << "service id# " << SelfId()
- << ", database# " << database
- << ", prev interval end# " << prevIntervalEnd
- << ", event interval end# " << record.GetIntervalEndUs());
-
- Send(ev->Sender, std::move(response), 0, ev->Cookie);
- return;
- }
-
- auto it = QueryLogs.find(database);
- if (it == QueryLogs.end()) {
- SVLOG_W("Handle TEvSysView::TEvGetIntervalMetricsRequest, no database: "
- << "service id# " << SelfId()
- << ", database# " << database
- << ", prev interval end# " << prevIntervalEnd);
-
- Send(ev->Sender, std::move(response), 0, ev->Cookie);
- return;
- }
-
- auto& log = it->second;
- log.Metrics.Report.FillMetrics(record, response->Record);
- log.TopByDuration.Report.FillStats(
- record.GetTopByDuration(), *response->Record.MutableTopByDuration());
- log.TopByReadBytes.Report.FillStats(
- record.GetTopByReadBytes(), *response->Record.MutableTopByReadBytes());
- log.TopByCpuTime.Report.FillStats(
- record.GetTopByCpuTime(), *response->Record.MutableTopByCpuTime());
- log.TopByRequestUnits.Report.FillStats(
- record.GetTopByRequestUnits(), *response->Record.MutableTopByRequestUnits());
-
- SVLOG_D("Handle TEvSysView::TEvGetIntervalMetricsRequest: "
- << "service id# " << SelfId()
- << ", database# " << database
- << ", prev interval end# " << prevIntervalEnd
- << ", metrics count# " << response->Record.MetricsSize()
- << ", texts count# " << response->Record.QueryTextsSize());
-
- Send(ev->Sender, std::move(response), 0, ev->Cookie);
- }
-
- void Handle(TEvPrivate::TEvProcessCounters::TPtr&) {
- SVLOG_D("Handle TEvPrivate::TEvProcessCounters: "
- << "service id# " << SelfId());
-
- for (auto& [database, dbCounters] : DatabaseCounters) {
- SendCounters(database);
- }
-
- Schedule(ProcessCountersInterval, new TEvPrivate::TEvProcessCounters());
- }
-
- void Handle(TEvPrivate::TEvRemoveDatabase::TPtr& ev) {
- auto database = ev->Get()->Database;
- auto pathId = ev->Get()->PathId;
-
- SVLOG_D("Handle TEvPrivate::TEvRemoveDatabase: "
- << "database# " << database
- << ", pathId# " << pathId);
-
- QueryLogs.erase(database);
- if (auto it = ProcessorIds.find(database); it != ProcessorIds.end()) {
- if (it->second) {
- ProcessorIdToDatabase.erase(*it->second);
- }
- }
- ProcessorIds.erase(database);
- Attempts.erase(database);
- DatabaseCounters.erase(database);
- UnresolvedTabletCounters.erase(pathId);
- }
-
- void Handle(TEvSysView::TEvSendDbCountersResponse::TPtr& ev) {
- const auto& record = ev->Get()->Record;
- const auto& database = record.GetDatabase();
- const auto generation = record.GetGeneration();
-
- auto it = DatabaseCounters.find(database);
- if (it == DatabaseCounters.end()) {
- SVLOG_W("Handle TEvSysView::TEvSendDbCountersResponse: "
- << "service id# " << SelfId()
- << ", unknown database# " << database);
- return;
- }
-
- auto& dbCounters = it->second;
- if (generation != dbCounters.Generation) {
- SVLOG_W("Handle TEvSysView::TEvSendDbCountersResponse, wrong generation: "
- << "service id# " << SelfId()
- << ", database# " << database
- << ", generation# " << generation
- << ", service generation# " << dbCounters.Generation);
- return;
- }
-
- dbCounters.IsConfirmed = true;
- for (auto& [_, state] : dbCounters.States) {
- state.Confirmed.Swap(state.Current);
- }
-
- if (dbCounters.IsRetrying) {
- SendCounters(database);
- }
-
- SVLOG_D("Handle TEvSysView::TEvSendDbCountersResponse: "
- << "service id# " << SelfId()
- << ", database# " << database
- << ", generation# " << generation);
- }
-
- void Handle(TEvSysView::TEvRegisterDbCounters::TPtr& ev) {
- const auto service = ev->Get()->Service;
-
- if (service == NKikimrSysView::TABLETS) { // register by path id
- auto pathId = ev->Get()->PathId;
- UnresolvedTabletCounters[pathId] = ev->Get()->Counters;
- RequestDatabaseName(pathId);
-
- SVLOG_D("Handle TEvSysView::TEvRegisterDbCounters: "
- << "service id# " << SelfId()
- << ", path id# " << pathId
- << ", service# " << (int)service);
-
- } else { // register by database name
- const auto& database = ev->Get()->Database;
- RegisterDbCounters(database, service, ev->Get()->Counters);
-
- SVLOG_D("Handle TEvSysView::TEvRegisterDbCounters: "
- << "service id# " << SelfId()
- << ", database# " << database
- << ", service# " << (int)service);
- }
- }
-
- void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) {
- ui64 processorId = ev->Get()->TabletId;
- TString database;
- auto it = ProcessorIdToDatabase.find(processorId);
- if (it != ProcessorIdToDatabase.end()) {
- database = it->second;
- RequestProcessorId(database);
- }
-
- SVLOG_W("Summary delivery problem: service id# " << SelfId()
- << ", processor id# " << processorId
- << ", database# " << database);
-
- if (!database) {
- return;
- }
-
- auto attemptIt = Attempts.find(database);
- if (attemptIt == Attempts.end()) {
- return;
- }
-
- auto& attempt = attemptIt->second;
- if (++attempt.second > SummaryRetryAttempts) {
- return;
- }
-
- auto summaryEnd = attempt.first;
- auto deadline = Now() + SummaryRetryInterval;
- Schedule(deadline, new TEvPrivate::TEvSendSummary(summaryEnd, database));
- }
-
- void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) {
- using TNavigate = NSchemeCache::TSchemeCacheNavigate;
-
- THolder<TNavigate> request(ev->Get()->Request.Release());
- Y_VERIFY(request->ResultSet.size() == 1);
- auto& entry = request->ResultSet.back();
-
- if (entry.RequestType == TNavigate::TEntry::ERequestType::ByTableId) {
- auto pathId = entry.TableId.PathId;
-
- if (entry.Status != TNavigate::EStatus::Ok) {
- SVLOG_W("Navigate by path id failed: service id# " << SelfId()
- << ", path id# " << pathId
- << ", status# " << entry.Status);
- return;
- }
-
- auto database = CanonizePath(entry.Path);
- auto it = UnresolvedTabletCounters.find(pathId);
- if (it == UnresolvedTabletCounters.end()) {
- return;
- }
-
- RegisterDbCounters(database, NKikimrSysView::TABLETS, it->second);
- UnresolvedTabletCounters.erase(it);
-
- SVLOG_I("Navigate by path id succeeded: service id# " << SelfId()
- << ", path id# " << pathId
- << ", database# " << database);
-
- } else {
- auto database = CanonizePath(entry.Path);
-
- if (entry.Status != TNavigate::EStatus::Ok) {
- SVLOG_W("Navigate by database failed: service id# " << SelfId()
- << ", database# " << database
- << ", status# " << entry.Status);
- ProcessorIds.erase(database);
- return;
- }
-
- if (entry.DomainInfo->Params.HasSysViewProcessor()) {
- auto processorId = entry.DomainInfo->Params.GetSysViewProcessor();
- ProcessorIds[database] = processorId;
- ProcessorIdToDatabase[processorId] = database;
-
- SVLOG_I("Navigate by database succeeded: service id# " << SelfId()
- << ", database# " << database
- << ", processor id# " << processorId);
- } else {
- ProcessorIds[database] = 0;
-
- SVLOG_I("Navigate by database succeeded: service id# " << SelfId()
- << ", database# " << database
- << ", no sysview processor");
- }
- }
- }
-
- void Handle(TEvTenantPool::TEvTenantPoolStatus::TPtr &ev) {
- auto& record = ev->Get()->Record;
- SVLOG_D("Handle TEvTenantPool::TEvTenantPoolStatus: service id# " << SelfId());
-
- Tenants.clear();
- for (auto& slot : record.GetSlots()) {
- Tenants.insert(slot.GetAssignedTenant());
- }
- HasTenants = true;
-
- TopByDuration1Minute->Clear();
- TopByDuration1Hour->Clear();
- TopByReadBytes1Minute->Clear();
- TopByReadBytes1Hour->Clear();
- TopByCpuTime1Minute->Clear();
- TopByCpuTime1Hour->Clear();
- TopByRequestUnits1Minute->Clear();
- TopByRequestUnits1Hour->Clear();
- }
-
- void Handle(TEvSysView::TEvCollectQueryStats::TPtr& ev) {
- const auto& database = ev->Get()->Database;
-
- auto stats = std::make_shared<NKikimrSysView::TQueryStats>();
- stats->Swap(&ev->Get()->QueryStats);
-
- SVLOG_T("Collect query stats: service id# " << SelfId()
- << ", database# " << database
- << ", query hash# " << stats->GetQueryTextHash()
- << ", cpu time# " << stats->GetTotalCpuTimeUs());
-
+ Send(ev->Sender, std::move(response), 0, ev->Cookie);
+ return;
+ }
+
+ const auto& record = ev->Get()->Record;
+ response->Record.SetIntervalEndUs(record.GetIntervalEndUs());
+ const auto& database = record.GetDatabase();
+
+ auto prevIntervalEnd = IntervalEnd - TotalInterval;
+ if (record.GetIntervalEndUs() != prevIntervalEnd.MicroSeconds()) {
+ SVLOG_W("Handle TEvSysView::TEvGetIntervalMetricsRequest, time mismatch: "
+ << "service id# " << SelfId()
+ << ", database# " << database
+ << ", prev interval end# " << prevIntervalEnd
+ << ", event interval end# " << record.GetIntervalEndUs());
+
+ Send(ev->Sender, std::move(response), 0, ev->Cookie);
+ return;
+ }
+
+ auto it = QueryLogs.find(database);
+ if (it == QueryLogs.end()) {
+ SVLOG_W("Handle TEvSysView::TEvGetIntervalMetricsRequest, no database: "
+ << "service id# " << SelfId()
+ << ", database# " << database
+ << ", prev interval end# " << prevIntervalEnd);
+
+ Send(ev->Sender, std::move(response), 0, ev->Cookie);
+ return;
+ }
+
+ auto& log = it->second;
+ log.Metrics.Report.FillMetrics(record, response->Record);
+ log.TopByDuration.Report.FillStats(
+ record.GetTopByDuration(), *response->Record.MutableTopByDuration());
+ log.TopByReadBytes.Report.FillStats(
+ record.GetTopByReadBytes(), *response->Record.MutableTopByReadBytes());
+ log.TopByCpuTime.Report.FillStats(
+ record.GetTopByCpuTime(), *response->Record.MutableTopByCpuTime());
+ log.TopByRequestUnits.Report.FillStats(
+ record.GetTopByRequestUnits(), *response->Record.MutableTopByRequestUnits());
+
+ SVLOG_D("Handle TEvSysView::TEvGetIntervalMetricsRequest: "
+ << "service id# " << SelfId()
+ << ", database# " << database
+ << ", prev interval end# " << prevIntervalEnd
+ << ", metrics count# " << response->Record.MetricsSize()
+ << ", texts count# " << response->Record.QueryTextsSize());
+
+ Send(ev->Sender, std::move(response), 0, ev->Cookie);
+ }
+
+ void Handle(TEvPrivate::TEvProcessCounters::TPtr&) {
+ SVLOG_D("Handle TEvPrivate::TEvProcessCounters: "
+ << "service id# " << SelfId());
+
+ for (auto& [database, dbCounters] : DatabaseCounters) {
+ SendCounters(database);
+ }
+
+ Schedule(ProcessCountersInterval, new TEvPrivate::TEvProcessCounters());
+ }
+
+ void Handle(TEvPrivate::TEvRemoveDatabase::TPtr& ev) {
+ auto database = ev->Get()->Database;
+ auto pathId = ev->Get()->PathId;
+
+ SVLOG_D("Handle TEvPrivate::TEvRemoveDatabase: "
+ << "database# " << database
+ << ", pathId# " << pathId);
+
+ QueryLogs.erase(database);
+ if (auto it = ProcessorIds.find(database); it != ProcessorIds.end()) {
+ if (it->second) {
+ ProcessorIdToDatabase.erase(*it->second);
+ }
+ }
+ ProcessorIds.erase(database);
+ Attempts.erase(database);
+ DatabaseCounters.erase(database);
+ UnresolvedTabletCounters.erase(pathId);
+ }
+
+ void Handle(TEvSysView::TEvSendDbCountersResponse::TPtr& ev) {
+ const auto& record = ev->Get()->Record;
+ const auto& database = record.GetDatabase();
+ const auto generation = record.GetGeneration();
+
+ auto it = DatabaseCounters.find(database);
+ if (it == DatabaseCounters.end()) {
+ SVLOG_W("Handle TEvSysView::TEvSendDbCountersResponse: "
+ << "service id# " << SelfId()
+ << ", unknown database# " << database);
+ return;
+ }
+
+ auto& dbCounters = it->second;
+ if (generation != dbCounters.Generation) {
+ SVLOG_W("Handle TEvSysView::TEvSendDbCountersResponse, wrong generation: "
+ << "service id# " << SelfId()
+ << ", database# " << database
+ << ", generation# " << generation
+ << ", service generation# " << dbCounters.Generation);
+ return;
+ }
+
+ dbCounters.IsConfirmed = true;
+ for (auto& [_, state] : dbCounters.States) {
+ state.Confirmed.Swap(state.Current);
+ }
+
+ if (dbCounters.IsRetrying) {
+ SendCounters(database);
+ }
+
+ SVLOG_D("Handle TEvSysView::TEvSendDbCountersResponse: "
+ << "service id# " << SelfId()
+ << ", database# " << database
+ << ", generation# " << generation);
+ }
+
+ void Handle(TEvSysView::TEvRegisterDbCounters::TPtr& ev) {
+ const auto service = ev->Get()->Service;
+
+ if (service == NKikimrSysView::TABLETS) { // register by path id
+ auto pathId = ev->Get()->PathId;
+ UnresolvedTabletCounters[pathId] = ev->Get()->Counters;
+ RequestDatabaseName(pathId);
+
+ SVLOG_D("Handle TEvSysView::TEvRegisterDbCounters: "
+ << "service id# " << SelfId()
+ << ", path id# " << pathId
+ << ", service# " << (int)service);
+
+ } else { // register by database name
+ const auto& database = ev->Get()->Database;
+ RegisterDbCounters(database, service, ev->Get()->Counters);
+
+ SVLOG_D("Handle TEvSysView::TEvRegisterDbCounters: "
+ << "service id# " << SelfId()
+ << ", database# " << database
+ << ", service# " << (int)service);
+ }
+ }
+
+ void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) {
+ ui64 processorId = ev->Get()->TabletId;
+ TString database;
+ auto it = ProcessorIdToDatabase.find(processorId);
+ if (it != ProcessorIdToDatabase.end()) {
+ database = it->second;
+ RequestProcessorId(database);
+ }
+
+ SVLOG_W("Summary delivery problem: service id# " << SelfId()
+ << ", processor id# " << processorId
+ << ", database# " << database);
+
+ if (!database) {
+ return;
+ }
+
+ auto attemptIt = Attempts.find(database);
+ if (attemptIt == Attempts.end()) {
+ return;
+ }
+
+ auto& attempt = attemptIt->second;
+ if (++attempt.second > SummaryRetryAttempts) {
+ return;
+ }
+
+ auto summaryEnd = attempt.first;
+ auto deadline = Now() + SummaryRetryInterval;
+ Schedule(deadline, new TEvPrivate::TEvSendSummary(summaryEnd, database));
+ }
+
+ void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) {
+ using TNavigate = NSchemeCache::TSchemeCacheNavigate;
+
+ THolder<TNavigate> request(ev->Get()->Request.Release());
+ Y_VERIFY(request->ResultSet.size() == 1);
+ auto& entry = request->ResultSet.back();
+
+ if (entry.RequestType == TNavigate::TEntry::ERequestType::ByTableId) {
+ auto pathId = entry.TableId.PathId;
+
+ if (entry.Status != TNavigate::EStatus::Ok) {
+ SVLOG_W("Navigate by path id failed: service id# " << SelfId()
+ << ", path id# " << pathId
+ << ", status# " << entry.Status);
+ return;
+ }
+
+ auto database = CanonizePath(entry.Path);
+ auto it = UnresolvedTabletCounters.find(pathId);
+ if (it == UnresolvedTabletCounters.end()) {
+ return;
+ }
+
+ RegisterDbCounters(database, NKikimrSysView::TABLETS, it->second);
+ UnresolvedTabletCounters.erase(it);
+
+ SVLOG_I("Navigate by path id succeeded: service id# " << SelfId()
+ << ", path id# " << pathId
+ << ", database# " << database);
+
+ } else {
+ auto database = CanonizePath(entry.Path);
+
+ if (entry.Status != TNavigate::EStatus::Ok) {
+ SVLOG_W("Navigate by database failed: service id# " << SelfId()
+ << ", database# " << database
+ << ", status# " << entry.Status);
+ ProcessorIds.erase(database);
+ return;
+ }
+
+ if (entry.DomainInfo->Params.HasSysViewProcessor()) {
+ auto processorId = entry.DomainInfo->Params.GetSysViewProcessor();
+ ProcessorIds[database] = processorId;
+ ProcessorIdToDatabase[processorId] = database;
+
+ SVLOG_I("Navigate by database succeeded: service id# " << SelfId()
+ << ", database# " << database
+ << ", processor id# " << processorId);
+ } else {
+ ProcessorIds[database] = 0;
+
+ SVLOG_I("Navigate by database succeeded: service id# " << SelfId()
+ << ", database# " << database
+ << ", no sysview processor");
+ }
+ }
+ }
+
+ void Handle(TEvTenantPool::TEvTenantPoolStatus::TPtr &ev) {
+ auto& record = ev->Get()->Record;
+ SVLOG_D("Handle TEvTenantPool::TEvTenantPoolStatus: service id# " << SelfId());
+
+ Tenants.clear();
+ for (auto& slot : record.GetSlots()) {
+ Tenants.insert(slot.GetAssignedTenant());
+ }
+ HasTenants = true;
+
+ TopByDuration1Minute->Clear();
+ TopByDuration1Hour->Clear();
+ TopByReadBytes1Minute->Clear();
+ TopByReadBytes1Hour->Clear();
+ TopByCpuTime1Minute->Clear();
+ TopByCpuTime1Hour->Clear();
+ TopByRequestUnits1Minute->Clear();
+ TopByRequestUnits1Hour->Clear();
+ }
+
+ void Handle(TEvSysView::TEvCollectQueryStats::TPtr& ev) {
+ const auto& database = ev->Get()->Database;
+
+ auto stats = std::make_shared<NKikimrSysView::TQueryStats>();
+ stats->Swap(&ev->Get()->QueryStats);
+
+ SVLOG_T("Collect query stats: service id# " << SelfId()
+ << ", database# " << database
+ << ", query hash# " << stats->GetQueryTextHash()
+ << ", cpu time# " << stats->GetTotalCpuTimeUs());
+
if (AppData()->FeatureFlags.GetEnablePersistentQueryStats() && !database.empty()) {
- auto queryEnd = TInstant::MilliSeconds(stats->GetEndTimeMs());
- if (queryEnd < IntervalEnd - TotalInterval) {
- return;
- }
- if (queryEnd >= IntervalEnd) {
- Rotate();
- }
-
- QueryLogs[database].Add(stats);
-
- if (ProcessorIds.find(database) == ProcessorIds.end()) {
- ProcessorIds[database];
- RequestProcessorId(database);
-
- if (DbWatcherActorId) {
- auto evWatch = MakeHolder<NSysView::TEvSysView::TEvWatchDatabase>(database);
- Send(DbWatcherActorId, evWatch.Release());
- }
- }
- }
-
- // gather old style stats while migrating
- // TODO: remove later
- TopByDuration1Minute->Add(stats);
- TopByDuration1Hour->Add(stats);
- TopByReadBytes1Minute->Add(stats);
- TopByReadBytes1Hour->Add(stats);
- TopByCpuTime1Minute->Add(stats);
- TopByCpuTime1Hour->Add(stats);
- TopByRequestUnits1Minute->Add(stats);
- TopByRequestUnits1Hour->Add(stats);
- }
-
- void Handle(TEvSysView::TEvGetQueryStats::TPtr& ev) {
- auto& record = ev->Get()->Record;
- auto result = MakeHolder<TEvSysView::TEvGetQueryStatsResult>();
-
- ui64 startBucket = 0;
- if (record.HasStartBucket()) {
- startBucket = record.GetStartBucket();
- }
-
- if (!HasTenants ||
- !record.HasTenantName() ||
- !Tenants.contains(record.GetTenantName()))
- {
- Send(ev->Sender, std::move(result), 0, ev->Cookie);
- return;
- }
-
- switch (record.GetStatsType()) {
- case NKikimrSysView::TOP_DURATION_ONE_MINUTE:
- TopByDuration1Minute->ToProto(startBucket, result->Record);
- break;
- case NKikimrSysView::TOP_DURATION_ONE_HOUR:
- TopByDuration1Hour->ToProto(startBucket, result->Record);
- break;
- case NKikimrSysView::TOP_READ_BYTES_ONE_MINUTE:
- TopByReadBytes1Minute->ToProto(startBucket, result->Record);
- break;
- case NKikimrSysView::TOP_READ_BYTES_ONE_HOUR:
- TopByReadBytes1Hour->ToProto(startBucket, result->Record);
- break;
- case NKikimrSysView::TOP_CPU_TIME_ONE_MINUTE:
- TopByCpuTime1Minute->ToProto(startBucket, result->Record);
- break;
- case NKikimrSysView::TOP_CPU_TIME_ONE_HOUR:
- TopByCpuTime1Hour->ToProto(startBucket, result->Record);
- break;
- case NKikimrSysView::TOP_REQUEST_UNITS_ONE_MINUTE:
- TopByRequestUnits1Minute->ToProto(startBucket, result->Record);
- break;
- case NKikimrSysView::TOP_REQUEST_UNITS_ONE_HOUR:
- TopByRequestUnits1Hour->ToProto(startBucket, result->Record);
- break;
- default:
- SVLOG_CRIT("NSysView::TSysViewService: unexpected query stats type# "
- << (size_t)record.GetStatsType());
- // send empty result
- break;
- }
-
- Send(ev->Sender, std::move(result), 0, ev->Cookie);
- }
-
- void Handle(TEvSysView::TEvGetScanLimiter::TPtr& ev) {
- auto result = MakeHolder<TEvSysView::TEvGetScanLimiterResult>();
- result->ScanLimiter = ScanLimiter;
- Send(ev->Sender, std::move(result));
- }
-
- void PassAway() override {
- Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(0));
- TBase::PassAway();
- }
-
-private:
- TExtCountersConfig Config;
- const bool HasExternalCounters;
- const TDuration TotalInterval;
- const TDuration CollectInterval;
- const TDuration SendInterval;
-
- template <typename TInterval>
- struct TDbWindow {
- TInterval Collect;
- TInterval Report;
-
- void Clear() {
- Collect.Clear();
- Report.Clear();
- }
-
- void Rotate() {
- Report.Clear();
- Report.Swap(Collect);
- }
- };
-
- struct TDbQueryLog {
- TDbWindow<TQueryInterval> Metrics;
- TDbWindow<TQueryStatsDedupBucket<TDurationGreater>> TopByDuration;
- TDbWindow<TQueryStatsDedupBucket<TReadBytesGreater>> TopByReadBytes;
- TDbWindow<TQueryStatsDedupBucket<TCpuTimeGreater>> TopByCpuTime;
- TDbWindow<TQueryStatsDedupBucket<TRequestUnitsGreater>> TopByRequestUnits;
-
- void Clear() {
- Metrics.Clear();
- TopByDuration.Clear();
- TopByReadBytes.Clear();
- TopByCpuTime.Clear();
- TopByRequestUnits.Clear();
- }
-
- void Rotate() {
- Metrics.Rotate();
- TopByDuration.Rotate();
- TopByReadBytes.Rotate();
- TopByCpuTime.Rotate();
- TopByRequestUnits.Rotate();
- }
-
- void Add(TQueryStatsPtr stats) {
- Metrics.Collect.Add(stats);
- TopByDuration.Collect.Add(stats);
- TopByReadBytes.Collect.Add(stats);
- TopByCpuTime.Collect.Add(stats);
- TopByRequestUnits.Collect.Add(stats);
- }
- };
- std::unordered_map<TString, TDbQueryLog> QueryLogs;
-
- TInstant IntervalEnd;
-
- std::unordered_map<TString, TMaybe<ui64>> ProcessorIds;
- std::unordered_map<ui64, TString> ProcessorIdToDatabase;
-
- std::unordered_map<TString, std::pair<TInstant, size_t>> Attempts;
-
- THolder<TServiceQueryHistory<TDurationGreater>> TopByDuration1Minute;
- THolder<TServiceQueryHistory<TDurationGreater>> TopByDuration1Hour;
-
- THolder<TServiceQueryHistory<TReadBytesGreater>> TopByReadBytes1Minute;
- THolder<TServiceQueryHistory<TReadBytesGreater>> TopByReadBytes1Hour;
-
- THolder<TServiceQueryHistory<TCpuTimeGreater>> TopByCpuTime1Minute;
- THolder<TServiceQueryHistory<TCpuTimeGreater>> TopByCpuTime1Hour;
-
- THolder<TServiceQueryHistory<TRequestUnitsGreater>> TopByRequestUnits1Minute;
- THolder<TServiceQueryHistory<TRequestUnitsGreater>> TopByRequestUnits1Hour;
-
- bool HasTenants = false;
- THashSet<TString> Tenants;
-
- struct TDbCountersState {
- TIntrusivePtr<IDbCounters> Counters;
- NKikimr::NSysView::TDbServiceCounters Current;
- NKikimr::NSysView::TDbServiceCounters Confirmed;
- };
-
- struct TDbCounters {
- std::unordered_map<NKikimrSysView::EDbCountersService, TDbCountersState> States;
- ui64 Generation;
- bool IsConfirmed = true;
- bool IsRetrying = false;
-
- TDbCounters()
- : Generation(RandomNumber<ui64>())
- {}
- };
-
- std::unordered_map<TString, TDbCounters> DatabaseCounters;
- THashMap<TPathId, TIntrusivePtr<IDbCounters>> UnresolvedTabletCounters;
- TActorId DbWatcherActorId;
-
- static constexpr i64 ConcurrentScansLimit = 5;
- TIntrusivePtr<TScanLimiter> ScanLimiter;
-
- static constexpr TDuration SummaryRetryInterval = TDuration::Seconds(2);
- static constexpr size_t SummaryRetryAttempts = 5;
-
- static constexpr TDuration ProcessCountersInterval = TDuration::Seconds(5);
-};
-
-THolder<IActor> CreateSysViewService(
- TExtCountersConfig&& config, bool hasExternalCounters)
-{
- return MakeHolder<TSysViewService>(
- std::move(config), hasExternalCounters, EProcessorMode::MINUTE);
-}
-
-THolder<IActor> CreateSysViewServiceForTests() {
- return MakeHolder<TSysViewService>(
- TExtCountersConfig(), true, EProcessorMode::FAST);
-}
-
-} // NSysView
-} // NKikimr
+ auto queryEnd = TInstant::MilliSeconds(stats->GetEndTimeMs());
+ if (queryEnd < IntervalEnd - TotalInterval) {
+ return;
+ }
+ if (queryEnd >= IntervalEnd) {
+ Rotate();
+ }
+
+ QueryLogs[database].Add(stats);
+
+ if (ProcessorIds.find(database) == ProcessorIds.end()) {
+ ProcessorIds[database];
+ RequestProcessorId(database);
+
+ if (DbWatcherActorId) {
+ auto evWatch = MakeHolder<NSysView::TEvSysView::TEvWatchDatabase>(database);
+ Send(DbWatcherActorId, evWatch.Release());
+ }
+ }
+ }
+
+ // gather old style stats while migrating
+ // TODO: remove later
+ TopByDuration1Minute->Add(stats);
+ TopByDuration1Hour->Add(stats);
+ TopByReadBytes1Minute->Add(stats);
+ TopByReadBytes1Hour->Add(stats);
+ TopByCpuTime1Minute->Add(stats);
+ TopByCpuTime1Hour->Add(stats);
+ TopByRequestUnits1Minute->Add(stats);
+ TopByRequestUnits1Hour->Add(stats);
+ }
+
+ void Handle(TEvSysView::TEvGetQueryStats::TPtr& ev) {
+ auto& record = ev->Get()->Record;
+ auto result = MakeHolder<TEvSysView::TEvGetQueryStatsResult>();
+
+ ui64 startBucket = 0;
+ if (record.HasStartBucket()) {
+ startBucket = record.GetStartBucket();
+ }
+
+ if (!HasTenants ||
+ !record.HasTenantName() ||
+ !Tenants.contains(record.GetTenantName()))
+ {
+ Send(ev->Sender, std::move(result), 0, ev->Cookie);
+ return;
+ }
+
+ switch (record.GetStatsType()) {
+ case NKikimrSysView::TOP_DURATION_ONE_MINUTE:
+ TopByDuration1Minute->ToProto(startBucket, result->Record);
+ break;
+ case NKikimrSysView::TOP_DURATION_ONE_HOUR:
+ TopByDuration1Hour->ToProto(startBucket, result->Record);
+ break;
+ case NKikimrSysView::TOP_READ_BYTES_ONE_MINUTE:
+ TopByReadBytes1Minute->ToProto(startBucket, result->Record);
+ break;
+ case NKikimrSysView::TOP_READ_BYTES_ONE_HOUR:
+ TopByReadBytes1Hour->ToProto(startBucket, result->Record);
+ break;
+ case NKikimrSysView::TOP_CPU_TIME_ONE_MINUTE:
+ TopByCpuTime1Minute->ToProto(startBucket, result->Record);
+ break;
+ case NKikimrSysView::TOP_CPU_TIME_ONE_HOUR:
+ TopByCpuTime1Hour->ToProto(startBucket, result->Record);
+ break;
+ case NKikimrSysView::TOP_REQUEST_UNITS_ONE_MINUTE:
+ TopByRequestUnits1Minute->ToProto(startBucket, result->Record);
+ break;
+ case NKikimrSysView::TOP_REQUEST_UNITS_ONE_HOUR:
+ TopByRequestUnits1Hour->ToProto(startBucket, result->Record);
+ break;
+ default:
+ SVLOG_CRIT("NSysView::TSysViewService: unexpected query stats type# "
+ << (size_t)record.GetStatsType());
+ // send empty result
+ break;
+ }
+
+ Send(ev->Sender, std::move(result), 0, ev->Cookie);
+ }
+
+ void Handle(TEvSysView::TEvGetScanLimiter::TPtr& ev) {
+ auto result = MakeHolder<TEvSysView::TEvGetScanLimiterResult>();
+ result->ScanLimiter = ScanLimiter;
+ Send(ev->Sender, std::move(result));
+ }
+
+ void PassAway() override {
+ Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(0));
+ TBase::PassAway();
+ }
+
+private:
+ TExtCountersConfig Config;
+ const bool HasExternalCounters;
+ const TDuration TotalInterval;
+ const TDuration CollectInterval;
+ const TDuration SendInterval;
+
+ template <typename TInterval>
+ struct TDbWindow {
+ TInterval Collect;
+ TInterval Report;
+
+ void Clear() {
+ Collect.Clear();
+ Report.Clear();
+ }
+
+ void Rotate() {
+ Report.Clear();
+ Report.Swap(Collect);
+ }
+ };
+
+ struct TDbQueryLog {
+ TDbWindow<TQueryInterval> Metrics;
+ TDbWindow<TQueryStatsDedupBucket<TDurationGreater>> TopByDuration;
+ TDbWindow<TQueryStatsDedupBucket<TReadBytesGreater>> TopByReadBytes;
+ TDbWindow<TQueryStatsDedupBucket<TCpuTimeGreater>> TopByCpuTime;
+ TDbWindow<TQueryStatsDedupBucket<TRequestUnitsGreater>> TopByRequestUnits;
+
+ void Clear() {
+ Metrics.Clear();
+ TopByDuration.Clear();
+ TopByReadBytes.Clear();
+ TopByCpuTime.Clear();
+ TopByRequestUnits.Clear();
+ }
+
+ void Rotate() {
+ Metrics.Rotate();
+ TopByDuration.Rotate();
+ TopByReadBytes.Rotate();
+ TopByCpuTime.Rotate();
+ TopByRequestUnits.Rotate();
+ }
+
+ void Add(TQueryStatsPtr stats) {
+ Metrics.Collect.Add(stats);
+ TopByDuration.Collect.Add(stats);
+ TopByReadBytes.Collect.Add(stats);
+ TopByCpuTime.Collect.Add(stats);
+ TopByRequestUnits.Collect.Add(stats);
+ }
+ };
+ std::unordered_map<TString, TDbQueryLog> QueryLogs;
+
+ TInstant IntervalEnd;
+
+ std::unordered_map<TString, TMaybe<ui64>> ProcessorIds;
+ std::unordered_map<ui64, TString> ProcessorIdToDatabase;
+
+ std::unordered_map<TString, std::pair<TInstant, size_t>> Attempts;
+
+ THolder<TServiceQueryHistory<TDurationGreater>> TopByDuration1Minute;
+ THolder<TServiceQueryHistory<TDurationGreater>> TopByDuration1Hour;
+
+ THolder<TServiceQueryHistory<TReadBytesGreater>> TopByReadBytes1Minute;
+ THolder<TServiceQueryHistory<TReadBytesGreater>> TopByReadBytes1Hour;
+
+ THolder<TServiceQueryHistory<TCpuTimeGreater>> TopByCpuTime1Minute;
+ THolder<TServiceQueryHistory<TCpuTimeGreater>> TopByCpuTime1Hour;
+
+ THolder<TServiceQueryHistory<TRequestUnitsGreater>> TopByRequestUnits1Minute;
+ THolder<TServiceQueryHistory<TRequestUnitsGreater>> TopByRequestUnits1Hour;
+
+ bool HasTenants = false;
+ THashSet<TString> Tenants;
+
+ struct TDbCountersState {
+ TIntrusivePtr<IDbCounters> Counters;
+ NKikimr::NSysView::TDbServiceCounters Current;
+ NKikimr::NSysView::TDbServiceCounters Confirmed;
+ };
+
+ struct TDbCounters {
+ std::unordered_map<NKikimrSysView::EDbCountersService, TDbCountersState> States;
+ ui64 Generation;
+ bool IsConfirmed = true;
+ bool IsRetrying = false;
+
+ TDbCounters()
+ : Generation(RandomNumber<ui64>())
+ {}
+ };
+
+ std::unordered_map<TString, TDbCounters> DatabaseCounters;
+ THashMap<TPathId, TIntrusivePtr<IDbCounters>> UnresolvedTabletCounters;
+ TActorId DbWatcherActorId;
+
+ static constexpr i64 ConcurrentScansLimit = 5;
+ TIntrusivePtr<TScanLimiter> ScanLimiter;
+
+ static constexpr TDuration SummaryRetryInterval = TDuration::Seconds(2);
+ static constexpr size_t SummaryRetryAttempts = 5;
+
+ static constexpr TDuration ProcessCountersInterval = TDuration::Seconds(5);
+};
+
+THolder<IActor> CreateSysViewService(
+ TExtCountersConfig&& config, bool hasExternalCounters)
+{
+ return MakeHolder<TSysViewService>(
+ std::move(config), hasExternalCounters, EProcessorMode::MINUTE);
+}
+
+THolder<IActor> CreateSysViewServiceForTests() {
+ return MakeHolder<TSysViewService>(
+ TExtCountersConfig(), true, EProcessorMode::FAST);
+}
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/service/sysview_service.h b/ydb/core/sys_view/service/sysview_service.h
index 430dfb0261a..fd923c2ffa3 100644
--- a/ydb/core/sys_view/service/sysview_service.h
+++ b/ydb/core/sys_view/service/sysview_service.h
@@ -1,28 +1,28 @@
-#pragma once
-
-#include "ext_counters.h"
-
+#pragma once
+
+#include "ext_counters.h"
+
#include <ydb/core/base/defs.h>
#include <ydb/core/protos/kqp.pb.h>
#include <ydb/core/protos/kqp_stats.pb.h>
-
-namespace NKikimr {
-namespace NSysView {
-
+
+namespace NKikimr {
+namespace NSysView {
+
inline TActorId MakeSysViewServiceID(ui32 node) {
- const char x[12] = "SysViewSvc!";
+ const char x[12] = "SysViewSvc!";
return TActorId(node, TStringBuf(x, 12));
-}
-
-void CollectQueryStats(const TActorContext& ctx, const NKqpProto::TKqpStatsQuery* queryStats,
- TDuration queryDuration, const TString& queryText,
- const TString& userSID, ui64 parametersSize, const TString& database,
- const NKikimrKqp::EQueryType type, ui64 requestUnits);
-
-THolder<IActor> CreateSysViewService(
- TExtCountersConfig&& config, bool hasExternalCounters);
-
-THolder<IActor> CreateSysViewServiceForTests();
-
-} // NSysView
-} // NKikimr
+}
+
+void CollectQueryStats(const TActorContext& ctx, const NKqpProto::TKqpStatsQuery* queryStats,
+ TDuration queryDuration, const TString& queryText,
+ const TString& userSID, ui64 parametersSize, const TString& database,
+ const NKikimrKqp::EQueryType type, ui64 requestUnits);
+
+THolder<IActor> CreateSysViewService(
+ TExtCountersConfig&& config, bool hasExternalCounters);
+
+THolder<IActor> CreateSysViewServiceForTests();
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/service/ut/ya.make b/ydb/core/sys_view/service/ut/ya.make
index 60f2fd4b7a7..a7b7eac2e0c 100644
--- a/ydb/core/sys_view/service/ut/ya.make
+++ b/ydb/core/sys_view/service/ut/ya.make
@@ -1,20 +1,20 @@
UNITTEST_FOR(ydb/core/sys_view/service)
-
-OWNER(
- monster
- g:kikimr
-)
-
-FORK_SUBTESTS()
-SIZE(MEDIUM)
-TIMEOUT(600)
-
-PEERDIR(
+
+OWNER(
+ monster
+ g:kikimr
+)
+
+FORK_SUBTESTS()
+SIZE(MEDIUM)
+TIMEOUT(600)
+
+PEERDIR(
library/cpp/testing/unittest
-)
-
-SRCS(
- query_history_ut.cpp
-)
-
-END()
+)
+
+SRCS(
+ query_history_ut.cpp
+)
+
+END()
diff --git a/ydb/core/sys_view/service/ya.make b/ydb/core/sys_view/service/ya.make
index 133557dbe10..f8f680ec0a0 100644
--- a/ydb/core/sys_view/service/ya.make
+++ b/ydb/core/sys_view/service/ya.make
@@ -1,30 +1,30 @@
-LIBRARY()
-
-OWNER(
- monster
- g:kikimr
-)
-
-SRCS(
- db_counters.h
- db_counters.cpp
- ext_counters.h
- ext_counters.cpp
- query_history.h
- query_interval.h
- query_interval.cpp
- sysview_service.h
- sysview_service.cpp
-)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(
+ monster
+ g:kikimr
+)
+
+SRCS(
+ db_counters.h
+ db_counters.cpp
+ ext_counters.h
+ ext_counters.cpp
+ query_history.h
+ query_interval.h
+ query_interval.cpp
+ sysview_service.h
+ sysview_service.cpp
+)
+
+PEERDIR(
library/cpp/actors/core
ydb/core/base
ydb/core/protos
ydb/library/aclib/protos
-)
-
-END()
+)
+
+END()
RECURSE_FOR_TESTS(
ut
diff --git a/ydb/core/sys_view/storage/groups.cpp b/ydb/core/sys_view/storage/groups.cpp
index dabd46f9dc7..45e80fc597b 100644
--- a/ydb/core/sys_view/storage/groups.cpp
+++ b/ydb/core/sys_view/storage/groups.cpp
@@ -1,22 +1,22 @@
#include "groups.h"
#include "base.h"
-
+
namespace NKikimr::NSysView {
-
+
template<> void SetField<0>(NKikimrSysView::TGroupKey& key, ui32 value) { key.SetGroupId(value); }
class TGroupsScan : public TStorageScanBase<TGroupsScan, TEvSysView::TEvGetGroupsResponse> {
-public:
+public:
using TStorageScanBase::TStorageScanBase;
-
+
static constexpr const char *GetName() { return "TGroupsScan"; }
-
+
TEvSysView::TEvGetGroupsRequest *CreateQuery() {
- auto request = MakeHolder<TEvSysView::TEvGetGroupsRequest>();
- ConvertKeyRange<NKikimrSysView::TEvGetGroupsRequest, ui32>(request->Record, TableRange);
+ auto request = MakeHolder<TEvSysView::TEvGetGroupsRequest>();
+ ConvertKeyRange<NKikimrSysView::TEvGetGroupsRequest, ui32>(request->Record, TableRange);
return request.Release();
- }
-
+ }
+
static const TFieldMap& GetFieldMap() {
using T = Schema::Groups;
using E = NKikimrSysView::TGroupEntry;
@@ -36,15 +36,15 @@ public:
{T::PutTabletLogLatency::ColumnId, {E::kInfoFieldNumber, V::kPutTabletLogLatencyFieldNumber}},
{T::PutUserDataLatency::ColumnId, {E::kInfoFieldNumber, V::kPutUserDataLatencyFieldNumber}},
{T::GetFastLatency::ColumnId, {E::kInfoFieldNumber, V::kGetFastLatencyFieldNumber}},
- };
+ };
return fieldMap;
- }
-
-};
+ }
+
+};
THolder<IActor> CreateGroupsScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId,
- const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns)
-{
+ const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns)
+{
return MakeHolder<TGroupsScan>(ownerId, scanId, tableId, tableRange, columns);
-}
-
+}
+
} // NKikimr::NSysView
diff --git a/ydb/core/sys_view/storage/groups.h b/ydb/core/sys_view/storage/groups.h
index ab861d94a84..af0f8380282 100644
--- a/ydb/core/sys_view/storage/groups.h
+++ b/ydb/core/sys_view/storage/groups.h
@@ -1,12 +1,12 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/kqp/runtime/kqp_compute.h>
-
-namespace NKikimr {
-namespace NSysView {
-
+
+namespace NKikimr {
+namespace NSysView {
+
THolder<IActor> CreateGroupsScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId,
- const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns);
-
-} // NSysView
-} // NKikimr
+ const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns);
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/storage/pdisks.cpp b/ydb/core/sys_view/storage/pdisks.cpp
index 47f84ae9aa5..39fa7e4ac75 100644
--- a/ydb/core/sys_view/storage/pdisks.cpp
+++ b/ydb/core/sys_view/storage/pdisks.cpp
@@ -1,23 +1,23 @@
-#include "pdisks.h"
+#include "pdisks.h"
#include "base.h"
-
+
namespace NKikimr::NSysView {
-
+
template<> void SetField<0>(NKikimrSysView::TPDiskKey& key, ui32 value) { key.SetNodeId(value); }
template<> void SetField<1>(NKikimrSysView::TPDiskKey& key, ui32 value) { key.SetPDiskId(value); }
class TPDisksScan : public TStorageScanBase<TPDisksScan, TEvSysView::TEvGetPDisksResponse> {
-public:
+public:
using TStorageScanBase::TStorageScanBase;
-
+
static constexpr const char *GetName() { return "TPDisksScan"; }
-
+
TEvSysView::TEvGetPDisksRequest *CreateQuery() {
- auto request = MakeHolder<TEvSysView::TEvGetPDisksRequest>();
- ConvertKeyRange<NKikimrSysView::TEvGetPDisksRequest, ui32, ui32>(request->Record, TableRange);
+ auto request = MakeHolder<TEvSysView::TEvGetPDisksRequest>();
+ ConvertKeyRange<NKikimrSysView::TEvGetPDisksRequest, ui32, ui32>(request->Record, TableRange);
return request.Release();
- }
-
+ }
+
static const TFieldMap& GetFieldMap() {
using T = Schema::PDisks;
using E = NKikimrSysView::TPDiskEntry;
@@ -39,15 +39,15 @@ public:
{T::StatusChangeTimestamp::ColumnId, {E::kInfoFieldNumber, V::kStatusChangeTimestampFieldNumber}},
{T::ExpectedSlotCount::ColumnId, {E::kInfoFieldNumber, V::kExpectedSlotCountFieldNumber}},
{T::NumActiveSlots::ColumnId, {E::kInfoFieldNumber, V::kNumActiveSlotsFieldNumber}},
- };
+ };
return fieldMap;
- }
-};
-
+ }
+};
+
THolder<IActor> CreatePDisksScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId,
- const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns)
-{
+ const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns)
+{
return MakeHolder<TPDisksScan>(ownerId, scanId, tableId, tableRange, columns);
-}
-
+}
+
} // NKikimr::NSysView
diff --git a/ydb/core/sys_view/storage/pdisks.h b/ydb/core/sys_view/storage/pdisks.h
index 4172cb333fd..7457f09c96b 100644
--- a/ydb/core/sys_view/storage/pdisks.h
+++ b/ydb/core/sys_view/storage/pdisks.h
@@ -1,12 +1,12 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/kqp/runtime/kqp_compute.h>
-
-namespace NKikimr {
-namespace NSysView {
-
+
+namespace NKikimr {
+namespace NSysView {
+
THolder<IActor> CreatePDisksScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId,
- const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns);
-
-} // NSysView
-} // NKikimr
+ const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns);
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/storage/storage_pools.cpp b/ydb/core/sys_view/storage/storage_pools.cpp
index 66896a69995..71e5a61bafd 100644
--- a/ydb/core/sys_view/storage/storage_pools.cpp
+++ b/ydb/core/sys_view/storage/storage_pools.cpp
@@ -1,23 +1,23 @@
#include "storage_pools.h"
#include "base.h"
-
+
namespace NKikimr::NSysView {
-
+
template<> void SetField<0>(NKikimrSysView::TStoragePoolKey& key, ui64 value) { key.SetBoxId(value); }
template<> void SetField<1>(NKikimrSysView::TStoragePoolKey& key, ui64 value) { key.SetStoragePoolId(value); }
class TStoragePoolsScan : public TStorageScanBase<TStoragePoolsScan, TEvSysView::TEvGetStoragePoolsResponse> {
-public:
+public:
using TStorageScanBase::TStorageScanBase;
-
+
static constexpr const char *GetName() { return "TStoragePoolsScan"; }
-
+
TEvSysView::TEvGetStoragePoolsRequest *CreateQuery() {
- auto request = MakeHolder<TEvSysView::TEvGetStoragePoolsRequest>();
- ConvertKeyRange<NKikimrSysView::TEvGetStoragePoolsRequest, ui64, ui64>(request->Record, TableRange);
+ auto request = MakeHolder<TEvSysView::TEvGetStoragePoolsRequest>();
+ ConvertKeyRange<NKikimrSysView::TEvGetStoragePoolsRequest, ui64, ui64>(request->Record, TableRange);
return request.Release();
- }
-
+ }
+
static const TFieldMap& GetFieldMap() {
using T = Schema::StoragePools;
using E = NKikimrSysView::TStoragePoolEntry;
@@ -35,15 +35,15 @@ public:
{T::EncryptionMode::ColumnId, {E::kInfoFieldNumber, V::kEncryptionModeFieldNumber}},
{T::SchemeshardId::ColumnId, {E::kInfoFieldNumber, V::kSchemeshardIdFieldNumber}},
{T::PathId::ColumnId, {E::kInfoFieldNumber, V::kPathIdFieldNumber}},
- };
+ };
return fieldMap;
- }
-};
-
+ }
+};
+
THolder<IActor> CreateStoragePoolsScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId,
- const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns)
-{
+ const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns)
+{
return MakeHolder<TStoragePoolsScan>(ownerId, scanId, tableId, tableRange, columns);
-}
-
+}
+
} // NKikimr::NSysView
diff --git a/ydb/core/sys_view/storage/storage_pools.h b/ydb/core/sys_view/storage/storage_pools.h
index 7ced359d00b..3cc4a0b530b 100644
--- a/ydb/core/sys_view/storage/storage_pools.h
+++ b/ydb/core/sys_view/storage/storage_pools.h
@@ -1,12 +1,12 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/kqp/runtime/kqp_compute.h>
-
-namespace NKikimr {
-namespace NSysView {
-
+
+namespace NKikimr {
+namespace NSysView {
+
THolder<IActor> CreateStoragePoolsScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId,
- const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns);
-
-} // NSysView
-} // NKikimr
+ const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns);
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/storage/vslots.cpp b/ydb/core/sys_view/storage/vslots.cpp
index 987eb2750de..f907a256350 100644
--- a/ydb/core/sys_view/storage/vslots.cpp
+++ b/ydb/core/sys_view/storage/vslots.cpp
@@ -1,24 +1,24 @@
-#include "vslots.h"
+#include "vslots.h"
#include "base.h"
-
+
namespace NKikimr::NSysView {
-
+
template<> void SetField<0>(NKikimrSysView::TVSlotKey& key, ui32 value) { key.SetNodeId(value); }
template<> void SetField<1>(NKikimrSysView::TVSlotKey& key, ui32 value) { key.SetPDiskId(value); }
template<> void SetField<2>(NKikimrSysView::TVSlotKey& key, ui32 value) { key.SetVSlotId(value); }
class TVSlotsScan : public TStorageScanBase<TVSlotsScan, TEvSysView::TEvGetVSlotsResponse> {
-public:
+public:
using TStorageScanBase::TStorageScanBase;
-
+
static constexpr const char *GetName() { return "TVSlotsScan"; }
-
+
TEvSysView::TEvGetVSlotsRequest *CreateQuery() {
- auto request = MakeHolder<TEvSysView::TEvGetVSlotsRequest>();
- ConvertKeyRange<NKikimrSysView::TEvGetVSlotsRequest, ui32, ui32, ui32>(request->Record, TableRange);
+ auto request = MakeHolder<TEvSysView::TEvGetVSlotsRequest>();
+ ConvertKeyRange<NKikimrSysView::TEvGetVSlotsRequest, ui32, ui32, ui32>(request->Record, TableRange);
return request.Release();
- }
-
+ }
+
static const TFieldMap& GetFieldMap() {
using T = Schema::VSlots;
using E = NKikimrSysView::TVSlotEntry;
@@ -37,15 +37,15 @@ public:
{T::AvailableSize::ColumnId, {E::kInfoFieldNumber, V::kAvailableSizeFieldNumber}},
{T::Status::ColumnId, {E::kInfoFieldNumber, V::kStatusV2FieldNumber}},
{T::Kind::ColumnId, {E::kInfoFieldNumber, V::kKindFieldNumber}},
- };
+ };
return fieldMap;
- }
-};
-
+ }
+};
+
THolder<IActor> CreateVSlotsScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId,
- const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns)
-{
+ const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns)
+{
return MakeHolder<TVSlotsScan>(ownerId, scanId, tableId, tableRange, columns);
-}
-
+}
+
} // NKikimr::NSysView
diff --git a/ydb/core/sys_view/storage/vslots.h b/ydb/core/sys_view/storage/vslots.h
index 34f38f9e0f8..3b6026ea461 100644
--- a/ydb/core/sys_view/storage/vslots.h
+++ b/ydb/core/sys_view/storage/vslots.h
@@ -1,12 +1,12 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/kqp/runtime/kqp_compute.h>
-
-namespace NKikimr {
-namespace NSysView {
-
+
+namespace NKikimr {
+namespace NSysView {
+
THolder<IActor> CreateVSlotsScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId,
- const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns);
-
-} // NSysView
-} // NKikimr
+ const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns);
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/storage/ya.make b/ydb/core/sys_view/storage/ya.make
index 324b36b7986..7136a617f7e 100644
--- a/ydb/core/sys_view/storage/ya.make
+++ b/ydb/core/sys_view/storage/ya.make
@@ -1,30 +1,30 @@
-LIBRARY()
-
-OWNER(
- monster
- g:kikimr
-)
-
-SRCS(
- groups.h
- groups.cpp
- pdisks.h
- pdisks.cpp
- storage_pools.h
- storage_pools.cpp
+LIBRARY()
+
+OWNER(
+ monster
+ g:kikimr
+)
+
+SRCS(
+ groups.h
+ groups.cpp
+ pdisks.h
+ pdisks.cpp
+ storage_pools.h
+ storage_pools.cpp
storage_stats.h
storage_stats.cpp
- vslots.h
- vslots.cpp
-)
-
-PEERDIR(
+ vslots.h
+ vslots.cpp
+)
+
+PEERDIR(
library/cpp/actors/core
ydb/core/base
ydb/core/kqp/runtime
ydb/core/sys_view/common
-)
-
+)
+
YQL_LAST_ABI_VERSION()
-END()
+END()
diff --git a/ydb/core/sys_view/tablets/tablets.cpp b/ydb/core/sys_view/tablets/tablets.cpp
index 88627748d8c..73da45318df 100644
--- a/ydb/core/sys_view/tablets/tablets.cpp
+++ b/ydb/core/sys_view/tablets/tablets.cpp
@@ -1,77 +1,77 @@
-#include "tablets.h"
-
+#include "tablets.h"
+
#include <ydb/core/sys_view/common/events.h>
#include <ydb/core/sys_view/common/schema.h>
#include <ydb/core/sys_view/common/scan_actor_base_impl.h>
#include <ydb/core/base/tablet_pipecache.h>
#include <ydb/core/mind/hive/hive.h>
#include <ydb/core/mind/hive/tablet_info.h>
-
+
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h>
-#include <library/cpp/actors/core/hfunc.h>
-
-namespace NKikimr::NSysView {
-
-using namespace NActors;
-
-class TTabletsScan : public TScanActorBase<TTabletsScan> {
-public:
- using TBase = TScanActorBase<TTabletsScan>;
-
+#include <library/cpp/actors/core/hfunc.h>
+
+namespace NKikimr::NSysView {
+
+using namespace NActors;
+
+class TTabletsScan : public TScanActorBase<TTabletsScan> {
+public:
+ using TBase = TScanActorBase<TTabletsScan>;
+
static constexpr auto ActorActivityType() {
- return NKikimrServices::TActivity::KQP_SYSTEM_VIEW_SCAN;
- }
-
+ return NKikimrServices::TActivity::KQP_SYSTEM_VIEW_SCAN;
+ }
+
TTabletsScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId,
- const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns)
- : TBase(ownerId, scanId, tableId, tableRange, columns)
- {
- }
-
- STFUNC(StateScan) {
- switch (ev->GetTypeRewrite()) {
- hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, Handle);
- hFunc(TEvSysView::TEvGetTabletIdsResponse, Handle);
- hFunc(TEvSysView::TEvGetTabletsResponse, Handle);
- hFunc(TEvPipeCache::TEvDeliveryProblem, Handle);
- hFunc(NKqp::TEvKqp::TEvAbortExecution, HandleAbortExecution);
- cFunc(TEvents::TEvWakeup::EventType, HandleTimeout);
- cFunc(TEvents::TEvPoison::EventType, PassAway);
- default:
- LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS,
- "NSysView::TTabletsScan: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite());
- }
- }
-
-private:
- void ProceedToScan() override {
- Become(&TThis::StateScan);
- if (AckReceived) {
- RequestTabletIds();
- }
- }
-
- void Handle(NKqp::TEvKqpCompute::TEvScanDataAck::TPtr&) {
- if (TabletIds.empty()) {
- RequestTabletIds();
- } else {
- RequestBatch();
- }
- }
-
+ const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns)
+ : TBase(ownerId, scanId, tableId, tableRange, columns)
+ {
+ }
+
+ STFUNC(StateScan) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, Handle);
+ hFunc(TEvSysView::TEvGetTabletIdsResponse, Handle);
+ hFunc(TEvSysView::TEvGetTabletsResponse, Handle);
+ hFunc(TEvPipeCache::TEvDeliveryProblem, Handle);
+ hFunc(NKqp::TEvKqp::TEvAbortExecution, HandleAbortExecution);
+ cFunc(TEvents::TEvWakeup::EventType, HandleTimeout);
+ cFunc(TEvents::TEvPoison::EventType, PassAway);
+ default:
+ LOG_CRIT(ctx, NKikimrServices::SYSTEM_VIEWS,
+ "NSysView::TTabletsScan: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite());
+ }
+ }
+
+private:
+ void ProceedToScan() override {
+ Become(&TThis::StateScan);
+ if (AckReceived) {
+ RequestTabletIds();
+ }
+ }
+
+ void Handle(NKqp::TEvKqpCompute::TEvScanDataAck::TPtr&) {
+ if (TabletIds.empty()) {
+ RequestTabletIds();
+ } else {
+ RequestBatch();
+ }
+ }
+
bool CalculateRangeFrom() {
/*
* Please note that TabletId and FollowerId do not have NULLs in columns
*/
- const auto& cellsFrom = TableRange.From.GetCells();
-
+ const auto& cellsFrom = TableRange.From.GetCells();
+
// Empty means that we read from +inf, it is impossible
if (cellsFrom.empty()) {
YQL_ENSURE(false, "Range starts from +inf, can't read anything.");
return false;
}
-
+
if (cellsFrom[0].IsNull()) {
return true;
}
@@ -85,11 +85,11 @@ private:
if (cellsFrom.size() == 2) {
if (!cellsFrom[1].IsNull()) {
FromFollowerId = cellsFrom[1].AsValue<ui32>();
- }
-
+ }
+
if (TableRange.FromInclusive) {
return true;
- }
+ }
// The range start from NULL exclusive. So, the next value after NULL will be used.
if (!FromFollowerId.has_value()) {
@@ -103,8 +103,8 @@ private:
}
FromFollowerId.reset();
- }
-
+ }
+
if (FromTabletId < Max<ui64>()) {
++FromTabletId;
return true;
@@ -114,12 +114,12 @@ private:
}
bool CalculateRangeTo() {
- const auto& cellsTo = TableRange.To.GetCells();
-
+ const auto& cellsTo = TableRange.To.GetCells();
+
if (cellsTo.empty()) {
return true;
}
-
+
YQL_ENSURE(!cellsTo[0].IsNull(), "Read to -inf range");
ToTabletId = cellsTo[0].AsValue<ui64>();
@@ -140,11 +140,11 @@ private:
if (cellsTo.size() == 2) {
if (!cellsTo[1].IsNull()) {
ToFollowerId = cellsTo[1].AsValue<ui32>();
- }
-
+ }
+
if (TableRange.ToInclusive) {
return true;
- }
+ }
// The range ends at NULL exclusive. So, the value before NULL will be used.
if (!ToFollowerId.has_value()) {
@@ -158,8 +158,8 @@ private:
}
ToFollowerId.reset();
- }
-
+ }
+
return decreaseTabletId();
}
@@ -171,196 +171,196 @@ private:
return;
}
- if (ToTabletId < FromTabletId) {
- ReplyEmptyAndDie();
- return;
- }
-
- auto& record = request->Record;
- record.SetFrom(FromTabletId);
- record.SetTo(ToTabletId);
-
- auto pipeCache = MakePipePeNodeCacheID(false);
- Send(pipeCache, new TEvPipeCache::TEvForward(request.Release(), HiveId, true),
- IEventHandle::FlagTrackDelivery);
- }
-
- void RequestBatch() {
- if (BatchRequested) {
- return;
- }
- BatchRequested = true;
-
- auto request = MakeHolder<TEvSysView::TEvGetTabletsRequest>();
- auto& record = request->Record;
-
- auto it = FromIterator;
- for (size_t count = 0; count < BatchSize && it != TabletIds.end(); ++count, ++it) {
- record.AddTabletIds(*it);
- }
- FromIterator = it;
-
- record.SetBatchSizeLimit(BatchSize);
-
- auto pipeCache = MakePipePeNodeCacheID(false);
- Send(pipeCache, new TEvPipeCache::TEvForward(request.Release(), HiveId, true),
- IEventHandle::FlagTrackDelivery);
- }
-
- void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr&) {
- ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "Delivery problem in TTabletsScan");
- }
-
- void Handle(TEvSysView::TEvGetTabletIdsResponse::TPtr& ev) {
- const auto& record = ev->Get()->Record;
- if (record.TabletIdsSize() == 0) {
- ReplyEmptyAndDie();
- return;
- }
-
- TabletIds.reserve(record.TabletIdsSize());
- for (const auto& id : record.GetTabletIds()) {
- TabletIds.push_back(id);
- }
- std::sort(TabletIds.begin(), TabletIds.end());
-
- FromIterator = TabletIds.begin();
- RequestBatch();
- }
-
- void Handle(TEvSysView::TEvGetTabletsResponse::TPtr& ev) {
- const auto& record = ev->Get()->Record;
-
- using TEntry = NKikimrSysView::TTabletEntry;
- using TExtractor = std::function<TCell(const TEntry&)>;
- using TSchema = Schema::Tablets;
-
- struct TExtractorsMap : public THashMap<NTable::TTag, TExtractor> {
- TExtractorsMap() {
- insert({TSchema::TabletId::ColumnId, [] (const TEntry& entry) {
- return TCell::Make<ui64>(entry.GetTabletId());
- }});
+ if (ToTabletId < FromTabletId) {
+ ReplyEmptyAndDie();
+ return;
+ }
+
+ auto& record = request->Record;
+ record.SetFrom(FromTabletId);
+ record.SetTo(ToTabletId);
+
+ auto pipeCache = MakePipePeNodeCacheID(false);
+ Send(pipeCache, new TEvPipeCache::TEvForward(request.Release(), HiveId, true),
+ IEventHandle::FlagTrackDelivery);
+ }
+
+ void RequestBatch() {
+ if (BatchRequested) {
+ return;
+ }
+ BatchRequested = true;
+
+ auto request = MakeHolder<TEvSysView::TEvGetTabletsRequest>();
+ auto& record = request->Record;
+
+ auto it = FromIterator;
+ for (size_t count = 0; count < BatchSize && it != TabletIds.end(); ++count, ++it) {
+ record.AddTabletIds(*it);
+ }
+ FromIterator = it;
+
+ record.SetBatchSizeLimit(BatchSize);
+
+ auto pipeCache = MakePipePeNodeCacheID(false);
+ Send(pipeCache, new TEvPipeCache::TEvForward(request.Release(), HiveId, true),
+ IEventHandle::FlagTrackDelivery);
+ }
+
+ void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr&) {
+ ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, "Delivery problem in TTabletsScan");
+ }
+
+ void Handle(TEvSysView::TEvGetTabletIdsResponse::TPtr& ev) {
+ const auto& record = ev->Get()->Record;
+ if (record.TabletIdsSize() == 0) {
+ ReplyEmptyAndDie();
+ return;
+ }
+
+ TabletIds.reserve(record.TabletIdsSize());
+ for (const auto& id : record.GetTabletIds()) {
+ TabletIds.push_back(id);
+ }
+ std::sort(TabletIds.begin(), TabletIds.end());
+
+ FromIterator = TabletIds.begin();
+ RequestBatch();
+ }
+
+ void Handle(TEvSysView::TEvGetTabletsResponse::TPtr& ev) {
+ const auto& record = ev->Get()->Record;
+
+ using TEntry = NKikimrSysView::TTabletEntry;
+ using TExtractor = std::function<TCell(const TEntry&)>;
+ using TSchema = Schema::Tablets;
+
+ struct TExtractorsMap : public THashMap<NTable::TTag, TExtractor> {
+ TExtractorsMap() {
+ insert({TSchema::TabletId::ColumnId, [] (const TEntry& entry) {
+ return TCell::Make<ui64>(entry.GetTabletId());
+ }});
insert({TSchema::FollowerId::ColumnId, [] (const TEntry& entry) {
return TCell::Make<ui32>(entry.GetFollowerId());
- }});
- insert({TSchema::TypeCol::ColumnId, [] (const TEntry& entry) {
- const auto& type = entry.GetType();
- return TCell(type.data(), type.size());
- }});
- insert({TSchema::State::ColumnId, [] (const TEntry& entry) {
- if (!entry.HasState()) {
- return TCell();
- }
- const auto& state = entry.GetState();
- return TCell(state.data(), state.size());
- }});
- insert({TSchema::VolatileState::ColumnId, [] (const TEntry& entry) {
- const auto& state = entry.GetVolatileState();
- return TCell(state.data(), state.size());
- }});
- insert({TSchema::BootState::ColumnId, [] (const TEntry& entry) {
- const auto& state = entry.GetBootState();
- return TCell(state.data(), state.size());
- }});
- insert({TSchema::Generation::ColumnId, [] (const TEntry& entry) {
- if (!entry.HasGeneration()) {
- return TCell();
- }
+ }});
+ insert({TSchema::TypeCol::ColumnId, [] (const TEntry& entry) {
+ const auto& type = entry.GetType();
+ return TCell(type.data(), type.size());
+ }});
+ insert({TSchema::State::ColumnId, [] (const TEntry& entry) {
+ if (!entry.HasState()) {
+ return TCell();
+ }
+ const auto& state = entry.GetState();
+ return TCell(state.data(), state.size());
+ }});
+ insert({TSchema::VolatileState::ColumnId, [] (const TEntry& entry) {
+ const auto& state = entry.GetVolatileState();
+ return TCell(state.data(), state.size());
+ }});
+ insert({TSchema::BootState::ColumnId, [] (const TEntry& entry) {
+ const auto& state = entry.GetBootState();
+ return TCell(state.data(), state.size());
+ }});
+ insert({TSchema::Generation::ColumnId, [] (const TEntry& entry) {
+ if (!entry.HasGeneration()) {
+ return TCell();
+ }
return TCell::Make<ui32>(entry.GetGeneration());
- }});
- insert({TSchema::NodeId::ColumnId, [] (const TEntry& entry) {
- return TCell::Make<ui32>(entry.GetNodeId());
- }});
- insert({TSchema::CPU::ColumnId, [] (const TEntry& entry) {
- return TCell::Make<double>(entry.GetCPU() / 1000000.);
- }});
- insert({TSchema::Memory::ColumnId, [] (const TEntry& entry) {
- return TCell::Make<ui64>(entry.GetMemory());
- }});
- insert({TSchema::Network::ColumnId, [] (const TEntry& entry) {
- return TCell::Make<ui64>(entry.GetNetwork());
- }});
- }
- };
- static TExtractorsMap extractors;
-
- size_t index = 0;
+ }});
+ insert({TSchema::NodeId::ColumnId, [] (const TEntry& entry) {
+ return TCell::Make<ui32>(entry.GetNodeId());
+ }});
+ insert({TSchema::CPU::ColumnId, [] (const TEntry& entry) {
+ return TCell::Make<double>(entry.GetCPU() / 1000000.);
+ }});
+ insert({TSchema::Memory::ColumnId, [] (const TEntry& entry) {
+ return TCell::Make<ui64>(entry.GetMemory());
+ }});
+ insert({TSchema::Network::ColumnId, [] (const TEntry& entry) {
+ return TCell::Make<ui64>(entry.GetNetwork());
+ }});
+ }
+ };
+ static TExtractorsMap extractors;
+
+ size_t index = 0;
ui32 fromFollowerId = FromFollowerId.value_or(Min<ui32>());
if (record.EntriesSize() > 0
- && record.GetEntries(0).GetTabletId() == FromTabletId)
- {
- for (; index < record.EntriesSize(); ++index) {
- const auto& entry = record.GetEntries(index);
+ && record.GetEntries(0).GetTabletId() == FromTabletId)
+ {
+ for (; index < record.EntriesSize(); ++index) {
+ const auto& entry = record.GetEntries(index);
if (entry.GetTabletId() != FromTabletId || entry.GetFollowerId() >= fromFollowerId) {
- break;
- }
- }
- }
-
- auto batch = MakeHolder<NKqp::TEvKqpCompute::TEvScanData>(ScanId);
- TVector<TCell> cells;
+ break;
+ }
+ }
+ }
+
+ auto batch = MakeHolder<NKqp::TEvKqpCompute::TEvScanData>(ScanId);
+ TVector<TCell> cells;
ui32 toFollowerId = ToFollowerId.value_or(Max<ui32>());
-
- for (; index < record.EntriesSize(); ++index) {
- const auto& entry = record.GetEntries(index);
-
+
+ for (; index < record.EntriesSize(); ++index) {
+ const auto& entry = record.GetEntries(index);
+
if (entry.GetTabletId() == ToTabletId && entry.GetFollowerId() > toFollowerId) {
- break;
- }
-
- for (auto& column : Columns) {
- auto extractor = extractors.find(column.Tag);
- if (extractor == extractors.end()) {
- cells.push_back(TCell());
- } else {
- cells.push_back(extractor->second(entry));
- }
- }
- TArrayRef<const TCell> ref(cells);
+ break;
+ }
+
+ for (auto& column : Columns) {
+ auto extractor = extractors.find(column.Tag);
+ if (extractor == extractors.end()) {
+ cells.push_back(TCell());
+ } else {
+ cells.push_back(extractor->second(entry));
+ }
+ }
+ TArrayRef<const TCell> ref(cells);
batch->Rows.emplace_back(TOwnedCellVec::Make(ref));
- cells.clear();
- }
-
- if (record.HasNextTabletId()) {
- FromIterator = std::lower_bound(TabletIds.begin(), TabletIds.end(),
- record.GetNextTabletId());
- }
-
- if (FromIterator == TabletIds.end()) {
- batch->Finished = true;
- }
-
- BatchRequested = false;
-
- SendBatch(std::move(batch));
- }
-
- void PassAway() override {
- Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(0));
- TBase::PassAway();
- }
-
-private:
- static constexpr size_t BatchSize = 10000;
-
- ui64 FromTabletId = 0;
+ cells.clear();
+ }
+
+ if (record.HasNextTabletId()) {
+ FromIterator = std::lower_bound(TabletIds.begin(), TabletIds.end(),
+ record.GetNextTabletId());
+ }
+
+ if (FromIterator == TabletIds.end()) {
+ batch->Finished = true;
+ }
+
+ BatchRequested = false;
+
+ SendBatch(std::move(batch));
+ }
+
+ void PassAway() override {
+ Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(0));
+ TBase::PassAway();
+ }
+
+private:
+ static constexpr size_t BatchSize = 10000;
+
+ ui64 FromTabletId = 0;
std::optional<ui32> FromFollowerId;
-
- ui64 ToTabletId = Max<ui64>();
+
+ ui64 ToTabletId = Max<ui64>();
std::optional<ui32> ToFollowerId;
-
- TVector<ui64> TabletIds;
- TVector<ui64>::const_iterator FromIterator;
-
- bool BatchRequested = false;
-};
-
+
+ TVector<ui64> TabletIds;
+ TVector<ui64>::const_iterator FromIterator;
+
+ bool BatchRequested = false;
+};
+
THolder<IActor> CreateTabletsScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId,
- const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns)
-{
+ const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns)
+{
return MakeHolder<TTabletsScan>(ownerId, scanId, tableId, tableRange, columns);
-}
-
-} // NKikimr::NSysView
+}
+
+} // NKikimr::NSysView
diff --git a/ydb/core/sys_view/tablets/tablets.h b/ydb/core/sys_view/tablets/tablets.h
index ca54b08d3e1..321bd9b081c 100644
--- a/ydb/core/sys_view/tablets/tablets.h
+++ b/ydb/core/sys_view/tablets/tablets.h
@@ -1,12 +1,12 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/kqp/runtime/kqp_compute.h>
-
-namespace NKikimr {
-namespace NSysView {
-
+
+namespace NKikimr {
+namespace NSysView {
+
THolder<IActor> CreateTabletsScan(const TActorId& ownerId, ui32 scanId, const TTableId& tableId,
- const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns);
-
-} // NSysView
-} // NKikimr
+ const TTableRange& tableRange, const TArrayRef<NMiniKQL::TKqpComputeContextBase::TColumn>& columns);
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/tablets/ya.make b/ydb/core/sys_view/tablets/ya.make
index 9da54e07123..95680b0ace5 100644
--- a/ydb/core/sys_view/tablets/ya.make
+++ b/ydb/core/sys_view/tablets/ya.make
@@ -1,22 +1,22 @@
-LIBRARY()
-
-OWNER(
- monster
- g:kikimr
-)
-
-SRCS(
- tablets.h
- tablets.cpp
-)
-
-PEERDIR(
- library/cpp/actors/core
+LIBRARY()
+
+OWNER(
+ monster
+ g:kikimr
+)
+
+SRCS(
+ tablets.h
+ tablets.cpp
+)
+
+PEERDIR(
+ library/cpp/actors/core
ydb/core/base
ydb/core/kqp/runtime
ydb/core/sys_view/common
-)
-
+)
+
YQL_LAST_ABI_VERSION()
-END()
+END()
diff --git a/ydb/core/sys_view/ut_common.cpp b/ydb/core/sys_view/ut_common.cpp
index 6165258568c..a0ef0dffa4b 100644
--- a/ydb/core/sys_view/ut_common.cpp
+++ b/ydb/core/sys_view/ut_common.cpp
@@ -1,92 +1,92 @@
-#include "ut_common.h"
-
-namespace NKikimr {
-namespace NSysView {
-
-NKikimrSubDomains::TSubDomainSettings GetSubDomainDeclareSettings(const TString &name, const TStoragePools &pools) {
- NKikimrSubDomains::TSubDomainSettings subdomain;
- subdomain.SetName(name);
- for (auto& pool: pools) {
- *subdomain.AddStoragePools() = pool;
- }
- return subdomain;
-}
-
-NKikimrSubDomains::TSubDomainSettings GetSubDomainDefaultSettings(const TString &name, const TStoragePools &pools) {
- NKikimrSubDomains::TSubDomainSettings subdomain;
- subdomain.SetName(name);
- subdomain.SetCoordinators(2);
- subdomain.SetMediators(2);
- subdomain.SetPlanResolution(50);
- subdomain.SetTimeCastBucketsPerMediator(2);
- for (auto& pool: pools) {
- *subdomain.AddStoragePools() = pool;
- }
- return subdomain;
-}
-
-TTestEnv::TTestEnv(ui32 staticNodes, ui32 dynamicNodes, ui32 storagePools, bool enableSVP) {
- auto mbusPort = PortManager.GetPort();
- auto grpcPort = PortManager.GetPort();
-
- TVector<NKikimrKqp::TKqpSetting> kqpSettings;
- NKikimrKqp::TKqpSetting setting;
- setting.SetName("_KqpAllowNewEngine");
- setting.SetValue("true");
- kqpSettings.push_back(setting);
-
- Settings = new Tests::TServerSettings(mbusPort);
- Settings->SetDomainName("Root");
- Settings->SetNodeCount(staticNodes);
- Settings->SetDynamicNodeCount(dynamicNodes);
- Settings->SetKqpSettings(kqpSettings);
-
- if (enableSVP) {
- Settings->SetEnablePersistentQueryStats(true);
- Settings->SetEnableDbCounters(true);
- }
-
- for (ui32 i : xrange(storagePools)) {
- TString poolName = Sprintf("test%d", i);
- Settings->AddStoragePool(poolName, TString("/Root:") + poolName, 2);
- }
-
- Server = new Tests::TServer(*Settings);
- Server->EnableGRpc(grpcPort);
-
- Client = MakeHolder<Tests::TClient>(*Settings);
-
- Tenants = MakeHolder<Tests::TTenants>(Server);
-
- Client->InitRootScheme("Root");
-
- Endpoint = "localhost:" + ToString(grpcPort);
- DriverConfig = NYdb::TDriverConfig().SetEndpoint(Endpoint);
- Driver = MakeHolder<NYdb::TDriver>(DriverConfig);
-
- Server->GetRuntime()->SetLogPriority(NKikimrServices::SYSTEM_VIEWS, NActors::NLog::PRI_DEBUG);
-}
-
-TTestEnv::~TTestEnv() {
- Driver->Stop(true);
-}
-
-TStoragePools TTestEnv::GetPools() const {
- TStoragePools pools;
- for (const auto& [kind, pool] : Settings->StoragePoolTypes) {
- pools.emplace_back(pool.GetName(), kind);
- }
- return pools;
-}
-
-TStoragePools TTestEnv::CreatePoolsForTenant(const TString& tenant) {
- TStoragePools result;
- for (auto& poolType: Settings->StoragePoolTypes) {
- auto& poolKind = poolType.first;
- result.emplace_back(Client->CreateStoragePool(poolKind, tenant), poolKind);
- }
- return result;
-}
-
-} // NSysView
-} // NKikimr
+#include "ut_common.h"
+
+namespace NKikimr {
+namespace NSysView {
+
+NKikimrSubDomains::TSubDomainSettings GetSubDomainDeclareSettings(const TString &name, const TStoragePools &pools) {
+ NKikimrSubDomains::TSubDomainSettings subdomain;
+ subdomain.SetName(name);
+ for (auto& pool: pools) {
+ *subdomain.AddStoragePools() = pool;
+ }
+ return subdomain;
+}
+
+NKikimrSubDomains::TSubDomainSettings GetSubDomainDefaultSettings(const TString &name, const TStoragePools &pools) {
+ NKikimrSubDomains::TSubDomainSettings subdomain;
+ subdomain.SetName(name);
+ subdomain.SetCoordinators(2);
+ subdomain.SetMediators(2);
+ subdomain.SetPlanResolution(50);
+ subdomain.SetTimeCastBucketsPerMediator(2);
+ for (auto& pool: pools) {
+ *subdomain.AddStoragePools() = pool;
+ }
+ return subdomain;
+}
+
+TTestEnv::TTestEnv(ui32 staticNodes, ui32 dynamicNodes, ui32 storagePools, bool enableSVP) {
+ auto mbusPort = PortManager.GetPort();
+ auto grpcPort = PortManager.GetPort();
+
+ TVector<NKikimrKqp::TKqpSetting> kqpSettings;
+ NKikimrKqp::TKqpSetting setting;
+ setting.SetName("_KqpAllowNewEngine");
+ setting.SetValue("true");
+ kqpSettings.push_back(setting);
+
+ Settings = new Tests::TServerSettings(mbusPort);
+ Settings->SetDomainName("Root");
+ Settings->SetNodeCount(staticNodes);
+ Settings->SetDynamicNodeCount(dynamicNodes);
+ Settings->SetKqpSettings(kqpSettings);
+
+ if (enableSVP) {
+ Settings->SetEnablePersistentQueryStats(true);
+ Settings->SetEnableDbCounters(true);
+ }
+
+ for (ui32 i : xrange(storagePools)) {
+ TString poolName = Sprintf("test%d", i);
+ Settings->AddStoragePool(poolName, TString("/Root:") + poolName, 2);
+ }
+
+ Server = new Tests::TServer(*Settings);
+ Server->EnableGRpc(grpcPort);
+
+ Client = MakeHolder<Tests::TClient>(*Settings);
+
+ Tenants = MakeHolder<Tests::TTenants>(Server);
+
+ Client->InitRootScheme("Root");
+
+ Endpoint = "localhost:" + ToString(grpcPort);
+ DriverConfig = NYdb::TDriverConfig().SetEndpoint(Endpoint);
+ Driver = MakeHolder<NYdb::TDriver>(DriverConfig);
+
+ Server->GetRuntime()->SetLogPriority(NKikimrServices::SYSTEM_VIEWS, NActors::NLog::PRI_DEBUG);
+}
+
+TTestEnv::~TTestEnv() {
+ Driver->Stop(true);
+}
+
+TStoragePools TTestEnv::GetPools() const {
+ TStoragePools pools;
+ for (const auto& [kind, pool] : Settings->StoragePoolTypes) {
+ pools.emplace_back(pool.GetName(), kind);
+ }
+ return pools;
+}
+
+TStoragePools TTestEnv::CreatePoolsForTenant(const TString& tenant) {
+ TStoragePools result;
+ for (auto& poolType: Settings->StoragePoolTypes) {
+ auto& poolKind = poolType.first;
+ result.emplace_back(Client->CreateStoragePool(poolKind, tenant), poolKind);
+ }
+ return result;
+}
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/ut_common.h b/ydb/core/sys_view/ut_common.h
index e142b969095..175ec2a5aba 100644
--- a/ydb/core/sys_view/ut_common.h
+++ b/ydb/core/sys_view/ut_common.h
@@ -1,63 +1,63 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/testlib/test_client.h>
#include <ydb/public/sdk/cpp/client/ydb_result/result.h>
#include <ydb/public/sdk/cpp/client/ydb_table/table.h>
#include <ydb/public/sdk/cpp/client/ydb_scheme/scheme.h>
-
+
#include <library/cpp/testing/unittest/registar.h>
-
-namespace NKikimr {
-namespace NSysView {
-
-NKikimrSubDomains::TSubDomainSettings GetSubDomainDeclareSettings(
- const TString &name, const TStoragePools &pools = {});
-
-NKikimrSubDomains::TSubDomainSettings GetSubDomainDefaultSettings(
- const TString &name, const TStoragePools &pools = {});
-
-class TTestEnv {
-public:
- TTestEnv(ui32 staticNodes = 1, ui32 dynamicNodes = 4, ui32 storagePools = 0,
- bool enableSVP = false);
- ~TTestEnv();
-
- Tests::TServer& GetServer() const {
- return *Server;
- }
-
- Tests::TClient& GetClient() const {
- return *Client;
- }
-
- Tests::TTenants& GetTenants() const {
- return *Tenants;
- }
-
- NYdb::TDriver& GetDriver() const {
- return *Driver;
- }
-
- const TString& GetEndpoint() const {
- return Endpoint;
- }
-
- TStoragePools GetPools() const;
- TStoragePools CreatePoolsForTenant(const TString& tenant);
-
-
-private:
- TPortManager PortManager;
-
- Tests::TServerSettings::TPtr Settings;
- Tests::TServer::TPtr Server;
- THolder<Tests::TClient> Client;
- THolder<Tests::TTenants> Tenants;
-
- TString Endpoint;
- NYdb::TDriverConfig DriverConfig;
- THolder<NYdb::TDriver> Driver;
-};
-
-} // NSysView
-} // NKikimr
+
+namespace NKikimr {
+namespace NSysView {
+
+NKikimrSubDomains::TSubDomainSettings GetSubDomainDeclareSettings(
+ const TString &name, const TStoragePools &pools = {});
+
+NKikimrSubDomains::TSubDomainSettings GetSubDomainDefaultSettings(
+ const TString &name, const TStoragePools &pools = {});
+
+class TTestEnv {
+public:
+ TTestEnv(ui32 staticNodes = 1, ui32 dynamicNodes = 4, ui32 storagePools = 0,
+ bool enableSVP = false);
+ ~TTestEnv();
+
+ Tests::TServer& GetServer() const {
+ return *Server;
+ }
+
+ Tests::TClient& GetClient() const {
+ return *Client;
+ }
+
+ Tests::TTenants& GetTenants() const {
+ return *Tenants;
+ }
+
+ NYdb::TDriver& GetDriver() const {
+ return *Driver;
+ }
+
+ const TString& GetEndpoint() const {
+ return Endpoint;
+ }
+
+ TStoragePools GetPools() const;
+ TStoragePools CreatePoolsForTenant(const TString& tenant);
+
+
+private:
+ TPortManager PortManager;
+
+ Tests::TServerSettings::TPtr Settings;
+ Tests::TServer::TPtr Server;
+ THolder<Tests::TClient> Client;
+ THolder<Tests::TTenants> Tenants;
+
+ TString Endpoint;
+ NYdb::TDriverConfig DriverConfig;
+ THolder<NYdb::TDriver> Driver;
+};
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/ut_counters.cpp b/ydb/core/sys_view/ut_counters.cpp
index 234f52478f0..7f7b6b76c3d 100644
--- a/ydb/core/sys_view/ut_counters.cpp
+++ b/ydb/core/sys_view/ut_counters.cpp
@@ -1,181 +1,181 @@
-#include "ut_common.h"
-
+#include "ut_common.h"
+
#include <ydb/core/base/counters.h>
#include <ydb/core/kqp/ut/common/kqp_ut_common.h>
-
-namespace NKikimr {
-namespace NSysView {
-
-using namespace NYdb;
-using namespace NYdb::NTable;
-using namespace NYdb::NScheme;
-
-namespace {
-
-void CreateDatabase(TTestEnv& env, const TString& databaseName) {
- auto subdomain = GetSubDomainDeclareSettings(databaseName);
- UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK,
- env.GetClient().CreateExtSubdomain("/Root", subdomain));
-
- env.GetTenants().Run("/Root/" + databaseName, 1);
-
- auto subdomainSettings = GetSubDomainDefaultSettings(databaseName, env.GetPools());
- subdomainSettings.SetExternalSysViewProcessor(true);
- subdomainSettings.SetExternalSchemeShard(true);
- UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK,
- env.GetClient().AlterExtSubdomain("/Root", subdomainSettings));
-}
-
-void CreateDatabases(TTestEnv& env) {
- CreateDatabase(env, "Database1");
- CreateDatabase(env, "Database2");
-}
-
-void CreateTables(TTestEnv& env) {
- TTableClient client(env.GetDriver());
- auto session = client.CreateSession().GetValueSync().GetSession();
-
- NKqp::AssertSuccessResult(session.ExecuteSchemeQuery(R"(
- CREATE TABLE `Root/Database1/Table1` (
- Key Uint64,
- Value String,
- PRIMARY KEY (Key)
- );
- )").GetValueSync());
-
- NKqp::AssertSuccessResult(session.ExecuteDataQuery(R"(
- REPLACE INTO `Root/Database1/Table1` (Key, Value) VALUES
- (1u, "A"),
- (2u, "B"),
- (3u, "C");
- )", TTxControl::BeginTx().CommitTx()).GetValueSync());
-
- NKqp::AssertSuccessResult(session.ExecuteSchemeQuery(R"(
- CREATE TABLE `Root/Database2/Table2` (
- Key Uint64,
- Value String,
- PRIMARY KEY (Key)
- );
- )").GetValueSync());
-
- NKqp::AssertSuccessResult(session.ExecuteDataQuery(R"(
- REPLACE INTO `Root/Database2/Table2` (Key, Value) VALUES
- (4u, "D"),
- (5u, "E");
- )", TTxControl::BeginTx().CommitTx()).GetValueSync());
-}
-
-void CreateDatabasesAndTables(TTestEnv& env) {
- CreateDatabases(env);
- CreateTables(env);
-}
-
-} // namespace
-
-Y_UNIT_TEST_SUITE(DbCounters) {
-
- Y_UNIT_TEST(TabletsSimple) {
- TTestEnv env(1, 2, 0, true);
- CreateDatabasesAndTables(env);
-
- for (size_t iter = 0; iter < 30; ++iter) {
- Cerr << "iteration " << iter << Endl;
-
- auto checkTabletCounters = [] (NMonitoring::TDynamicCounterPtr databaseGroup,
- const char* databaseName)
- {
- auto checkCounter = [databaseName] (NMonitoring::TDynamicCounterPtr group,
- const char* sensorName, bool isDerivative)
- {
- auto value = group->GetCounter(sensorName, isDerivative)->Val();
- Cerr << "Database " << databaseName << ", sensor " << sensorName << ", value " << value << Endl;
- return (value > 0);
- };
-
- bool isGood = true;
-
- auto tabletGroup = databaseGroup->GetSubgroup("host", "")->GetSubgroup("group", "tablets");
- auto datashardGroup = tabletGroup->GetSubgroup("type", "DataShard");
- {
- auto executorGroup = datashardGroup->GetSubgroup("category", "executor");
- {
- TStringStream ss;
- executorGroup->OutputHtml(ss);
- Cerr << ss.Str() << Endl;
- }
-
- isGood &= checkCounter(executorGroup, "SUM(UsedTabletMemory)", false);
- isGood &= checkCounter(executorGroup, "MAX(UsedTabletMemory)", false);
-
- isGood &= checkCounter(executorGroup, "TabletBytesWritten", true);
-
- auto appGroup = datashardGroup->GetSubgroup("category", "app");
- {
- TStringStream ss;
- appGroup->OutputHtml(ss);
- Cerr << ss.Str() << Endl;
- }
-
- isGood &= checkCounter(appGroup, "DataShard/EngineHostRowUpdateBytes", true);
- isGood &= checkCounter(appGroup, "MAX(DataShard/EngineHostRowUpdateBytes)", false);
- }
-
- auto schemeshardGroup = tabletGroup->GetSubgroup("type", "SchemeShard");
- {
- auto executorGroup = schemeshardGroup->GetSubgroup("category", "executor");
- {
- TStringStream ss;
- executorGroup->OutputHtml(ss);
- Cerr << ss.Str() << Endl;
- }
-
- isGood &= checkCounter(executorGroup, "SUM(UsedTabletMemory)", false);
- isGood &= checkCounter(executorGroup, "MAX(UsedTabletMemory)", false);
-
- isGood &= checkCounter(executorGroup, "TabletBytesWritten", true);
-
- auto appGroup = schemeshardGroup->GetSubgroup("category", "app");
- {
- TStringStream ss;
- appGroup->OutputHtml(ss);
- Cerr << ss.Str() << Endl;
- }
-
- isGood &= checkCounter(appGroup, "SUM(SchemeShard/Tables)", false);
- isGood &= checkCounter(appGroup, "MAX(SchemeShard/Tables)", false);
-
- isGood &= checkCounter(appGroup, "SchemeShard/FinishedOps/CreateTable", true);
- }
-
- return isGood;
- };
-
- bool checkDb1 = false, checkDb2 = false;
-
- for (ui32 nodeId = 0; nodeId < env.GetServer().GetRuntime()->GetNodeCount(); ++nodeId) {
- auto counters = env.GetServer().GetRuntime()->GetAppData(nodeId).Counters;
- auto dbGroup = GetServiceCounters(counters, "db", false);
-
- auto databaseGroup1 = dbGroup->FindSubgroup("database", "/Root/Database1");
- if (databaseGroup1) {
- checkDb1 = checkTabletCounters(databaseGroup1, "/Root/Database1");
- }
- auto databaseGroup2 = dbGroup->FindSubgroup("database", "/Root/Database2");
- if (databaseGroup2) {
- checkDb2 = checkTabletCounters(databaseGroup2, "/Root/Database2");
- }
- }
-
- if (checkDb1 && checkDb2) {
- return;
- }
-
- Sleep(TDuration::Seconds(5));
- }
-
- UNIT_ASSERT_C(false, "out of iterations");
- }
-}
-
-} // NSysView
-} // NKikimr
+
+namespace NKikimr {
+namespace NSysView {
+
+using namespace NYdb;
+using namespace NYdb::NTable;
+using namespace NYdb::NScheme;
+
+namespace {
+
+void CreateDatabase(TTestEnv& env, const TString& databaseName) {
+ auto subdomain = GetSubDomainDeclareSettings(databaseName);
+ UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK,
+ env.GetClient().CreateExtSubdomain("/Root", subdomain));
+
+ env.GetTenants().Run("/Root/" + databaseName, 1);
+
+ auto subdomainSettings = GetSubDomainDefaultSettings(databaseName, env.GetPools());
+ subdomainSettings.SetExternalSysViewProcessor(true);
+ subdomainSettings.SetExternalSchemeShard(true);
+ UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK,
+ env.GetClient().AlterExtSubdomain("/Root", subdomainSettings));
+}
+
+void CreateDatabases(TTestEnv& env) {
+ CreateDatabase(env, "Database1");
+ CreateDatabase(env, "Database2");
+}
+
+void CreateTables(TTestEnv& env) {
+ TTableClient client(env.GetDriver());
+ auto session = client.CreateSession().GetValueSync().GetSession();
+
+ NKqp::AssertSuccessResult(session.ExecuteSchemeQuery(R"(
+ CREATE TABLE `Root/Database1/Table1` (
+ Key Uint64,
+ Value String,
+ PRIMARY KEY (Key)
+ );
+ )").GetValueSync());
+
+ NKqp::AssertSuccessResult(session.ExecuteDataQuery(R"(
+ REPLACE INTO `Root/Database1/Table1` (Key, Value) VALUES
+ (1u, "A"),
+ (2u, "B"),
+ (3u, "C");
+ )", TTxControl::BeginTx().CommitTx()).GetValueSync());
+
+ NKqp::AssertSuccessResult(session.ExecuteSchemeQuery(R"(
+ CREATE TABLE `Root/Database2/Table2` (
+ Key Uint64,
+ Value String,
+ PRIMARY KEY (Key)
+ );
+ )").GetValueSync());
+
+ NKqp::AssertSuccessResult(session.ExecuteDataQuery(R"(
+ REPLACE INTO `Root/Database2/Table2` (Key, Value) VALUES
+ (4u, "D"),
+ (5u, "E");
+ )", TTxControl::BeginTx().CommitTx()).GetValueSync());
+}
+
+void CreateDatabasesAndTables(TTestEnv& env) {
+ CreateDatabases(env);
+ CreateTables(env);
+}
+
+} // namespace
+
+Y_UNIT_TEST_SUITE(DbCounters) {
+
+ Y_UNIT_TEST(TabletsSimple) {
+ TTestEnv env(1, 2, 0, true);
+ CreateDatabasesAndTables(env);
+
+ for (size_t iter = 0; iter < 30; ++iter) {
+ Cerr << "iteration " << iter << Endl;
+
+ auto checkTabletCounters = [] (NMonitoring::TDynamicCounterPtr databaseGroup,
+ const char* databaseName)
+ {
+ auto checkCounter = [databaseName] (NMonitoring::TDynamicCounterPtr group,
+ const char* sensorName, bool isDerivative)
+ {
+ auto value = group->GetCounter(sensorName, isDerivative)->Val();
+ Cerr << "Database " << databaseName << ", sensor " << sensorName << ", value " << value << Endl;
+ return (value > 0);
+ };
+
+ bool isGood = true;
+
+ auto tabletGroup = databaseGroup->GetSubgroup("host", "")->GetSubgroup("group", "tablets");
+ auto datashardGroup = tabletGroup->GetSubgroup("type", "DataShard");
+ {
+ auto executorGroup = datashardGroup->GetSubgroup("category", "executor");
+ {
+ TStringStream ss;
+ executorGroup->OutputHtml(ss);
+ Cerr << ss.Str() << Endl;
+ }
+
+ isGood &= checkCounter(executorGroup, "SUM(UsedTabletMemory)", false);
+ isGood &= checkCounter(executorGroup, "MAX(UsedTabletMemory)", false);
+
+ isGood &= checkCounter(executorGroup, "TabletBytesWritten", true);
+
+ auto appGroup = datashardGroup->GetSubgroup("category", "app");
+ {
+ TStringStream ss;
+ appGroup->OutputHtml(ss);
+ Cerr << ss.Str() << Endl;
+ }
+
+ isGood &= checkCounter(appGroup, "DataShard/EngineHostRowUpdateBytes", true);
+ isGood &= checkCounter(appGroup, "MAX(DataShard/EngineHostRowUpdateBytes)", false);
+ }
+
+ auto schemeshardGroup = tabletGroup->GetSubgroup("type", "SchemeShard");
+ {
+ auto executorGroup = schemeshardGroup->GetSubgroup("category", "executor");
+ {
+ TStringStream ss;
+ executorGroup->OutputHtml(ss);
+ Cerr << ss.Str() << Endl;
+ }
+
+ isGood &= checkCounter(executorGroup, "SUM(UsedTabletMemory)", false);
+ isGood &= checkCounter(executorGroup, "MAX(UsedTabletMemory)", false);
+
+ isGood &= checkCounter(executorGroup, "TabletBytesWritten", true);
+
+ auto appGroup = schemeshardGroup->GetSubgroup("category", "app");
+ {
+ TStringStream ss;
+ appGroup->OutputHtml(ss);
+ Cerr << ss.Str() << Endl;
+ }
+
+ isGood &= checkCounter(appGroup, "SUM(SchemeShard/Tables)", false);
+ isGood &= checkCounter(appGroup, "MAX(SchemeShard/Tables)", false);
+
+ isGood &= checkCounter(appGroup, "SchemeShard/FinishedOps/CreateTable", true);
+ }
+
+ return isGood;
+ };
+
+ bool checkDb1 = false, checkDb2 = false;
+
+ for (ui32 nodeId = 0; nodeId < env.GetServer().GetRuntime()->GetNodeCount(); ++nodeId) {
+ auto counters = env.GetServer().GetRuntime()->GetAppData(nodeId).Counters;
+ auto dbGroup = GetServiceCounters(counters, "db", false);
+
+ auto databaseGroup1 = dbGroup->FindSubgroup("database", "/Root/Database1");
+ if (databaseGroup1) {
+ checkDb1 = checkTabletCounters(databaseGroup1, "/Root/Database1");
+ }
+ auto databaseGroup2 = dbGroup->FindSubgroup("database", "/Root/Database2");
+ if (databaseGroup2) {
+ checkDb2 = checkTabletCounters(databaseGroup2, "/Root/Database2");
+ }
+ }
+
+ if (checkDb1 && checkDb2) {
+ return;
+ }
+
+ Sleep(TDuration::Seconds(5));
+ }
+
+ UNIT_ASSERT_C(false, "out of iterations");
+ }
+}
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/ut_kqp.cpp b/ydb/core/sys_view/ut_kqp.cpp
index 5e90eb59927..eafa84f4fcb 100644
--- a/ydb/core/sys_view/ut_kqp.cpp
+++ b/ydb/core/sys_view/ut_kqp.cpp
@@ -1,142 +1,142 @@
-#include "ut_common.h"
-
+#include "ut_common.h"
+
#include <ydb/core/kqp/ut/common/kqp_ut_common.h>
-
+
#include <ydb/core/sys_view/common/events.h>
#include <ydb/core/sys_view/service/sysview_service.h>
#include <ydb/core/tx/datashard/datashard.h>
-
+
#include <ydb/public/sdk/cpp/client/draft/ydb_scripting.h>
-
+
#include <library/cpp/yson/node/node_io.h>
-
-namespace NKikimr {
-namespace NSysView {
-
-using namespace NYdb;
-using namespace NYdb::NTable;
-using namespace NYdb::NScheme;
-
-namespace {
-
-void CreateTenant(TTestEnv& env, const TString& tenantName, bool extSchemeShard = true) {
- auto subdomain = GetSubDomainDeclareSettings(tenantName);
- if (extSchemeShard) {
- UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK,
- env.GetClient().CreateExtSubdomain("/Root", subdomain));
- } else {
- UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK,
- env.GetClient().CreateSubdomain("/Root", subdomain));
- }
-
- env.GetTenants().Run("/Root/" + tenantName, 2);
-
- auto subdomainSettings = GetSubDomainDefaultSettings(tenantName, env.GetPools());
- subdomainSettings.SetExternalSysViewProcessor(true);
-
- if (extSchemeShard) {
- subdomainSettings.SetExternalSchemeShard(true);
- UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK,
- env.GetClient().AlterExtSubdomain("/Root", subdomainSettings));
- } else {
- UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK,
- env.GetClient().AlterSubdomain("/Root", subdomainSettings));
- }
-}
-
-void CreateTenants(TTestEnv& env, bool extSchemeShard = true) {
- CreateTenant(env, "Tenant1", extSchemeShard);
- CreateTenant(env, "Tenant2", extSchemeShard);
-}
-
-void CreateTables(TTestEnv& env) {
- TTableClient client(env.GetDriver());
- auto session = client.CreateSession().GetValueSync().GetSession();
-
- NKqp::AssertSuccessResult(session.ExecuteSchemeQuery(R"(
- CREATE TABLE `Root/Table0` (
- Key Uint64,
- Value String,
- PRIMARY KEY (Key)
- );
- )").GetValueSync());
-
- NKqp::AssertSuccessResult(session.ExecuteDataQuery(R"(
- REPLACE INTO `Root/Table0` (Key, Value) VALUES
- (0u, "Z");
- )", TTxControl::BeginTx().CommitTx()).GetValueSync());
-
- NKqp::AssertSuccessResult(session.ExecuteSchemeQuery(R"(
- CREATE TABLE `Root/Tenant1/Table1` (
- Key Uint64,
- Value String,
- PRIMARY KEY (Key)
- );
- )").GetValueSync());
-
- NKqp::AssertSuccessResult(session.ExecuteDataQuery(R"(
- REPLACE INTO `Root/Tenant1/Table1` (Key, Value) VALUES
- (1u, "A"),
- (2u, "B"),
- (3u, "C");
- )", TTxControl::BeginTx().CommitTx()).GetValueSync());
-
- NKqp::AssertSuccessResult(session.ExecuteSchemeQuery(R"(
- CREATE TABLE `Root/Tenant2/Table2` (
- Key Uint64,
- Value String,
- PRIMARY KEY (Key)
- );
- )").GetValueSync());
-
- NKqp::AssertSuccessResult(session.ExecuteDataQuery(R"(
- REPLACE INTO `Root/Tenant2/Table2` (Key, Value) VALUES
- (4u, "D"),
- (5u, "E");
- )", TTxControl::BeginTx().CommitTx()).GetValueSync());
-}
-
-void CreateTenantsAndTables(TTestEnv& env, bool extSchemeShard = true) {
- CreateTenants(env, extSchemeShard);
- CreateTables(env);
-}
-
-void CreateRootTable(TTestEnv& env, ui64 partitionCount = 1) {
- env.GetClient().CreateTable("/Root", Sprintf(R"(
- Name: "Table0"
- Columns { Name: "Key", Type: "Uint64" }
- Columns { Name: "Value", Type: "String" }
- KeyColumnNames: ["Key"]
- UniformPartitionsCount: %lu
- )", partitionCount));
-}
-
-class TYsonFieldChecker {
- NYT::TNode Root;
- NYT::TNode::TListType::const_iterator RowIterator;
-
-private:
- const NYT::TNode& ExtractOptional(const NYT::TNode& opt) {
- UNIT_ASSERT(opt.IsList());
- UNIT_ASSERT_VALUES_EQUAL(opt.AsList().size(), 1);
- return opt.AsList().front();
- };
-
-public:
- TYsonFieldChecker(const TString& ysonString, size_t fieldCount) {
+
+namespace NKikimr {
+namespace NSysView {
+
+using namespace NYdb;
+using namespace NYdb::NTable;
+using namespace NYdb::NScheme;
+
+namespace {
+
+void CreateTenant(TTestEnv& env, const TString& tenantName, bool extSchemeShard = true) {
+ auto subdomain = GetSubDomainDeclareSettings(tenantName);
+ if (extSchemeShard) {
+ UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK,
+ env.GetClient().CreateExtSubdomain("/Root", subdomain));
+ } else {
+ UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK,
+ env.GetClient().CreateSubdomain("/Root", subdomain));
+ }
+
+ env.GetTenants().Run("/Root/" + tenantName, 2);
+
+ auto subdomainSettings = GetSubDomainDefaultSettings(tenantName, env.GetPools());
+ subdomainSettings.SetExternalSysViewProcessor(true);
+
+ if (extSchemeShard) {
+ subdomainSettings.SetExternalSchemeShard(true);
+ UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK,
+ env.GetClient().AlterExtSubdomain("/Root", subdomainSettings));
+ } else {
+ UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK,
+ env.GetClient().AlterSubdomain("/Root", subdomainSettings));
+ }
+}
+
+void CreateTenants(TTestEnv& env, bool extSchemeShard = true) {
+ CreateTenant(env, "Tenant1", extSchemeShard);
+ CreateTenant(env, "Tenant2", extSchemeShard);
+}
+
+void CreateTables(TTestEnv& env) {
+ TTableClient client(env.GetDriver());
+ auto session = client.CreateSession().GetValueSync().GetSession();
+
+ NKqp::AssertSuccessResult(session.ExecuteSchemeQuery(R"(
+ CREATE TABLE `Root/Table0` (
+ Key Uint64,
+ Value String,
+ PRIMARY KEY (Key)
+ );
+ )").GetValueSync());
+
+ NKqp::AssertSuccessResult(session.ExecuteDataQuery(R"(
+ REPLACE INTO `Root/Table0` (Key, Value) VALUES
+ (0u, "Z");
+ )", TTxControl::BeginTx().CommitTx()).GetValueSync());
+
+ NKqp::AssertSuccessResult(session.ExecuteSchemeQuery(R"(
+ CREATE TABLE `Root/Tenant1/Table1` (
+ Key Uint64,
+ Value String,
+ PRIMARY KEY (Key)
+ );
+ )").GetValueSync());
+
+ NKqp::AssertSuccessResult(session.ExecuteDataQuery(R"(
+ REPLACE INTO `Root/Tenant1/Table1` (Key, Value) VALUES
+ (1u, "A"),
+ (2u, "B"),
+ (3u, "C");
+ )", TTxControl::BeginTx().CommitTx()).GetValueSync());
+
+ NKqp::AssertSuccessResult(session.ExecuteSchemeQuery(R"(
+ CREATE TABLE `Root/Tenant2/Table2` (
+ Key Uint64,
+ Value String,
+ PRIMARY KEY (Key)
+ );
+ )").GetValueSync());
+
+ NKqp::AssertSuccessResult(session.ExecuteDataQuery(R"(
+ REPLACE INTO `Root/Tenant2/Table2` (Key, Value) VALUES
+ (4u, "D"),
+ (5u, "E");
+ )", TTxControl::BeginTx().CommitTx()).GetValueSync());
+}
+
+void CreateTenantsAndTables(TTestEnv& env, bool extSchemeShard = true) {
+ CreateTenants(env, extSchemeShard);
+ CreateTables(env);
+}
+
+void CreateRootTable(TTestEnv& env, ui64 partitionCount = 1) {
+ env.GetClient().CreateTable("/Root", Sprintf(R"(
+ Name: "Table0"
+ Columns { Name: "Key", Type: "Uint64" }
+ Columns { Name: "Value", Type: "String" }
+ KeyColumnNames: ["Key"]
+ UniformPartitionsCount: %lu
+ )", partitionCount));
+}
+
+class TYsonFieldChecker {
+ NYT::TNode Root;
+ NYT::TNode::TListType::const_iterator RowIterator;
+
+private:
+ const NYT::TNode& ExtractOptional(const NYT::TNode& opt) {
+ UNIT_ASSERT(opt.IsList());
+ UNIT_ASSERT_VALUES_EQUAL(opt.AsList().size(), 1);
+ return opt.AsList().front();
+ };
+
+public:
+ TYsonFieldChecker(const TString& ysonString, size_t fieldCount) {
Root = NYT::NodeFromYsonString(ysonString, ::NYson::EYsonType::Node);
- UNIT_ASSERT(Root.IsList());
- UNIT_ASSERT_VALUES_EQUAL(Root.AsList().size(), 1);
-
- const auto& rowNode = Root.AsList().front();
- UNIT_ASSERT(rowNode.IsList());
-
- const auto& row = rowNode.AsList();
- UNIT_ASSERT_VALUES_EQUAL(row.size(), fieldCount);
-
- RowIterator = row.begin();
- }
-
+ UNIT_ASSERT(Root.IsList());
+ UNIT_ASSERT_VALUES_EQUAL(Root.AsList().size(), 1);
+
+ const auto& rowNode = Root.AsList().front();
+ UNIT_ASSERT(rowNode.IsList());
+
+ const auto& row = rowNode.AsList();
+ UNIT_ASSERT_VALUES_EQUAL(row.size(), fieldCount);
+
+ RowIterator = row.begin();
+ }
+
bool SkipNull() {
if (RowIterator->IsNull()) {
++RowIterator;
@@ -147,492 +147,492 @@ public:
}
void Null() {
- const auto& value = *RowIterator++;
+ const auto& value = *RowIterator++;
UNIT_ASSERT(value.IsNull());
- }
-
- void Bool(bool expected) {
- const auto& value = ExtractOptional(*RowIterator++);
- UNIT_ASSERT(value.IsBool());
- UNIT_ASSERT_VALUES_EQUAL(value.AsBool(), expected);
- }
-
+ }
+
+ void Bool(bool expected) {
+ const auto& value = ExtractOptional(*RowIterator++);
+ UNIT_ASSERT(value.IsBool());
+ UNIT_ASSERT_VALUES_EQUAL(value.AsBool(), expected);
+ }
+
void Uint64(ui64 expected, bool orNull = false) {
if (!orNull || !SkipNull()) {
const auto& value = ExtractOptional(*RowIterator++);
UNIT_ASSERT(value.IsUint64());
UNIT_ASSERT_VALUES_EQUAL(value.AsUint64(), expected);
}
- }
-
- void Uint64Greater(ui64 expected) {
- const auto& value = ExtractOptional(*RowIterator++);
- UNIT_ASSERT(value.IsUint64());
- UNIT_ASSERT_GT(value.AsUint64(), expected);
- }
-
- void Uint64GreaterOrEquals(ui64 expected) {
- const auto& value = ExtractOptional(*RowIterator++);
- UNIT_ASSERT(value.IsUint64());
- UNIT_ASSERT_GE(value.AsUint64(), expected);
- }
-
+ }
+
+ void Uint64Greater(ui64 expected) {
+ const auto& value = ExtractOptional(*RowIterator++);
+ UNIT_ASSERT(value.IsUint64());
+ UNIT_ASSERT_GT(value.AsUint64(), expected);
+ }
+
+ void Uint64GreaterOrEquals(ui64 expected) {
+ const auto& value = ExtractOptional(*RowIterator++);
+ UNIT_ASSERT(value.IsUint64());
+ UNIT_ASSERT_GE(value.AsUint64(), expected);
+ }
+
void Uint64LessOrEquals(ui64 expected) {
const auto& value = ExtractOptional(*RowIterator++);
UNIT_ASSERT(value.IsUint64());
UNIT_ASSERT_LE(value.AsUint64(), expected);
}
- void Int64(i64 expected) {
- const auto& value = ExtractOptional(*RowIterator++);
- UNIT_ASSERT(value.IsInt64());
- UNIT_ASSERT_VALUES_EQUAL(value.AsInt64(), expected);
- }
-
- void Int64Greater(i64 expected) {
- const auto& value = ExtractOptional(*RowIterator++);
- UNIT_ASSERT(value.IsInt64());
- UNIT_ASSERT_GT(value.AsInt64(), expected);
- }
-
- void Int64GreaterOrEquals(i64 expected) {
- const auto& value = ExtractOptional(*RowIterator++);
- UNIT_ASSERT(value.IsInt64());
- UNIT_ASSERT_GE(value.AsInt64(), expected);
- }
-
- void Double(double expected) {
- const auto& value = ExtractOptional(*RowIterator++);
- UNIT_ASSERT(value.IsDouble());
- UNIT_ASSERT_VALUES_EQUAL(value.AsDouble(), expected);
- }
-
- void DoubleGreaterOrEquals(double expected) {
- const auto& value = ExtractOptional(*RowIterator++);
- UNIT_ASSERT(value.IsDouble());
- UNIT_ASSERT_GE(value.AsDouble(), expected);
- }
-
- void String(const TString& expected) {
- const auto& value = ExtractOptional(*RowIterator++);
- UNIT_ASSERT(value.IsString());
- UNIT_ASSERT_STRINGS_EQUAL(value.AsString(), expected);
- }
-
- void StringContains(const TString& substr) {
- const auto& value = ExtractOptional(*RowIterator++);
- UNIT_ASSERT(value.IsString());
- UNIT_ASSERT(value.AsString().Contains(substr));
- }
-};
-
-} // namespace
-
-Y_UNIT_TEST_SUITE(SystemView) {
-
- Y_UNIT_TEST(PartitionStatsOneSchemeShard) {
- TTestEnv env;
- CreateTenantsAndTables(env, false);
- TTableClient client(env.GetDriver());
- {
- auto it = client.StreamExecuteScanQuery(R"(
- SELECT PathId, PartIdx, Path FROM `Root/.sys/partition_stats`;
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- NKqp::CompareYson(R"([
+ void Int64(i64 expected) {
+ const auto& value = ExtractOptional(*RowIterator++);
+ UNIT_ASSERT(value.IsInt64());
+ UNIT_ASSERT_VALUES_EQUAL(value.AsInt64(), expected);
+ }
+
+ void Int64Greater(i64 expected) {
+ const auto& value = ExtractOptional(*RowIterator++);
+ UNIT_ASSERT(value.IsInt64());
+ UNIT_ASSERT_GT(value.AsInt64(), expected);
+ }
+
+ void Int64GreaterOrEquals(i64 expected) {
+ const auto& value = ExtractOptional(*RowIterator++);
+ UNIT_ASSERT(value.IsInt64());
+ UNIT_ASSERT_GE(value.AsInt64(), expected);
+ }
+
+ void Double(double expected) {
+ const auto& value = ExtractOptional(*RowIterator++);
+ UNIT_ASSERT(value.IsDouble());
+ UNIT_ASSERT_VALUES_EQUAL(value.AsDouble(), expected);
+ }
+
+ void DoubleGreaterOrEquals(double expected) {
+ const auto& value = ExtractOptional(*RowIterator++);
+ UNIT_ASSERT(value.IsDouble());
+ UNIT_ASSERT_GE(value.AsDouble(), expected);
+ }
+
+ void String(const TString& expected) {
+ const auto& value = ExtractOptional(*RowIterator++);
+ UNIT_ASSERT(value.IsString());
+ UNIT_ASSERT_STRINGS_EQUAL(value.AsString(), expected);
+ }
+
+ void StringContains(const TString& substr) {
+ const auto& value = ExtractOptional(*RowIterator++);
+ UNIT_ASSERT(value.IsString());
+ UNIT_ASSERT(value.AsString().Contains(substr));
+ }
+};
+
+} // namespace
+
+Y_UNIT_TEST_SUITE(SystemView) {
+
+ Y_UNIT_TEST(PartitionStatsOneSchemeShard) {
+ TTestEnv env;
+ CreateTenantsAndTables(env, false);
+ TTableClient client(env.GetDriver());
+ {
+ auto it = client.StreamExecuteScanQuery(R"(
+ SELECT PathId, PartIdx, Path FROM `Root/.sys/partition_stats`;
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ NKqp::CompareYson(R"([
[[4u];[0u];["/Root/Table0"]]
- ])", NKqp::StreamResultToYson(it));
- }
- {
- auto it = client.StreamExecuteScanQuery(R"(
- SELECT PathId, PartIdx, Path FROM `Root/Tenant1/.sys/partition_stats`;
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- NKqp::CompareYson(R"([
+ ])", NKqp::StreamResultToYson(it));
+ }
+ {
+ auto it = client.StreamExecuteScanQuery(R"(
+ SELECT PathId, PartIdx, Path FROM `Root/Tenant1/.sys/partition_stats`;
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ NKqp::CompareYson(R"([
[[5u];[0u];["/Root/Tenant1/Table1"]]
- ])", NKqp::StreamResultToYson(it));
- }
- {
- auto it = client.StreamExecuteScanQuery(R"(
- SELECT PathId, PartIdx, Path FROM `Root/Tenant2/.sys/partition_stats`;
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- NKqp::CompareYson(R"([
+ ])", NKqp::StreamResultToYson(it));
+ }
+ {
+ auto it = client.StreamExecuteScanQuery(R"(
+ SELECT PathId, PartIdx, Path FROM `Root/Tenant2/.sys/partition_stats`;
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ NKqp::CompareYson(R"([
[[6u];[0u];["/Root/Tenant2/Table2"]]
- ])", NKqp::StreamResultToYson(it));
- }
- }
-
- Y_UNIT_TEST(PartitionStatsOneSchemeShardDataQueryNewEngine) {
- TTestEnv env;
- CreateTenantsAndTables(env, false);
-
+ ])", NKqp::StreamResultToYson(it));
+ }
+ }
+
+ Y_UNIT_TEST(PartitionStatsOneSchemeShardDataQueryNewEngine) {
+ TTestEnv env;
+ CreateTenantsAndTables(env, false);
+
env.GetServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_EXECUTER, NActors::NLog::PRI_DEBUG);
- TTableClient client(env.GetDriver());
- auto session = client.CreateSession().GetValueSync().GetSession();
- {
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- SELECT PathId, PartIdx, Path FROM `Root/.sys/partition_stats`;
- )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
-
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
- NKqp::CompareYson(R"([
- [[4u];[0u];["/Root/Table0"]]
- ])", FormatResultSetYson(result.GetResultSet(0)));
- }
- {
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- SELECT PathId, PartIdx, Path FROM `Root/Tenant1/.sys/partition_stats`;
- )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
-
- UNIT_ASSERT(result.IsSuccess());
- NKqp::CompareYson(R"([
- [[5u];[0u];["/Root/Tenant1/Table1"]]
- ])", FormatResultSetYson(result.GetResultSet(0)));
- }
- {
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- SELECT PathId, PartIdx, Path FROM `Root/Tenant2/.sys/partition_stats`;
- )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
-
- UNIT_ASSERT(result.IsSuccess());
- NKqp::CompareYson(R"([
- [[6u];[0u];["/Root/Tenant2/Table2"]]
- ])", FormatResultSetYson(result.GetResultSet(0)));
- }
- }
-
- Y_UNIT_TEST(Nodes) {
- return; // table is currenty switched off
-
- TTestEnv env;
+ TTableClient client(env.GetDriver());
+ auto session = client.CreateSession().GetValueSync().GetSession();
+ {
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ SELECT PathId, PartIdx, Path FROM `Root/.sys/partition_stats`;
+ )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ NKqp::CompareYson(R"([
+ [[4u];[0u];["/Root/Table0"]]
+ ])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+ {
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ SELECT PathId, PartIdx, Path FROM `Root/Tenant1/.sys/partition_stats`;
+ )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+
+ UNIT_ASSERT(result.IsSuccess());
+ NKqp::CompareYson(R"([
+ [[5u];[0u];["/Root/Tenant1/Table1"]]
+ ])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+ {
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ SELECT PathId, PartIdx, Path FROM `Root/Tenant2/.sys/partition_stats`;
+ )", TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+
+ UNIT_ASSERT(result.IsSuccess());
+ NKqp::CompareYson(R"([
+ [[6u];[0u];["/Root/Tenant2/Table2"]]
+ ])", FormatResultSetYson(result.GetResultSet(0)));
+ }
+ }
+
+ Y_UNIT_TEST(Nodes) {
+ return; // table is currenty switched off
+
+ TTestEnv env;
CreateTenantsAndTables(env, false);
- TTableClient client(env.GetDriver());
- {
- auto it = client.StreamExecuteScanQuery(R"(
- SELECT Host, NodeId
- FROM `Root/Tenant1/.sys/nodes`;
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- ui32 offset = env.GetServer().GetRuntime()->GetNodeId(0);
- auto expected = Sprintf(R"([
- [["::1"];[%du]];
- [["::1"];[%du]];
- ])", offset + 3, offset + 4);
-
- NKqp::CompareYson(expected, NKqp::StreamResultToYson(it));
- }
- {
- auto it = client.StreamExecuteScanQuery(R"(
- SELECT Host, NodeId
- FROM `Root/Tenant2/.sys/nodes`;
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- ui32 offset = env.GetServer().GetRuntime()->GetNodeId(0);
- auto expected = Sprintf(R"([
- [["::1"];[%du]];
- [["::1"];[%du]];
- ])", offset + 1, offset + 2);
-
- NKqp::CompareYson(expected, NKqp::StreamResultToYson(it));
- }
- {
- auto it = client.StreamExecuteScanQuery(R"(
- SELECT Host, NodeId
- FROM `Root/.sys/nodes`;
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- ui32 offset = env.GetServer().GetRuntime()->GetNodeId(0);
- auto expected = Sprintf(R"([
- [["::1"];[%du]];
- [["::1"];[%du]];
- [["::1"];[%du]];
- [["::1"];[%du]];
- [["::1"];[%du]];
- ])", offset, offset + 1, offset + 2, offset + 3, offset + 4);
-
- NKqp::CompareYson(expected, NKqp::StreamResultToYson(it));
- }
- }
-
- Y_UNIT_TEST(QueryStats) {
- TTestEnv env;
- CreateTenants(env);
-
- auto* runtime = env.GetServer().GetRuntime();
- runtime ->SetLogPriority(NKikimrServices::KQP_YQL, NActors::NLog::PRI_DEBUG);
-
- auto oneMinute = TDuration::Minutes(1);
- auto oneMinuteUs = oneMinute.MicroSeconds();
-
- auto instant = TAppData::TimeProvider->Now() + oneMinute;
- auto instantRounded = instant.MicroSeconds() / oneMinuteUs * oneMinuteUs;
-
- std::vector<ui64> buckets;
- for (size_t i : xrange(4)) {
- Y_UNUSED(i);
- buckets.push_back(instantRounded);
- instantRounded += oneMinuteUs;
- }
-
- auto& tenant1Nodes = env.GetTenants().List("/Root/Tenant1");
- auto& tenant2Nodes = env.GetTenants().List("/Root/Tenant2");
-
- UNIT_ASSERT_EQUAL(tenant1Nodes.size(), 2);
- UNIT_ASSERT_EQUAL(tenant2Nodes.size(), 2);
-
- auto tenant1Node0 = tenant1Nodes[0];
- auto tenant1Node1 = tenant1Nodes[1];
- auto tenant2Node = tenant2Nodes.front();
- ui32 staticNode = 0;
-
- auto makeQueryEvent = [&runtime] (ui32 nodeIdx, ui64 endTimeUs, const TString& queryText, ui64 readBytes) {
- auto stats = MakeHolder<NSysView::TEvSysView::TEvCollectQueryStats>();
- stats->QueryStats.MutableStats()->SetReadBytes(readBytes);
- stats->QueryStats.SetQueryText(queryText);
- stats->QueryStats.SetQueryTextHash(MurmurHash<ui64>(queryText.data(), queryText.size()));
- stats->QueryStats.SetDurationMs(1);
- stats->QueryStats.SetEndTimeMs(endTimeUs / 1000);
-
- auto serviceId = MakeSysViewServiceID(runtime->GetNodeId(nodeIdx));
+ TTableClient client(env.GetDriver());
+ {
+ auto it = client.StreamExecuteScanQuery(R"(
+ SELECT Host, NodeId
+ FROM `Root/Tenant1/.sys/nodes`;
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ ui32 offset = env.GetServer().GetRuntime()->GetNodeId(0);
+ auto expected = Sprintf(R"([
+ [["::1"];[%du]];
+ [["::1"];[%du]];
+ ])", offset + 3, offset + 4);
+
+ NKqp::CompareYson(expected, NKqp::StreamResultToYson(it));
+ }
+ {
+ auto it = client.StreamExecuteScanQuery(R"(
+ SELECT Host, NodeId
+ FROM `Root/Tenant2/.sys/nodes`;
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ ui32 offset = env.GetServer().GetRuntime()->GetNodeId(0);
+ auto expected = Sprintf(R"([
+ [["::1"];[%du]];
+ [["::1"];[%du]];
+ ])", offset + 1, offset + 2);
+
+ NKqp::CompareYson(expected, NKqp::StreamResultToYson(it));
+ }
+ {
+ auto it = client.StreamExecuteScanQuery(R"(
+ SELECT Host, NodeId
+ FROM `Root/.sys/nodes`;
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ ui32 offset = env.GetServer().GetRuntime()->GetNodeId(0);
+ auto expected = Sprintf(R"([
+ [["::1"];[%du]];
+ [["::1"];[%du]];
+ [["::1"];[%du]];
+ [["::1"];[%du]];
+ [["::1"];[%du]];
+ ])", offset, offset + 1, offset + 2, offset + 3, offset + 4);
+
+ NKqp::CompareYson(expected, NKqp::StreamResultToYson(it));
+ }
+ }
+
+ Y_UNIT_TEST(QueryStats) {
+ TTestEnv env;
+ CreateTenants(env);
+
+ auto* runtime = env.GetServer().GetRuntime();
+ runtime ->SetLogPriority(NKikimrServices::KQP_YQL, NActors::NLog::PRI_DEBUG);
+
+ auto oneMinute = TDuration::Minutes(1);
+ auto oneMinuteUs = oneMinute.MicroSeconds();
+
+ auto instant = TAppData::TimeProvider->Now() + oneMinute;
+ auto instantRounded = instant.MicroSeconds() / oneMinuteUs * oneMinuteUs;
+
+ std::vector<ui64> buckets;
+ for (size_t i : xrange(4)) {
+ Y_UNUSED(i);
+ buckets.push_back(instantRounded);
+ instantRounded += oneMinuteUs;
+ }
+
+ auto& tenant1Nodes = env.GetTenants().List("/Root/Tenant1");
+ auto& tenant2Nodes = env.GetTenants().List("/Root/Tenant2");
+
+ UNIT_ASSERT_EQUAL(tenant1Nodes.size(), 2);
+ UNIT_ASSERT_EQUAL(tenant2Nodes.size(), 2);
+
+ auto tenant1Node0 = tenant1Nodes[0];
+ auto tenant1Node1 = tenant1Nodes[1];
+ auto tenant2Node = tenant2Nodes.front();
+ ui32 staticNode = 0;
+
+ auto makeQueryEvent = [&runtime] (ui32 nodeIdx, ui64 endTimeUs, const TString& queryText, ui64 readBytes) {
+ auto stats = MakeHolder<NSysView::TEvSysView::TEvCollectQueryStats>();
+ stats->QueryStats.MutableStats()->SetReadBytes(readBytes);
+ stats->QueryStats.SetQueryText(queryText);
+ stats->QueryStats.SetQueryTextHash(MurmurHash<ui64>(queryText.data(), queryText.size()));
+ stats->QueryStats.SetDurationMs(1);
+ stats->QueryStats.SetEndTimeMs(endTimeUs / 1000);
+
+ auto serviceId = MakeSysViewServiceID(runtime->GetNodeId(nodeIdx));
runtime->Send(new IEventHandle(serviceId, TActorId(), stats.Release()), nodeIdx);
- };
-
- makeQueryEvent(tenant1Node0, buckets[0], "a", 100);
- makeQueryEvent(tenant1Node1, buckets[1], "b", 200);
- makeQueryEvent(tenant1Node0, buckets[1], "c", 300);
- makeQueryEvent(tenant1Node0, buckets[1], "d", 400);
- makeQueryEvent(tenant1Node1, buckets[2], "e", 500);
-
- makeQueryEvent(tenant2Node, buckets[0], "f", 600);
-
- makeQueryEvent(staticNode, buckets[0], "g", 700);
-
- TTableClient client(env.GetDriver());
- {
- auto it = client.StreamExecuteScanQuery(R"(
- SELECT IntervalEnd, QueryText, Rank, ReadBytes
- FROM `Root/.sys/top_queries_by_read_bytes_one_minute`;
- )").GetValueSync();
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- TStringBuilder result;
- result << "[";
- result << "[[" << buckets[1] << "u];[\"g\"];[1u];[700u]];";
- result << "]";
-
- NKqp::CompareYson(result, NKqp::StreamResultToYson(it));
- }
- {
- auto it = client.StreamExecuteScanQuery(R"(
- SELECT IntervalEnd, QueryText, Rank, ReadBytes
- FROM `Root/Tenant1/.sys/top_queries_by_read_bytes_one_minute`;
- )").GetValueSync();
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- TStringBuilder result;
- result << "[";
- result << "[[" << buckets[1] << "u];[\"a\"];[1u];[100u]];";
- result << "[[" << buckets[2] << "u];[\"d\"];[1u];[400u]];";
- result << "[[" << buckets[2] << "u];[\"c\"];[2u];[300u]];";
- result << "[[" << buckets[2] << "u];[\"b\"];[3u];[200u]];";
- result << "[[" << buckets[3] << "u];[\"e\"];[1u];[500u]];";
- result << "]";
-
- NKqp::CompareYson(result, NKqp::StreamResultToYson(it));
- }
- {
- auto it = client.StreamExecuteScanQuery(R"(
- SELECT IntervalEnd, QueryText, Rank, ReadBytes
- FROM `Root/Tenant2/.sys/top_queries_by_read_bytes_one_minute`;
- )").GetValueSync();
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- TStringBuilder result;
- result << "[";
- result << "[[" << buckets[1] << "u];[\"f\"];[1u];[600u]];";
- result << "]";
-
- NKqp::CompareYson(result, NKqp::StreamResultToYson(it));
- }
- {
- TStringBuilder query;
- query << "SELECT IntervalEnd, QueryText, Rank, ReadBytes ";
- query << "FROM `Root/Tenant1/.sys/top_queries_by_read_bytes_one_minute` ";
- query << "WHERE IntervalEnd >= CAST(" << buckets[1] << "ul as Timestamp) ";
- query << "AND IntervalEnd < CAST(" << buckets[3] << "ul as Timestamp);";
-
- auto it = client.StreamExecuteScanQuery(query).GetValueSync();
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- TStringBuilder result;
- result << "[";
- result << "[[" << buckets[1] << "u];[\"a\"];[1u];[100u]];";
- result << "[[" << buckets[2] << "u];[\"d\"];[1u];[400u]];";
- result << "[[" << buckets[2] << "u];[\"c\"];[2u];[300u]];";
- result << "[[" << buckets[2] << "u];[\"b\"];[3u];[200u]];";
- result << "]";
-
- NKqp::CompareYson(result, NKqp::StreamResultToYson(it));
- }
- {
- TStringBuilder query;
- query << "SELECT IntervalEnd, QueryText, Rank, ReadBytes ";
- query << "FROM `Root/Tenant1/.sys/top_queries_by_read_bytes_one_minute` ";
- query << "WHERE IntervalEnd > CAST(" << buckets[1] << "ul as Timestamp) ";
- query << "AND IntervalEnd <= CAST(" << buckets[3] << "ul as Timestamp);";
-
- auto it = client.StreamExecuteScanQuery(query).GetValueSync();
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- TStringBuilder result;
- result << "[";
- result << "[[" << buckets[2] << "u];[\"d\"];[1u];[400u]];";
- result << "[[" << buckets[2] << "u];[\"c\"];[2u];[300u]];";
- result << "[[" << buckets[2] << "u];[\"b\"];[3u];[200u]];";
- result << "[[" << buckets[3] << "u];[\"e\"];[1u];[500u]];";
- result << "]";
-
- NKqp::CompareYson(result, NKqp::StreamResultToYson(it));
- }
- {
- TStringBuilder query;
- query << "SELECT IntervalEnd, QueryText, Rank, ReadBytes ";
- query << "FROM `Root/Tenant1/.sys/top_queries_by_read_bytes_one_minute` ";
- query << "WHERE IntervalEnd = CAST(" << buckets[2] << "ul as Timestamp) ";
- query << "AND Rank >= 1u AND Rank < 3u";
-
- auto it = client.StreamExecuteScanQuery(query).GetValueSync();
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- TStringBuilder result;
- result << "[";
- result << "[[" << buckets[2] << "u];[\"d\"];[1u];[400u]];";
- result << "[[" << buckets[2] << "u];[\"c\"];[2u];[300u]];";
- result << "]";
-
- NKqp::CompareYson(result, NKqp::StreamResultToYson(it));
- }
- {
- TStringBuilder query;
- query << "SELECT IntervalEnd, QueryText, Rank, ReadBytes ";
- query << "FROM `Root/Tenant1/.sys/top_queries_by_read_bytes_one_minute` ";
- query << "WHERE IntervalEnd = CAST(" << buckets[2] << "ul as Timestamp) ";
- query << "AND Rank > 1u AND Rank <= 3u";
-
- auto it = client.StreamExecuteScanQuery(query).GetValueSync();
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- TStringBuilder result;
- result << "[";
- result << "[[" << buckets[2] << "u];[\"c\"];[2u];[300u]];";
- result << "[[" << buckets[2] << "u];[\"b\"];[3u];[200u]];";
- result << "]";
-
- NKqp::CompareYson(result, NKqp::StreamResultToYson(it));
- }
- }
-
- Y_UNIT_TEST(QueryStatsFields) {
- TTestEnv env;
- CreateRootTable(env, 3);
-
- auto nowUs = TInstant::Now().MicroSeconds();
-
- TString queryText("SELECT * FROM `Root/Table0`");
-
- TTableClient client(env.GetDriver());
- auto session = client.CreateSession().GetValueSync().GetSession();
- NKqp::AssertSuccessResult(session.ExecuteDataQuery(
- queryText, TTxControl::BeginTx().CommitTx()
- ).GetValueSync());
-
- auto it = client.StreamExecuteScanQuery(R"(
- SELECT
- CPUTime,
- CompileCPUTime,
- CompileDuration,
- ComputeNodesCount,
- DeleteBytes,
- DeleteRows,
- Duration,
- EndTime,
- FromQueryCache,
- IntervalEnd,
- MaxComputeCPUTime,
- MaxShardCPUTime,
- MinComputeCPUTime,
- MinShardCPUTime,
- ParametersSize,
- Partitions,
- ProcessCPUTime,
- QueryText,
- Rank,
- ReadBytes,
- ReadRows,
- RequestUnits,
- ShardCount,
- SumComputeCPUTime,
- SumShardCPUTime,
- Type,
- UpdateBytes,
- UpdateRows,
- UserSID
- FROM `Root/.sys/top_queries_by_read_bytes_one_minute`;
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- auto ysonString = NKqp::StreamResultToYson(it);
-
- TYsonFieldChecker check(ysonString, 29);
-
- check.Uint64GreaterOrEquals(0); // CPUTime
- check.Uint64GreaterOrEquals(0); // CompileCPUTime
- check.Int64GreaterOrEquals(0); // CompileDuration
- check.Uint64(1); // ComputeNodesCount
- check.Uint64(0); // DeleteBytes
- check.Uint64(0); // DeleteRows
- check.Int64Greater(0); // Duration
- check.Uint64Greater(nowUs); // EndTime
- check.Bool(false); // FromQueryCache
- check.Uint64Greater(nowUs); // IntervalEnd
- check.Uint64GreaterOrEquals(0); // MaxComputeCPUTime
- check.Uint64GreaterOrEquals(0); // MaxShardCPUTime
- check.Uint64GreaterOrEquals(0); // MinComputeCPUTime
- check.Uint64GreaterOrEquals(0); // MinShardCPUTime
- check.Uint64(0); // ParametersSize
- check.Uint64(3); // Partitions
- check.Uint64GreaterOrEquals(0); // ProcessCPUTime
- check.String(queryText); // QueryText
- check.Uint64(1); // Rank
- check.Uint64(0); // ReadBytes
- check.Uint64(0); // ReadRows
- check.Uint64Greater(0); // RequestUnits
- check.Uint64(3); // ShardCount
- check.Uint64GreaterOrEquals(0); // SumComputeCPUTime
- check.Uint64GreaterOrEquals(0); // SumShardCPUTime
- check.String("data"); // Type
- check.Uint64(0); // UpdateBytes
- check.Uint64(0); // UpdateRows
+ };
+
+ makeQueryEvent(tenant1Node0, buckets[0], "a", 100);
+ makeQueryEvent(tenant1Node1, buckets[1], "b", 200);
+ makeQueryEvent(tenant1Node0, buckets[1], "c", 300);
+ makeQueryEvent(tenant1Node0, buckets[1], "d", 400);
+ makeQueryEvent(tenant1Node1, buckets[2], "e", 500);
+
+ makeQueryEvent(tenant2Node, buckets[0], "f", 600);
+
+ makeQueryEvent(staticNode, buckets[0], "g", 700);
+
+ TTableClient client(env.GetDriver());
+ {
+ auto it = client.StreamExecuteScanQuery(R"(
+ SELECT IntervalEnd, QueryText, Rank, ReadBytes
+ FROM `Root/.sys/top_queries_by_read_bytes_one_minute`;
+ )").GetValueSync();
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ TStringBuilder result;
+ result << "[";
+ result << "[[" << buckets[1] << "u];[\"g\"];[1u];[700u]];";
+ result << "]";
+
+ NKqp::CompareYson(result, NKqp::StreamResultToYson(it));
+ }
+ {
+ auto it = client.StreamExecuteScanQuery(R"(
+ SELECT IntervalEnd, QueryText, Rank, ReadBytes
+ FROM `Root/Tenant1/.sys/top_queries_by_read_bytes_one_minute`;
+ )").GetValueSync();
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ TStringBuilder result;
+ result << "[";
+ result << "[[" << buckets[1] << "u];[\"a\"];[1u];[100u]];";
+ result << "[[" << buckets[2] << "u];[\"d\"];[1u];[400u]];";
+ result << "[[" << buckets[2] << "u];[\"c\"];[2u];[300u]];";
+ result << "[[" << buckets[2] << "u];[\"b\"];[3u];[200u]];";
+ result << "[[" << buckets[3] << "u];[\"e\"];[1u];[500u]];";
+ result << "]";
+
+ NKqp::CompareYson(result, NKqp::StreamResultToYson(it));
+ }
+ {
+ auto it = client.StreamExecuteScanQuery(R"(
+ SELECT IntervalEnd, QueryText, Rank, ReadBytes
+ FROM `Root/Tenant2/.sys/top_queries_by_read_bytes_one_minute`;
+ )").GetValueSync();
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ TStringBuilder result;
+ result << "[";
+ result << "[[" << buckets[1] << "u];[\"f\"];[1u];[600u]];";
+ result << "]";
+
+ NKqp::CompareYson(result, NKqp::StreamResultToYson(it));
+ }
+ {
+ TStringBuilder query;
+ query << "SELECT IntervalEnd, QueryText, Rank, ReadBytes ";
+ query << "FROM `Root/Tenant1/.sys/top_queries_by_read_bytes_one_minute` ";
+ query << "WHERE IntervalEnd >= CAST(" << buckets[1] << "ul as Timestamp) ";
+ query << "AND IntervalEnd < CAST(" << buckets[3] << "ul as Timestamp);";
+
+ auto it = client.StreamExecuteScanQuery(query).GetValueSync();
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ TStringBuilder result;
+ result << "[";
+ result << "[[" << buckets[1] << "u];[\"a\"];[1u];[100u]];";
+ result << "[[" << buckets[2] << "u];[\"d\"];[1u];[400u]];";
+ result << "[[" << buckets[2] << "u];[\"c\"];[2u];[300u]];";
+ result << "[[" << buckets[2] << "u];[\"b\"];[3u];[200u]];";
+ result << "]";
+
+ NKqp::CompareYson(result, NKqp::StreamResultToYson(it));
+ }
+ {
+ TStringBuilder query;
+ query << "SELECT IntervalEnd, QueryText, Rank, ReadBytes ";
+ query << "FROM `Root/Tenant1/.sys/top_queries_by_read_bytes_one_minute` ";
+ query << "WHERE IntervalEnd > CAST(" << buckets[1] << "ul as Timestamp) ";
+ query << "AND IntervalEnd <= CAST(" << buckets[3] << "ul as Timestamp);";
+
+ auto it = client.StreamExecuteScanQuery(query).GetValueSync();
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ TStringBuilder result;
+ result << "[";
+ result << "[[" << buckets[2] << "u];[\"d\"];[1u];[400u]];";
+ result << "[[" << buckets[2] << "u];[\"c\"];[2u];[300u]];";
+ result << "[[" << buckets[2] << "u];[\"b\"];[3u];[200u]];";
+ result << "[[" << buckets[3] << "u];[\"e\"];[1u];[500u]];";
+ result << "]";
+
+ NKqp::CompareYson(result, NKqp::StreamResultToYson(it));
+ }
+ {
+ TStringBuilder query;
+ query << "SELECT IntervalEnd, QueryText, Rank, ReadBytes ";
+ query << "FROM `Root/Tenant1/.sys/top_queries_by_read_bytes_one_minute` ";
+ query << "WHERE IntervalEnd = CAST(" << buckets[2] << "ul as Timestamp) ";
+ query << "AND Rank >= 1u AND Rank < 3u";
+
+ auto it = client.StreamExecuteScanQuery(query).GetValueSync();
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ TStringBuilder result;
+ result << "[";
+ result << "[[" << buckets[2] << "u];[\"d\"];[1u];[400u]];";
+ result << "[[" << buckets[2] << "u];[\"c\"];[2u];[300u]];";
+ result << "]";
+
+ NKqp::CompareYson(result, NKqp::StreamResultToYson(it));
+ }
+ {
+ TStringBuilder query;
+ query << "SELECT IntervalEnd, QueryText, Rank, ReadBytes ";
+ query << "FROM `Root/Tenant1/.sys/top_queries_by_read_bytes_one_minute` ";
+ query << "WHERE IntervalEnd = CAST(" << buckets[2] << "ul as Timestamp) ";
+ query << "AND Rank > 1u AND Rank <= 3u";
+
+ auto it = client.StreamExecuteScanQuery(query).GetValueSync();
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ TStringBuilder result;
+ result << "[";
+ result << "[[" << buckets[2] << "u];[\"c\"];[2u];[300u]];";
+ result << "[[" << buckets[2] << "u];[\"b\"];[3u];[200u]];";
+ result << "]";
+
+ NKqp::CompareYson(result, NKqp::StreamResultToYson(it));
+ }
+ }
+
+ Y_UNIT_TEST(QueryStatsFields) {
+ TTestEnv env;
+ CreateRootTable(env, 3);
+
+ auto nowUs = TInstant::Now().MicroSeconds();
+
+ TString queryText("SELECT * FROM `Root/Table0`");
+
+ TTableClient client(env.GetDriver());
+ auto session = client.CreateSession().GetValueSync().GetSession();
+ NKqp::AssertSuccessResult(session.ExecuteDataQuery(
+ queryText, TTxControl::BeginTx().CommitTx()
+ ).GetValueSync());
+
+ auto it = client.StreamExecuteScanQuery(R"(
+ SELECT
+ CPUTime,
+ CompileCPUTime,
+ CompileDuration,
+ ComputeNodesCount,
+ DeleteBytes,
+ DeleteRows,
+ Duration,
+ EndTime,
+ FromQueryCache,
+ IntervalEnd,
+ MaxComputeCPUTime,
+ MaxShardCPUTime,
+ MinComputeCPUTime,
+ MinShardCPUTime,
+ ParametersSize,
+ Partitions,
+ ProcessCPUTime,
+ QueryText,
+ Rank,
+ ReadBytes,
+ ReadRows,
+ RequestUnits,
+ ShardCount,
+ SumComputeCPUTime,
+ SumShardCPUTime,
+ Type,
+ UpdateBytes,
+ UpdateRows,
+ UserSID
+ FROM `Root/.sys/top_queries_by_read_bytes_one_minute`;
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ auto ysonString = NKqp::StreamResultToYson(it);
+
+ TYsonFieldChecker check(ysonString, 29);
+
+ check.Uint64GreaterOrEquals(0); // CPUTime
+ check.Uint64GreaterOrEquals(0); // CompileCPUTime
+ check.Int64GreaterOrEquals(0); // CompileDuration
+ check.Uint64(1); // ComputeNodesCount
+ check.Uint64(0); // DeleteBytes
+ check.Uint64(0); // DeleteRows
+ check.Int64Greater(0); // Duration
+ check.Uint64Greater(nowUs); // EndTime
+ check.Bool(false); // FromQueryCache
+ check.Uint64Greater(nowUs); // IntervalEnd
+ check.Uint64GreaterOrEquals(0); // MaxComputeCPUTime
+ check.Uint64GreaterOrEquals(0); // MaxShardCPUTime
+ check.Uint64GreaterOrEquals(0); // MinComputeCPUTime
+ check.Uint64GreaterOrEquals(0); // MinShardCPUTime
+ check.Uint64(0); // ParametersSize
+ check.Uint64(3); // Partitions
+ check.Uint64GreaterOrEquals(0); // ProcessCPUTime
+ check.String(queryText); // QueryText
+ check.Uint64(1); // Rank
+ check.Uint64(0); // ReadBytes
+ check.Uint64(0); // ReadRows
+ check.Uint64Greater(0); // RequestUnits
+ check.Uint64(3); // ShardCount
+ check.Uint64GreaterOrEquals(0); // SumComputeCPUTime
+ check.Uint64GreaterOrEquals(0); // SumShardCPUTime
+ check.String("data"); // Type
+ check.Uint64(0); // UpdateBytes
+ check.Uint64(0); // UpdateRows
check.Null(); // UserSID
- }
-
+ }
+
Y_UNIT_TEST(PartitionStatsTtlFields) {
TTestEnv env;
env.GetClient().CreateTable("/Root", R"(
@@ -687,844 +687,844 @@ Y_UNIT_TEST_SUITE(SystemView) {
check.Uint64(1u); // LastTtlRowsErased
}
- Y_UNIT_TEST(PartitionStatsFields) {
+ Y_UNIT_TEST(PartitionStatsFields) {
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
-
- auto nowUs = TInstant::Now().MicroSeconds();
-
- TTestEnv env;
- CreateRootTable(env);
-
- TTableClient client(env.GetDriver());
- auto session = client.CreateSession().GetValueSync().GetSession();
- NKqp::AssertSuccessResult(session.ExecuteDataQuery(
- "REPLACE INTO `Root/Table0` (Key, Value) VALUES (0u, \"A\");",
- TTxControl::BeginTx().CommitTx()
- ).GetValueSync());
-
- // wait for stats
- for (size_t iter = 0; iter < 30; ++iter) {
- auto it = client.StreamExecuteScanQuery(R"(
- SELECT AccessTime FROM `/Root/.sys/partition_stats`;
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- auto ysonString = NKqp::StreamResultToYson(it);
- if (ysonString != "[[#]]") {
- break;
- }
-
- Sleep(TDuration::Seconds(1));
- }
-
- auto it = client.StreamExecuteScanQuery(R"(
- SELECT
- AccessTime,
- CPUCores,
- CoordinatedTxCompleted,
- DataSize,
- ImmediateTxCompleted,
- IndexSize,
- InFlightTxCount,
- NodeId,
- OwnerId,
- PartIdx,
- Path,
- PathId,
- RangeReadRows,
- RangeReads,
- RowCount,
- RowDeletes,
- RowReads,
- RowUpdates,
- StartTime,
- TabletId,
- TxRejectedByOutOfStorage,
- TxRejectedByOverload,
- UpdateTime
- FROM `/Root/.sys/partition_stats`;
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- auto ysonString = NKqp::StreamResultToYson(it);
-
- TYsonFieldChecker check(ysonString, 23);
-
- check.Uint64GreaterOrEquals(nowUs); // AccessTime
- check.DoubleGreaterOrEquals(0.0); // CPUCores
- check.Uint64(1u); // CoordinatedTxCompleted
- check.Uint64(608u); // DataSize
- check.Uint64(1u); // ImmediateTxCompleted
- check.Uint64(0u); // IndexSize
- check.Uint64(0u); // InFlightTxCount
- check.Uint64Greater(0u); // NodeId
- check.Uint64(72057594046644480ull); // OwnerId
- check.Uint64(0u); // PartIdx
- check.String("/Root/Table0"); // Path
- check.Uint64(2u); // PathId
- check.Uint64(0u); // RangeReadRows
- check.Uint64(0u); // RangeReads
- check.Uint64(1u); // RowCount
- check.Uint64(0u); // RowDeletes
- check.Uint64(0u); // RowReads
- check.Uint64(1u); // RowUpdates
- check.Uint64GreaterOrEquals(nowUs); // StartTime
- check.Uint64Greater(0u); // TabletId
- check.Uint64(0u); // TxRejectedByOutOfStorage
- check.Uint64(0u); // TxRejectedByOverload
- check.Uint64GreaterOrEquals(nowUs); // UpdateTime
- }
-
- Y_UNIT_TEST(QueryStatsAllTables) {
- auto check = [&] (const TString& queryText) {
- TTestEnv env;
- CreateRootTable(env);
-
- TTableClient client(env.GetDriver());
- auto session = client.CreateSession().GetValueSync().GetSession();
- NKqp::AssertSuccessResult(session.ExecuteDataQuery(
- "SELECT * FROM `Root/Table0`", TTxControl::BeginTx().CommitTx()
- ).GetValueSync());
-
- auto it = client.StreamExecuteScanQuery(queryText).GetValueSync();
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- NKqp::CompareYson(R"([
- [[0u]]
- ])", NKqp::StreamResultToYson(it));
- };
-
- check("SELECT ReadBytes FROM `Root/.sys/top_queries_by_read_bytes_one_minute`");
- check("SELECT ReadBytes FROM `Root/.sys/top_queries_by_read_bytes_one_hour`");
- check("SELECT ReadBytes FROM `Root/.sys/top_queries_by_duration_one_minute`");
- check("SELECT ReadBytes FROM `Root/.sys/top_queries_by_duration_one_hour`");
- check("SELECT ReadBytes FROM `Root/.sys/top_queries_by_cpu_time_one_minute`");
- check("SELECT ReadBytes FROM `Root/.sys/top_queries_by_cpu_time_one_hour`");
- check("SELECT ReadBytes FROM `Root/.sys/top_queries_by_request_units_one_minute`");
- check("SELECT ReadBytes FROM `Root/.sys/top_queries_by_request_units_one_hour`");
- }
-
- Y_UNIT_TEST(QueryStatsRetries) {
- TTestEnv env;
- CreateRootTable(env);
-
- TString queryText("SELECT * FROM `Root/Table0`");
-
- TTableClient client(env.GetDriver());
- auto session = client.CreateSession().GetValueSync().GetSession();
- NKqp::AssertSuccessResult(session.ExecuteDataQuery(
- queryText, TTxControl::BeginTx().CommitTx()
- ).GetValueSync());
-
- auto serviceToKill = MakeSysViewServiceID(env.GetServer().GetRuntime()->GetNodeId(2));
+
+ auto nowUs = TInstant::Now().MicroSeconds();
+
+ TTestEnv env;
+ CreateRootTable(env);
+
+ TTableClient client(env.GetDriver());
+ auto session = client.CreateSession().GetValueSync().GetSession();
+ NKqp::AssertSuccessResult(session.ExecuteDataQuery(
+ "REPLACE INTO `Root/Table0` (Key, Value) VALUES (0u, \"A\");",
+ TTxControl::BeginTx().CommitTx()
+ ).GetValueSync());
+
+ // wait for stats
+ for (size_t iter = 0; iter < 30; ++iter) {
+ auto it = client.StreamExecuteScanQuery(R"(
+ SELECT AccessTime FROM `/Root/.sys/partition_stats`;
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ auto ysonString = NKqp::StreamResultToYson(it);
+ if (ysonString != "[[#]]") {
+ break;
+ }
+
+ Sleep(TDuration::Seconds(1));
+ }
+
+ auto it = client.StreamExecuteScanQuery(R"(
+ SELECT
+ AccessTime,
+ CPUCores,
+ CoordinatedTxCompleted,
+ DataSize,
+ ImmediateTxCompleted,
+ IndexSize,
+ InFlightTxCount,
+ NodeId,
+ OwnerId,
+ PartIdx,
+ Path,
+ PathId,
+ RangeReadRows,
+ RangeReads,
+ RowCount,
+ RowDeletes,
+ RowReads,
+ RowUpdates,
+ StartTime,
+ TabletId,
+ TxRejectedByOutOfStorage,
+ TxRejectedByOverload,
+ UpdateTime
+ FROM `/Root/.sys/partition_stats`;
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ auto ysonString = NKqp::StreamResultToYson(it);
+
+ TYsonFieldChecker check(ysonString, 23);
+
+ check.Uint64GreaterOrEquals(nowUs); // AccessTime
+ check.DoubleGreaterOrEquals(0.0); // CPUCores
+ check.Uint64(1u); // CoordinatedTxCompleted
+ check.Uint64(608u); // DataSize
+ check.Uint64(1u); // ImmediateTxCompleted
+ check.Uint64(0u); // IndexSize
+ check.Uint64(0u); // InFlightTxCount
+ check.Uint64Greater(0u); // NodeId
+ check.Uint64(72057594046644480ull); // OwnerId
+ check.Uint64(0u); // PartIdx
+ check.String("/Root/Table0"); // Path
+ check.Uint64(2u); // PathId
+ check.Uint64(0u); // RangeReadRows
+ check.Uint64(0u); // RangeReads
+ check.Uint64(1u); // RowCount
+ check.Uint64(0u); // RowDeletes
+ check.Uint64(0u); // RowReads
+ check.Uint64(1u); // RowUpdates
+ check.Uint64GreaterOrEquals(nowUs); // StartTime
+ check.Uint64Greater(0u); // TabletId
+ check.Uint64(0u); // TxRejectedByOutOfStorage
+ check.Uint64(0u); // TxRejectedByOverload
+ check.Uint64GreaterOrEquals(nowUs); // UpdateTime
+ }
+
+ Y_UNIT_TEST(QueryStatsAllTables) {
+ auto check = [&] (const TString& queryText) {
+ TTestEnv env;
+ CreateRootTable(env);
+
+ TTableClient client(env.GetDriver());
+ auto session = client.CreateSession().GetValueSync().GetSession();
+ NKqp::AssertSuccessResult(session.ExecuteDataQuery(
+ "SELECT * FROM `Root/Table0`", TTxControl::BeginTx().CommitTx()
+ ).GetValueSync());
+
+ auto it = client.StreamExecuteScanQuery(queryText).GetValueSync();
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ NKqp::CompareYson(R"([
+ [[0u]]
+ ])", NKqp::StreamResultToYson(it));
+ };
+
+ check("SELECT ReadBytes FROM `Root/.sys/top_queries_by_read_bytes_one_minute`");
+ check("SELECT ReadBytes FROM `Root/.sys/top_queries_by_read_bytes_one_hour`");
+ check("SELECT ReadBytes FROM `Root/.sys/top_queries_by_duration_one_minute`");
+ check("SELECT ReadBytes FROM `Root/.sys/top_queries_by_duration_one_hour`");
+ check("SELECT ReadBytes FROM `Root/.sys/top_queries_by_cpu_time_one_minute`");
+ check("SELECT ReadBytes FROM `Root/.sys/top_queries_by_cpu_time_one_hour`");
+ check("SELECT ReadBytes FROM `Root/.sys/top_queries_by_request_units_one_minute`");
+ check("SELECT ReadBytes FROM `Root/.sys/top_queries_by_request_units_one_hour`");
+ }
+
+ Y_UNIT_TEST(QueryStatsRetries) {
+ TTestEnv env;
+ CreateRootTable(env);
+
+ TString queryText("SELECT * FROM `Root/Table0`");
+
+ TTableClient client(env.GetDriver());
+ auto session = client.CreateSession().GetValueSync().GetSession();
+ NKqp::AssertSuccessResult(session.ExecuteDataQuery(
+ queryText, TTxControl::BeginTx().CommitTx()
+ ).GetValueSync());
+
+ auto serviceToKill = MakeSysViewServiceID(env.GetServer().GetRuntime()->GetNodeId(2));
env.GetServer().GetRuntime()->Send(new IEventHandle(serviceToKill, TActorId(), new TEvents::TEvPoison()));
-
- auto it = client.StreamExecuteScanQuery(R"(
- SELECT
- ReadBytes
- FROM `Root/.sys/top_queries_by_read_bytes_one_minute`;
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- NKqp::CompareYson(R"([
- [[0u]]
- ])", NKqp::StreamResultToYson(it));
- }
-
- Y_UNIT_TEST(ConcurrentScans) {
- TTestEnv env;
- CreateRootTable(env);
- TTableClient client(env.GetDriver());
-
- TVector<TAsyncScanQueryPartIterator> futures;
- for (size_t i = 0; i < 20; ++i) {
- auto future0 = client.StreamExecuteScanQuery(R"(
- SELECT
- ReadBytes
- FROM `Root/.sys/top_queries_by_read_bytes_one_minute`;
- )");
- futures.push_back(future0);
- }
-
- for (auto& future0 : futures) {
- auto it = future0.GetValueSync();
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- auto streamPart = it.ReadNext().GetValueSync();
- if (streamPart.IsSuccess()) {
- UNIT_ASSERT_VALUES_EQUAL(streamPart.GetStatus(), EStatus::SUCCESS);
- Cerr << "SUCCESS" << Endl;
- } else {
- UNIT_ASSERT_VALUES_EQUAL(streamPart.GetStatus(), EStatus::GENERIC_ERROR);
- Cerr << "FAIL " << streamPart.GetIssues().ToString() << Endl;
- }
- }
- }
-
- Y_UNIT_TEST(PDisksFields) {
- TTestEnv env(1, 0);
-
- TTableClient client(env.GetDriver());
- size_t rowCount = 0;
- TString ysonString;
-
- for (size_t iter = 0; iter < 30 && !rowCount; ++iter) {
- auto it = client.StreamExecuteScanQuery(R"(
- SELECT
- AvailableSize,
- BoxId,
- Guid,
- Kind,
- NodeId,
- PDiskId,
- Path,
- ReadCentric,
- SharedWithOS,
- Status,
+
+ auto it = client.StreamExecuteScanQuery(R"(
+ SELECT
+ ReadBytes
+ FROM `Root/.sys/top_queries_by_read_bytes_one_minute`;
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ NKqp::CompareYson(R"([
+ [[0u]]
+ ])", NKqp::StreamResultToYson(it));
+ }
+
+ Y_UNIT_TEST(ConcurrentScans) {
+ TTestEnv env;
+ CreateRootTable(env);
+ TTableClient client(env.GetDriver());
+
+ TVector<TAsyncScanQueryPartIterator> futures;
+ for (size_t i = 0; i < 20; ++i) {
+ auto future0 = client.StreamExecuteScanQuery(R"(
+ SELECT
+ ReadBytes
+ FROM `Root/.sys/top_queries_by_read_bytes_one_minute`;
+ )");
+ futures.push_back(future0);
+ }
+
+ for (auto& future0 : futures) {
+ auto it = future0.GetValueSync();
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ auto streamPart = it.ReadNext().GetValueSync();
+ if (streamPart.IsSuccess()) {
+ UNIT_ASSERT_VALUES_EQUAL(streamPart.GetStatus(), EStatus::SUCCESS);
+ Cerr << "SUCCESS" << Endl;
+ } else {
+ UNIT_ASSERT_VALUES_EQUAL(streamPart.GetStatus(), EStatus::GENERIC_ERROR);
+ Cerr << "FAIL " << streamPart.GetIssues().ToString() << Endl;
+ }
+ }
+ }
+
+ Y_UNIT_TEST(PDisksFields) {
+ TTestEnv env(1, 0);
+
+ TTableClient client(env.GetDriver());
+ size_t rowCount = 0;
+ TString ysonString;
+
+ for (size_t iter = 0; iter < 30 && !rowCount; ++iter) {
+ auto it = client.StreamExecuteScanQuery(R"(
+ SELECT
+ AvailableSize,
+ BoxId,
+ Guid,
+ Kind,
+ NodeId,
+ PDiskId,
+ Path,
+ ReadCentric,
+ SharedWithOS,
+ Status,
StatusChangeTimestamp,
- TotalSize,
+ TotalSize,
Type,
ExpectedSlotCount,
NumActiveSlots
- FROM `/Root/.sys/ds_pdisks`;
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- ysonString = NKqp::StreamResultToYson(it);
-
+ FROM `/Root/.sys/ds_pdisks`;
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ ysonString = NKqp::StreamResultToYson(it);
+
auto node = NYT::NodeFromYsonString(ysonString, ::NYson::EYsonType::Node);
- UNIT_ASSERT(node.IsList());
- rowCount = node.AsList().size();
-
- if (!rowCount) {
- Sleep(TDuration::Seconds(1));
- }
- }
-
+ UNIT_ASSERT(node.IsList());
+ rowCount = node.AsList().size();
+
+ if (!rowCount) {
+ Sleep(TDuration::Seconds(1));
+ }
+ }
+
TYsonFieldChecker check(ysonString, 15);
-
- check.Uint64(0u); // AvailableSize
- check.Uint64(999u); // BoxId
- check.Uint64(123u); // Guid
- check.Uint64(0u); // Kind
- check.Uint64(env.GetServer().GetRuntime()->GetNodeId(0)); // NodeId
- check.Uint64(1u); // PDiskId
- check.StringContains("pdisk_1.dat"); // Path
- check.Bool(false); // ReadCentric
- check.Bool(false); // SharedWithOS
- check.String("ACTIVE"); // Status
+
+ check.Uint64(0u); // AvailableSize
+ check.Uint64(999u); // BoxId
+ check.Uint64(123u); // Guid
+ check.Uint64(0u); // Kind
+ check.Uint64(env.GetServer().GetRuntime()->GetNodeId(0)); // NodeId
+ check.Uint64(1u); // PDiskId
+ check.StringContains("pdisk_1.dat"); // Path
+ check.Bool(false); // ReadCentric
+ check.Bool(false); // SharedWithOS
+ check.String("ACTIVE"); // Status
check.Null(); // StatusChangeTimestamp
- check.Uint64(0u); // TotalSize
- check.String("ROT"); // Type
+ check.Uint64(0u); // TotalSize
+ check.String("ROT"); // Type
check.Uint64(16); // ExpectedSlotCount
check.Uint64(2); // NumActiveSlots
- }
-
- Y_UNIT_TEST(VSlotsFields) {
- TTestEnv env(1, 0);
-
- TTableClient client(env.GetDriver());
- size_t rowCount = 0;
- TString ysonString;
-
- for (size_t iter = 0; iter < 30 && !rowCount; ++iter) {
- auto it = client.StreamExecuteScanQuery(R"(
- SELECT
- AllocatedSize,
- AvailableSize,
- FailDomain,
- FailRealm,
- GroupGeneration,
- GroupId,
- Kind,
- NodeId,
- PDiskId,
- Status,
- VDisk,
- VSlotId
+ }
+
+ Y_UNIT_TEST(VSlotsFields) {
+ TTestEnv env(1, 0);
+
+ TTableClient client(env.GetDriver());
+ size_t rowCount = 0;
+ TString ysonString;
+
+ for (size_t iter = 0; iter < 30 && !rowCount; ++iter) {
+ auto it = client.StreamExecuteScanQuery(R"(
+ SELECT
+ AllocatedSize,
+ AvailableSize,
+ FailDomain,
+ FailRealm,
+ GroupGeneration,
+ GroupId,
+ Kind,
+ NodeId,
+ PDiskId,
+ Status,
+ VDisk,
+ VSlotId
FROM `/Root/.sys/ds_vslots` WHERE GroupId >= 0x80000000;
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- ysonString = NKqp::StreamResultToYson(it);
-
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ ysonString = NKqp::StreamResultToYson(it);
+
auto node = NYT::NodeFromYsonString(ysonString, ::NYson::EYsonType::Node);
- UNIT_ASSERT(node.IsList());
- rowCount = node.AsList().size();
-
- if (!rowCount) {
- Sleep(TDuration::Seconds(1));
- }
- }
-
+ UNIT_ASSERT(node.IsList());
+ rowCount = node.AsList().size();
+
+ if (!rowCount) {
+ Sleep(TDuration::Seconds(1));
+ }
+ }
+
TYsonFieldChecker check(ysonString, 12);
-
+
check.Uint64(0u, true); // AllocatedSize
check.Uint64(0u, true); // AvailableSize
- check.Uint64(0u); // FailDomain
- check.Uint64(0u); // FailRealm
- check.Uint64(1u); // GroupGeneration
- check.Uint64(2181038080u); // GroupId
- check.String("Default"); // Kind
- check.Uint64(env.GetServer().GetRuntime()->GetNodeId(0)); // NodeId
- check.Uint64(1u); // PDiskId
+ check.Uint64(0u); // FailDomain
+ check.Uint64(0u); // FailRealm
+ check.Uint64(1u); // GroupGeneration
+ check.Uint64(2181038080u); // GroupId
+ check.String("Default"); // Kind
+ check.Uint64(env.GetServer().GetRuntime()->GetNodeId(0)); // NodeId
+ check.Uint64(1u); // PDiskId
check.String("INIT_PENDING"); // Status
- check.Uint64(0u); // VDisk
- check.Uint64(1000u); // VSlotId
- }
-
- Y_UNIT_TEST(GroupsFields) {
- TTestEnv env;
-
- TTableClient client(env.GetDriver());
- size_t rowCount = 0;
- TString ysonString;
-
- for (size_t iter = 0; iter < 30 && !rowCount; ++iter) {
- auto it = client.StreamExecuteScanQuery(R"(
- SELECT
- AllocatedSize,
- AvailableSize,
- BoxId,
- EncryptionMode,
- ErasureSpecies,
- Generation,
- GetFastLatency,
- GroupId,
- LifeCyclePhase,
- PutTabletLogLatency,
- PutUserDataLatency,
+ check.Uint64(0u); // VDisk
+ check.Uint64(1000u); // VSlotId
+ }
+
+ Y_UNIT_TEST(GroupsFields) {
+ TTestEnv env;
+
+ TTableClient client(env.GetDriver());
+ size_t rowCount = 0;
+ TString ysonString;
+
+ for (size_t iter = 0; iter < 30 && !rowCount; ++iter) {
+ auto it = client.StreamExecuteScanQuery(R"(
+ SELECT
+ AllocatedSize,
+ AvailableSize,
+ BoxId,
+ EncryptionMode,
+ ErasureSpecies,
+ Generation,
+ GetFastLatency,
+ GroupId,
+ LifeCyclePhase,
+ PutTabletLogLatency,
+ PutUserDataLatency,
StoragePoolId
FROM `/Root/.sys/ds_groups` WHERE GroupId >= 0x80000000;
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- ysonString = NKqp::StreamResultToYson(it);
-
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ ysonString = NKqp::StreamResultToYson(it);
+
auto node = NYT::NodeFromYsonString(ysonString, ::NYson::EYsonType::Node);
- UNIT_ASSERT(node.IsList());
- rowCount = node.AsList().size();
-
- if (!rowCount) {
- Sleep(TDuration::Seconds(1));
- }
- }
-
+ UNIT_ASSERT(node.IsList());
+ rowCount = node.AsList().size();
+
+ if (!rowCount) {
+ Sleep(TDuration::Seconds(1));
+ }
+ }
+
TYsonFieldChecker check(ysonString, 12);
-
- check.Uint64(0u); // AllocatedSize
- check.Uint64(0u); // AvailableSize
- check.Uint64(999u); // BoxId
- check.Uint64(0u); // EncryptionMode
- check.String("none"); // ErasureSpecies
- check.Uint64(1u); // Generation
+
+ check.Uint64(0u); // AllocatedSize
+ check.Uint64(0u); // AvailableSize
+ check.Uint64(999u); // BoxId
+ check.Uint64(0u); // EncryptionMode
+ check.String("none"); // ErasureSpecies
+ check.Uint64(1u); // Generation
check.Null(); // GetFastLatency
- check.Uint64(2181038080u); // GroupId
- check.Uint64(0u); // LifeCyclePhase
+ check.Uint64(2181038080u); // GroupId
+ check.Uint64(0u); // LifeCyclePhase
check.Null(); // PutTabletLogLatency
check.Null(); // PutUserDataLatency
- check.Uint64(2u); // StoragePoolId
- }
-
- Y_UNIT_TEST(StoragePoolsFields) {
- TTestEnv env(1, 0);
-
- TTableClient client(env.GetDriver());
- size_t rowCount = 0;
- TString ysonString;
-
- for (size_t iter = 0; iter < 30 && !rowCount; ++iter) {
- auto it = client.StreamExecuteScanQuery(R"(
- SELECT
- BoxId,
- EncryptionMode,
- ErasureSpecies,
- Generation,
- Kind,
- Name,
- NumGroups,
- PathId,
- SchemeshardId,
- StoragePoolId,
- VDiskKind
- FROM `/Root/.sys/ds_storage_pools`;
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- ysonString = NKqp::StreamResultToYson(it);
-
+ check.Uint64(2u); // StoragePoolId
+ }
+
+ Y_UNIT_TEST(StoragePoolsFields) {
+ TTestEnv env(1, 0);
+
+ TTableClient client(env.GetDriver());
+ size_t rowCount = 0;
+ TString ysonString;
+
+ for (size_t iter = 0; iter < 30 && !rowCount; ++iter) {
+ auto it = client.StreamExecuteScanQuery(R"(
+ SELECT
+ BoxId,
+ EncryptionMode,
+ ErasureSpecies,
+ Generation,
+ Kind,
+ Name,
+ NumGroups,
+ PathId,
+ SchemeshardId,
+ StoragePoolId,
+ VDiskKind
+ FROM `/Root/.sys/ds_storage_pools`;
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ ysonString = NKqp::StreamResultToYson(it);
+
auto node = NYT::NodeFromYsonString(ysonString, ::NYson::EYsonType::Node);
- UNIT_ASSERT(node.IsList());
- rowCount = node.AsList().size();
-
- if (!rowCount) {
- Sleep(TDuration::Seconds(1));
- }
- }
-
- TYsonFieldChecker check(ysonString, 11);
-
- check.Uint64(999u); // BoxId
- check.Uint64(0u); // EncryptionMode
- check.String("none"); // ErasureSpecies
- check.Uint64(1u); // Generation
- check.String("test"); // Kind
- check.String("/Root:test"); // Name
- check.Uint64(1u); // NumGroups
+ UNIT_ASSERT(node.IsList());
+ rowCount = node.AsList().size();
+
+ if (!rowCount) {
+ Sleep(TDuration::Seconds(1));
+ }
+ }
+
+ TYsonFieldChecker check(ysonString, 11);
+
+ check.Uint64(999u); // BoxId
+ check.Uint64(0u); // EncryptionMode
+ check.String("none"); // ErasureSpecies
+ check.Uint64(1u); // Generation
+ check.String("test"); // Kind
+ check.String("/Root:test"); // Name
+ check.Uint64(1u); // NumGroups
check.Null(); // PathId
check.Null(); // SchemeshardId
- check.Uint64(2u); // StoragePoolId
- check.String("Default"); // VDiskKind
- }
-
- Y_UNIT_TEST(StoragePoolsRanges) {
- TTestEnv env(1, 0, 3);
-
- TTableClient client(env.GetDriver());
- size_t rowCount = 0;
- for (size_t iter = 0; iter < 30 && !rowCount; ++iter) {
- auto it = client.StreamExecuteScanQuery(R"(
- SELECT BoxId FROM `/Root/.sys/ds_storage_pools`;
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- auto ysonString = NKqp::StreamResultToYson(it);
+ check.Uint64(2u); // StoragePoolId
+ check.String("Default"); // VDiskKind
+ }
+
+ Y_UNIT_TEST(StoragePoolsRanges) {
+ TTestEnv env(1, 0, 3);
+
+ TTableClient client(env.GetDriver());
+ size_t rowCount = 0;
+ for (size_t iter = 0; iter < 30 && !rowCount; ++iter) {
+ auto it = client.StreamExecuteScanQuery(R"(
+ SELECT BoxId FROM `/Root/.sys/ds_storage_pools`;
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ auto ysonString = NKqp::StreamResultToYson(it);
auto node = NYT::NodeFromYsonString(ysonString, ::NYson::EYsonType::Node);
- UNIT_ASSERT(node.IsList());
- rowCount = node.AsList().size();
-
- if (!rowCount) {
- Sleep(TDuration::Seconds(1));
- }
- }
- {
- auto it = client.StreamExecuteScanQuery(R"(
- SELECT BoxId, StoragePoolId
- FROM `/Root/.sys/ds_storage_pools`
- WHERE BoxId = 999u AND StoragePoolId > 3u;
- )").GetValueSync();
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- NKqp::CompareYson(R"([
- [[999u];[4u]];
- [[999u];[5u]];
- ])", NKqp::StreamResultToYson(it));
- }
- {
- auto it = client.StreamExecuteScanQuery(R"(
- SELECT BoxId, StoragePoolId
- FROM `/Root/.sys/ds_storage_pools`
- WHERE BoxId = 999u AND StoragePoolId >= 3u;
- )").GetValueSync();
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- NKqp::CompareYson(R"([
- [[999u];[3u]];
- [[999u];[4u]];
- [[999u];[5u]];
- ])", NKqp::StreamResultToYson(it));
- }
- {
- auto it = client.StreamExecuteScanQuery(R"(
- SELECT BoxId, StoragePoolId
- FROM `/Root/.sys/ds_storage_pools`
- WHERE BoxId = 999u AND StoragePoolId < 4u;
- )").GetValueSync();
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- NKqp::CompareYson(R"([
- [[999u];[2u]];
- [[999u];[3u]];
- ])", NKqp::StreamResultToYson(it));
- }
- {
- auto it = client.StreamExecuteScanQuery(R"(
- SELECT BoxId, StoragePoolId
- FROM `/Root/.sys/ds_storage_pools`
- WHERE BoxId = 999u AND StoragePoolId <= 4u;
- )").GetValueSync();
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- NKqp::CompareYson(R"([
- [[999u];[2u]];
- [[999u];[3u]];
- [[999u];[4u]];
- ])", NKqp::StreamResultToYson(it));
- }
- }
-
- Y_UNIT_TEST(OldEngineSystemView) {
- TTestEnv env;
- CreateRootTable(env);
-
- TTableClient client(env.GetDriver());
- auto session = client.CreateSession().GetValueSync().GetSession();
- auto result = session.ExecuteDataQuery(
- "SELECT * from `/Root/.sys/partition_stats`", TTxControl::BeginTx().CommitTx()
- ).GetValueSync();
-
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR);
- result.GetIssues().PrintTo(Cerr);
- }
-
- Y_UNIT_TEST(Describe) {
- TTestEnv env;
- CreateRootTable(env);
-
- TTableClient client(env.GetDriver());
- auto session = client.CreateSession().GetValueSync().GetSession();
- {
- auto settings = TDescribeTableSettings()
- .WithKeyShardBoundary(true)
- .WithTableStatistics(true)
- .WithPartitionStatistics(true);
-
- auto result = session.DescribeTable("/Root/.sys/partition_stats", settings).GetValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- const auto& table = result.GetTableDescription();
- const auto& columns = table.GetTableColumns();
- const auto& keyColumns = table.GetPrimaryKeyColumns();
-
+ UNIT_ASSERT(node.IsList());
+ rowCount = node.AsList().size();
+
+ if (!rowCount) {
+ Sleep(TDuration::Seconds(1));
+ }
+ }
+ {
+ auto it = client.StreamExecuteScanQuery(R"(
+ SELECT BoxId, StoragePoolId
+ FROM `/Root/.sys/ds_storage_pools`
+ WHERE BoxId = 999u AND StoragePoolId > 3u;
+ )").GetValueSync();
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ NKqp::CompareYson(R"([
+ [[999u];[4u]];
+ [[999u];[5u]];
+ ])", NKqp::StreamResultToYson(it));
+ }
+ {
+ auto it = client.StreamExecuteScanQuery(R"(
+ SELECT BoxId, StoragePoolId
+ FROM `/Root/.sys/ds_storage_pools`
+ WHERE BoxId = 999u AND StoragePoolId >= 3u;
+ )").GetValueSync();
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ NKqp::CompareYson(R"([
+ [[999u];[3u]];
+ [[999u];[4u]];
+ [[999u];[5u]];
+ ])", NKqp::StreamResultToYson(it));
+ }
+ {
+ auto it = client.StreamExecuteScanQuery(R"(
+ SELECT BoxId, StoragePoolId
+ FROM `/Root/.sys/ds_storage_pools`
+ WHERE BoxId = 999u AND StoragePoolId < 4u;
+ )").GetValueSync();
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ NKqp::CompareYson(R"([
+ [[999u];[2u]];
+ [[999u];[3u]];
+ ])", NKqp::StreamResultToYson(it));
+ }
+ {
+ auto it = client.StreamExecuteScanQuery(R"(
+ SELECT BoxId, StoragePoolId
+ FROM `/Root/.sys/ds_storage_pools`
+ WHERE BoxId = 999u AND StoragePoolId <= 4u;
+ )").GetValueSync();
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ NKqp::CompareYson(R"([
+ [[999u];[2u]];
+ [[999u];[3u]];
+ [[999u];[4u]];
+ ])", NKqp::StreamResultToYson(it));
+ }
+ }
+
+ Y_UNIT_TEST(OldEngineSystemView) {
+ TTestEnv env;
+ CreateRootTable(env);
+
+ TTableClient client(env.GetDriver());
+ auto session = client.CreateSession().GetValueSync().GetSession();
+ auto result = session.ExecuteDataQuery(
+ "SELECT * from `/Root/.sys/partition_stats`", TTxControl::BeginTx().CommitTx()
+ ).GetValueSync();
+
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR);
+ result.GetIssues().PrintTo(Cerr);
+ }
+
+ Y_UNIT_TEST(Describe) {
+ TTestEnv env;
+ CreateRootTable(env);
+
+ TTableClient client(env.GetDriver());
+ auto session = client.CreateSession().GetValueSync().GetSession();
+ {
+ auto settings = TDescribeTableSettings()
+ .WithKeyShardBoundary(true)
+ .WithTableStatistics(true)
+ .WithPartitionStatistics(true);
+
+ auto result = session.DescribeTable("/Root/.sys/partition_stats", settings).GetValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ const auto& table = result.GetTableDescription();
+ const auto& columns = table.GetTableColumns();
+ const auto& keyColumns = table.GetPrimaryKeyColumns();
+
UNIT_ASSERT_VALUES_EQUAL(columns.size(), 26);
- UNIT_ASSERT_STRINGS_EQUAL(columns[0].Name, "OwnerId");
- UNIT_ASSERT_STRINGS_EQUAL(FormatType(columns[0].Type), "Uint64?");
-
- UNIT_ASSERT_VALUES_EQUAL(keyColumns.size(), 3);
- UNIT_ASSERT_STRINGS_EQUAL(keyColumns[0], "OwnerId");
-
- UNIT_ASSERT_VALUES_EQUAL(table.GetPartitionStats().size(), 0);
- UNIT_ASSERT_VALUES_EQUAL(table.GetPartitionsCount(), 0);
- }
-
- TSchemeClient schemeClient(env.GetDriver());
-
- {
- auto result = schemeClient.DescribePath("/Root/.sys/partition_stats").GetValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- auto entry = result.GetEntry();
- UNIT_ASSERT_VALUES_EQUAL(entry.Name, "partition_stats");
- UNIT_ASSERT_VALUES_EQUAL(entry.Type, ESchemeEntryType::Table);
- }
- {
- auto result = schemeClient.ListDirectory("/Root/.sys/partition_stats").GetValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- auto entry = result.GetEntry();
- UNIT_ASSERT_VALUES_EQUAL(entry.Name, "partition_stats");
- UNIT_ASSERT_VALUES_EQUAL(entry.Type, ESchemeEntryType::Table);
- }
- }
-
- Y_UNIT_TEST(SystemViewFailOps) {
- TTestEnv env;
-
- TTableClient client(env.GetDriver());
- auto session = client.CreateSession().GetValueSync().GetSession();
-
- {
- auto desc = TTableBuilder()
- .AddNullableColumn("Column1", EPrimitiveType::Uint64)
- .SetPrimaryKeyColumn("Column1")
- .Build();
-
- auto result = session.CreateTable("/Root/.sys/partition_stats", std::move(desc)).GetValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR);
- result.GetIssues().PrintTo(Cerr);
- }
- {
- auto result = session.CopyTable("/Root/.sys/partition_stats", "/Root/Table0").GetValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR);
- result.GetIssues().PrintTo(Cerr);
- }
- {
- auto settings = TAlterTableSettings()
- .AppendDropColumns("OwnerId");
-
- auto result = session.AlterTable("/Root/.sys/partition_stats", settings).GetValueSync();
+ UNIT_ASSERT_STRINGS_EQUAL(columns[0].Name, "OwnerId");
+ UNIT_ASSERT_STRINGS_EQUAL(FormatType(columns[0].Type), "Uint64?");
+
+ UNIT_ASSERT_VALUES_EQUAL(keyColumns.size(), 3);
+ UNIT_ASSERT_STRINGS_EQUAL(keyColumns[0], "OwnerId");
+
+ UNIT_ASSERT_VALUES_EQUAL(table.GetPartitionStats().size(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(table.GetPartitionsCount(), 0);
+ }
+
+ TSchemeClient schemeClient(env.GetDriver());
+
+ {
+ auto result = schemeClient.DescribePath("/Root/.sys/partition_stats").GetValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ auto entry = result.GetEntry();
+ UNIT_ASSERT_VALUES_EQUAL(entry.Name, "partition_stats");
+ UNIT_ASSERT_VALUES_EQUAL(entry.Type, ESchemeEntryType::Table);
+ }
+ {
+ auto result = schemeClient.ListDirectory("/Root/.sys/partition_stats").GetValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ auto entry = result.GetEntry();
+ UNIT_ASSERT_VALUES_EQUAL(entry.Name, "partition_stats");
+ UNIT_ASSERT_VALUES_EQUAL(entry.Type, ESchemeEntryType::Table);
+ }
+ }
+
+ Y_UNIT_TEST(SystemViewFailOps) {
+ TTestEnv env;
+
+ TTableClient client(env.GetDriver());
+ auto session = client.CreateSession().GetValueSync().GetSession();
+
+ {
+ auto desc = TTableBuilder()
+ .AddNullableColumn("Column1", EPrimitiveType::Uint64)
+ .SetPrimaryKeyColumn("Column1")
+ .Build();
+
+ auto result = session.CreateTable("/Root/.sys/partition_stats", std::move(desc)).GetValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR);
- result.GetIssues().PrintTo(Cerr);
- }
- {
- auto result = session.DropTable("/Root/.sys/partition_stats").GetValueSync();
+ result.GetIssues().PrintTo(Cerr);
+ }
+ {
+ auto result = session.CopyTable("/Root/.sys/partition_stats", "/Root/Table0").GetValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR);
- result.GetIssues().PrintTo(Cerr);
- }
- {
- auto result = session.ExecuteSchemeQuery(R"(
- DROP TABLE `/Root/.sys/partition_stats`;
- )").GetValueSync();
+ result.GetIssues().PrintTo(Cerr);
+ }
+ {
+ auto settings = TAlterTableSettings()
+ .AppendDropColumns("OwnerId");
+
+ auto result = session.AlterTable("/Root/.sys/partition_stats", settings).GetValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR);
- result.GetIssues().PrintTo(Cerr);
- }
- {
- auto result = session.ReadTable("/Root/.sys/partition_stats").GetValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
-
- TReadTableResultPart streamPart = result.ReadNext().GetValueSync();
- UNIT_ASSERT_VALUES_EQUAL(streamPart.GetStatus(), EStatus::SCHEME_ERROR);
- streamPart.GetIssues().PrintTo(Cerr);
- }
- {
- TValueBuilder rows;
- rows.BeginList().EndList();
- auto result = client.BulkUpsert("/Root/.sys/partition_stats", rows.Build()).GetValueSync();
+ result.GetIssues().PrintTo(Cerr);
+ }
+ {
+ auto result = session.DropTable("/Root/.sys/partition_stats").GetValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR);
- result.GetIssues().PrintTo(Cerr);
- }
-
- TSchemeClient schemeClient(env.GetDriver());
-
- {
- auto result = schemeClient.MakeDirectory("/Root/.sys").GetValueSync();
+ result.GetIssues().PrintTo(Cerr);
+ }
+ {
+ auto result = session.ExecuteSchemeQuery(R"(
+ DROP TABLE `/Root/.sys/partition_stats`;
+ )").GetValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR);
- result.GetIssues().PrintTo(Cerr);
- }
- {
- auto result = schemeClient.MakeDirectory("/Root/.sys/partition_stats").GetValueSync();
+ result.GetIssues().PrintTo(Cerr);
+ }
+ {
+ auto result = session.ReadTable("/Root/.sys/partition_stats").GetValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+
+ TReadTableResultPart streamPart = result.ReadNext().GetValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(streamPart.GetStatus(), EStatus::SCHEME_ERROR);
+ streamPart.GetIssues().PrintTo(Cerr);
+ }
+ {
+ TValueBuilder rows;
+ rows.BeginList().EndList();
+ auto result = client.BulkUpsert("/Root/.sys/partition_stats", rows.Build()).GetValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR);
+ result.GetIssues().PrintTo(Cerr);
+ }
+
+ TSchemeClient schemeClient(env.GetDriver());
+
+ {
+ auto result = schemeClient.MakeDirectory("/Root/.sys").GetValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR);
+ result.GetIssues().PrintTo(Cerr);
+ }
+ {
+ auto result = schemeClient.MakeDirectory("/Root/.sys/partition_stats").GetValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR);
- result.GetIssues().PrintTo(Cerr);
- }
- {
- auto result = schemeClient.RemoveDirectory("/Root/.sys").GetValueSync();
+ result.GetIssues().PrintTo(Cerr);
+ }
+ {
+ auto result = schemeClient.RemoveDirectory("/Root/.sys").GetValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR);
- result.GetIssues().PrintTo(Cerr);
- }
- {
- auto result = schemeClient.RemoveDirectory("/Root/.sys/partition_stats").GetValueSync();
+ result.GetIssues().PrintTo(Cerr);
+ }
+ {
+ auto result = schemeClient.RemoveDirectory("/Root/.sys/partition_stats").GetValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR);
- result.GetIssues().PrintTo(Cerr);
- }
- {
- TModifyPermissionsSettings settings;
- auto result = schemeClient.ModifyPermissions("/Root/.sys/partition_stats", settings).GetValueSync();
+ result.GetIssues().PrintTo(Cerr);
+ }
+ {
+ TModifyPermissionsSettings settings;
+ auto result = schemeClient.ModifyPermissions("/Root/.sys/partition_stats", settings).GetValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR);
- result.GetIssues().PrintTo(Cerr);
- }
- }
-
- Y_UNIT_TEST(DescribeSystemFolder) {
- TTestEnv env;
+ result.GetIssues().PrintTo(Cerr);
+ }
+ }
+
+ Y_UNIT_TEST(DescribeSystemFolder) {
+ TTestEnv env;
CreateTenantsAndTables(env, false);
-
- TSchemeClient schemeClient(env.GetDriver());
- {
- auto result = schemeClient.ListDirectory("/Root").GetValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- auto entry = result.GetEntry();
- UNIT_ASSERT_STRINGS_EQUAL(entry.Name, "Root");
- UNIT_ASSERT_VALUES_EQUAL(entry.Type, ESchemeEntryType::Directory);
-
- auto children = result.GetChildren();
- UNIT_ASSERT_VALUES_EQUAL(children.size(), 4);
- UNIT_ASSERT_STRINGS_EQUAL(children[0].Name, "Table0");
- UNIT_ASSERT_STRINGS_EQUAL(children[1].Name, "Tenant1");
- UNIT_ASSERT_STRINGS_EQUAL(children[2].Name, "Tenant2");
- UNIT_ASSERT_STRINGS_EQUAL(children[3].Name, ".sys");
- }
- {
- auto result = schemeClient.ListDirectory("/Root/Tenant1").GetValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- auto entry = result.GetEntry();
+
+ TSchemeClient schemeClient(env.GetDriver());
+ {
+ auto result = schemeClient.ListDirectory("/Root").GetValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ auto entry = result.GetEntry();
+ UNIT_ASSERT_STRINGS_EQUAL(entry.Name, "Root");
+ UNIT_ASSERT_VALUES_EQUAL(entry.Type, ESchemeEntryType::Directory);
+
+ auto children = result.GetChildren();
+ UNIT_ASSERT_VALUES_EQUAL(children.size(), 4);
+ UNIT_ASSERT_STRINGS_EQUAL(children[0].Name, "Table0");
+ UNIT_ASSERT_STRINGS_EQUAL(children[1].Name, "Tenant1");
+ UNIT_ASSERT_STRINGS_EQUAL(children[2].Name, "Tenant2");
+ UNIT_ASSERT_STRINGS_EQUAL(children[3].Name, ".sys");
+ }
+ {
+ auto result = schemeClient.ListDirectory("/Root/Tenant1").GetValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ auto entry = result.GetEntry();
UNIT_ASSERT_VALUES_EQUAL(entry.Name, "Tenant1");
- UNIT_ASSERT_VALUES_EQUAL(entry.Type, ESchemeEntryType::SubDomain);
-
- auto children = result.GetChildren();
- UNIT_ASSERT_VALUES_EQUAL(children.size(), 2);
- UNIT_ASSERT_STRINGS_EQUAL(children[0].Name, "Table1");
- UNIT_ASSERT_STRINGS_EQUAL(children[1].Name, ".sys");
- }
- {
- auto result = schemeClient.ListDirectory("/Root/.sys").GetValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- auto entry = result.GetEntry();
- UNIT_ASSERT_VALUES_EQUAL(entry.Name, ".sys");
- UNIT_ASSERT_VALUES_EQUAL(entry.Type, ESchemeEntryType::Directory);
-
- auto children = result.GetChildren();
+ UNIT_ASSERT_VALUES_EQUAL(entry.Type, ESchemeEntryType::SubDomain);
+
+ auto children = result.GetChildren();
+ UNIT_ASSERT_VALUES_EQUAL(children.size(), 2);
+ UNIT_ASSERT_STRINGS_EQUAL(children[0].Name, "Table1");
+ UNIT_ASSERT_STRINGS_EQUAL(children[1].Name, ".sys");
+ }
+ {
+ auto result = schemeClient.ListDirectory("/Root/.sys").GetValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ auto entry = result.GetEntry();
+ UNIT_ASSERT_VALUES_EQUAL(entry.Name, ".sys");
+ UNIT_ASSERT_VALUES_EQUAL(entry.Type, ESchemeEntryType::Directory);
+
+ auto children = result.GetChildren();
UNIT_ASSERT_VALUES_EQUAL(children.size(), 16);
-
- THashSet<TString> names;
- for (const auto& child : children) {
- names.insert(child.Name);
- UNIT_ASSERT_VALUES_EQUAL(child.Type, ESchemeEntryType::Table);
- }
- UNIT_ASSERT(names.contains("partition_stats"));
- }
- {
- auto result = schemeClient.ListDirectory("/Root/Tenant1/.sys").GetValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- auto entry = result.GetEntry();
- UNIT_ASSERT_VALUES_EQUAL(entry.Name, ".sys");
- UNIT_ASSERT_VALUES_EQUAL(entry.Type, ESchemeEntryType::Directory);
-
- auto children = result.GetChildren();
- UNIT_ASSERT_VALUES_EQUAL(children.size(), 10);
-
- THashSet<TString> names;
- for (const auto& child : children) {
- names.insert(child.Name);
- UNIT_ASSERT_VALUES_EQUAL(child.Type, ESchemeEntryType::Table);
- }
- UNIT_ASSERT(names.contains("partition_stats"));
- }
- {
- auto result = schemeClient.ListDirectory("/Root/Tenant1/Table1/.sys").GetValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR);
- result.GetIssues().PrintTo(Cerr);
- }
- }
-
- Y_UNIT_TEST(DescribeAccessDenied) {
- TTestEnv env;
+
+ THashSet<TString> names;
+ for (const auto& child : children) {
+ names.insert(child.Name);
+ UNIT_ASSERT_VALUES_EQUAL(child.Type, ESchemeEntryType::Table);
+ }
+ UNIT_ASSERT(names.contains("partition_stats"));
+ }
+ {
+ auto result = schemeClient.ListDirectory("/Root/Tenant1/.sys").GetValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ auto entry = result.GetEntry();
+ UNIT_ASSERT_VALUES_EQUAL(entry.Name, ".sys");
+ UNIT_ASSERT_VALUES_EQUAL(entry.Type, ESchemeEntryType::Directory);
+
+ auto children = result.GetChildren();
+ UNIT_ASSERT_VALUES_EQUAL(children.size(), 10);
+
+ THashSet<TString> names;
+ for (const auto& child : children) {
+ names.insert(child.Name);
+ UNIT_ASSERT_VALUES_EQUAL(child.Type, ESchemeEntryType::Table);
+ }
+ UNIT_ASSERT(names.contains("partition_stats"));
+ }
+ {
+ auto result = schemeClient.ListDirectory("/Root/Tenant1/Table1/.sys").GetValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR);
+ result.GetIssues().PrintTo(Cerr);
+ }
+ }
+
+ Y_UNIT_TEST(DescribeAccessDenied) {
+ TTestEnv env;
CreateTenantsAndTables(env, false);
-
- auto driverConfig = TDriverConfig()
- .SetEndpoint(env.GetEndpoint())
- .SetAuthToken("user0@builtin");
- auto driver = TDriver(driverConfig);
-
- TSchemeClient schemeClient(driver);
- {
- auto result = schemeClient.ListDirectory("/Root").GetValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNAUTHORIZED);
- result.GetIssues().PrintTo(Cerr);
- }
- {
- auto result = schemeClient.ListDirectory("/Root/Tenant1").GetValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNAUTHORIZED);
- result.GetIssues().PrintTo(Cerr);
- }
- {
- auto result = schemeClient.ListDirectory("/Root/.sys").GetValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNAUTHORIZED);
- result.GetIssues().PrintTo(Cerr);
- }
- {
- auto result = schemeClient.ListDirectory("/Root/Tenant1/.sys").GetValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNAUTHORIZED);
- result.GetIssues().PrintTo(Cerr);
- }
- {
- auto result = schemeClient.DescribePath("/Root/.sys/partition_stats").GetValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNAUTHORIZED);
- result.GetIssues().PrintTo(Cerr);
- }
- {
- auto result = schemeClient.DescribePath("/Root/Tenant1/.sys/partition_stats").GetValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNAUTHORIZED);
- result.GetIssues().PrintTo(Cerr);
- }
- }
-
- Y_UNIT_TEST(TabletsFields) {
- TTestEnv env(1, 0);
- CreateRootTable(env);
-
- TTableClient client(env.GetDriver());
- auto it = client.StreamExecuteScanQuery(R"(
- SELECT
- BootState,
- CPU,
- Generation,
- Memory,
- Network,
- NodeId,
+
+ auto driverConfig = TDriverConfig()
+ .SetEndpoint(env.GetEndpoint())
+ .SetAuthToken("user0@builtin");
+ auto driver = TDriver(driverConfig);
+
+ TSchemeClient schemeClient(driver);
+ {
+ auto result = schemeClient.ListDirectory("/Root").GetValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNAUTHORIZED);
+ result.GetIssues().PrintTo(Cerr);
+ }
+ {
+ auto result = schemeClient.ListDirectory("/Root/Tenant1").GetValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNAUTHORIZED);
+ result.GetIssues().PrintTo(Cerr);
+ }
+ {
+ auto result = schemeClient.ListDirectory("/Root/.sys").GetValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNAUTHORIZED);
+ result.GetIssues().PrintTo(Cerr);
+ }
+ {
+ auto result = schemeClient.ListDirectory("/Root/Tenant1/.sys").GetValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNAUTHORIZED);
+ result.GetIssues().PrintTo(Cerr);
+ }
+ {
+ auto result = schemeClient.DescribePath("/Root/.sys/partition_stats").GetValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNAUTHORIZED);
+ result.GetIssues().PrintTo(Cerr);
+ }
+ {
+ auto result = schemeClient.DescribePath("/Root/Tenant1/.sys/partition_stats").GetValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNAUTHORIZED);
+ result.GetIssues().PrintTo(Cerr);
+ }
+ }
+
+ Y_UNIT_TEST(TabletsFields) {
+ TTestEnv env(1, 0);
+ CreateRootTable(env);
+
+ TTableClient client(env.GetDriver());
+ auto it = client.StreamExecuteScanQuery(R"(
+ SELECT
+ BootState,
+ CPU,
+ Generation,
+ Memory,
+ Network,
+ NodeId,
FollowerId,
- State,
- TabletId,
- Type,
- VolatileState
- FROM `/Root/.sys/hive_tablets`;
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- auto ysonString = NKqp::StreamResultToYson(it);
-
- TYsonFieldChecker check(ysonString, 11);
-
+ State,
+ TabletId,
+ Type,
+ VolatileState
+ FROM `/Root/.sys/hive_tablets`;
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ auto ysonString = NKqp::StreamResultToYson(it);
+
+ TYsonFieldChecker check(ysonString, 11);
+
check.String("Running"); // BootState
- check.DoubleGreaterOrEquals(0.0); // CPU
- check.Uint64(1u); // Generation
- check.Uint64GreaterOrEquals(0u); // Memory
- check.Uint64(0u); // Network
- check.Uint64(env.GetServer().GetRuntime()->GetNodeId(0)); // NodeId
+ check.DoubleGreaterOrEquals(0.0); // CPU
+ check.Uint64(1u); // Generation
+ check.Uint64GreaterOrEquals(0u); // Memory
+ check.Uint64(0u); // Network
+ check.Uint64(env.GetServer().GetRuntime()->GetNodeId(0)); // NodeId
check.Uint64(0u); // FollowerId
- check.String("ReadyToWork"); // State
- check.Uint64(72075186224037888ul); // TabletId
- check.String("DataShard"); // Type
- check.String("Running"); // VolatileState
- }
-
- Y_UNIT_TEST(TabletsShards) {
- TTestEnv env(1, 0);
- CreateRootTable(env, 3);
-
- TTableClient client(env.GetDriver());
- auto it = client.StreamExecuteScanQuery(R"(
+ check.String("ReadyToWork"); // State
+ check.Uint64(72075186224037888ul); // TabletId
+ check.String("DataShard"); // Type
+ check.String("Running"); // VolatileState
+ }
+
+ Y_UNIT_TEST(TabletsShards) {
+ TTestEnv env(1, 0);
+ CreateRootTable(env, 3);
+
+ TTableClient client(env.GetDriver());
+ auto it = client.StreamExecuteScanQuery(R"(
SELECT FollowerId, TabletId, Type
- FROM `/Root/.sys/hive_tablets`;
- )").GetValueSync();
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- NKqp::CompareYson(R"([
- [[0u];[72075186224037888u];["DataShard"]];
- [[0u];[72075186224037889u];["DataShard"]];
- [[0u];[72075186224037890u];["DataShard"]];
- ])", NKqp::StreamResultToYson(it));
- }
-
+ FROM `/Root/.sys/hive_tablets`;
+ )").GetValueSync();
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ NKqp::CompareYson(R"([
+ [[0u];[72075186224037888u];["DataShard"]];
+ [[0u];[72075186224037889u];["DataShard"]];
+ [[0u];[72075186224037890u];["DataShard"]];
+ ])", NKqp::StreamResultToYson(it));
+ }
+
Y_UNIT_TEST(TabletsFollowers) {
- TTestEnv env(1, 0);
-
- TTableClient client(env.GetDriver());
- auto session = client.CreateSession().GetValueSync().GetSession();
-
- auto desc = TTableBuilder()
- .AddNullableColumn("Column1", EPrimitiveType::Uint64)
- .SetPrimaryKeyColumn("Column1")
- .Build();
-
- auto settings = TCreateTableSettings()
- .ReplicationPolicy(TReplicationPolicy().ReplicasCount(3));
-
- auto result = session.CreateTable("/Root/Table0",
- std::move(desc), settings).GetValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- auto it = client.StreamExecuteScanQuery(R"(
+ TTestEnv env(1, 0);
+
+ TTableClient client(env.GetDriver());
+ auto session = client.CreateSession().GetValueSync().GetSession();
+
+ auto desc = TTableBuilder()
+ .AddNullableColumn("Column1", EPrimitiveType::Uint64)
+ .SetPrimaryKeyColumn("Column1")
+ .Build();
+
+ auto settings = TCreateTableSettings()
+ .ReplicationPolicy(TReplicationPolicy().ReplicasCount(3));
+
+ auto result = session.CreateTable("/Root/Table0",
+ std::move(desc), settings).GetValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ auto it = client.StreamExecuteScanQuery(R"(
SELECT FollowerId, TabletId, Type
- FROM `/Root/.sys/hive_tablets`;
- )").GetValueSync();
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- NKqp::CompareYson(R"([
- [[0u];[72075186224037888u];["DataShard"]];
- [[1u];[72075186224037888u];["DataShard"]];
- [[2u];[72075186224037888u];["DataShard"]];
- [[3u];[72075186224037888u];["DataShard"]];
- ])", NKqp::StreamResultToYson(it));
- }
-
- Y_UNIT_TEST(TabletsRanges) {
- TTestEnv env(1, 0);
-
- TTableClient client(env.GetDriver());
- auto session = client.CreateSession().GetValueSync().GetSession();
-
- auto desc = TTableBuilder()
- .AddNullableColumn("Column1", EPrimitiveType::Uint64)
- .SetPrimaryKeyColumn("Column1")
- .Build();
-
- auto settings = TCreateTableSettings()
- .ReplicationPolicy(TReplicationPolicy().ReplicasCount(3))
- .PartitioningPolicy(TPartitioningPolicy().UniformPartitions(3));
-
- auto result = session.CreateTable("/Root/Table0",
- std::move(desc), settings).GetValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
+ FROM `/Root/.sys/hive_tablets`;
+ )").GetValueSync();
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ NKqp::CompareYson(R"([
+ [[0u];[72075186224037888u];["DataShard"]];
+ [[1u];[72075186224037888u];["DataShard"]];
+ [[2u];[72075186224037888u];["DataShard"]];
+ [[3u];[72075186224037888u];["DataShard"]];
+ ])", NKqp::StreamResultToYson(it));
+ }
+
+ Y_UNIT_TEST(TabletsRanges) {
+ TTestEnv env(1, 0);
+
+ TTableClient client(env.GetDriver());
+ auto session = client.CreateSession().GetValueSync().GetSession();
+
+ auto desc = TTableBuilder()
+ .AddNullableColumn("Column1", EPrimitiveType::Uint64)
+ .SetPrimaryKeyColumn("Column1")
+ .Build();
+
+ auto settings = TCreateTableSettings()
+ .ReplicationPolicy(TReplicationPolicy().ReplicasCount(3))
+ .PartitioningPolicy(TPartitioningPolicy().UniformPartitions(3));
+
+ auto result = session.CreateTable("/Root/Table0",
+ std::move(desc), settings).GetValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
std::vector<std::pair<TString, TString>> testData = {
{
"TabletId = 72075186224037888ul AND FollowerId > 1u",
@@ -1574,27 +1574,27 @@ Y_UNIT_TEST_SUITE(SystemView) {
for (auto& data: testData) {
TString query = R"(
SELECT FollowerId, TabletId
- FROM `/Root/.sys/hive_tablets`
+ FROM `/Root/.sys/hive_tablets`
WHERE <PREDICATE>;
)";
SubstGlobal(query, "<PREDICATE>", data.first);
auto it = client.StreamExecuteScanQuery(enablePredicateExtractor + query).GetValueSync();
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
auto streamed = NKqp::StreamResultToYson(it);
it = client.StreamExecuteScanQuery(query).GetValueSync();
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
auto expected = NKqp::StreamResultToYson(it);
// Compare two ways of execution
NKqp::CompareYson(expected, streamed);
// And check with expected result from test description
NKqp::CompareYson(data.second, streamed);
- }
- }
-
+ }
+ }
+
Y_UNIT_TEST(TabletsRangesPredicateExtractDisabled) {
TTestEnv env(1, 0);
@@ -1638,110 +1638,110 @@ Y_UNIT_TEST_SUITE(SystemView) {
NKqp::CompareYson(expected, NKqp::StreamResultToYson(it));
}
- void TestQueryType(
- std::function<void(const TTestEnv&, const TString&)> execQuery,
- const TString& type)
- {
- TTestEnv env(1, 0);
- CreateRootTable(env);
-
- TString query("SELECT * FROM `Root/Table0`");
- execQuery(env, query);
-
- TTableClient client(env.GetDriver());
- auto it = client.StreamExecuteScanQuery(R"(
- SELECT QueryText, Type
- FROM `Root/.sys/top_queries_by_read_bytes_one_minute`;
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- NKqp::CompareYson(
- Sprintf("[[[\"%s\"];[\"%s\"]]]", query.c_str(), type.c_str()),
- NKqp::StreamResultToYson(it));
- }
-
- Y_UNIT_TEST(CollectPreparedQueries) {
- TestQueryType([](const TTestEnv& env, const TString& query) {
- TTableClient client(env.GetDriver());
- auto session = client.CreateSession().GetValueSync().GetSession();
- auto prepareResult = session.PrepareDataQuery(query).GetValueSync();
- UNIT_ASSERT_C(prepareResult.IsSuccess(), prepareResult.GetIssues().ToString());
- auto prepared = prepareResult.GetQuery();
- auto result = prepared.Execute(TTxControl::BeginTx().CommitTx()).GetValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
- }, "data");
- }
-
- Y_UNIT_TEST(CollectScanQueries) {
- TestQueryType([](const TTestEnv& env, const TString& query) {
- TTableClient client(env.GetDriver());
- auto it = client.StreamExecuteScanQuery(query).GetValueSync();
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- NKqp::StreamResultToYson(it);
- }, "scan");
- }
-
- Y_UNIT_TEST(CollectScriptingQueries) {
- TestQueryType([](const TTestEnv& env, const TString& query) {
- auto scriptingClient = NYdb::NScripting::TScriptingClient(env.GetDriver());
- auto result = scriptingClient.ExecuteYqlScript(query).GetValueSync();
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
- }, "script");
- }
-
- // TODO: make a test when tenant support is provided
- void QueryMetricsSimple() {
- TTestEnv env(1, 2);
- CreateTenant(env, "Tenant1", true);
- {
- TTableClient client(env.GetDriver());
- auto session = client.CreateSession().GetValueSync().GetSession();
-
- NKqp::AssertSuccessResult(session.ExecuteSchemeQuery(R"(
- CREATE TABLE `Root/Tenant1/Table1` (
- Key Uint64,
- Value String,
- PRIMARY KEY (Key)
- );
- )").GetValueSync());
- }
-
- auto driverConfig = TDriverConfig()
- .SetEndpoint(env.GetEndpoint())
- .SetDatabase("/Root/Tenant1");
- auto driver = TDriver(driverConfig);
-
- TTableClient client(driver);
- auto session = client.CreateSession().GetValueSync().GetSession();
- NKqp::AssertSuccessResult(session.ExecuteDataQuery(
- "SELECT * FROM `Root/Tenant1/Table1`", TTxControl::BeginTx().CommitTx()
- ).GetValueSync());
-
- size_t rowCount = 0;
- TString ysonString;
-
- for (size_t iter = 0; iter < 30 && !rowCount; ++iter) {
- auto it = client.StreamExecuteScanQuery(R"(
- SELECT SumReadBytes FROM `Root/Tenant1/.sys/query_metrics`;
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- ysonString = NKqp::StreamResultToYson(it);
-
+ void TestQueryType(
+ std::function<void(const TTestEnv&, const TString&)> execQuery,
+ const TString& type)
+ {
+ TTestEnv env(1, 0);
+ CreateRootTable(env);
+
+ TString query("SELECT * FROM `Root/Table0`");
+ execQuery(env, query);
+
+ TTableClient client(env.GetDriver());
+ auto it = client.StreamExecuteScanQuery(R"(
+ SELECT QueryText, Type
+ FROM `Root/.sys/top_queries_by_read_bytes_one_minute`;
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ NKqp::CompareYson(
+ Sprintf("[[[\"%s\"];[\"%s\"]]]", query.c_str(), type.c_str()),
+ NKqp::StreamResultToYson(it));
+ }
+
+ Y_UNIT_TEST(CollectPreparedQueries) {
+ TestQueryType([](const TTestEnv& env, const TString& query) {
+ TTableClient client(env.GetDriver());
+ auto session = client.CreateSession().GetValueSync().GetSession();
+ auto prepareResult = session.PrepareDataQuery(query).GetValueSync();
+ UNIT_ASSERT_C(prepareResult.IsSuccess(), prepareResult.GetIssues().ToString());
+ auto prepared = prepareResult.GetQuery();
+ auto result = prepared.Execute(TTxControl::BeginTx().CommitTx()).GetValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ }, "data");
+ }
+
+ Y_UNIT_TEST(CollectScanQueries) {
+ TestQueryType([](const TTestEnv& env, const TString& query) {
+ TTableClient client(env.GetDriver());
+ auto it = client.StreamExecuteScanQuery(query).GetValueSync();
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ NKqp::StreamResultToYson(it);
+ }, "scan");
+ }
+
+ Y_UNIT_TEST(CollectScriptingQueries) {
+ TestQueryType([](const TTestEnv& env, const TString& query) {
+ auto scriptingClient = NYdb::NScripting::TScriptingClient(env.GetDriver());
+ auto result = scriptingClient.ExecuteYqlScript(query).GetValueSync();
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ }, "script");
+ }
+
+ // TODO: make a test when tenant support is provided
+ void QueryMetricsSimple() {
+ TTestEnv env(1, 2);
+ CreateTenant(env, "Tenant1", true);
+ {
+ TTableClient client(env.GetDriver());
+ auto session = client.CreateSession().GetValueSync().GetSession();
+
+ NKqp::AssertSuccessResult(session.ExecuteSchemeQuery(R"(
+ CREATE TABLE `Root/Tenant1/Table1` (
+ Key Uint64,
+ Value String,
+ PRIMARY KEY (Key)
+ );
+ )").GetValueSync());
+ }
+
+ auto driverConfig = TDriverConfig()
+ .SetEndpoint(env.GetEndpoint())
+ .SetDatabase("/Root/Tenant1");
+ auto driver = TDriver(driverConfig);
+
+ TTableClient client(driver);
+ auto session = client.CreateSession().GetValueSync().GetSession();
+ NKqp::AssertSuccessResult(session.ExecuteDataQuery(
+ "SELECT * FROM `Root/Tenant1/Table1`", TTxControl::BeginTx().CommitTx()
+ ).GetValueSync());
+
+ size_t rowCount = 0;
+ TString ysonString;
+
+ for (size_t iter = 0; iter < 30 && !rowCount; ++iter) {
+ auto it = client.StreamExecuteScanQuery(R"(
+ SELECT SumReadBytes FROM `Root/Tenant1/.sys/query_metrics`;
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ ysonString = NKqp::StreamResultToYson(it);
+
auto node = NYT::NodeFromYsonString(ysonString, ::NYson::EYsonType::Node);
- UNIT_ASSERT(node.IsList());
- rowCount = node.AsList().size();
-
- if (!rowCount) {
- Sleep(TDuration::Seconds(1));
- }
- }
-
- NKqp::CompareYson(R"([
- [[0u]];
- ])", ysonString);
- }
-}
-
-} // NSysView
-} // NKikimr
+ UNIT_ASSERT(node.IsList());
+ rowCount = node.AsList().size();
+
+ if (!rowCount) {
+ Sleep(TDuration::Seconds(1));
+ }
+ }
+
+ NKqp::CompareYson(R"([
+ [[0u]];
+ ])", ysonString);
+ }
+}
+
+} // NSysView
+} // NKikimr
diff --git a/ydb/core/sys_view/ut_kqp/ya.make b/ydb/core/sys_view/ut_kqp/ya.make
index 3b75f4e6feb..9c19748dea4 100644
--- a/ydb/core/sys_view/ut_kqp/ya.make
+++ b/ydb/core/sys_view/ut_kqp/ya.make
@@ -1,35 +1,35 @@
UNITTEST_FOR(ydb/core/sys_view)
-
-OWNER(
- monster
- g:kikimr
-)
-
-FORK_SUBTESTS()
-
-IF (WITH_VALGRIND)
- TIMEOUT(3600)
- SIZE(LARGE)
- TAG(ya:fat)
-ELSE()
- TIMEOUT(600)
- SIZE(MEDIUM)
-ENDIF()
-
-PEERDIR(
+
+OWNER(
+ monster
+ g:kikimr
+)
+
+FORK_SUBTESTS()
+
+IF (WITH_VALGRIND)
+ TIMEOUT(3600)
+ SIZE(LARGE)
+ TAG(ya:fat)
+ELSE()
+ TIMEOUT(600)
+ SIZE(MEDIUM)
+ENDIF()
+
+PEERDIR(
library/cpp/testing/unittest
library/cpp/yson/node
ydb/core/kqp/ut/common
ydb/core/testlib
ydb/public/sdk/cpp/client/draft
-)
-
+)
+
YQL_LAST_ABI_VERSION()
-SRCS(
- ut_kqp.cpp
- ut_common.cpp
- ut_counters.cpp
-)
-
-END()
+SRCS(
+ ut_kqp.cpp
+ ut_common.cpp
+ ut_counters.cpp
+)
+
+END()
diff --git a/ydb/core/sys_view/ya.make b/ydb/core/sys_view/ya.make
index edaa0470bf1..46b32d93351 100644
--- a/ydb/core/sys_view/ya.make
+++ b/ydb/core/sys_view/ya.make
@@ -1,16 +1,16 @@
-LIBRARY()
-
-OWNER(
- monster
- g:kikimr
-)
-
-SRCS(
- scan.h
- scan.cpp
-)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(
+ monster
+ g:kikimr
+)
+
+SRCS(
+ scan.h
+ scan.cpp
+)
+
+PEERDIR(
ydb/core/base
ydb/core/kqp/runtime
ydb/core/sys_view/common
@@ -20,8 +20,8 @@ PEERDIR(
ydb/core/sys_view/service
ydb/core/sys_view/storage
ydb/core/sys_view/tablets
-)
-
+)
+
YQL_LAST_ABI_VERSION()
END()
diff --git a/ydb/core/tablet/node_whiteboard.cpp b/ydb/core/tablet/node_whiteboard.cpp
index 783758e99d1..8b23eb7f71f 100644
--- a/ydb/core/tablet/node_whiteboard.cpp
+++ b/ydb/core/tablet/node_whiteboard.cpp
@@ -798,7 +798,7 @@ protected:
if (ProcessStats.CGroupMemLim != 0) {
systemStatsUpdate->Record.SetMemoryLimit(ProcessStats.CGroupMemLim);
}
- systemStatsUpdate->Record.SetMemoryUsedInAlloc(TAllocState::GetAllocatedMemoryEstimate());
+ systemStatsUpdate->Record.SetMemoryUsedInAlloc(TAllocState::GetAllocatedMemoryEstimate());
ctx.Send(ctx.SelfID, systemStatsUpdate.Release());
ctx.Schedule(TDuration::Seconds(30), new TEvPrivate::TEvUpdateRuntimeStats());
}
diff --git a/ydb/core/tablet/tablet_counters_aggregator.cpp b/ydb/core/tablet/tablet_counters_aggregator.cpp
index 1885b26e09c..af724e00809 100644
--- a/ydb/core/tablet/tablet_counters_aggregator.cpp
+++ b/ydb/core/tablet/tablet_counters_aggregator.cpp
@@ -1,5 +1,5 @@
#include "tablet_counters_aggregator.h"
-#include "tablet_counters_app.h"
+#include "tablet_counters_app.h"
#include <library/cpp/actors/core/log.h>
#include <ydb/core/mon/mon.h>
#include <library/cpp/actors/core/mon.h>
@@ -64,15 +64,15 @@ using TCountersVector = TVector<NMonitoring::TDynamicCounters::TCounterPtr>;
struct THistogramCounter {
TVector<TTabletPercentileCounter::TRangeDef> Ranges;
TVector<NMonitoring::TDynamicCounters::TCounterPtr> Values;
- NMonitoring::THistogramPtr Histogram;
+ NMonitoring::THistogramPtr Histogram;
- THistogramCounter(
- const TVector<TTabletPercentileCounter::TRangeDef>& ranges,
- TVector<NMonitoring::TDynamicCounters::TCounterPtr>&& values,
- NMonitoring::THistogramPtr histogram)
+ THistogramCounter(
+ const TVector<TTabletPercentileCounter::TRangeDef>& ranges,
+ TVector<NMonitoring::TDynamicCounters::TCounterPtr>&& values,
+ NMonitoring::THistogramPtr histogram)
: Ranges(ranges)
, Values(values)
- , Histogram(histogram)
+ , Histogram(histogram)
{
Y_VERIFY(!Ranges.empty() && Ranges.size() == Values.size());
}
@@ -81,15 +81,15 @@ struct THistogramCounter {
for (const NMonitoring::TDynamicCounters::TCounterPtr& cnt : Values) {
*cnt = 0;
}
-
- Histogram->Reset();
+
+ Histogram->Reset();
}
void IncrementFor(ui64 value) {
const size_t i = Max<ssize_t>(0, std::upper_bound(Ranges.begin(), Ranges.end(), value) - Ranges.begin() - 1);
Values[i]->Inc();
-
- Histogram->Collect(value);
+
+ Histogram->Collect(value);
}
};
@@ -119,33 +119,33 @@ public:
HistSimpleCounters.emplace_back(std::move(percentileAggregate));
}
- ui64 GetSum(ui32 counterIndex) const {
- Y_VERIFY(counterIndex < SumSimpleCounters.size(),
- "inconsistent sum simple counters, %u >= %lu", counterIndex, SumSimpleCounters.size());
- return *SumSimpleCounters[counterIndex];
- }
-
- void SetSum(ui32 counterIndex, ui64 value) {
- Y_VERIFY(counterIndex < SumSimpleCounters.size(),
- "inconsistent sum simple counters, %u >= %lu", counterIndex, SumSimpleCounters.size());
- *SumSimpleCounters[counterIndex] = value;
- }
-
- ui64 GetMax(ui32 counterIndex) const {
- Y_VERIFY(counterIndex < MaxSimpleCounters.size(),
- "inconsistent max simple counters, %u >= %lu", counterIndex, MaxSimpleCounters.size());
- return *MaxSimpleCounters[counterIndex];
- }
-
- void SetMax(ui32 counterIndex, ui64 value) {
- Y_VERIFY(counterIndex < MaxSimpleCounters.size(),
- "inconsistent max simple counters, %u >= %lu", counterIndex, MaxSimpleCounters.size());
- *MaxSimpleCounters[counterIndex] = value;
- }
-
+ ui64 GetSum(ui32 counterIndex) const {
+ Y_VERIFY(counterIndex < SumSimpleCounters.size(),
+ "inconsistent sum simple counters, %u >= %lu", counterIndex, SumSimpleCounters.size());
+ return *SumSimpleCounters[counterIndex];
+ }
+
+ void SetSum(ui32 counterIndex, ui64 value) {
+ Y_VERIFY(counterIndex < SumSimpleCounters.size(),
+ "inconsistent sum simple counters, %u >= %lu", counterIndex, SumSimpleCounters.size());
+ *SumSimpleCounters[counterIndex] = value;
+ }
+
+ ui64 GetMax(ui32 counterIndex) const {
+ Y_VERIFY(counterIndex < MaxSimpleCounters.size(),
+ "inconsistent max simple counters, %u >= %lu", counterIndex, MaxSimpleCounters.size());
+ return *MaxSimpleCounters[counterIndex];
+ }
+
+ void SetMax(ui32 counterIndex, ui64 value) {
+ Y_VERIFY(counterIndex < MaxSimpleCounters.size(),
+ "inconsistent max simple counters, %u >= %lu", counterIndex, MaxSimpleCounters.size());
+ *MaxSimpleCounters[counterIndex] = value;
+ }
+
void SetValue(ui64 tabletID, ui32 counterIndex, ui64 value, TTabletTypes::EType tabletType) {
- Y_VERIFY(counterIndex < CountersByTabletID.size(),
- "inconsistent counters for tablet type %s", TTabletTypes::TypeToStr(tabletType));
+ Y_VERIFY(counterIndex < CountersByTabletID.size(),
+ "inconsistent counters for tablet type %s", TTabletTypes::TypeToStr(tabletType));
auto it = CountersByTabletID[counterIndex].find(tabletID);
if (it != CountersByTabletID[counterIndex].end()) {
if (it->second != value) {
@@ -234,18 +234,18 @@ public:
HistCumulativeCounters.emplace_back(std::move(percentileAggregate));
}
- ui64 GetMax(ui32 counterIndex) const {
- Y_VERIFY(counterIndex < MaxCumulativeCounters.size(),
- "inconsistent max cumulative counters, %u >= %lu", counterIndex, MaxCumulativeCounters.size());
- return *MaxCumulativeCounters[counterIndex];
- }
-
- void SetMax(ui32 counterIndex, ui64 value) {
- Y_VERIFY(counterIndex < MaxCumulativeCounters.size(),
- "inconsistent max cumulative counters, %u >= %lu", counterIndex, MaxCumulativeCounters.size());
- *MaxCumulativeCounters[counterIndex] = value;
- }
-
+ ui64 GetMax(ui32 counterIndex) const {
+ Y_VERIFY(counterIndex < MaxCumulativeCounters.size(),
+ "inconsistent max cumulative counters, %u >= %lu", counterIndex, MaxCumulativeCounters.size());
+ return *MaxCumulativeCounters[counterIndex];
+ }
+
+ void SetMax(ui32 counterIndex, ui64 value) {
+ Y_VERIFY(counterIndex < MaxCumulativeCounters.size(),
+ "inconsistent max cumulative counters, %u >= %lu", counterIndex, MaxCumulativeCounters.size());
+ *MaxCumulativeCounters[counterIndex] = value;
+ }
+
void SetValue(ui64 tabletID, ui32 counterIndex, ui64 value, TTabletTypes::EType tabletType) {
Y_VERIFY(counterIndex < CountersByTabletID.size(), "inconsistent counters for tablet type %s", TTabletTypes::TypeToStr(tabletType));
auto it = CountersByTabletID[counterIndex].find(tabletID);
@@ -474,31 +474,31 @@ public:
: Counters(GetServiceCounters(counters, isFollower ? "followers" : "tablets"))
, AllTypes(Counters.Get(), "type", "all", true)
, IsFollower(isFollower)
- , DbWatcherActorId(dbWatcherActorId)
+ , DbWatcherActorId(dbWatcherActorId)
{
if (!IsFollower) {
- YdbCounters = MakeIntrusive<TYdbTabletCounters>(GetServiceCounters(counters, "ydb"));
- }
+ YdbCounters = MakeIntrusive<TYdbTabletCounters>(GetServiceCounters(counters, "ydb"));
+ }
}
- void Apply(ui64 tabletID, TTabletTypes::EType tabletType, TPathId tenantPathId,
- const TTabletCountersBase* executorCounters, const TTabletCountersBase* appCounters,
- const TActorContext& ctx)
- {
- AllTypes.Apply(tabletID, executorCounters, nullptr, tabletType);
+ void Apply(ui64 tabletID, TTabletTypes::EType tabletType, TPathId tenantPathId,
+ const TTabletCountersBase* executorCounters, const TTabletCountersBase* appCounters,
+ const TActorContext& ctx)
+ {
+ AllTypes.Apply(tabletID, executorCounters, nullptr, tabletType);
//
- auto* typeCounters = GetOrAddCountersByTabletType(tabletType, CountersByTabletType, Counters);
- if (typeCounters) {
- typeCounters->Apply(tabletID, executorCounters, appCounters, tabletType);
+ auto* typeCounters = GetOrAddCountersByTabletType(tabletType, CountersByTabletType, Counters);
+ if (typeCounters) {
+ typeCounters->Apply(tabletID, executorCounters, appCounters, tabletType);
}
//
if (!IsFollower && AppData(ctx)->FeatureFlags.GetEnableDbCounters() && tenantPathId) {
- auto dbCounters = GetDbCounters(tenantPathId, ctx);
- if (dbCounters) {
- auto* limitedAppCounters = GetOrAddLimitedAppCounters(tabletType);
- dbCounters->Apply(tabletID, executorCounters, appCounters, tabletType, limitedAppCounters);
- }
- }
+ auto dbCounters = GetDbCounters(tenantPathId, ctx);
+ if (dbCounters) {
+ auto* limitedAppCounters = GetOrAddLimitedAppCounters(tabletType);
+ dbCounters->Apply(tabletID, executorCounters, appCounters, tabletType, limitedAppCounters);
+ }
+ }
//
auto& quietStats = QuietTabletCounters[tabletID];
@@ -553,17 +553,17 @@ public:
}
}
- void ForgetTablet(ui64 tabletID, TTabletTypes::EType tabletType, TPathId tenantPathId) {
+ void ForgetTablet(ui64 tabletID, TTabletTypes::EType tabletType, TPathId tenantPathId) {
AllTypes.Forget(tabletID);
// and now erase from every other path
auto iterTabletType = CountersByTabletType.find(tabletType);
if (iterTabletType != CountersByTabletType.end()) {
iterTabletType->second->Forget(tabletID);
}
- // from db counters
- if (auto itPath = CountersByPathId.find(tenantPathId); itPath != CountersByPathId.end()) {
- itPath->second->Forget(tabletID, tabletType);
- }
+ // from db counters
+ if (auto itPath = CountersByPathId.find(tenantPathId); itPath != CountersByPathId.end()) {
+ itPath->second->Forget(tabletID, tabletType);
+ }
//and from all labeledCounters that could have this tablet
auto iterTabletTypeAndGroup = LabeledCountersByTabletTypeAndGroup.lower_bound(std::make_pair(tabletType, TString()));
for (; iterTabletTypeAndGroup != LabeledCountersByTabletTypeAndGroup.end() && iterTabletTypeAndGroup->first.first == tabletType; ) {
@@ -687,17 +687,17 @@ public:
for (auto& c : CountersByTabletType) {
c.second->RecalcAll();
}
-
- if (YdbCounters) {
- YdbCounters->Initialize(Counters, CountersByTabletType);
- YdbCounters->Transform();
- }
- }
-
- void RemoveTabletsByPathId(TPathId pathId) {
- CountersByPathId.erase(pathId);
+
+ if (YdbCounters) {
+ YdbCounters->Initialize(Counters, CountersByTabletType);
+ YdbCounters->Transform();
+ }
}
+ void RemoveTabletsByPathId(TPathId pathId) {
+ CountersByPathId.erase(pathId);
+ }
+
private:
// subgroups
class TTabletCountersForTabletType {
@@ -711,12 +711,12 @@ private:
, TabletAppCounters(TabletAppCountersSection, doAggregateSimpleCountrers)
{}
- void Apply(ui64 tabletID,
- const TTabletCountersBase* executorCounters,
- const TTabletCountersBase* appCounters,
- TTabletTypes::EType tabletType,
- const TTabletCountersBase* limitedAppCounters = {})
- {
+ void Apply(ui64 tabletID,
+ const TTabletCountersBase* executorCounters,
+ const TTabletCountersBase* appCounters,
+ TTabletTypes::EType tabletType,
+ const TTabletCountersBase* limitedAppCounters = {})
+ {
Y_VERIFY(executorCounters);
if (executorCounters) {
@@ -728,7 +728,7 @@ private:
if (appCounters) {
if (!TabletAppCounters.IsInitialized) {
- TabletAppCounters.Initialize(limitedAppCounters ? limitedAppCounters : appCounters);
+ TabletAppCounters.Initialize(limitedAppCounters ? limitedAppCounters : appCounters);
}
TabletAppCounters.Apply(tabletID, appCounters, tabletType);
}
@@ -752,48 +752,48 @@ private:
}
}
- // db counters
-
- bool IsInitialized() const {
- return TabletExecutorCounters.IsInitialized;
- }
-
- void Initialize(const TTabletCountersBase* executorCounters, const TTabletCountersBase* appCounters) {
- Y_VERIFY(executorCounters);
-
- if (!TabletExecutorCounters.IsInitialized) {
- TabletExecutorCounters.Initialize(executorCounters);
- }
-
- if (appCounters && !TabletAppCounters.IsInitialized) {
- TabletAppCounters.Initialize(appCounters);
- }
- }
-
- void ToProto(NKikimrSysView::TDbTabletCounters& tabletCounters) {
- if (TabletExecutorCounters.IsInitialized) {
- TabletExecutorCounters.RecalcAll();
- TabletExecutorCounters.ToProto(*tabletCounters.MutableExecutorCounters(),
- *tabletCounters.MutableMaxExecutorCounters());
- }
- if (TabletAppCounters.IsInitialized) {
- TabletAppCounters.RecalcAll();
- TabletAppCounters.ToProto(*tabletCounters.MutableAppCounters(),
- *tabletCounters.MutableMaxAppCounters());
- }
- }
-
- void FromProto(NKikimrSysView::TDbTabletCounters& tabletCounters) {
- if (TabletExecutorCounters.IsInitialized) {
- TabletExecutorCounters.FromProto(*tabletCounters.MutableExecutorCounters(),
- *tabletCounters.MutableMaxExecutorCounters());
- }
- if (TabletAppCounters.IsInitialized) {
- TabletAppCounters.FromProto(*tabletCounters.MutableAppCounters(),
- *tabletCounters.MutableMaxAppCounters());
- }
- }
-
+ // db counters
+
+ bool IsInitialized() const {
+ return TabletExecutorCounters.IsInitialized;
+ }
+
+ void Initialize(const TTabletCountersBase* executorCounters, const TTabletCountersBase* appCounters) {
+ Y_VERIFY(executorCounters);
+
+ if (!TabletExecutorCounters.IsInitialized) {
+ TabletExecutorCounters.Initialize(executorCounters);
+ }
+
+ if (appCounters && !TabletAppCounters.IsInitialized) {
+ TabletAppCounters.Initialize(appCounters);
+ }
+ }
+
+ void ToProto(NKikimrSysView::TDbTabletCounters& tabletCounters) {
+ if (TabletExecutorCounters.IsInitialized) {
+ TabletExecutorCounters.RecalcAll();
+ TabletExecutorCounters.ToProto(*tabletCounters.MutableExecutorCounters(),
+ *tabletCounters.MutableMaxExecutorCounters());
+ }
+ if (TabletAppCounters.IsInitialized) {
+ TabletAppCounters.RecalcAll();
+ TabletAppCounters.ToProto(*tabletCounters.MutableAppCounters(),
+ *tabletCounters.MutableMaxAppCounters());
+ }
+ }
+
+ void FromProto(NKikimrSysView::TDbTabletCounters& tabletCounters) {
+ if (TabletExecutorCounters.IsInitialized) {
+ TabletExecutorCounters.FromProto(*tabletCounters.MutableExecutorCounters(),
+ *tabletCounters.MutableMaxExecutorCounters());
+ }
+ if (TabletAppCounters.IsInitialized) {
+ TabletAppCounters.FromProto(*tabletCounters.MutableAppCounters(),
+ *tabletCounters.MutableMaxAppCounters());
+ }
+ }
+
private:
//
class TSolomonCounters {
@@ -817,14 +817,14 @@ private:
THashMap<TString, THolder<THistogramCounter>> histogramAggregates;
// percentile counters
- FullSizePercentile = counters->Percentile().Size();
- for (ui32 i = 0; i < FullSizePercentile; ++i) {
+ FullSizePercentile = counters->Percentile().Size();
+ for (ui32 i = 0; i < FullSizePercentile; ++i) {
if (!counters->PercentileCounterName(i)) {
- DeprecatedPercentile.insert(i);
+ DeprecatedPercentile.insert(i);
continue;
}
- // old style
+ // old style
PercentileCounters.push_back(TVector<NMonitoring::TDynamicCounters::TCounterPtr>());
auto counterRBeginIter = PercentileCounters.rbegin();
@@ -833,44 +833,44 @@ private:
TStringBuf counterName(percentileCounterName);
TStringBuf simpleCounterName = GetHistogramAggregateSimpleName(counterName);
bool histogramAggregate = !simpleCounterName.empty();
-
- bool isDerivative = !histogramAggregate && !percentileCounter.GetIntegral();
-
- auto rangeCount = percentileCounter.GetRangeCount();
- for (ui32 r = 0; r < rangeCount; ++r) {
+
+ bool isDerivative = !histogramAggregate && !percentileCounter.GetIntegral();
+
+ auto rangeCount = percentileCounter.GetRangeCount();
+ for (ui32 r = 0; r < rangeCount; ++r) {
const char* rangeName = percentileCounter.GetRangeName(r);
auto subgroup = CounterGroup->GetSubgroup("range", rangeName);
- auto counter = subgroup->GetCounter(percentileCounterName, isDerivative);
+ auto counter = subgroup->GetCounter(percentileCounterName, isDerivative);
counterRBeginIter->push_back(counter);
}
-
- // new style
- NMonitoring::TBucketBounds bucketBounds;
- for (ui32 r = 1; r < rangeCount; ++r) { // values in proto are lower bounds, thus shift
- bucketBounds.push_back(percentileCounter.GetRangeBound(r));
- }
- auto histogram = CounterGroup->GetHistogram(
- percentileCounterName, NMonitoring::ExplicitHistogram(bucketBounds), isDerivative);
- Histograms.push_back(histogram);
-
+
+ // new style
+ NMonitoring::TBucketBounds bucketBounds;
+ for (ui32 r = 1; r < rangeCount; ++r) { // values in proto are lower bounds, thus shift
+ bucketBounds.push_back(percentileCounter.GetRangeBound(r));
+ }
+ auto histogram = CounterGroup->GetHistogram(
+ percentileCounterName, NMonitoring::ExplicitHistogram(bucketBounds), isDerivative);
+ Histograms.push_back(histogram);
+
if (histogramAggregate) {
- histogramAggregates.emplace(simpleCounterName, new THistogramCounter(
- percentileCounter.GetRanges(), std::move(*counterRBeginIter), histogram));
+ histogramAggregates.emplace(simpleCounterName, new THistogramCounter(
+ percentileCounter.GetRanges(), std::move(*counterRBeginIter), histogram));
}
}
// simple counters
- FullSizeSimple = counters->Simple().Size();
- for (ui32 i = 0; i < FullSizeSimple; ++i) {
+ FullSizeSimple = counters->Simple().Size();
+ for (ui32 i = 0; i < FullSizeSimple; ++i) {
const char* name = counters->SimpleCounterName(i);
if (!name) {
- DeprecatedSimple.insert(i);
+ DeprecatedSimple.insert(i);
continue;
}
if (DoAggregateSimpleCounters) {
- auto itHistogramAggregate = histogramAggregates.find(name);
- if (itHistogramAggregate != histogramAggregates.end()) {
- AggregatedSimpleCounters.AddSimpleCounter(name, std::move(itHistogramAggregate->second));
+ auto itHistogramAggregate = histogramAggregates.find(name);
+ if (itHistogramAggregate != histogramAggregates.end()) {
+ AggregatedSimpleCounters.AddSimpleCounter(name, std::move(itHistogramAggregate->second));
} else {
AggregatedSimpleCounters.AddSimpleCounter(name);
}
@@ -881,17 +881,17 @@ private:
}
// cumulative counters
- FullSizeCumulative = counters->Cumulative().Size();
- for (ui32 i = 0; i < FullSizeCumulative; ++i) {
+ FullSizeCumulative = counters->Cumulative().Size();
+ for (ui32 i = 0; i < FullSizeCumulative; ++i) {
const char* name = counters->CumulativeCounterName(i);
if (!name) {
- DeprecatedCumulative.insert(i);
+ DeprecatedCumulative.insert(i);
continue;
}
if (DoAggregateCumulativeCounters) {
- auto itHistogramAggregate = histogramAggregates.find(name);
- if (itHistogramAggregate != histogramAggregates.end()) {
- AggregatedCumulativeCounters.AddCumulativeCounter(name, std::move(itHistogramAggregate->second));
+ auto itHistogramAggregate = histogramAggregates.find(name);
+ if (itHistogramAggregate != histogramAggregates.end()) {
+ AggregatedCumulativeCounters.AddCumulativeCounter(name, std::move(itHistogramAggregate->second));
} else {
AggregatedCumulativeCounters.AddCumulativeCounter(name);
}
@@ -920,7 +920,7 @@ private:
// simple counters
ui32 nextSimpleOffset = 0;
- for (ui32 i = 0; i < FullSizeSimple; ++i) {
+ for (ui32 i = 0; i < FullSizeSimple; ++i) {
if (!counters->SimpleCounterName(i)) {
continue;
}
@@ -936,7 +936,7 @@ private:
// cumulative counters
ui32 nextCumulativeOffset = 0;
- for (ui32 i = 0; i < FullSizeCumulative; ++i) {
+ for (ui32 i = 0; i < FullSizeCumulative; ++i) {
if (!counters->CumulativeCounterName(i)) {
continue;
}
@@ -954,7 +954,7 @@ private:
// percentile counters
ui32 nextPercentileOffset = 0;
- for (ui32 i = 0; i < FullSizePercentile; ++i) {
+ for (ui32 i = 0; i < FullSizePercentile; ++i) {
if (!counters->PercentileCounterName(i)) {
continue;
}
@@ -968,34 +968,34 @@ private:
}
auto&& percentileCounter = counters->Percentile()[i];
- auto rangeCount = percentileCounter.GetRangeCount();
- Y_VERIFY(rangeCount <= pcx.size(),
- "inconsistent counters for tablet type %s", TTabletTypes::TypeToStr(tabletType));
-
- for (ui32 r = 0; r < rangeCount; ++r) {
+ auto rangeCount = percentileCounter.GetRangeCount();
+ Y_VERIFY(rangeCount <= pcx.size(),
+ "inconsistent counters for tablet type %s", TTabletTypes::TypeToStr(tabletType));
+
+ for (ui32 r = 0; r < rangeCount; ++r) {
if (percentileCounter.GetIntegral()) {
*pcx[r] = percentileCounter.GetRangeValue(r);
} else {
*pcx[r] += percentileCounter.GetRangeValue(r);
}
}
-
- if (rangeCount < 2) {
- continue;
- }
-
- auto& histogram = Histograms[offset];
- if (percentileCounter.GetIntegral()) {
- histogram->Reset();
- }
- for (ui32 r = 0; r < rangeCount - 1; ++r) {
- histogram->Collect(
- (NMonitoring::TBucketBound)percentileCounter.GetRangeBound(r + 1),
- percentileCounter.GetRangeValue(r));
- }
- histogram->Collect(
- Max<NMonitoring::TBucketBound>(),
- percentileCounter.GetRangeValue(rangeCount - 1));
+
+ if (rangeCount < 2) {
+ continue;
+ }
+
+ auto& histogram = Histograms[offset];
+ if (percentileCounter.GetIntegral()) {
+ histogram->Reset();
+ }
+ for (ui32 r = 0; r < rangeCount - 1; ++r) {
+ histogram->Collect(
+ (NMonitoring::TBucketBound)percentileCounter.GetRangeBound(r + 1),
+ percentileCounter.GetRangeValue(r));
+ }
+ histogram->Collect(
+ Max<NMonitoring::TBucketBound>(),
+ percentileCounter.GetRangeValue(rangeCount - 1));
}
}
@@ -1017,120 +1017,120 @@ private:
}
void RecalcAll() {
- if (DoAggregateSimpleCounters) {
+ if (DoAggregateSimpleCounters) {
AggregatedSimpleCounters.RecalcAll();
}
- if (DoAggregateCumulativeCounters) {
+ if (DoAggregateCumulativeCounters) {
AggregatedCumulativeCounters.RecalcAll();
}
}
- template <bool IsSaving>
- void Convert(NKikimrSysView::TDbCounters& sumCounters,
- NKikimrSysView::TDbCounters& maxCounters)
- {
- if (!DoAggregateSimpleCounters || !DoAggregateCumulativeCounters) {
- return;
- }
- // simple counters
- auto* simpleSum = sumCounters.MutableSimple();
- auto* simpleMax = maxCounters.MutableSimple();
- simpleSum->Resize(FullSizeSimple, 0);
- simpleMax->Resize(FullSizeSimple, 0);
- ui32 nextSimpleOffset = 0;
- for (ui32 i = 0; i < FullSizeSimple; ++i) {
- if (DeprecatedSimple.find(i) != DeprecatedSimple.end()) {
- if constexpr (IsSaving) {
- (*simpleSum)[i] = 0;
- (*simpleMax)[i] = 0;
- }
- continue;
- }
- const ui32 offset = nextSimpleOffset++;
- if constexpr (IsSaving) {
- (*simpleSum)[i] = AggregatedSimpleCounters.GetSum(offset);
- (*simpleMax)[i] = AggregatedSimpleCounters.GetMax(offset);
- } else {
- AggregatedSimpleCounters.SetSum(offset, (*simpleSum)[i]);
- AggregatedSimpleCounters.SetMax(offset, (*simpleMax)[i]);
- }
- }
- // cumulative counters
- auto* cumulativeSum = sumCounters.MutableCumulative();
- auto* cumulativeMax = maxCounters.MutableCumulative();
- cumulativeSum->Resize(FullSizeCumulative, 0);
- cumulativeMax->Resize(FullSizeCumulative, 0);
- ui32 nextCumulativeOffset = 0;
- for (ui32 i = 0; i < FullSizeCumulative; ++i) {
- if (DeprecatedCumulative.find(i) != DeprecatedCumulative.end()) {
- if constexpr (IsSaving) {
- (*cumulativeSum)[i] = 0;
- (*cumulativeMax)[i] = 0;
- }
- continue;
- }
- const ui32 offset = nextCumulativeOffset++;
- Y_VERIFY(offset < CumulativeCounters.size(),
- "inconsistent cumulative counters %u >= %lu", offset, CumulativeCounters.size());
- if constexpr (IsSaving) {
- (*cumulativeSum)[i] = *CumulativeCounters[offset];
- (*cumulativeMax)[i] = AggregatedCumulativeCounters.GetMax(offset);
- } else {
- *CumulativeCounters[offset] = (*cumulativeSum)[i];
- AggregatedCumulativeCounters.SetMax(offset, (*cumulativeMax)[i]);
- }
- }
- // percentile counters
- auto* histogramSum = sumCounters.MutableHistogram();
- if (sumCounters.HistogramSize() < FullSizePercentile) {
- auto missing = FullSizePercentile - sumCounters.HistogramSize();
- for (; missing > 0; --missing) {
- sumCounters.AddHistogram();
- }
- }
- ui32 nextPercentileOffset = 0;
- for (ui32 i = 0; i < FullSizePercentile; ++i) {
- if (DeprecatedPercentile.find(i) != DeprecatedPercentile.end()) {
- continue;
- }
- auto* buckets = (*histogramSum)[i].MutableBuckets();
- const ui32 offset = nextPercentileOffset++;
- auto& histogram = Histograms[offset];
- auto snapshot = histogram->Snapshot();
- auto count = snapshot->Count();
- buckets->Resize(count, 0);
- if constexpr (!IsSaving) {
- histogram->Reset();
- }
- for (ui32 r = 0; r < count; ++r) {
- if constexpr (IsSaving) {
- (*buckets)[r] = snapshot->Value(r);
- } else {
- histogram->Collect(snapshot->UpperBound(r), (*buckets)[r]);
- }
- }
- }
- }
-
- void ToProto(NKikimrSysView::TDbCounters& sumCounters,
- NKikimrSysView::TDbCounters& maxCounters)
- {
- Convert<true>(sumCounters, maxCounters);
- }
-
- void FromProto(NKikimrSysView::TDbCounters& sumCounters,
- NKikimrSysView::TDbCounters& maxCounters)
- {
- Convert<false>(sumCounters, maxCounters);
- }
-
+ template <bool IsSaving>
+ void Convert(NKikimrSysView::TDbCounters& sumCounters,
+ NKikimrSysView::TDbCounters& maxCounters)
+ {
+ if (!DoAggregateSimpleCounters || !DoAggregateCumulativeCounters) {
+ return;
+ }
+ // simple counters
+ auto* simpleSum = sumCounters.MutableSimple();
+ auto* simpleMax = maxCounters.MutableSimple();
+ simpleSum->Resize(FullSizeSimple, 0);
+ simpleMax->Resize(FullSizeSimple, 0);
+ ui32 nextSimpleOffset = 0;
+ for (ui32 i = 0; i < FullSizeSimple; ++i) {
+ if (DeprecatedSimple.find(i) != DeprecatedSimple.end()) {
+ if constexpr (IsSaving) {
+ (*simpleSum)[i] = 0;
+ (*simpleMax)[i] = 0;
+ }
+ continue;
+ }
+ const ui32 offset = nextSimpleOffset++;
+ if constexpr (IsSaving) {
+ (*simpleSum)[i] = AggregatedSimpleCounters.GetSum(offset);
+ (*simpleMax)[i] = AggregatedSimpleCounters.GetMax(offset);
+ } else {
+ AggregatedSimpleCounters.SetSum(offset, (*simpleSum)[i]);
+ AggregatedSimpleCounters.SetMax(offset, (*simpleMax)[i]);
+ }
+ }
+ // cumulative counters
+ auto* cumulativeSum = sumCounters.MutableCumulative();
+ auto* cumulativeMax = maxCounters.MutableCumulative();
+ cumulativeSum->Resize(FullSizeCumulative, 0);
+ cumulativeMax->Resize(FullSizeCumulative, 0);
+ ui32 nextCumulativeOffset = 0;
+ for (ui32 i = 0; i < FullSizeCumulative; ++i) {
+ if (DeprecatedCumulative.find(i) != DeprecatedCumulative.end()) {
+ if constexpr (IsSaving) {
+ (*cumulativeSum)[i] = 0;
+ (*cumulativeMax)[i] = 0;
+ }
+ continue;
+ }
+ const ui32 offset = nextCumulativeOffset++;
+ Y_VERIFY(offset < CumulativeCounters.size(),
+ "inconsistent cumulative counters %u >= %lu", offset, CumulativeCounters.size());
+ if constexpr (IsSaving) {
+ (*cumulativeSum)[i] = *CumulativeCounters[offset];
+ (*cumulativeMax)[i] = AggregatedCumulativeCounters.GetMax(offset);
+ } else {
+ *CumulativeCounters[offset] = (*cumulativeSum)[i];
+ AggregatedCumulativeCounters.SetMax(offset, (*cumulativeMax)[i]);
+ }
+ }
+ // percentile counters
+ auto* histogramSum = sumCounters.MutableHistogram();
+ if (sumCounters.HistogramSize() < FullSizePercentile) {
+ auto missing = FullSizePercentile - sumCounters.HistogramSize();
+ for (; missing > 0; --missing) {
+ sumCounters.AddHistogram();
+ }
+ }
+ ui32 nextPercentileOffset = 0;
+ for (ui32 i = 0; i < FullSizePercentile; ++i) {
+ if (DeprecatedPercentile.find(i) != DeprecatedPercentile.end()) {
+ continue;
+ }
+ auto* buckets = (*histogramSum)[i].MutableBuckets();
+ const ui32 offset = nextPercentileOffset++;
+ auto& histogram = Histograms[offset];
+ auto snapshot = histogram->Snapshot();
+ auto count = snapshot->Count();
+ buckets->Resize(count, 0);
+ if constexpr (!IsSaving) {
+ histogram->Reset();
+ }
+ for (ui32 r = 0; r < count; ++r) {
+ if constexpr (IsSaving) {
+ (*buckets)[r] = snapshot->Value(r);
+ } else {
+ histogram->Collect(snapshot->UpperBound(r), (*buckets)[r]);
+ }
+ }
+ }
+ }
+
+ void ToProto(NKikimrSysView::TDbCounters& sumCounters,
+ NKikimrSysView::TDbCounters& maxCounters)
+ {
+ Convert<true>(sumCounters, maxCounters);
+ }
+
+ void FromProto(NKikimrSysView::TDbCounters& sumCounters,
+ NKikimrSysView::TDbCounters& maxCounters)
+ {
+ Convert<false>(sumCounters, maxCounters);
+ }
+
private:
- ui32 FullSizeSimple = 0;
- THashSet<ui32> DeprecatedSimple;
- ui32 FullSizeCumulative = 0;
- THashSet<ui32> DeprecatedCumulative;
- ui32 FullSizePercentile = 0;
- THashSet<ui32> DeprecatedPercentile;
+ ui32 FullSizeSimple = 0;
+ THashSet<ui32> DeprecatedSimple;
+ ui32 FullSizeCumulative = 0;
+ THashSet<ui32> DeprecatedCumulative;
+ ui32 FullSizePercentile = 0;
+ THashSet<ui32> DeprecatedPercentile;
//
bool DoAggregateSimpleCounters;
TCountersVector SimpleCounters;
@@ -1141,8 +1141,8 @@ private:
TAggregatedCumulativeCounters AggregatedCumulativeCounters;
THashMap<ui64, TInstant> LastAggregateUpdateTime;
- TVector<TCountersVector> PercentileCounters; // old style
- TVector<NMonitoring::THistogramPtr> Histograms; // new style
+ TVector<TCountersVector> PercentileCounters; // old style
+ TVector<NMonitoring::THistogramPtr> Histograms; // new style
NMonitoring::TDynamicCounterPtr CounterGroup;
};
@@ -1157,59 +1157,59 @@ private:
TSolomonCounters TabletAppCounters;
};
- typedef TMap<TTabletTypes::EType, TAutoPtr<TTabletCountersForTabletType> > TCountersByTabletType;
-
- static TTabletCountersForTabletType* FindCountersByTabletType(
- TTabletTypes::EType tabletType,
- TCountersByTabletType& countersByTabletType)
- {
- auto iterTabletType = countersByTabletType.find(tabletType);
- if (iterTabletType != countersByTabletType.end()) {
- return iterTabletType->second.Get();
- }
- return {};
- }
-
- static TTabletCountersForTabletType* GetOrAddCountersByTabletType(
- TTabletTypes::EType tabletType,
- TCountersByTabletType& countersByTabletType,
- NMonitoring::TDynamicCounterPtr counters)
- {
- auto* typeCounters = FindCountersByTabletType(tabletType, countersByTabletType);
- if (!typeCounters) {
- TString tabletTypeStr = TTabletTypes::TypeToStr(tabletType);
- typeCounters = new TTabletCountersForTabletType(
- counters.Get(), "type", tabletTypeStr.data(), true);
- countersByTabletType.emplace(tabletType, typeCounters);
- }
- return typeCounters;
- }
-
- const TTabletCountersBase* GetOrAddLimitedAppCounters(TTabletTypes::EType tabletType) {
- if (auto it = LimitedAppCounters.find(tabletType); it != LimitedAppCounters.end()) {
- return it->second.Get();
- }
- auto appCounters = CreateAppCountersByTabletType(tabletType);
- return LimitedAppCounters.emplace(tabletType, std::move(appCounters)).first->second.Get();
- }
-
- class TYdbTabletCounters : public TThrRefBase {
- using TCounterPtr = NMonitoring::TDynamicCounters::TCounterPtr;
- using THistogramPtr = NMonitoring::THistogramPtr;
-
- private:
- TCounterPtr WriteRowCount;
- TCounterPtr WriteBytes;
- TCounterPtr ReadRowCount;
- TCounterPtr ReadBytes;
- TCounterPtr EraseRowCount;
- TCounterPtr EraseBytes;
- TCounterPtr BulkUpsertRowCount;
- TCounterPtr BulkUpsertBytes;
- TCounterPtr ScanRowCount;
- TCounterPtr ScanBytes;
- TCounterPtr DatashardRowCount;
- TCounterPtr DatashardSizeBytes;
+ typedef TMap<TTabletTypes::EType, TAutoPtr<TTabletCountersForTabletType> > TCountersByTabletType;
+
+ static TTabletCountersForTabletType* FindCountersByTabletType(
+ TTabletTypes::EType tabletType,
+ TCountersByTabletType& countersByTabletType)
+ {
+ auto iterTabletType = countersByTabletType.find(tabletType);
+ if (iterTabletType != countersByTabletType.end()) {
+ return iterTabletType->second.Get();
+ }
+ return {};
+ }
+
+ static TTabletCountersForTabletType* GetOrAddCountersByTabletType(
+ TTabletTypes::EType tabletType,
+ TCountersByTabletType& countersByTabletType,
+ NMonitoring::TDynamicCounterPtr counters)
+ {
+ auto* typeCounters = FindCountersByTabletType(tabletType, countersByTabletType);
+ if (!typeCounters) {
+ TString tabletTypeStr = TTabletTypes::TypeToStr(tabletType);
+ typeCounters = new TTabletCountersForTabletType(
+ counters.Get(), "type", tabletTypeStr.data(), true);
+ countersByTabletType.emplace(tabletType, typeCounters);
+ }
+ return typeCounters;
+ }
+
+ const TTabletCountersBase* GetOrAddLimitedAppCounters(TTabletTypes::EType tabletType) {
+ if (auto it = LimitedAppCounters.find(tabletType); it != LimitedAppCounters.end()) {
+ return it->second.Get();
+ }
+ auto appCounters = CreateAppCountersByTabletType(tabletType);
+ return LimitedAppCounters.emplace(tabletType, std::move(appCounters)).first->second.Get();
+ }
+
+ class TYdbTabletCounters : public TThrRefBase {
+ using TCounterPtr = NMonitoring::TDynamicCounters::TCounterPtr;
+ using THistogramPtr = NMonitoring::THistogramPtr;
+
+ private:
+ TCounterPtr WriteRowCount;
+ TCounterPtr WriteBytes;
+ TCounterPtr ReadRowCount;
+ TCounterPtr ReadBytes;
+ TCounterPtr EraseRowCount;
+ TCounterPtr EraseBytes;
+ TCounterPtr BulkUpsertRowCount;
+ TCounterPtr BulkUpsertBytes;
+ TCounterPtr ScanRowCount;
+ TCounterPtr ScanBytes;
+ TCounterPtr DatashardRowCount;
+ TCounterPtr DatashardSizeBytes;
TCounterPtr ResourcesStorageUsedBytes;
TCounterPtr ResourcesStorageLimitBytes;
TCounterPtr ResourcesStreamUsedShards;
@@ -1219,24 +1219,24 @@ private:
TCounterPtr ResourcesStreamReservedStorage;
TCounterPtr ResourcesStreamReservedStorageLimit;
- THistogramPtr ShardCpuUtilization;
-
- TCounterPtr RowUpdates;
- TCounterPtr RowUpdateBytes;
- TCounterPtr RowReads;
- TCounterPtr RangeReadRows;
- TCounterPtr RowReadBytes;
- TCounterPtr RangeReadBytes;
- TCounterPtr RowErases;
- TCounterPtr RowEraseBytes;
- TCounterPtr UploadRows;
- TCounterPtr UploadRowsBytes;
- TCounterPtr ScannedRows;
- TCounterPtr ScannedBytes;
- TCounterPtr DbUniqueRowsTotal;
- TCounterPtr DbUniqueDataBytes;
- THistogramPtr ConsumedCpuHistogram;
-
+ THistogramPtr ShardCpuUtilization;
+
+ TCounterPtr RowUpdates;
+ TCounterPtr RowUpdateBytes;
+ TCounterPtr RowReads;
+ TCounterPtr RangeReadRows;
+ TCounterPtr RowReadBytes;
+ TCounterPtr RangeReadBytes;
+ TCounterPtr RowErases;
+ TCounterPtr RowEraseBytes;
+ TCounterPtr UploadRows;
+ TCounterPtr UploadRowsBytes;
+ TCounterPtr ScannedRows;
+ TCounterPtr ScannedBytes;
+ TCounterPtr DbUniqueRowsTotal;
+ TCounterPtr DbUniqueDataBytes;
+ THistogramPtr ConsumedCpuHistogram;
+
TCounterPtr DiskSpaceTablesTotalBytes;
TCounterPtr DiskSpaceSoftQuotaBytes;
@@ -1247,39 +1247,39 @@ private:
TCounterPtr StreamReservedStorageLimit;
- public:
- explicit TYdbTabletCounters(const NMonitoring::TDynamicCounterPtr& ydbGroup) {
- WriteRowCount = ydbGroup->GetNamedCounter("name",
- "table.datashard.write.rows", true);
- WriteBytes = ydbGroup->GetNamedCounter("name",
- "table.datashard.write.bytes", true);
- ReadRowCount = ydbGroup->GetNamedCounter("name",
- "table.datashard.read.rows", true);
- ReadBytes = ydbGroup->GetNamedCounter("name",
- "table.datashard.read.bytes", true);
- EraseRowCount = ydbGroup->GetNamedCounter("name",
- "table.datashard.erase.rows", true);
- EraseBytes = ydbGroup->GetNamedCounter("name",
- "table.datashard.erase.bytes", true);
- BulkUpsertRowCount = ydbGroup->GetNamedCounter("name",
- "table.datashard.bulk_upsert.rows", true);
- BulkUpsertBytes = ydbGroup->GetNamedCounter("name",
- "table.datashard.bulk_upsert.bytes", true);
- ScanRowCount = ydbGroup->GetNamedCounter("name",
- "table.datashard.scan.rows", true);
- ScanBytes = ydbGroup->GetNamedCounter("name",
- "table.datashard.scan.bytes", true);
-
- DatashardRowCount = ydbGroup->GetNamedCounter("name",
- "table.datashard.row_count", false);
- DatashardSizeBytes = ydbGroup->GetNamedCounter("name",
- "table.datashard.size_bytes", false);
+ public:
+ explicit TYdbTabletCounters(const NMonitoring::TDynamicCounterPtr& ydbGroup) {
+ WriteRowCount = ydbGroup->GetNamedCounter("name",
+ "table.datashard.write.rows", true);
+ WriteBytes = ydbGroup->GetNamedCounter("name",
+ "table.datashard.write.bytes", true);
+ ReadRowCount = ydbGroup->GetNamedCounter("name",
+ "table.datashard.read.rows", true);
+ ReadBytes = ydbGroup->GetNamedCounter("name",
+ "table.datashard.read.bytes", true);
+ EraseRowCount = ydbGroup->GetNamedCounter("name",
+ "table.datashard.erase.rows", true);
+ EraseBytes = ydbGroup->GetNamedCounter("name",
+ "table.datashard.erase.bytes", true);
+ BulkUpsertRowCount = ydbGroup->GetNamedCounter("name",
+ "table.datashard.bulk_upsert.rows", true);
+ BulkUpsertBytes = ydbGroup->GetNamedCounter("name",
+ "table.datashard.bulk_upsert.bytes", true);
+ ScanRowCount = ydbGroup->GetNamedCounter("name",
+ "table.datashard.scan.rows", true);
+ ScanBytes = ydbGroup->GetNamedCounter("name",
+ "table.datashard.scan.bytes", true);
+
+ DatashardRowCount = ydbGroup->GetNamedCounter("name",
+ "table.datashard.row_count", false);
+ DatashardSizeBytes = ydbGroup->GetNamedCounter("name",
+ "table.datashard.size_bytes", false);
ResourcesStorageUsedBytes = ydbGroup->GetNamedCounter("name",
"resources.storage.used_bytes", false);
ResourcesStorageLimitBytes = ydbGroup->GetNamedCounter("name",
"resources.storage.limit_bytes", false);
-
+
ResourcesStreamUsedShards = ydbGroup->GetNamedCounter("name",
"resources.stream.used_shards", false);
ResourcesStreamLimitShards = ydbGroup->GetNamedCounter("name",
@@ -1297,40 +1297,40 @@ private:
ResourcesStreamReservedStorageLimit = ydbGroup->GetNamedCounter("name",
"resources.stream.storage.limit_bytes", false);
- ShardCpuUtilization = ydbGroup->GetNamedHistogram("name",
- "table.datashard.used_core_percents", NMonitoring::LinearHistogram(12, 0, 10), false);
- };
-
- void Initialize(
- NMonitoring::TDynamicCounterPtr counters,
- TCountersByTabletType& countersByTabletType)
- {
- auto datashard = FindCountersByTabletType(
- TTabletTypes::FLAT_DATASHARD, countersByTabletType);
-
- if (datashard && !RowUpdates) {
- auto datashardGroup = counters->GetSubgroup("type", "DataShard");
- auto appGroup = datashardGroup->GetSubgroup("category", "app");
-
- RowUpdates = appGroup->GetCounter("DataShard/EngineHostRowUpdates");
- RowUpdateBytes = appGroup->GetCounter("DataShard/EngineHostRowUpdateBytes");
- RowReads = appGroup->GetCounter("DataShard/EngineHostRowReads");
- RangeReadRows = appGroup->GetCounter("DataShard/EngineHostRangeReadRows");
- RowReadBytes = appGroup->GetCounter("DataShard/EngineHostRowReadBytes");
- RangeReadBytes = appGroup->GetCounter("DataShard/EngineHostRangeReadBytes");
- RowErases = appGroup->GetCounter("DataShard/EngineHostRowErases");
- RowEraseBytes = appGroup->GetCounter("DataShard/EngineHostRowEraseBytes");
- UploadRows = appGroup->GetCounter("DataShard/UploadRows");
- UploadRowsBytes = appGroup->GetCounter("DataShard/UploadRowsBytes");
- ScannedRows = appGroup->GetCounter("DataShard/ScannedRows");
- ScannedBytes = appGroup->GetCounter("DataShard/ScannedBytes");
-
- auto execGroup = datashardGroup->GetSubgroup("category", "executor");
-
- DbUniqueRowsTotal = execGroup->GetCounter("SUM(DbUniqueRowsTotal)");
- DbUniqueDataBytes = execGroup->GetCounter("SUM(DbUniqueDataBytes)");
- ConsumedCpuHistogram = execGroup->FindHistogram("HIST(ConsumedCPU)");
- }
+ ShardCpuUtilization = ydbGroup->GetNamedHistogram("name",
+ "table.datashard.used_core_percents", NMonitoring::LinearHistogram(12, 0, 10), false);
+ };
+
+ void Initialize(
+ NMonitoring::TDynamicCounterPtr counters,
+ TCountersByTabletType& countersByTabletType)
+ {
+ auto datashard = FindCountersByTabletType(
+ TTabletTypes::FLAT_DATASHARD, countersByTabletType);
+
+ if (datashard && !RowUpdates) {
+ auto datashardGroup = counters->GetSubgroup("type", "DataShard");
+ auto appGroup = datashardGroup->GetSubgroup("category", "app");
+
+ RowUpdates = appGroup->GetCounter("DataShard/EngineHostRowUpdates");
+ RowUpdateBytes = appGroup->GetCounter("DataShard/EngineHostRowUpdateBytes");
+ RowReads = appGroup->GetCounter("DataShard/EngineHostRowReads");
+ RangeReadRows = appGroup->GetCounter("DataShard/EngineHostRangeReadRows");
+ RowReadBytes = appGroup->GetCounter("DataShard/EngineHostRowReadBytes");
+ RangeReadBytes = appGroup->GetCounter("DataShard/EngineHostRangeReadBytes");
+ RowErases = appGroup->GetCounter("DataShard/EngineHostRowErases");
+ RowEraseBytes = appGroup->GetCounter("DataShard/EngineHostRowEraseBytes");
+ UploadRows = appGroup->GetCounter("DataShard/UploadRows");
+ UploadRowsBytes = appGroup->GetCounter("DataShard/UploadRowsBytes");
+ ScannedRows = appGroup->GetCounter("DataShard/ScannedRows");
+ ScannedBytes = appGroup->GetCounter("DataShard/ScannedBytes");
+
+ auto execGroup = datashardGroup->GetSubgroup("category", "executor");
+
+ DbUniqueRowsTotal = execGroup->GetCounter("SUM(DbUniqueRowsTotal)");
+ DbUniqueDataBytes = execGroup->GetCounter("SUM(DbUniqueDataBytes)");
+ ConsumedCpuHistogram = execGroup->FindHistogram("HIST(ConsumedCPU)");
+ }
auto schemeshard = FindCountersByTabletType(
TTabletTypes::FLAT_SCHEMESHARD, countersByTabletType);
@@ -1349,31 +1349,31 @@ private:
StreamReservedStorageLimit = appGroup->GetCounter("SUM(SchemeShard/StreamReservedStorageQuota)");
}
- }
-
- void Transform() {
- if (RowUpdates) {
- WriteRowCount->Set(RowUpdates->Val());
- WriteBytes->Set(RowUpdateBytes->Val());
- ReadRowCount->Set(RowReads->Val() + RangeReadRows->Val());
- ReadBytes->Set(RowReadBytes->Val() + RangeReadBytes->Val());
- EraseRowCount->Set(RowErases->Val());
- EraseBytes->Set(RowEraseBytes->Val());
- BulkUpsertRowCount->Set(UploadRows->Val());
- BulkUpsertBytes->Set(UploadRowsBytes->Val());
- ScanRowCount->Set(ScannedRows->Val());
- ScanBytes->Set(ScannedBytes->Val());
- DatashardRowCount->Set(DbUniqueRowsTotal->Val());
- DatashardSizeBytes->Set(DbUniqueDataBytes->Val());
-
- if (ConsumedCpuHistogram) {
- TransferBuckets(ShardCpuUtilization, ConsumedCpuHistogram);
- }
- }
+ }
+
+ void Transform() {
+ if (RowUpdates) {
+ WriteRowCount->Set(RowUpdates->Val());
+ WriteBytes->Set(RowUpdateBytes->Val());
+ ReadRowCount->Set(RowReads->Val() + RangeReadRows->Val());
+ ReadBytes->Set(RowReadBytes->Val() + RangeReadBytes->Val());
+ EraseRowCount->Set(RowErases->Val());
+ EraseBytes->Set(RowEraseBytes->Val());
+ BulkUpsertRowCount->Set(UploadRows->Val());
+ BulkUpsertBytes->Set(UploadRowsBytes->Val());
+ ScanRowCount->Set(ScannedRows->Val());
+ ScanBytes->Set(ScannedBytes->Val());
+ DatashardRowCount->Set(DbUniqueRowsTotal->Val());
+ DatashardSizeBytes->Set(DbUniqueDataBytes->Val());
+
+ if (ConsumedCpuHistogram) {
+ TransferBuckets(ShardCpuUtilization, ConsumedCpuHistogram);
+ }
+ }
if (DiskSpaceTablesTotalBytes) {
- ResourcesStorageUsedBytes->Set(DiskSpaceTablesTotalBytes->Val());
- ResourcesStorageLimitBytes->Set(DiskSpaceSoftQuotaBytes->Val());
+ ResourcesStorageUsedBytes->Set(DiskSpaceTablesTotalBytes->Val());
+ ResourcesStorageLimitBytes->Set(DiskSpaceSoftQuotaBytes->Val());
auto quota = StreamShardsQuota->Val();
ResourcesStreamUsedShards->Set(StreamShardsCount->Val());
@@ -1388,171 +1388,171 @@ private:
ResourcesStreamReservedStorage->Set(StreamReservedStorage->Val());
ResourcesStreamReservedStorageLimit->Set(StreamReservedStorageLimit->Val());
}
- }
-
- void TransferBuckets(THistogramPtr dst, THistogramPtr src) {
- auto srcSnapshot = src->Snapshot();
- auto srcCount = srcSnapshot->Count();
- auto dstSnapshot = dst->Snapshot();
- auto dstCount = dstSnapshot->Count();
-
- dst->Reset();
- for (ui32 b = 0; b < std::min(srcCount, dstCount); ++b) {
- dst->Collect(dstSnapshot->UpperBound(b), srcSnapshot->Value(b));
- }
- }
- };
-
- using TYdbTabletCountersPtr = TIntrusivePtr<TYdbTabletCounters>;
-
-public:
- class TTabletCountersForDb : public NSysView::IDbCounters {
- public:
- TTabletCountersForDb()
- : SolomonCounters(new NMonitoring::TDynamicCounters)
- , AllTypes(SolomonCounters.Get(), "type", "all", true)
- {}
-
- TTabletCountersForDb(NMonitoring::TDynamicCounterPtr externalGroup,
- NMonitoring::TDynamicCounterPtr internalGroup,
- THolder<TTabletCountersBase> executorCounters)
- : SolomonCounters(internalGroup->GetSubgroup("group", "tablets"))
- , ExecutorCounters(std::move(executorCounters))
- , AllTypes(SolomonCounters.Get(), "type", "all", true)
- {
- YdbCounters = MakeIntrusive<TYdbTabletCounters>(externalGroup);
- }
-
- void ToProto(NKikimr::NSysView::TDbServiceCounters& counters) override {
- auto* proto = counters.FindOrAddTabletCounters(TTabletTypes::Unknown);
- AllTypes.ToProto(*proto);
-
- for (auto& [type, tabletCounters] : CountersByTabletType) {
- auto* proto = counters.FindOrAddTabletCounters(type);
- tabletCounters->ToProto(*proto);
- }
- }
-
- void FromProto(NKikimr::NSysView::TDbServiceCounters& counters) override {
- for (auto& proto : *counters.Proto().MutableTabletCounters()) {
- auto type = proto.GetType();
- TTabletCountersForTabletType* tabletCounters = {};
- if (type == TTabletTypes::Unknown) {
- tabletCounters = &AllTypes;
- } else {
- tabletCounters = GetOrAddCountersByTabletType(type, CountersByTabletType, SolomonCounters);
- }
- if (tabletCounters) {
- if (!tabletCounters->IsInitialized()) {
- Y_VERIFY(ExecutorCounters.Get());
- auto appCounters = CreateAppCountersByTabletType(type);
- tabletCounters->Initialize(ExecutorCounters.Get(), appCounters.Get());
- }
- tabletCounters->FromProto(proto);
- }
- }
- if (YdbCounters) {
- YdbCounters->Initialize(SolomonCounters, CountersByTabletType);
- YdbCounters->Transform();
- }
- }
-
- void Apply(ui64 tabletId, const TTabletCountersBase* executorCounters,
- const TTabletCountersBase* appCounters, TTabletTypes::EType type,
- const TTabletCountersBase* limitedAppCounters)
- {
- AllTypes.Apply(tabletId, executorCounters, nullptr, type);
- auto* tabletCounters = GetOrAddCountersByTabletType(type, CountersByTabletType, SolomonCounters);
- if (tabletCounters) {
- tabletCounters->Apply(tabletId, executorCounters, appCounters, type, limitedAppCounters);
- }
- }
-
- void Forget(ui64 tabletId, TTabletTypes::EType type) {
- if (auto it = CountersByTabletType.find(type); it != CountersByTabletType.end()) {
- it->second->Forget(tabletId);
- }
- }
-
- private:
- NMonitoring::TDynamicCounterPtr SolomonCounters;
- THolder<TTabletCountersBase> ExecutorCounters;
-
- TTabletCountersForTabletType AllTypes;
- TCountersByTabletType CountersByTabletType;
-
- TYdbTabletCountersPtr YdbCounters;
- };
-
- class TTabletsDbWatcherCallback : public NKikimr::NSysView::TDbWatcherCallback {
- TActorSystem* ActorSystem = {};
-
- public:
- explicit TTabletsDbWatcherCallback(TActorSystem* actorSystem)
- : ActorSystem(actorSystem)
- {}
-
- void OnDatabaseRemoved(const TString&, TPathId pathId) override {
- auto evRemove = MakeHolder<TEvTabletCounters::TEvRemoveDatabase>(pathId);
- auto aggregator = MakeTabletCountersAggregatorID(ActorSystem->NodeId, false);
- ActorSystem->Send(aggregator, evRemove.Release());
- }
- };
-
-private:
- TIntrusivePtr<TTabletCountersForDb> GetDbCounters(TPathId pathId, const TActorContext& ctx) {
- auto it = CountersByPathId.find(pathId);
- if (it != CountersByPathId.end()) {
- return it->second;
- }
-
- auto dbCounters = MakeIntrusive<TTabletMon::TTabletCountersForDb>();
- CountersByPathId[pathId] = dbCounters;
-
- auto evRegister = MakeHolder<NSysView::TEvSysView::TEvRegisterDbCounters>(
- NKikimrSysView::TABLETS, pathId, dbCounters);
- ctx.Send(NSysView::MakeSysViewServiceID(ctx.SelfID.NodeId()), evRegister.Release());
-
- if (DbWatcherActorId) {
- auto evWatch = MakeHolder<NSysView::TEvSysView::TEvWatchDatabase>(pathId);
- ctx.Send(DbWatcherActorId, evWatch.Release());
- }
-
- return dbCounters;
- }
-
-private:
+ }
+
+ void TransferBuckets(THistogramPtr dst, THistogramPtr src) {
+ auto srcSnapshot = src->Snapshot();
+ auto srcCount = srcSnapshot->Count();
+ auto dstSnapshot = dst->Snapshot();
+ auto dstCount = dstSnapshot->Count();
+
+ dst->Reset();
+ for (ui32 b = 0; b < std::min(srcCount, dstCount); ++b) {
+ dst->Collect(dstSnapshot->UpperBound(b), srcSnapshot->Value(b));
+ }
+ }
+ };
+
+ using TYdbTabletCountersPtr = TIntrusivePtr<TYdbTabletCounters>;
+
+public:
+ class TTabletCountersForDb : public NSysView::IDbCounters {
+ public:
+ TTabletCountersForDb()
+ : SolomonCounters(new NMonitoring::TDynamicCounters)
+ , AllTypes(SolomonCounters.Get(), "type", "all", true)
+ {}
+
+ TTabletCountersForDb(NMonitoring::TDynamicCounterPtr externalGroup,
+ NMonitoring::TDynamicCounterPtr internalGroup,
+ THolder<TTabletCountersBase> executorCounters)
+ : SolomonCounters(internalGroup->GetSubgroup("group", "tablets"))
+ , ExecutorCounters(std::move(executorCounters))
+ , AllTypes(SolomonCounters.Get(), "type", "all", true)
+ {
+ YdbCounters = MakeIntrusive<TYdbTabletCounters>(externalGroup);
+ }
+
+ void ToProto(NKikimr::NSysView::TDbServiceCounters& counters) override {
+ auto* proto = counters.FindOrAddTabletCounters(TTabletTypes::Unknown);
+ AllTypes.ToProto(*proto);
+
+ for (auto& [type, tabletCounters] : CountersByTabletType) {
+ auto* proto = counters.FindOrAddTabletCounters(type);
+ tabletCounters->ToProto(*proto);
+ }
+ }
+
+ void FromProto(NKikimr::NSysView::TDbServiceCounters& counters) override {
+ for (auto& proto : *counters.Proto().MutableTabletCounters()) {
+ auto type = proto.GetType();
+ TTabletCountersForTabletType* tabletCounters = {};
+ if (type == TTabletTypes::Unknown) {
+ tabletCounters = &AllTypes;
+ } else {
+ tabletCounters = GetOrAddCountersByTabletType(type, CountersByTabletType, SolomonCounters);
+ }
+ if (tabletCounters) {
+ if (!tabletCounters->IsInitialized()) {
+ Y_VERIFY(ExecutorCounters.Get());
+ auto appCounters = CreateAppCountersByTabletType(type);
+ tabletCounters->Initialize(ExecutorCounters.Get(), appCounters.Get());
+ }
+ tabletCounters->FromProto(proto);
+ }
+ }
+ if (YdbCounters) {
+ YdbCounters->Initialize(SolomonCounters, CountersByTabletType);
+ YdbCounters->Transform();
+ }
+ }
+
+ void Apply(ui64 tabletId, const TTabletCountersBase* executorCounters,
+ const TTabletCountersBase* appCounters, TTabletTypes::EType type,
+ const TTabletCountersBase* limitedAppCounters)
+ {
+ AllTypes.Apply(tabletId, executorCounters, nullptr, type);
+ auto* tabletCounters = GetOrAddCountersByTabletType(type, CountersByTabletType, SolomonCounters);
+ if (tabletCounters) {
+ tabletCounters->Apply(tabletId, executorCounters, appCounters, type, limitedAppCounters);
+ }
+ }
+
+ void Forget(ui64 tabletId, TTabletTypes::EType type) {
+ if (auto it = CountersByTabletType.find(type); it != CountersByTabletType.end()) {
+ it->second->Forget(tabletId);
+ }
+ }
+
+ private:
+ NMonitoring::TDynamicCounterPtr SolomonCounters;
+ THolder<TTabletCountersBase> ExecutorCounters;
+
+ TTabletCountersForTabletType AllTypes;
+ TCountersByTabletType CountersByTabletType;
+
+ TYdbTabletCountersPtr YdbCounters;
+ };
+
+ class TTabletsDbWatcherCallback : public NKikimr::NSysView::TDbWatcherCallback {
+ TActorSystem* ActorSystem = {};
+
+ public:
+ explicit TTabletsDbWatcherCallback(TActorSystem* actorSystem)
+ : ActorSystem(actorSystem)
+ {}
+
+ void OnDatabaseRemoved(const TString&, TPathId pathId) override {
+ auto evRemove = MakeHolder<TEvTabletCounters::TEvRemoveDatabase>(pathId);
+ auto aggregator = MakeTabletCountersAggregatorID(ActorSystem->NodeId, false);
+ ActorSystem->Send(aggregator, evRemove.Release());
+ }
+ };
+
+private:
+ TIntrusivePtr<TTabletCountersForDb> GetDbCounters(TPathId pathId, const TActorContext& ctx) {
+ auto it = CountersByPathId.find(pathId);
+ if (it != CountersByPathId.end()) {
+ return it->second;
+ }
+
+ auto dbCounters = MakeIntrusive<TTabletMon::TTabletCountersForDb>();
+ CountersByPathId[pathId] = dbCounters;
+
+ auto evRegister = MakeHolder<NSysView::TEvSysView::TEvRegisterDbCounters>(
+ NKikimrSysView::TABLETS, pathId, dbCounters);
+ ctx.Send(NSysView::MakeSysViewServiceID(ctx.SelfID.NodeId()), evRegister.Release());
+
+ if (DbWatcherActorId) {
+ auto evWatch = MakeHolder<NSysView::TEvSysView::TEvWatchDatabase>(pathId);
+ ctx.Send(DbWatcherActorId, evWatch.Release());
+ }
+
+ return dbCounters;
+ }
+
+private:
//
NMonitoring::TDynamicCounterPtr Counters;
TTabletCountersForTabletType AllTypes;
bool IsFollower = false;
- typedef THashMap<TPathId, TIntrusivePtr<TTabletCountersForDb>> TCountersByPathId;
- typedef TMap<TTabletTypes::EType, THolder<TTabletCountersBase>> TAppCountersByTabletType;
+ typedef THashMap<TPathId, TIntrusivePtr<TTabletCountersForDb>> TCountersByPathId;
+ typedef TMap<TTabletTypes::EType, THolder<TTabletCountersBase>> TAppCountersByTabletType;
typedef TMap<std::pair<TTabletTypes::EType, TString>, TAutoPtr<TAggregatedLabeledCounters> > TLabeledCountersByTabletTypeAndGroup;
TCountersByTabletType CountersByTabletType;
- TCountersByPathId CountersByPathId;
- TActorId DbWatcherActorId;
- TAppCountersByTabletType LimitedAppCounters; // without txs
-
- TYdbTabletCountersPtr YdbCounters;
-
+ TCountersByPathId CountersByPathId;
+ TActorId DbWatcherActorId;
+ TAppCountersByTabletType LimitedAppCounters; // without txs
+
+ TYdbTabletCountersPtr YdbCounters;
+
TLabeledCountersByTabletTypeAndGroup LabeledCountersByTabletTypeAndGroup;
THashMap<ui64, std::pair<TAutoPtr<TTabletCountersBase>, TAutoPtr<TTabletCountersBase>>> QuietTabletCounters;
};
-
-TIntrusivePtr<NSysView::IDbCounters> CreateTabletDbCounters(
- NMonitoring::TDynamicCounterPtr externalGroup,
- NMonitoring::TDynamicCounterPtr internalGroup,
- THolder<TTabletCountersBase> executorCounters)
-{
- return MakeIntrusive<TTabletMon::TTabletCountersForDb>(
- externalGroup, internalGroup, std::move(executorCounters));
-}
-
+
+TIntrusivePtr<NSysView::IDbCounters> CreateTabletDbCounters(
+ NMonitoring::TDynamicCounterPtr externalGroup,
+ NMonitoring::TDynamicCounterPtr internalGroup,
+ THolder<TTabletCountersBase> executorCounters)
+{
+ return MakeIntrusive<TTabletMon::TTabletCountersForDb>(
+ externalGroup, internalGroup, std::move(executorCounters));
+}
+
////////////////////////////////////////////
/// The TTabletCountersAggregatorActor class
////////////////////////////////////////////
@@ -1582,11 +1582,11 @@ private:
void HandleWork(TEvTabletCounters::TEvTabletLabeledCountersResponse::TPtr &ev, const TActorContext &ctx);//from cluster aggregator
void HandleWork(NMon::TEvHttpInfo::TPtr& ev, const TActorContext &ctx);
void HandleWakeup(const TActorContext &ctx);
- void HandleWork(TEvTabletCounters::TEvRemoveDatabase::TPtr& ev);
+ void HandleWork(TEvTabletCounters::TEvRemoveDatabase::TPtr& ev);
//
TAutoPtr<TTabletMon> TabletMon;
- TActorId DbWatcherActorId;
+ TActorId DbWatcherActorId;
THashMap<TActorId, std::pair<TActorId, TAutoPtr<NMon::TEvHttpInfo>>> HttpRequestHandlers;
THashSet<ui32> TabletTypeOfReceivedLabeledCounters;
bool Follower;
@@ -1610,12 +1610,12 @@ TTabletCountersAggregatorActor::Bootstrap(const TActorContext &ctx) {
TAppData* appData = AppData(ctx);
Y_VERIFY(!TabletMon);
-
+
if (AppData(ctx)->FeatureFlags.GetEnableDbCounters() && !Follower) {
- auto callback = MakeIntrusive<TTabletMon::TTabletsDbWatcherCallback>(ctx.ActorSystem());
- DbWatcherActorId = ctx.Register(NSysView::CreateDbWatcherActor(callback));
- }
-
+ auto callback = MakeIntrusive<TTabletMon::TTabletsDbWatcherCallback>(ctx.ActorSystem());
+ DbWatcherActorId = ctx.Register(NSysView::CreateDbWatcherActor(callback));
+ }
+
TabletMon = new TTabletMon(appData->Counters, Follower, DbWatcherActorId);
auto mon = appData->Mon;
if (mon) {
@@ -1634,7 +1634,7 @@ void
TTabletCountersAggregatorActor::HandleWork(TEvTabletCounters::TEvTabletAddCounters::TPtr &ev, const TActorContext &ctx) {
Y_UNUSED(ctx);
TEvTabletCounters::TEvTabletAddCounters* msg = ev->Get();
- TabletMon->Apply(msg->TabletID, msg->TabletType, msg->TenantPathId, msg->ExecutorCounters.Get(), msg->AppCounters.Get(), ctx);
+ TabletMon->Apply(msg->TabletID, msg->TabletType, msg->TenantPathId, msg->ExecutorCounters.Get(), msg->AppCounters.Get(), ctx);
}
////////////////////////////////////////////
@@ -1652,7 +1652,7 @@ void
TTabletCountersAggregatorActor::HandleWork(TEvTabletCounters::TEvTabletCountersForgetTablet::TPtr &ev, const TActorContext &ctx) {
Y_UNUSED(ctx);
TEvTabletCounters::TEvTabletCountersForgetTablet* msg = ev->Get();
- TabletMon->ForgetTablet(msg->TabletID, msg->TabletType, msg->TenantPathId);
+ TabletMon->ForgetTablet(msg->TabletID, msg->TabletType, msg->TenantPathId);
}
////////////////////////////////////////////
@@ -1786,12 +1786,12 @@ TTabletCountersAggregatorActor::HandleWork(TEvTabletCounters::TEvTabletLabeledCo
////////////////////////////////////////////
-void TTabletCountersAggregatorActor::HandleWork(TEvTabletCounters::TEvRemoveDatabase::TPtr& ev) {
-
- TabletMon->RemoveTabletsByPathId(ev->Get()->PathId);
-}
-
-////////////////////////////////////////////
+void TTabletCountersAggregatorActor::HandleWork(TEvTabletCounters::TEvRemoveDatabase::TPtr& ev) {
+
+ TabletMon->RemoveTabletsByPathId(ev->Get()->PathId);
+}
+
+////////////////////////////////////////////
void
TTabletCountersAggregatorActor::HandleWork(NMon::TEvHttpInfo::TPtr &ev, const TActorContext &ctx) {
@@ -1839,7 +1839,7 @@ STFUNC(TTabletCountersAggregatorActor::StateWork) {
HFunc(TEvTabletCounters::TEvTabletAddLabeledCounters, HandleWork);
HFunc(TEvTabletCounters::TEvTabletLabeledCountersRequest, HandleWork);
HFunc(TEvTabletCounters::TEvTabletLabeledCountersResponse, HandleWork); //from cluster aggregator, for http requests
- hFunc(TEvTabletCounters::TEvRemoveDatabase, HandleWork);
+ hFunc(TEvTabletCounters::TEvRemoveDatabase, HandleWork);
HFunc(NMon::TEvHttpInfo, HandleWork);
CFunc(TEvents::TSystem::Wakeup, HandleWakeup);
@@ -1858,7 +1858,7 @@ CreateTabletCountersAggregator(bool follower) {
void TabletCountersForgetTablet(ui64 tabletId, TTabletTypes::EType tabletType, TPathId tenantPathId, bool follower, TActorIdentity identity) {
const TActorId countersAggregator = MakeTabletCountersAggregatorID(identity.NodeId(), follower);
- identity.Send(countersAggregator, new TEvTabletCounters::TEvTabletCountersForgetTablet(tabletId, tabletType, tenantPathId));
+ identity.Send(countersAggregator, new TEvTabletCounters::TEvTabletCountersForgetTablet(tabletId, tabletType, tenantPathId));
}
///////////////////////////////////////////
diff --git a/ydb/core/tablet/tablet_counters_aggregator.h b/ydb/core/tablet/tablet_counters_aggregator.h
index db350d41cbf..cfc645305c3 100644
--- a/ydb/core/tablet/tablet_counters_aggregator.h
+++ b/ydb/core/tablet/tablet_counters_aggregator.h
@@ -26,14 +26,14 @@ struct TEvTabletCounters {
//
enum EEv {
EvTabletAddCounters = EventSpaceBegin(TKikimrEvents::ES_TABLET_COUNTERS_AGGREGATOR),
- EvDeprecated1,
+ EvDeprecated1,
EvTabletCountersForgetTablet,
EvTabletCountersRequest,
EvTabletCountersResponse,
EvTabletAddLabeledCounters,
EvTabletLabeledCountersRequest,
EvTabletLabeledCountersResponse,
- EvRemoveDatabase,
+ EvRemoveDatabase,
EvEnd
};
@@ -46,16 +46,16 @@ struct TEvTabletCounters {
//
const ui64 TabletID;
const TTabletTypes::EType TabletType;
- const TPathId TenantPathId;
+ const TPathId TenantPathId;
TAutoPtr<TTabletCountersBase> ExecutorCounters;
TAutoPtr<TTabletCountersBase> AppCounters;
TIntrusivePtr<TInFlightCookie> InFlightCounter; // Used to detect when previous event has been consumed by the aggregator
- TEvTabletAddCounters(TIntrusivePtr<TInFlightCookie> inFlightCounter, ui64 tabletID, TTabletTypes::EType tabletType, TPathId tenantPathId,
- TAutoPtr<TTabletCountersBase> executorCounters, TAutoPtr<TTabletCountersBase> appCounters)
+ TEvTabletAddCounters(TIntrusivePtr<TInFlightCookie> inFlightCounter, ui64 tabletID, TTabletTypes::EType tabletType, TPathId tenantPathId,
+ TAutoPtr<TTabletCountersBase> executorCounters, TAutoPtr<TTabletCountersBase> appCounters)
: TabletID(tabletID)
, TabletType(tabletType)
- , TenantPathId(tenantPathId)
+ , TenantPathId(tenantPathId)
, ExecutorCounters(executorCounters)
, AppCounters(appCounters)
, InFlightCounter(inFlightCounter)
@@ -81,12 +81,12 @@ struct TEvTabletCounters {
//
const ui64 TabletID;
const TTabletTypes::EType TabletType;
- const TPathId TenantPathId;
+ const TPathId TenantPathId;
- TEvTabletCountersForgetTablet(ui64 tabletID, TTabletTypes::EType tabletType, TPathId tenantPathId)
+ TEvTabletCountersForgetTablet(ui64 tabletID, TTabletTypes::EType tabletType, TPathId tenantPathId)
: TabletID(tabletID)
, TabletType(tabletType)
- , TenantPathId(tenantPathId)
+ , TenantPathId(tenantPathId)
{}
};
@@ -104,29 +104,29 @@ struct TEvTabletCounters {
struct TEvTabletLabeledCountersResponse : public TEventPB<TEvTabletLabeledCountersResponse, NKikimrTabletCountersAggregator::TEvTabletLabeledCountersResponse, EvTabletLabeledCountersResponse> {
};
- struct TEvRemoveDatabase : public TEventLocal<TEvRemoveDatabase, EvRemoveDatabase> {
- const TPathId PathId;
-
- explicit TEvRemoveDatabase(TPathId pathId)
- : PathId(pathId)
- {}
- };
+ struct TEvRemoveDatabase : public TEventLocal<TEvRemoveDatabase, EvRemoveDatabase> {
+ const TPathId PathId;
+ explicit TEvRemoveDatabase(TPathId pathId)
+ : PathId(pathId)
+ {}
+ };
+
};
////////////////////////////////////////////
void TabletCountersForgetTablet(ui64 tabletId, TTabletTypes::EType tabletType, TPathId tenantPathId, bool follower, TActorIdentity identity);
-
+
TStringBuf GetHistogramAggregateSimpleName(TStringBuf name);
bool IsHistogramAggregateSimpleName(TStringBuf name);
////////////////////////////////////////////
-TIntrusivePtr<NSysView::IDbCounters> CreateTabletDbCounters(
- NMonitoring::TDynamicCounterPtr externalGroup,
- NMonitoring::TDynamicCounterPtr internalGroup,
- THolder<TTabletCountersBase> executorCounters);
-
-////////////////////////////////////////////
+TIntrusivePtr<NSysView::IDbCounters> CreateTabletDbCounters(
+ NMonitoring::TDynamicCounterPtr externalGroup,
+ NMonitoring::TDynamicCounterPtr internalGroup,
+ THolder<TTabletCountersBase> executorCounters);
+
+////////////////////////////////////////////
IActor* CreateTabletCountersAggregator(bool follower);
diff --git a/ydb/core/tablet/tablet_counters_app.cpp b/ydb/core/tablet/tablet_counters_app.cpp
index bfed7ffd13c..c7f09ffcfef 100644
--- a/ydb/core/tablet/tablet_counters_app.cpp
+++ b/ydb/core/tablet/tablet_counters_app.cpp
@@ -1,43 +1,43 @@
-#include "tablet_counters_app.h"
-
-#include "tablet_counters_protobuf.h"
-
+#include "tablet_counters_app.h"
+
+#include "tablet_counters_protobuf.h"
+
#include <ydb/core/protos/counters_schemeshard.pb.h>
#include <ydb/core/protos/counters_datashard.pb.h>
#include <ydb/core/protos/counters_hive.pb.h>
#include <ydb/core/protos/counters_kesus.pb.h>
-
-namespace NKikimr {
-
-THolder<TTabletCountersBase> CreateAppCountersByTabletType(TTabletTypes::EType type) {
- switch (type) {
- case TTabletTypes::SchemeShard:
+
+namespace NKikimr {
+
+THolder<TTabletCountersBase> CreateAppCountersByTabletType(TTabletTypes::EType type) {
+ switch (type) {
+ case TTabletTypes::SchemeShard:
return MakeHolder<TAppProtobufTabletCounters<
NSchemeShard::ESimpleCounters_descriptor,
NSchemeShard::ECumulativeCounters_descriptor,
NSchemeShard::EPercentileCounters_descriptor
>>();
- case TTabletTypes::DataShard:
+ case TTabletTypes::DataShard:
return MakeHolder<TAppProtobufTabletCounters<
NDataShard::ESimpleCounters_descriptor,
NDataShard::ECumulativeCounters_descriptor,
NDataShard::EPercentileCounters_descriptor
>>();
- case TTabletTypes::Hive:
+ case TTabletTypes::Hive:
return MakeHolder<TAppProtobufTabletCounters<
- NHive::ESimpleCounters_descriptor,
- NHive::ECumulativeCounters_descriptor,
- NHive::EPercentileCounters_descriptor
+ NHive::ESimpleCounters_descriptor,
+ NHive::ECumulativeCounters_descriptor,
+ NHive::EPercentileCounters_descriptor
>>();
- case TTabletTypes::Kesus:
+ case TTabletTypes::Kesus:
return MakeHolder<TAppProtobufTabletCounters<
- NKesus::ESimpleCounters_descriptor,
- NKesus::ECumulativeCounters_descriptor,
- NKesus::EPercentileCounters_descriptor
+ NKesus::ESimpleCounters_descriptor,
+ NKesus::ECumulativeCounters_descriptor,
+ NKesus::EPercentileCounters_descriptor
>>();
- default:
- return {};
- }
-}
-
-}
+ default:
+ return {};
+ }
+}
+
+}
diff --git a/ydb/core/tablet/tablet_counters_app.h b/ydb/core/tablet/tablet_counters_app.h
index a8b473126e9..9b4efe098e0 100644
--- a/ydb/core/tablet/tablet_counters_app.h
+++ b/ydb/core/tablet/tablet_counters_app.h
@@ -1,11 +1,11 @@
-#pragma once
-
-#include "tablet_counters.h"
-
+#pragma once
+
+#include "tablet_counters.h"
+
#include <ydb/core/base/tablet_types.h>
-
-namespace NKikimr {
-
-THolder<TTabletCountersBase> CreateAppCountersByTabletType(TTabletTypes::EType type);
-
-}
+
+namespace NKikimr {
+
+THolder<TTabletCountersBase> CreateAppCountersByTabletType(TTabletTypes::EType type);
+
+}
diff --git a/ydb/core/tablet/tablet_resolver.cpp b/ydb/core/tablet/tablet_resolver.cpp
index c0a39a2216b..67cce605101 100644
--- a/ydb/core/tablet/tablet_resolver.cpp
+++ b/ydb/core/tablet/tablet_resolver.cpp
@@ -88,7 +88,7 @@ class TTabletResolver : public TActorBootstrapped<TTabletResolver> {
}
}
- struct TQueueEntry {
+ struct TQueueEntry {
TInstant AddInstant;
TEvTabletResolver::TEvForward::TPtr Ev;
diff --git a/ydb/core/tablet/ya.make b/ydb/core/tablet/ya.make
index 25e611b4085..461d50af093 100644
--- a/ydb/core/tablet/ya.make
+++ b/ydb/core/tablet/ya.make
@@ -20,8 +20,8 @@ SRCS(
tablet_counters.h
tablet_counters_aggregator.cpp
tablet_counters_aggregator.h
- tablet_counters_app.cpp
- tablet_counters_app.h
+ tablet_counters_app.cpp
+ tablet_counters_app.h
tablet_counters_protobuf.h
tablet_exception.h
tablet_impl.h
diff --git a/ydb/core/tablet_flat/flat_executor.cpp b/ydb/core/tablet_flat/flat_executor.cpp
index 59afc462158..0ffcd492ec5 100644
--- a/ydb/core/tablet_flat/flat_executor.cpp
+++ b/ydb/core/tablet_flat/flat_executor.cpp
@@ -135,7 +135,7 @@ void TExecutor::Broken() {
BootLogic->Cancel();
if (Owner) {
- TabletCountersForgetTablet(Owner->TabletID(), Owner->TabletType(),
+ TabletCountersForgetTablet(Owner->TabletID(), Owner->TabletType(),
Owner->Info()->TenantPathId, Stats->IsFollower, SelfId());
Owner->Detach(OwnerCtx());
}
@@ -332,7 +332,7 @@ void TExecutor::ActivateFollower(const TActorContext &ctx) {
CounterCacheMemTable = new NMonitoring::TCounterForPtr;
ResourceMetrics = MakeHolder<NMetrics::TResourceMetrics>(Owner->TabletID(), FollowerId, Launcher);
-
+
PendingBlobQueue.Config.TabletID = Owner->TabletID();
PendingBlobQueue.Config.Generation = Generation();
PendingBlobQueue.Config.Follower = true;
@@ -617,9 +617,9 @@ TExecutorCaches TExecutor::CleanupState() {
void TExecutor::Boot(TEvTablet::TEvBoot::TPtr &ev, const TActorContext &ctx) {
if (Stats->IsFollower) {
- TabletCountersForgetTablet(Owner->TabletID(), Owner->TabletType(),
+ TabletCountersForgetTablet(Owner->TabletID(), Owner->TabletType(),
Owner->Info()->TenantPathId, Stats->IsFollower, SelfId());
- }
+ }
RegisterTabletFlatProbes();
@@ -696,7 +696,7 @@ void TExecutor::Restored(TEvTablet::TEvRestored::TPtr &ev, const TActorContext &
}
void TExecutor::DetachTablet(const TActorContext &) {
- TabletCountersForgetTablet(Owner->TabletID(), Owner->TabletType(),
+ TabletCountersForgetTablet(Owner->TabletID(), Owner->TabletType(),
Owner->Info()->TenantPathId, Stats->IsFollower, SelfId());
return PassAway();
}
@@ -3342,11 +3342,11 @@ void TExecutor::UpdateCounters(const TActorContext &ctx) {
// tablet id + tablet type
ui64 tabletId = Owner->TabletID();
auto tabletType = Owner->TabletType();
- auto tenantPathId = Owner->Info()->TenantPathId;
+ auto tenantPathId = Owner->Info()->TenantPathId;
TActorId countersAggregator = MakeTabletCountersAggregatorID(SelfId().NodeId(), Stats->IsFollower);
Send(countersAggregator, new TEvTabletCounters::TEvTabletAddCounters(
- CounterEventsInFlight, tabletId, tabletType, tenantPathId, executorCounters, externalTabletCounters));
+ CounterEventsInFlight, tabletId, tabletType, tenantPathId, executorCounters, externalTabletCounters));
if (ResourceMetrics) {
ResourceMetrics->TryUpdate(ctx);
diff --git a/ydb/core/tablet_flat/flat_executor.h b/ydb/core/tablet_flat/flat_executor.h
index df08b3a338a..7aeabff1b51 100644
--- a/ydb/core/tablet_flat/flat_executor.h
+++ b/ydb/core/tablet_flat/flat_executor.h
@@ -355,7 +355,7 @@ class TExecutor
TAutoPtr<NUtil::ILogger> Logger;
ui32 FollowerId = 0;
-
+
using TActivationQueue = TOneOneQueueInplace<TSeat *, 64>;
THolder<TActivationQueue, TActivationQueue::TPtrCleanDestructor> ActivationQueue;
THolder<TActivationQueue, TActivationQueue::TPtrCleanDestructor> PendingQueue;
diff --git a/ydb/core/tablet_flat/flat_range_cache.h b/ydb/core/tablet_flat/flat_range_cache.h
index 2005831095e..851ce9b8df8 100644
--- a/ydb/core/tablet_flat/flat_range_cache.h
+++ b/ydb/core/tablet_flat/flat_range_cache.h
@@ -6,8 +6,8 @@
#include <ydb/core/base/row_version.h>
#include <ydb/core/scheme/scheme_tablecell.h>
-#include <library/cpp/actors/core/memory_track.h>
-
+#include <library/cpp/actors/core/memory_track.h>
+
#include <util/generic/deque.h>
#include <util/generic/intrlist.h>
@@ -16,8 +16,8 @@
namespace NKikimr {
namespace NTable {
-static constexpr char MemoryLabelKeyRangeCache[] = "Tablet/TKeyRangeCache";
-
+static constexpr char MemoryLabelKeyRangeCache[] = "Tablet/TKeyRangeCache";
+
struct TKeyRangeEntry {
TArrayRef<TCell> FromKey;
TArrayRef<TCell> ToKey;
@@ -190,10 +190,10 @@ private:
return Used() == 0;
}
- size_t AllocSize() const noexcept {
- return Next_ - (char*)this + Left_;
- }
-
+ size_t AllocSize() const noexcept {
+ return Next_ - (char*)this + Left_;
+ }
+
private:
char* Next_;
size_t Left_;
@@ -288,12 +288,12 @@ private:
void* ptr = ::operator new(allocSize);
TChunk* chunk = new(ptr) TChunk(allocSize - sizeof(TChunk));
TotalAllocated_ += allocSize;
- NActors::NMemory::TLabel<MemoryLabelKeyRangeCache>::Add(allocSize);
+ NActors::NMemory::TLabel<MemoryLabelKeyRangeCache>::Add(allocSize);
return chunk;
}
void DeallocateChunk(TChunk* chunk) {
- NActors::NMemory::TLabel<MemoryLabelKeyRangeCache>::Sub(chunk->AllocSize());
+ NActors::NMemory::TLabel<MemoryLabelKeyRangeCache>::Sub(chunk->AllocSize());
chunk->~TChunk();
::operator delete((void*)chunk);
}
@@ -335,17 +335,17 @@ private:
{ }
T* allocate(size_t n) {
- auto size = sizeof(T) * n;
- *UsedMemory += size;
- NActors::NMemory::TLabel<MemoryLabelKeyRangeCache>::Add(size);
+ auto size = sizeof(T) * n;
+ *UsedMemory += size;
+ NActors::NMemory::TLabel<MemoryLabelKeyRangeCache>::Add(size);
return TBase::allocate(n);
}
void deallocate(T* ptr, size_t n) noexcept {
- auto size = sizeof(T) * n;
- Y_VERIFY_DEBUG(*UsedMemory >= size);
- *UsedMemory -= size;
- NActors::NMemory::TLabel<MemoryLabelKeyRangeCache>::Sub(size);
+ auto size = sizeof(T) * n;
+ Y_VERIFY_DEBUG(*UsedMemory >= size);
+ *UsedMemory -= size;
+ NActors::NMemory::TLabel<MemoryLabelKeyRangeCache>::Sub(size);
return TBase::deallocate(ptr, n);
}
diff --git a/ydb/core/tablet_flat/tablet_flat_executed.h b/ydb/core/tablet_flat/tablet_flat_executed.h
index 117d7532bb3..8126ff6ef7d 100644
--- a/ydb/core/tablet_flat/tablet_flat_executed.h
+++ b/ydb/core/tablet_flat/tablet_flat_executed.h
@@ -21,7 +21,7 @@ protected:
TTabletExecutedFlat(TTabletStorageInfo *info, const TActorId &tablet, IMiniKQLFactory *factory);
IExecutor* Executor() const { return Executor0; }
- const TInstant StartTime() const { return StartTime0; }
+ const TInstant StartTime() const { return StartTime0; }
void Execute(TAutoPtr<ITransaction> transaction, const TActorContext &ctx);
void Execute(TAutoPtr<ITransaction> transaction);
diff --git a/ydb/core/testlib/basics/helpers.h b/ydb/core/testlib/basics/helpers.h
index 7a70063f7a3..964f8aad551 100644
--- a/ydb/core/testlib/basics/helpers.h
+++ b/ydb/core/testlib/basics/helpers.h
@@ -51,7 +51,7 @@ namespace NFake {
void SetupNodeTabletMonitor(TTestActorRuntime& runtime, ui32 nodeIndex);
void SetupSchemeCache(TTestActorRuntime& runtime, ui32 nodeIndex, const TString& root);
void SetupQuoterService(TTestActorRuntime& runtime, ui32 nodeIndex);
- void SetupSysViewService(TTestActorRuntime& runtime, ui32 nodeIndex);
+ void SetupSysViewService(TTestActorRuntime& runtime, ui32 nodeIndex);
// StateStorage, NodeWarden, TabletResolver, ResourceBroker, SharedPageCache
void SetupBasicServices(TTestActorRuntime &runtime, TAppPrepare &app, bool mockDisk = false,
diff --git a/ydb/core/testlib/tablet_helpers.cpp b/ydb/core/testlib/tablet_helpers.cpp
index a853040b932..4a7644d9021 100644
--- a/ydb/core/testlib/tablet_helpers.cpp
+++ b/ydb/core/testlib/tablet_helpers.cpp
@@ -1183,8 +1183,8 @@ namespace NKikimr {
bootstrapperActorId = Boot(ctx, type, [=](const TActorId& tablet, TTabletStorageInfo* info) {
return new TFakeHive(tablet, info, state, &TFakeHive::DefaultGetTabletCreationFunc);
}, DataGroupErasure);
- } else if (type == defaultTabletTypes.SysViewProcessor) {
- bootstrapperActorId = Boot(ctx, type, &NSysView::CreateSysViewProcessor, DataGroupErasure);
+ } else if (type == defaultTabletTypes.SysViewProcessor) {
+ bootstrapperActorId = Boot(ctx, type, &NSysView::CreateSysViewProcessor, DataGroupErasure);
} else if (type == defaultTabletTypes.SequenceShard) {
bootstrapperActorId = Boot(ctx, type, &NSequenceShard::CreateSequenceShard, DataGroupErasure);
} else if (type == defaultTabletTypes.ReplicationController) {
diff --git a/ydb/core/testlib/tenant_runtime.cpp b/ydb/core/testlib/tenant_runtime.cpp
index 974cc166d0d..f047a9bca95 100644
--- a/ydb/core/testlib/tenant_runtime.cpp
+++ b/ydb/core/testlib/tenant_runtime.cpp
@@ -431,8 +431,8 @@ class TFakeHive : public TActor<TFakeHive>, public TTabletExecutedFlat {
bootstrapperActorId = Boot(ctx, type, &CreateFlatTxSchemeShard, DataGroupErasure);
} else if (type == defaultTabletTypes.Hive) {
bootstrapperActorId = Boot(ctx, type, &CreateDefaultHive, DataGroupErasure);
- } else if (type == defaultTabletTypes.SysViewProcessor) {
- bootstrapperActorId = Boot(ctx, type, &NSysView::CreateSysViewProcessor, DataGroupErasure);
+ } else if (type == defaultTabletTypes.SysViewProcessor) {
+ bootstrapperActorId = Boot(ctx, type, &NSysView::CreateSysViewProcessor, DataGroupErasure);
} else if (type == defaultTabletTypes.SequenceShard) {
bootstrapperActorId = Boot(ctx, type, &NSequenceShard::CreateSequenceShard, DataGroupErasure);
} else if (type == defaultTabletTypes.ReplicationController) {
diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp
index d4907c26f1e..f2f0ed782f2 100644
--- a/ydb/core/testlib/test_client.cpp
+++ b/ydb/core/testlib/test_client.cpp
@@ -537,10 +537,10 @@ namespace Tests {
TLocalConfig::TTabletClassInfo(new TTabletSetupInfo(
&CreateDefaultHive, TMailboxType::Revolving, appData.UserPoolId,
TMailboxType::Revolving, appData.SystemPoolId));
- localConfig.TabletClassInfo[appData.DefaultTabletTypes.SysViewProcessor] =
- TLocalConfig::TTabletClassInfo(new TTabletSetupInfo(
- &NSysView::CreateSysViewProcessorForTests, TMailboxType::Revolving, appData.UserPoolId,
- TMailboxType::Revolving, appData.SystemPoolId));
+ localConfig.TabletClassInfo[appData.DefaultTabletTypes.SysViewProcessor] =
+ TLocalConfig::TTabletClassInfo(new TTabletSetupInfo(
+ &NSysView::CreateSysViewProcessorForTests, TMailboxType::Revolving, appData.UserPoolId,
+ TMailboxType::Revolving, appData.SystemPoolId));
localConfig.TabletClassInfo[appData.DefaultTabletTypes.SequenceShard] =
TLocalConfig::TTabletClassInfo(new TTabletSetupInfo(
&NSequenceShard::CreateSequenceShard, TMailboxType::Revolving, appData.UserPoolId,
@@ -568,13 +568,13 @@ namespace Tests {
Runtime->RegisterService(NConsole::MakeConfigsDispatcherID(Runtime->GetNodeId(nodeIdx)), aid);
}
Runtime->Register(CreateLabelsMaintainer({}), nodeIdx, appData.SystemPoolId, TMailboxType::Revolving, 0);
-
- auto sysViewService = NSysView::CreateSysViewServiceForTests();
+
+ auto sysViewService = NSysView::CreateSysViewServiceForTests();
TActorId sysViewServiceId = Runtime->Register(sysViewService.Release(), nodeIdx);
- Runtime->RegisterService(NSysView::MakeSysViewServiceID(Runtime->GetNodeId(nodeIdx)), sysViewServiceId, nodeIdx);
-
- auto tenantPublisher = CreateTenantNodeEnumerationPublisher();
- Runtime->Register(tenantPublisher, nodeIdx);
+ Runtime->RegisterService(NSysView::MakeSysViewServiceID(Runtime->GetNodeId(nodeIdx)), sysViewServiceId, nodeIdx);
+
+ auto tenantPublisher = CreateTenantNodeEnumerationPublisher();
+ Runtime->Register(tenantPublisher, nodeIdx);
}
void TServer::SetupConfigurators(ui32 nodeIdx) {
diff --git a/ydb/core/testlib/test_client.h b/ydb/core/testlib/test_client.h
index 2064752ab45..0734a1b4df6 100644
--- a/ydb/core/testlib/test_client.h
+++ b/ydb/core/testlib/test_client.h
@@ -153,7 +153,7 @@ namespace Tests {
TServerSettings& SetDomainPlanResolution(ui64 resolution) { DomainPlanResolution = resolution; return *this; }
TServerSettings& SetFeatureFlags(const NKikimrConfig::TFeatureFlags& value) { FeatureFlags = value; return *this; }
TServerSettings& SetCompactionConfig(const NKikimrConfig::TCompactionConfig& value) { CompactionConfig = value; return *this; }
- TServerSettings& SetEnableDbCounters(bool value) { FeatureFlags.SetEnableDbCounters(value); return *this; }
+ TServerSettings& SetEnableDbCounters(bool value) { FeatureFlags.SetEnableDbCounters(value); return *this; }
TServerSettings& SetEnableYq(bool value) { EnableYq = value; return *this; }
TServerSettings& SetKeepSnapshotTimeout(TDuration value) { KeepSnapshotTimeout = value; return *this; }
TServerSettings& SetChangesQueueItemsLimit(ui64 value) { ChangesQueueItemsLimit = value; return *this; }
diff --git a/ydb/core/tx/coordinator/coordinator__plan_step.cpp b/ydb/core/tx/coordinator/coordinator__plan_step.cpp
index c1fac2ffa58..313393d4dab 100644
--- a/ydb/core/tx/coordinator/coordinator__plan_step.cpp
+++ b/ydb/core/tx/coordinator/coordinator__plan_step.cpp
@@ -72,9 +72,9 @@ struct TTxCoordinator::TTxPlanStep : public TTransactionBase<TTxCoordinator> {
Y_VERIFY(txId);
Self->MonCounters.StepConsideredTx->Inc();
- auto durationMs = (ExecStartMoment - proposal->AcceptMoment).MilliSeconds();
- Self->MonCounters.LegacyTxFromReceiveToPlan.Add(durationMs);
- Self->MonCounters.TxFromReceiveToPlan->Collect(durationMs);
+ auto durationMs = (ExecStartMoment - proposal->AcceptMoment).MilliSeconds();
+ Self->MonCounters.LegacyTxFromReceiveToPlan.Add(durationMs);
+ Self->MonCounters.TxFromReceiveToPlan->Collect(durationMs);
if (proposal->MaxStep < PlanOnStep) {
Self->MonCounters.StepOutdatedTx->Inc();
@@ -213,9 +213,9 @@ struct TTxCoordinator::TTxPlanStep : public TTransactionBase<TTxCoordinator> {
}
void Complete(const TActorContext &ctx) override {
- auto durationMs = (ctx.Now() - ExecStartMoment).MilliSeconds();
- Self->MonCounters.LegacyTxPlanLatency.Add(durationMs);
- Self->MonCounters.TxPlanLatency->Collect(durationMs);
+ auto durationMs = (ctx.Now() - ExecStartMoment).MilliSeconds();
+ Self->MonCounters.LegacyTxPlanLatency.Add(durationMs);
+ Self->MonCounters.TxPlanLatency->Collect(durationMs);
for (auto &cx : StepsToConfirm) {
const ui64 mediatorId = cx.first;
diff --git a/ydb/core/tx/coordinator/coordinator_impl.cpp b/ydb/core/tx/coordinator/coordinator_impl.cpp
index b4eeb184e29..ed21df845d6 100644
--- a/ydb/core/tx/coordinator/coordinator_impl.cpp
+++ b/ydb/core/tx/coordinator/coordinator_impl.cpp
@@ -334,12 +334,12 @@ void TTxCoordinator::TryInitMonCounters(const TActorContext &ctx) {
MonCounters.StepPlannedTx = MonCounters.Coordinator->GetCounter("Step/PlannedTx", true);
MonCounters.StepDeclinedNoSpaceTx = MonCounters.Coordinator->GetCounter("Step/DeclinedNoSpaceTx", true);
- MonCounters.LegacyTxFromReceiveToPlan.Init(MonCounters.Coordinator.Get(), "TxFromReceiveToPlan", "ms", 1, 20);
- MonCounters.TxFromReceiveToPlan = MonCounters.Coordinator->GetHistogram(
- "TxFromReceiveToPlanMs", NMonitoring::ExponentialHistogram(20, 2, 1));
- MonCounters.LegacyTxPlanLatency.Init(MonCounters.Coordinator.Get(), "TxPlanLatency", "ms", 1, 20);
- MonCounters.TxPlanLatency = MonCounters.Coordinator->GetHistogram(
- "TxPlanLatencyMs", NMonitoring::ExponentialHistogram(20, 2, 1));
+ MonCounters.LegacyTxFromReceiveToPlan.Init(MonCounters.Coordinator.Get(), "TxFromReceiveToPlan", "ms", 1, 20);
+ MonCounters.TxFromReceiveToPlan = MonCounters.Coordinator->GetHistogram(
+ "TxFromReceiveToPlanMs", NMonitoring::ExponentialHistogram(20, 2, 1));
+ MonCounters.LegacyTxPlanLatency.Init(MonCounters.Coordinator.Get(), "TxPlanLatency", "ms", 1, 20);
+ MonCounters.TxPlanLatency = MonCounters.Coordinator->GetHistogram(
+ "TxPlanLatencyMs", NMonitoring::ExponentialHistogram(20, 2, 1));
}
void TTxCoordinator::SendMediatorStep(TMediator &mediator, const TActorContext &ctx) {
diff --git a/ydb/core/tx/coordinator/coordinator_impl.h b/ydb/core/tx/coordinator/coordinator_impl.h
index 39018aff1e7..3fec141a614 100644
--- a/ydb/core/tx/coordinator/coordinator_impl.h
+++ b/ydb/core/tx/coordinator/coordinator_impl.h
@@ -64,7 +64,7 @@ struct TTransactionProposal {
};
struct TCoordinatorStepConfirmations {
- struct TEntry {
+ struct TEntry {
TTxId TxId;
TActorId ProxyId;
TEvTxProxy::TEvProposeTransactionStatus::EStatus Status;
@@ -396,10 +396,10 @@ private:
NMonitoring::TDynamicCounters::TCounterPtr StepPlannedTx;
NMonitoring::TDynamicCounters::TCounterPtr StepDeclinedNoSpaceTx;
- NMon::THistogramCounterHelper LegacyTxFromReceiveToPlan;
- NMonitoring::THistogramPtr TxFromReceiveToPlan;
- NMon::THistogramCounterHelper LegacyTxPlanLatency;
- NMonitoring::THistogramPtr TxPlanLatency;
+ NMon::THistogramCounterHelper LegacyTxFromReceiveToPlan;
+ NMonitoring::THistogramPtr TxFromReceiveToPlan;
+ NMon::THistogramCounterHelper LegacyTxPlanLatency;
+ NMonitoring::THistogramPtr TxPlanLatency;
i64 CurrentTxInFly;
};
diff --git a/ydb/core/tx/datashard/datashard.cpp b/ydb/core/tx/datashard/datashard.cpp
index d86568b4a15..789b81b796d 100644
--- a/ydb/core/tx/datashard/datashard.cpp
+++ b/ydb/core/tx/datashard/datashard.cpp
@@ -1062,7 +1062,7 @@ TUserTable::TSpecialUpdate TDataShard::SpecialUpdates(const NTable::TDatabase& d
Y_VERIFY(tableId.PathId.OwnerId == PathOwnerId, "%" PRIu64 " vs %" PRIu64,
tableId.PathId.OwnerId, PathOwnerId);
- auto it = TableInfos.find(tableId.PathId.LocalPathId);
+ auto it = TableInfos.find(tableId.PathId.LocalPathId);
Y_VERIFY(it != TableInfos.end());
const TUserTable& tableInfo = *it->second;
Y_VERIFY(tableInfo.LocalTid != Max<ui32>());
@@ -1092,14 +1092,14 @@ TUserTable::TSpecialUpdate TDataShard::SpecialUpdates(const NTable::TDatabase& d
void TDataShard::SetTableAccessTime(const TTableId& tableId, TInstant ts) {
Y_VERIFY(!TSysTables::IsSystemTable(tableId));
- auto iter = TableInfos.find(tableId.PathId.LocalPathId);
+ auto iter = TableInfos.find(tableId.PathId.LocalPathId);
Y_VERIFY(iter != TableInfos.end());
iter->second->Stats.AccessTime = ts;
}
void TDataShard::SetTableUpdateTime(const TTableId& tableId, TInstant ts) {
Y_VERIFY(!TSysTables::IsSystemTable(tableId));
- auto iter = TableInfos.find(tableId.PathId.LocalPathId);
+ auto iter = TableInfos.find(tableId.PathId.LocalPathId);
Y_VERIFY(iter != TableInfos.end());
iter->second->Stats.AccessTime = ts;
iter->second->Stats.UpdateTime = ts;
diff --git a/ydb/core/tx/datashard/datashard__engine_host.cpp b/ydb/core/tx/datashard/datashard__engine_host.cpp
index 876c3370933..fd5b7f9320b 100644
--- a/ydb/core/tx/datashard/datashard__engine_host.cpp
+++ b/ydb/core/tx/datashard/datashard__engine_host.cpp
@@ -195,7 +195,7 @@ public:
: TableId(tableId)
, Self(self)
{
- switch (TableId.PathId.LocalPathId) {
+ switch (TableId.PathId.LocalPathId) {
case TSysTables::SysTableLocks:
TSysTables::TLocksTable::GetInfo(Columns, KeyTypes, false);
break;
@@ -217,10 +217,10 @@ public:
auto lock = Self->SysLocksTable().GetLock(row);
Y_VERIFY(!lock.IsError());
- if (TableId.PathId.LocalPathId == TSysTables::SysTableLocks2) {
+ if (TableId.PathId.LocalPathId == TSysTables::SysTableLocks2) {
return result.CreateResult(lock.MakeRow(true), holderFactory);
}
- Y_VERIFY(TableId.PathId.LocalPathId == TSysTables::SysTableLocks);
+ Y_VERIFY(TableId.PathId.LocalPathId == TSysTables::SysTableLocks);
return result.CreateResult(lock.MakeRow(false), holderFactory);
}
@@ -262,11 +262,11 @@ public:
, Locks2(TTableId(TSysTables::SysSchemeShard, TSysTables::SysTableLocks2), self)
{}
- const TDataShardSysTable& Get(const TTableId& tableId) const {
- if (tableId.PathId.LocalPathId == TSysTables::SysTableLocks2)
+ const TDataShardSysTable& Get(const TTableId& tableId) const {
+ if (tableId.PathId.LocalPathId == TSysTables::SysTableLocks2)
return Locks2;
- if (tableId.PathId.LocalPathId == TSysTables::SysTableLocks)
+ if (tableId.PathId.LocalPathId == TSysTables::SysTableLocks)
return Locks;
Y_FAIL("unexpected sys table id");
@@ -455,7 +455,7 @@ public:
if (TSysTables::IsSystemTable(tableId))
return DataShardSysTable(tableId).IsMyKey(row);
- auto iter = Self->TableInfos.find(tableId.PathId.LocalPathId);
+ auto iter = Self->TableInfos.find(tableId.PathId.LocalPathId);
if (iter == Self->TableInfos.end()) {
// TODO: can this happen?
return false;
@@ -492,7 +492,7 @@ public:
}
private:
- const TDataShardSysTable& DataShardSysTable(const TTableId& tableId) const {
+ const TDataShardSysTable& DataShardSysTable(const TTableId& tableId) const {
return static_cast<const TDataShardSysTables *>(Self->GetDataShardSysTables())->Get(tableId);
}
diff --git a/ydb/core/tx/datashard/datashard__read_columns.cpp b/ydb/core/tx/datashard/datashard__read_columns.cpp
index 2b56a192d10..42a30e90b7f 100644
--- a/ydb/core/tx/datashard/datashard__read_columns.cpp
+++ b/ydb/core/tx/datashard/datashard__read_columns.cpp
@@ -182,8 +182,8 @@ private:
TSmallVec<TRawTypeValue> KeyTo;
bool InclusiveFrom;
bool InclusiveTo;
- ui64 RowsLimit = 100000;
- ui64 BytesLimit = 1024*1024;
+ ui64 RowsLimit = 100000;
+ ui64 BytesLimit = 1024*1024;
ui64 Restarts = 0;
TRowVersion ReadVersion = TRowVersion::Max();
diff --git a/ydb/core/tx/datashard/datashard_active_transaction.cpp b/ydb/core/tx/datashard/datashard_active_transaction.cpp
index 0226bd42e25..5af7903fa4f 100644
--- a/ydb/core/tx/datashard/datashard_active_transaction.cpp
+++ b/ydb/core/tx/datashard/datashard_active_transaction.cpp
@@ -7,8 +7,8 @@
#include "datashard_failpoints.h"
#include "key_conflicts.h"
-#include <library/cpp/actors/core/memory_track.h>
-
+#include <library/cpp/actors/core/memory_track.h>
+
namespace NKikimr {
namespace NDataShard {
@@ -38,9 +38,9 @@ TValidatedDataTx::TValidatedDataTx(TDataShard *self,
return;
}
- ComputeTxSize();
- NActors::NMemory::TLabel<MemoryLabelValidatedDataTx>::Add(TxSize);
-
+ ComputeTxSize();
+ NActors::NMemory::TLabel<MemoryLabelValidatedDataTx>::Add(TxSize);
+
Y_VERIFY(Tx.HasMiniKQL() || Tx.HasReadTableTransaction() || Tx.HasKqpTransaction(),
"One of the fields should be set: MiniKQL, ReadTableTransaction, KqpTransaction");
@@ -195,10 +195,10 @@ TValidatedDataTx::TValidatedDataTx(TDataShard *self,
ComputeDeadline();
}
-TValidatedDataTx::~TValidatedDataTx() {
- NActors::NMemory::TLabel<MemoryLabelValidatedDataTx>::Sub(TxSize);
-}
-
+TValidatedDataTx::~TValidatedDataTx() {
+ NActors::NMemory::TLabel<MemoryLabelValidatedDataTx>::Sub(TxSize);
+}
+
const google::protobuf::RepeatedPtrField<NYql::NDqProto::TDqTask>& TValidatedDataTx::GetKqpTasks() const {
Y_VERIFY(IsKqpTx());
return Tx.GetKqpTransaction().GetTasks();
@@ -309,10 +309,10 @@ void TValidatedDataTx::ReleaseTxData() {
Tx.SetLockTxId(lock);
EngineBay.DestroyEngine();
IsReleased = true;
-
- NActors::NMemory::TLabel<MemoryLabelValidatedDataTx>::Sub(TxSize);
- ComputeTxSize();
- NActors::NMemory::TLabel<MemoryLabelValidatedDataTx>::Add(TxSize);
+
+ NActors::NMemory::TLabel<MemoryLabelValidatedDataTx>::Sub(TxSize);
+ ComputeTxSize();
+ NActors::NMemory::TLabel<MemoryLabelValidatedDataTx>::Add(TxSize);
}
void TValidatedDataTx::ComputeTxSize() {
@@ -335,7 +335,7 @@ TActiveTransaction::TActiveTransaction(const TBasicOpInfo &op,
TValidatedDataTx::TPtr dataTx)
: TActiveTransaction(op)
{
- TrackMemory();
+ TrackMemory();
FillTxData(dataTx);
}
@@ -349,15 +349,15 @@ TActiveTransaction::TActiveTransaction(TDataShard *self,
ui64 artifactFlags)
: TActiveTransaction(op)
{
- TrackMemory();
+ TrackMemory();
FillTxData(self, txc, ctx, target, txBody, locks, artifactFlags);
}
-TActiveTransaction::~TActiveTransaction()
-{
- UntrackMemory();
-}
-
+TActiveTransaction::~TActiveTransaction()
+{
+ UntrackMemory();
+}
+
void TActiveTransaction::FillTxData(TValidatedDataTx::TPtr dataTx)
{
Y_VERIFY(!DataTx);
@@ -378,8 +378,8 @@ void TActiveTransaction::FillTxData(TDataShard *self,
const TVector<TSysTables::TLocksTable::TLock> &locks,
ui64 artifactFlags)
{
- UntrackMemory();
-
+ UntrackMemory();
+
Y_VERIFY(!DataTx);
Y_VERIFY(TxBody.empty());
@@ -406,8 +406,8 @@ void TActiveTransaction::FillTxData(TDataShard *self,
} else if (IsCommitWritesTx()) {
BuildCommitWritesTx();
}
-
- TrackMemory();
+
+ TrackMemory();
}
TValidatedDataTx::TPtr TActiveTransaction::BuildDataTx(TDataShard *self,
@@ -546,11 +546,11 @@ void TActiveTransaction::ReleaseTxData(NTabletFlatExecutor::TTxMemoryProviderBas
DataTx->ReleaseTxData();
// Immediate transactions have no body stored.
- if (!IsImmediate()) {
- UntrackMemory();
+ if (!IsImmediate()) {
+ UntrackMemory();
TxBody.clear();
- TrackMemory();
- }
+ TrackMemory();
+ }
//InReadSets.clear();
OutReadSets().clear();
@@ -613,8 +613,8 @@ ERestoreDataStatus TActiveTransaction::RestoreTxData(
return ERestoreDataStatus::Ok;
}
- UntrackMemory();
-
+ UntrackMemory();
+
// For immediate transactions we should restore just
// from the TxBody. For planned transaction we should
// restore from local database.
@@ -632,8 +632,8 @@ ERestoreDataStatus TActiveTransaction::RestoreTxData(
Y_VERIFY(TxBody);
}
- TrackMemory();
-
+ TrackMemory();
+
for (auto &lock : locks)
LocksCache().Locks[lock.LockId] = lock;
@@ -912,12 +912,12 @@ void TActiveTransaction::KillAsyncJobActor(const TActorContext& ctx) {
SetAsyncJobActor(TActorId());
}
-void TActiveTransaction::TrackMemory() const {
- NActors::NMemory::TLabel<MemoryLabelActiveTransactionBody>::Add(TxBody.size());
-}
-
-void TActiveTransaction::UntrackMemory() const {
- NActors::NMemory::TLabel<MemoryLabelActiveTransactionBody>::Sub(TxBody.size());
-}
-
+void TActiveTransaction::TrackMemory() const {
+ NActors::NMemory::TLabel<MemoryLabelActiveTransactionBody>::Add(TxBody.size());
+}
+
+void TActiveTransaction::UntrackMemory() const {
+ NActors::NMemory::TLabel<MemoryLabelActiveTransactionBody>::Sub(TxBody.size());
+}
+
}}
diff --git a/ydb/core/tx/datashard/datashard_active_transaction.h b/ydb/core/tx/datashard/datashard_active_transaction.h
index 43abd525e15..15b732398a1 100644
--- a/ydb/core/tx/datashard/datashard_active_transaction.h
+++ b/ydb/core/tx/datashard/datashard_active_transaction.h
@@ -16,9 +16,9 @@ class TBalanceCoverageBuilder;
namespace NDataShard {
-static constexpr char MemoryLabelValidatedDataTx[] = "Datashard/TValidatedDataTx";
-static constexpr char MemoryLabelActiveTransactionBody[] = "Datashard/TActiveTransaction/TxBody";
-
+static constexpr char MemoryLabelValidatedDataTx[] = "Datashard/TValidatedDataTx";
+static constexpr char MemoryLabelActiveTransactionBody[] = "Datashard/TActiveTransaction/TxBody";
+
using NTabletFlatExecutor::TTransactionContext;
using NTabletFlatExecutor::TTableSnapshotContext;
@@ -122,8 +122,8 @@ public:
TInstant receivedAt,
const TString &txBody);
- ~TValidatedDataTx();
-
+ ~TValidatedDataTx();
+
static constexpr ui64 MaxReorderTxKeys() { return 100; }
NKikimrTxDataShard::TError::EKind Code() const { return ErrCode; }
@@ -321,9 +321,9 @@ public:
, SchemeTxType(TSchemaOperation::ETypeUnknown)
, ScanSnapshotId(0)
, ScanTask(0)
- {
- TrackMemory();
- }
+ {
+ TrackMemory();
+ }
TActiveTransaction(const TBasicOpInfo &op,
TValidatedDataTx::TPtr savedTx);
@@ -336,8 +336,8 @@ public:
const TVector<TSysTables::TLocksTable::TLock> &locks,
ui64 artifactFlags);
- ~TActiveTransaction();
-
+ ~TActiveTransaction();
+
void FillTxData(TValidatedDataTx::TPtr dataTx);
void FillTxData(TDataShard *self,
TTransactionContext &txc,
@@ -348,16 +348,16 @@ public:
ui64 artifactFlags);
const TString &GetTxBody() const { return TxBody; }
- void SetTxBody(const TString &txBody) {
- UntrackMemory();
- TxBody = txBody;
- TrackMemory();
- }
- void ClearTxBody() {
- UntrackMemory();
- TxBody.clear();
- TrackMemory();
- }
+ void SetTxBody(const TString &txBody) {
+ UntrackMemory();
+ TxBody = txBody;
+ TrackMemory();
+ }
+ void ClearTxBody() {
+ UntrackMemory();
+ TxBody.clear();
+ TrackMemory();
+ }
ui64 GetSchemeShardId() const { return SchemeShardId; }
void SetSchemeShardId(ui64 id) { SchemeShardId = id; }
@@ -534,10 +534,10 @@ public:
}
private:
- void TrackMemory() const;
- void UntrackMemory() const;
-
-private:
+ void TrackMemory() const;
+ void UntrackMemory() const;
+
+private:
TValidatedDataTx::TPtr DataTx;
THolder<NKikimrTxDataShard::TFlatSchemeTransaction> SchemeTx;
THolder<NKikimrTxDataShard::TSnapshotTransaction> SnapshotTx;
diff --git a/ydb/core/tx/datashard/datashard_impl.h b/ydb/core/tx/datashard/datashard_impl.h
index 11313474f80..5ac43aaa45f 100644
--- a/ydb/core/tx/datashard/datashard_impl.h
+++ b/ydb/core/tx/datashard/datashard_impl.h
@@ -1188,7 +1188,7 @@ public:
ui64 GetLocalTableId(const TTableId& tableId) const {
Y_VERIFY(!TSysTables::IsSystemTable(tableId));
- auto it = TableInfos.find(tableId.PathId.LocalPathId);
+ auto it = TableInfos.find(tableId.PathId.LocalPathId);
return it == TableInfos.end() ? 0 : it->second->LocalTid;
}
@@ -2398,9 +2398,9 @@ protected:
ev->Record.AddSysTablesPartOwners(pi);
}
- ev->Record.SetNodeId(ctx.ExecutorThread.ActorSystem->NodeId);
- ev->Record.SetStartTime(StartTime().MilliSeconds());
-
+ ev->Record.SetNodeId(ctx.ExecutorThread.ActorSystem->NodeId);
+ ev->Record.SetStartTime(StartTime().MilliSeconds());
+
NTabletPipe::SendData(ctx, DbStatsReportPipe, ev.Release());
}
diff --git a/ydb/core/tx/datashard/datashard_ut_locks.cpp b/ydb/core/tx/datashard/datashard_ut_locks.cpp
index 1d25c39f99d..d8940334069 100644
--- a/ydb/core/tx/datashard/datashard_ut_locks.cpp
+++ b/ydb/core/tx/datashard/datashard_ut_locks.cpp
@@ -88,7 +88,7 @@ using namespace NDataShard;
namespace NTest {
bool TFakeDataShard::IsUserTable(const TTableId& tableId) const {
- return tableId.PathId.LocalPathId >= NumSysTables();
+ return tableId.PathId.LocalPathId >= NumSysTables();
}
///
@@ -140,7 +140,7 @@ namespace NTest {
: TableId(tableId)
, Locks(&DataShard)
{
- ui64 tid = tableId.PathId.LocalPathId;
+ ui64 tid = tableId.PathId.LocalPathId;
ui64 sid = tableId.PathId.OwnerId;
TmpLock.PathId = tid;
diff --git a/ydb/core/tx/datashard/sys_tables.h b/ydb/core/tx/datashard/sys_tables.h
index e9887bd8dbb..18885f54f76 100644
--- a/ydb/core/tx/datashard/sys_tables.h
+++ b/ydb/core/tx/datashard/sys_tables.h
@@ -31,9 +31,9 @@ struct TSysTables {
SysTableMAX = 999
};
- static bool IsSystemTable(const TTableId& table) { return table.PathId.OwnerId == SysSchemeShard; }
- static bool IsLocksTable(const TTableId& table) {
- return IsSystemTable(table) && (table.PathId.LocalPathId == SysTableLocks || table.PathId.LocalPathId == SysTableLocks2);
+ static bool IsSystemTable(const TTableId& table) { return table.PathId.OwnerId == SysSchemeShard; }
+ static bool IsLocksTable(const TTableId& table) {
+ return IsSystemTable(table) && (table.PathId.LocalPathId == SysTableLocks || table.PathId.LocalPathId == SysTableLocks2);
}
struct TLocksTable {
diff --git a/ydb/core/tx/scheme_board/cache.cpp b/ydb/core/tx/scheme_board/cache.cpp
index 8e778a4a1d8..b175c1d4903 100644
--- a/ydb/core/tx/scheme_board/cache.cpp
+++ b/ydb/core/tx/scheme_board/cache.cpp
@@ -1265,7 +1265,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
Kind = TNavigate::KindPath;
Created = true;
PathId = TPathId(TSysTables::SysSchemeShard, 0);
- Path = "/sys";
+ Path = "/sys";
IsVirtual = true;
}
@@ -1278,7 +1278,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
Kind = TNavigate::KindTable;
Created = true;
PathId = TPathId(TSysTables::SysSchemeShard, v2 ? TSysTables::SysTableLocks2 : TSysTables::SysTableLocks);
- Path = v2 ? "/sys/locks2" : "/sys/locks";
+ Path = v2 ? "/sys/locks2" : "/sys/locks";
TVector<ui32> keyColumnTypes;
TSysTables::TLocksTable::GetInfo(Columns, keyColumnTypes, v2);
@@ -1309,9 +1309,9 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
Y_VERIFY(notify.DescribeSchemeResult.HasPathDescription());
auto& pathDesc = *notify.DescribeSchemeResult.MutablePathDescription();
- Y_VERIFY(notify.DescribeSchemeResult.HasPath());
- Path = notify.DescribeSchemeResult.GetPath();
-
+ Y_VERIFY(notify.DescribeSchemeResult.HasPath());
+ Path = notify.DescribeSchemeResult.GetPath();
+
const auto& abandoned = pathDesc.GetAbandonedTenantsSchemeShards();
AbandonedSchemeShardsIds = TSet<ui64>(abandoned.begin(), abandoned.end());
@@ -1523,38 +1523,38 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
return AbandonedSchemeShardsIds;
}
- void FillSystemViewEntry(TNavigateContext* context, TNavigate::TEntry& entry,
- NSysView::ISystemViewResolver::ETarget target) const
- {
- auto sysViewInfo = entry.TableId.SysViewInfo;
-
- if (sysViewInfo == NSysView::SysPathName) {
- if (entry.Operation == TNavigate::OpTable) {
+ void FillSystemViewEntry(TNavigateContext* context, TNavigate::TEntry& entry,
+ NSysView::ISystemViewResolver::ETarget target) const
+ {
+ auto sysViewInfo = entry.TableId.SysViewInfo;
+
+ if (sysViewInfo == NSysView::SysPathName) {
+ if (entry.Operation == TNavigate::OpTable) {
return SetError(context, entry, TNavigate::EStatus::PathNotTable);
- }
-
- auto listNodeEntry = MakeIntrusive<TNavigate::TListNodeEntry>();
-
- auto names = Owner->SystemViewResolver->GetSystemViewNames(target);
- std::sort(names.begin(), names.end());
-
- listNodeEntry->Kind = TNavigate::KindPath;
- listNodeEntry->Children.reserve(names.size());
- for (const auto& name : names) {
+ }
+
+ auto listNodeEntry = MakeIntrusive<TNavigate::TListNodeEntry>();
+
+ auto names = Owner->SystemViewResolver->GetSystemViewNames(target);
+ std::sort(names.begin(), names.end());
+
+ listNodeEntry->Kind = TNavigate::KindPath;
+ listNodeEntry->Children.reserve(names.size());
+ for (const auto& name : names) {
listNodeEntry->Children.emplace_back(name, TPathId(), TNavigate::KindTable);
- }
-
- entry.Kind = TNavigate::KindPath;
- entry.ListNodeEntry = listNodeEntry;
- entry.TableId = TTableId(PathId.OwnerId, InvalidLocalPathId, sysViewInfo);
-
- } else {
- auto schema = Owner->SystemViewResolver->GetSystemViewSchema(sysViewInfo, target);
- if (!schema) {
+ }
+
+ entry.Kind = TNavigate::KindPath;
+ entry.ListNodeEntry = listNodeEntry;
+ entry.TableId = TTableId(PathId.OwnerId, InvalidLocalPathId, sysViewInfo);
+
+ } else {
+ auto schema = Owner->SystemViewResolver->GetSystemViewSchema(sysViewInfo, target);
+ if (!schema) {
return SetError(context, entry, TNavigate::EStatus::PathErrorUnknown);
- }
-
- entry.Kind = TNavigate::KindTable;
+ }
+
+ entry.Kind = TNavigate::KindTable;
if (target == NSysView::ISystemViewResolver::ETarget::OlapStore ||
target == NSysView::ISystemViewResolver::ETarget::OlapTable)
{
@@ -1562,23 +1562,23 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
entry.Kind =TNavigate::KindOlapTable;
}
- entry.Columns = std::move(schema->Columns);
-
- if (entry.RequestType == TNavigate::TEntry::ERequestType::ByPath) {
- entry.TableId = TTableId(PathId.OwnerId, PathId.LocalPathId, sysViewInfo);
- } else {
- entry.Path = SplitPath(Path);
- entry.Path.emplace_back(NKikimr::NSysView::SysPathName);
- entry.Path.emplace_back(sysViewInfo);
- }
- }
-
- entry.Status = TNavigate::EStatus::Ok;
-
- entry.SecurityObject = SecurityObject;
- entry.DomainInfo = DomainInfo;
- }
-
+ entry.Columns = std::move(schema->Columns);
+
+ if (entry.RequestType == TNavigate::TEntry::ERequestType::ByPath) {
+ entry.TableId = TTableId(PathId.OwnerId, PathId.LocalPathId, sysViewInfo);
+ } else {
+ entry.Path = SplitPath(Path);
+ entry.Path.emplace_back(NKikimr::NSysView::SysPathName);
+ entry.Path.emplace_back(sysViewInfo);
+ }
+ }
+
+ entry.Status = TNavigate::EStatus::Ok;
+
+ entry.SecurityObject = SecurityObject;
+ entry.DomainInfo = DomainInfo;
+ }
+
void FillEntry(TNavigateContext* context, TNavigate::TEntry& entry, const TResponseProps& props = TResponseProps()) const {
SBC_LOG_D("FillEntry for TNavigate"
<< ": self# " << Owner->SelfId()
@@ -1598,13 +1598,13 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
return SetError(context, entry, TNavigate::EStatus::LookupError);
}
- if (!entry.TableId.SysViewInfo.empty()) {
- if (Kind == TNavigate::KindPath) {
- auto split = SplitPath(Path);
- if (split.size() == 1) {
+ if (!entry.TableId.SysViewInfo.empty()) {
+ if (Kind == TNavigate::KindPath) {
+ auto split = SplitPath(Path);
+ if (split.size() == 1) {
return FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ETarget::Domain);
- }
- } else if (Kind == TNavigate::KindSubdomain || Kind == TNavigate::KindExtSubdomain) {
+ }
+ } else if (Kind == TNavigate::KindSubdomain || Kind == TNavigate::KindExtSubdomain) {
return FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ETarget::SubDomain);
} else if (Kind == TNavigate::KindOlapStore) {
FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ETarget::OlapStore);
@@ -1615,11 +1615,11 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
entry.OlapStoreInfo = OlapStoreInfo;
entry.OlapTableInfo = OlapTableInfo;
return;
- }
+ }
return SetError(context, entry, TNavigate::EStatus::PathErrorUnknown);
- }
-
+ }
+
const bool isTable = Kind == TNavigate::KindTable || Kind == TNavigate::KindOlapTable;
if (entry.Operation == TNavigate::OpTable && !isTable) {
return SetError(context, entry, TNavigate::EStatus::PathNotTable);
@@ -1647,14 +1647,14 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
entry.Kind = Kind;
entry.CreateStep = CreateStep;
- if (entry.RequestType == TNavigate::TEntry::ERequestType::ByPath) {
- if (Kind == TNavigate::KindTable) {
+ if (entry.RequestType == TNavigate::TEntry::ERequestType::ByPath) {
+ if (Kind == TNavigate::KindTable) {
entry.TableId = TTableId(PathId.OwnerId, PathId.LocalPathId, SchemaVersion);
- } else {
+ } else {
entry.TableId = TTableId(PathId.OwnerId, PathId.LocalPathId);
- }
+ }
} else {
- entry.Path = SplitPath(Path);
+ entry.Path = SplitPath(Path);
// update schema version
entry.TableId.SchemaVersion = SchemaVersion;
@@ -1683,25 +1683,25 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
}
bool CheckColumns(TResolveContext* context, TResolve::TEntry& entry,
- const TVector<NScheme::TTypeId>& keyColumnTypes,
- const THashMap<ui32, TSysTables::TTableColumnInfo>& columns) const
- {
+ const TVector<NScheme::TTypeId>& keyColumnTypes,
+ const THashMap<ui32, TSysTables::TTableColumnInfo>& columns) const
+ {
TKeyDesc& keyDesc = *entry.KeyDescription;
- // check key types
- if (keyDesc.KeyColumnTypes.size() > keyColumnTypes.size()) {
+ // check key types
+ if (keyDesc.KeyColumnTypes.size() > keyColumnTypes.size()) {
SetError(context, entry, TResolve::EStatus::TypeCheckError, TKeyDesc::EStatus::TypeCheckFailed);
- return false;
+ return false;
}
for (ui32 i : xrange(keyDesc.KeyColumnTypes.size())) {
- if (keyDesc.KeyColumnTypes[i] != keyColumnTypes[i]) {
+ if (keyDesc.KeyColumnTypes[i] != keyColumnTypes[i]) {
SetError(context, entry, TResolve::EStatus::TypeCheckError, TKeyDesc::EStatus::TypeCheckFailed);
- return false;
+ return false;
}
}
- // check operations
+ // check operations
keyDesc.ColumnInfos.clear();
keyDesc.ColumnInfos.reserve(keyDesc.Columns.size());
for (auto& columnOp : keyDesc.Columns) {
@@ -1711,10 +1711,10 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
if (IsSysTable() && columnOp.Operation == TKeyDesc::EColumnOperation::InplaceUpdate) {
SetError(context, entry, TResolve::EStatus::TypeCheckError, TKeyDesc::EStatus::OperationNotSupported);
- return false;
+ return false;
}
- const auto* column = columns.FindPtr(columnOp.Column);
+ const auto* column = columns.FindPtr(columnOp.Column);
if (!column) {
entry.Status = TResolve::EStatus::TypeCheckError;
@@ -1744,36 +1744,36 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
}
}
- return true;
- }
-
- void FillSystemViewEntry(TResolveContext* context, TResolve::TEntry& entry,
- NSysView::ISystemViewResolver::ETarget target) const
- {
- TKeyDesc& keyDesc = *entry.KeyDescription;
- auto sysViewInfo = keyDesc.TableId.SysViewInfo;
-
- auto schema = Owner->SystemViewResolver->GetSystemViewSchema(sysViewInfo, target);
- if (!schema) {
+ return true;
+ }
+
+ void FillSystemViewEntry(TResolveContext* context, TResolve::TEntry& entry,
+ NSysView::ISystemViewResolver::ETarget target) const
+ {
+ TKeyDesc& keyDesc = *entry.KeyDescription;
+ auto sysViewInfo = keyDesc.TableId.SysViewInfo;
+
+ auto schema = Owner->SystemViewResolver->GetSystemViewSchema(sysViewInfo, target);
+ if (!schema) {
return SetError(context, entry, TResolve::EStatus::PathErrorNotExist, TKeyDesc::EStatus::NotExists);
- }
-
+ }
+
if (!CheckColumns(context, entry, schema->KeyColumnTypes, schema->Columns)) {
- return;
- }
-
- if (keyDesc.Status != TKeyDesc::EStatus::Unknown) {
- ++context->Request->ErrorCount;
- return;
- }
-
- entry.DomainInfo = DomainInfo;
- keyDesc.SecurityObject = SecurityObject;
-
- entry.Status = TResolve::EStatus::OkData;
- keyDesc.Status = TKeyDesc::EStatus::Ok;
- }
-
+ return;
+ }
+
+ if (keyDesc.Status != TKeyDesc::EStatus::Unknown) {
+ ++context->Request->ErrorCount;
+ return;
+ }
+
+ entry.DomainInfo = DomainInfo;
+ keyDesc.SecurityObject = SecurityObject;
+
+ entry.Status = TResolve::EStatus::OkData;
+ keyDesc.Status = TKeyDesc::EStatus::Ok;
+ }
+
void FillEntry(TResolveContext* context, TResolve::TEntry& entry, const TResponseProps& props = TResponseProps()) const {
SBC_LOG_D("FillEntry for TResolve"
<< ": self# " << Owner->SelfId()
@@ -1781,27 +1781,27 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
<< ", entry# " << entry.ToString()
<< ", props# " << props.ToString());
- TKeyDesc& keyDesc = *entry.KeyDescription;
-
+ TKeyDesc& keyDesc = *entry.KeyDescription;
+
if (props.IsSync && props.Partial) {
return SetError(context, entry, TResolve::EStatus::LookupError, TKeyDesc::EStatus::NotExists);
}
if (Status && Status == NKikimrScheme::StatusPathDoesNotExist) {
return SetError(context, entry, TResolve::EStatus::PathErrorNotExist, TKeyDesc::EStatus::NotExists);
- }
-
+ }
+
if (!Status || Status != NKikimrScheme::StatusSuccess) {
return SetError(context, entry, TResolve::EStatus::LookupError, TKeyDesc::EStatus::NotExists);
- }
-
+ }
+
if (!keyDesc.TableId.SysViewInfo.empty()) {
- if (Kind == TNavigate::KindPath) {
- auto split = SplitPath(Path);
- if (split.size() == 1) {
+ if (Kind == TNavigate::KindPath) {
+ auto split = SplitPath(Path);
+ if (split.size() == 1) {
return FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ETarget::Domain);
- }
- } else if (Kind == TNavigate::KindSubdomain || Kind == TNavigate::KindExtSubdomain) {
+ }
+ } else if (Kind == TNavigate::KindSubdomain || Kind == TNavigate::KindExtSubdomain) {
return FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ETarget::SubDomain);
} else if (Kind == TNavigate::KindOlapStore) {
FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ETarget::OlapStore);
@@ -1819,23 +1819,23 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
keyDesc.Partitions.back().Range = TKeyDesc::TPartitionRangeInfo();
}
return;
- }
+ }
return SetError(context, entry, TResolve::EStatus::PathErrorNotExist, TKeyDesc::EStatus::NotExists);
- }
-
- if (!Created) {
+ }
+
+ if (!Created) {
return SetError(context, entry, TResolve::EStatus::NotMaterialized, TKeyDesc::EStatus::NotExists);
- }
-
+ }
+
entry.Kind = TableKind;
- entry.DomainInfo = DomainInfo;
-
+ entry.DomainInfo = DomainInfo;
+
if (!CheckColumns(context, entry, KeyColumnTypes, Columns)) {
- return;
- }
-
- // fill partition info
+ return;
+ }
+
+ // fill partition info
if (IsSysTable()) {
if (keyDesc.Range.Point) {
ui64 shard = 0;
@@ -1866,7 +1866,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
keyDesc.Status = TKeyDesc::EStatus::OperationNotSupported;
}
- // fill ACL info
+ // fill ACL info
keyDesc.SecurityObject = SecurityObject;
if (keyDesc.Status != TKeyDesc::EStatus::Unknown) {
@@ -1893,7 +1893,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
bool Created;
ui64 CreateStep;
TPathId PathId;
- TString Path;
+ TString Path;
TSet<ui64> AbandonedSchemeShardsIds;
TIntrusivePtr<TSecurityObject> SecurityObject;
NSchemeCache::TDomainInfo::TPtr DomainInfo;
@@ -2010,19 +2010,19 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
return CreateSubscriber(pathId, pathId.OwnerId, domainOwnerId);
}
- template <typename TContextPtr, typename TEntry, typename TPathExtractor, typename TTabletIdExtractor>
- void HandleEntry(TContextPtr context, TEntry& entry, size_t index,
- TPathExtractor pathExtractor, TTabletIdExtractor tabletIdExtractor)
- {
- auto path = pathExtractor(entry);
- TCacheItem* cacheItem = Cache.FindPtr(path);
-
- if (!cacheItem) {
- const EPathType pathType = PathType(path);
- switch (pathType) {
- case EPathType::RegularPath:
- {
- const ui64 tabletId = tabletIdExtractor(entry);
+ template <typename TContextPtr, typename TEntry, typename TPathExtractor, typename TTabletIdExtractor>
+ void HandleEntry(TContextPtr context, TEntry& entry, size_t index,
+ TPathExtractor pathExtractor, TTabletIdExtractor tabletIdExtractor)
+ {
+ auto path = pathExtractor(entry);
+ TCacheItem* cacheItem = Cache.FindPtr(path);
+
+ if (!cacheItem) {
+ const EPathType pathType = PathType(path);
+ switch (pathType) {
+ case EPathType::RegularPath:
+ {
+ const ui64 tabletId = tabletIdExtractor(entry);
const ui32 domainId = AppData()->DomainsInfo->GetDomainUidByTabletId(tabletId);
if (tabletId == ui64(NSchemeShard::InvalidTabletId) || domainId == TDomainsInfo::BadDomainId) {
return SetRootUnknown(context.Get(), entry);
@@ -2053,33 +2053,33 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
}
cacheItem = &Cache.Upsert(path, TCacheItem(this, CreateSubscriber(path, tabletId, domainOwnerId), false));
- break;
+ break;
}
- case EPathType::SysPath:
- cacheItem = &Cache.Upsert(path, TCacheItem(this, TSubscriber(), true));
- cacheItem->FillAsSysPath();
- break;
- case EPathType::SysLocksV1:
- case EPathType::SysLocksV2:
- cacheItem = &Cache.Upsert(path, TCacheItem(this, TSubscriber(), true));
- cacheItem->FillAsSysLocks(pathType == EPathType::SysLocksV2);
- break;
+ case EPathType::SysPath:
+ cacheItem = &Cache.Upsert(path, TCacheItem(this, TSubscriber(), true));
+ cacheItem->FillAsSysPath();
+ break;
+ case EPathType::SysLocksV1:
+ case EPathType::SysLocksV2:
+ cacheItem = &Cache.Upsert(path, TCacheItem(this, TSubscriber(), true));
+ cacheItem->FillAsSysLocks(pathType == EPathType::SysLocksV2);
+ break;
}
}
- Cache.Promote(path);
-
+ Cache.Promote(path);
+
if (cacheItem->IsFilled() && !entry.SyncVersion) {
cacheItem->FillEntry(context.Get(), entry);
}
- if (entry.SyncVersion) {
- cacheItem->AddInFlight(context, index, true);
- }
-
- if (!cacheItem->IsFilled()) {
- cacheItem->AddInFlight(context, index, false);
+ if (entry.SyncVersion) {
+ cacheItem->AddInFlight(context, index, true);
}
+
+ if (!cacheItem->IsFilled()) {
+ cacheItem->AddInFlight(context, index, false);
+ }
}
TCacheItem* SwapSubscriberAndUpsert(TCacheItem* byPath, const TPathId& notifyPathId, const TString& notifyPath) {
@@ -2369,59 +2369,59 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
void Handle(TEvTxProxySchemeCache::TEvNavigateKeySet::TPtr& ev) {
SBC_LOG_D("Handle TEvTxProxySchemeCache::TEvNavigateKeySet"
<< ": self# " << SelfId()
- << ", request# " << ev->Get()->Request->ToString(*AppData()->TypeRegistry));
+ << ", request# " << ev->Get()->Request->ToString(*AppData()->TypeRegistry));
if (MaybeRunDbResolver(ev)) {
return;
}
TIntrusivePtr<TNavigateContext> context(new TNavigateContext(ev->Sender, ev->Get()->Request, Now()));
+
+ for (size_t i = 0; i < context->Request->ResultSet.size(); ++i) {
+ auto& entry = context->Request->ResultSet[i];
- for (size_t i = 0; i < context->Request->ResultSet.size(); ++i) {
- auto& entry = context->Request->ResultSet[i];
-
- if (entry.RequestType == TNavigate::TEntry::ERequestType::ByPath) {
- auto pathExtractor = [this](TNavigate::TEntry& entry) {
+ if (entry.RequestType == TNavigate::TEntry::ERequestType::ByPath) {
+ auto pathExtractor = [this](TNavigate::TEntry& entry) {
if (AppData()->FeatureFlags.GetEnableSystemViews()
- && (entry.Operation == TNavigate::OpPath || entry.Operation == TNavigate::OpTable))
- {
- NSysView::ISystemViewResolver::TSystemViewPath sysViewPath;
- if (SystemViewResolver->IsSystemViewPath(entry.Path, sysViewPath)) {
- entry.TableId.SysViewInfo = sysViewPath.ViewName;
- return CanonizePath(sysViewPath.Parent);
- }
- }
-
- TString path = CanonizePath(entry.Path);
- return path ? path : TString("/");
- };
-
- auto tabletIdExtractor = [this](const TNavigate::TEntry& entry) {
- if (entry.Path.empty()) {
+ && (entry.Operation == TNavigate::OpPath || entry.Operation == TNavigate::OpTable))
+ {
+ NSysView::ISystemViewResolver::TSystemViewPath sysViewPath;
+ if (SystemViewResolver->IsSystemViewPath(entry.Path, sysViewPath)) {
+ entry.TableId.SysViewInfo = sysViewPath.ViewName;
+ return CanonizePath(sysViewPath.Parent);
+ }
+ }
+
+ TString path = CanonizePath(entry.Path);
+ return path ? path : TString("/");
+ };
+
+ auto tabletIdExtractor = [this](const TNavigate::TEntry& entry) {
+ if (entry.Path.empty()) {
return ui64(NSchemeShard::InvalidTabletId);
- }
-
- auto it = Roots.find(entry.Path.front());
- if (it == Roots.end()) {
+ }
+
+ auto it = Roots.find(entry.Path.front());
+ if (it == Roots.end()) {
return ui64(NSchemeShard::InvalidTabletId);
- }
-
- return it->second;
- };
-
- HandleEntry(context, entry, i, pathExtractor, tabletIdExtractor);
- } else {
- auto pathExtractor = [](const TNavigate::TEntry& entry) {
- return entry.TableId.PathId;
- };
-
- auto tabletIdExtractor = [](const TNavigate::TEntry& entry) {
- return entry.TableId.PathId.OwnerId;
- };
-
- HandleEntry(context, entry, i, pathExtractor, tabletIdExtractor);
- }
- }
+ }
+
+ return it->second;
+ };
+
+ HandleEntry(context, entry, i, pathExtractor, tabletIdExtractor);
+ } else {
+ auto pathExtractor = [](const TNavigate::TEntry& entry) {
+ return entry.TableId.PathId;
+ };
+
+ auto tabletIdExtractor = [](const TNavigate::TEntry& entry) {
+ return entry.TableId.PathId.OwnerId;
+ };
+
+ HandleEntry(context, entry, i, pathExtractor, tabletIdExtractor);
+ }
+ }
MaybeComplete(context);
}
@@ -2436,22 +2436,22 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
}
TIntrusivePtr<TResolveContext> context(new TResolveContext(ev->Sender, ev->Get()->Request, Now()));
-
+
auto pathExtractor = [](const TResolve::TEntry& entry) {
const TKeyDesc* keyDesc = entry.KeyDescription.Get();
Y_VERIFY(keyDesc != nullptr);
- return TPathId(keyDesc->TableId.PathId);
+ return TPathId(keyDesc->TableId.PathId);
};
auto tabletIdExtractor = [](const TResolve::TEntry& entry) {
- return entry.KeyDescription->TableId.PathId.OwnerId;
+ return entry.KeyDescription->TableId.PathId.OwnerId;
};
- for (size_t i = 0; i < context->Request->ResultSet.size(); ++i) {
- auto& entry = context->Request->ResultSet[i];
- HandleEntry(context, entry, i, pathExtractor, tabletIdExtractor);
- }
-
+ for (size_t i = 0; i < context->Request->ResultSet.size(); ++i) {
+ auto& entry = context->Request->ResultSet[i];
+ HandleEntry(context, entry, i, pathExtractor, tabletIdExtractor);
+ }
+
MaybeComplete(context);
}
@@ -2538,7 +2538,7 @@ public:
TSchemeCache(NSchemeCache::TSchemeCacheConfig* config)
: Counters(config->Counters)
, Cache(TDuration::Minutes(2), TThis::Now)
- , SystemViewResolver(NSysView::CreateSystemViewResolver())
+ , SystemViewResolver(NSysView::CreateSystemViewResolver())
{
for (const auto& root : config->Roots) {
Roots.emplace(root.Name, root.RootSchemeShard);
@@ -2578,7 +2578,7 @@ private:
TCounters Counters;
TDoubleIndexedCache<TString, TPathId, TCacheItem, TMerger, TEvicter> Cache;
- THolder<NSysView::ISystemViewResolver> SystemViewResolver;
+ THolder<NSysView::ISystemViewResolver> SystemViewResolver;
TActorId WatchCache;
diff --git a/ydb/core/tx/scheme_board/cache_ut.cpp b/ydb/core/tx/scheme_board/cache_ut.cpp
index c27b3517b34..969cf901fba 100644
--- a/ydb/core/tx/scheme_board/cache_ut.cpp
+++ b/ydb/core/tx/scheme_board/cache_ut.cpp
@@ -47,9 +47,9 @@ public:
UNIT_TEST(RacyRecreateAndSync);
UNIT_TEST(RacyCreateAndSync);
UNIT_TEST(CheckAccess);
- UNIT_TEST(CheckSystemViewAccess);
- UNIT_TEST(SystemView);
- UNIT_TEST(SysLocks);
+ UNIT_TEST(CheckSystemViewAccess);
+ UNIT_TEST(SystemView);
+ UNIT_TEST(SysLocks);
UNIT_TEST(TableSchemaVersion);
UNIT_TEST(MigrationCommon);
UNIT_TEST(MigrationCommit);
@@ -66,9 +66,9 @@ public:
void RacyRecreateAndSync();
void RacyCreateAndSync();
void CheckAccess();
- void CheckSystemViewAccess();
- void SystemView();
- void SysLocks();
+ void CheckSystemViewAccess();
+ void SystemView();
+ void SysLocks();
void TableSchemaVersion();
void MigrationCommon();
void MigrationCommit();
@@ -78,18 +78,18 @@ public:
void WatchRoot();
protected:
- TNavigate::TEntry TestNavigateImpl(THolder<TNavigate> request, TNavigate::EStatus expectedStatus,
+ TNavigate::TEntry TestNavigateImpl(THolder<TNavigate> request, TNavigate::EStatus expectedStatus,
const TString& sid, TNavigate::EOp op, bool showPrivatePath, bool redirectRequired);
-
+
TNavigate::TEntry TestNavigate(const TString& path, TNavigate::EStatus expectedStatus = TNavigate::EStatus::Ok,
const TString& sid = TString(), TNavigate::EOp op = TNavigate::EOp::OpPath,
bool showPrivatePath = false, bool redirectRequired = true, bool syncVersion = false);
- TNavigate::TEntry TestNavigateByTableId(const TTableId& tableId, TNavigate::EStatus expectedStatus,
- const TString& expectedPath, const TString& sid = TString(),
+ TNavigate::TEntry TestNavigateByTableId(const TTableId& tableId, TNavigate::EStatus expectedStatus,
+ const TString& expectedPath, const TString& sid = TString(),
TNavigate::EOp op = TNavigate::EOp::OpPath, bool showPrivatePath = false);
-
- TResolve::TEntry TestResolve(const TTableId& tableId, TResolve::EStatus expectedStatus = TResolve::EStatus::OkData,
+
+ TResolve::TEntry TestResolve(const TTableId& tableId, TResolve::EStatus expectedStatus = TResolve::EStatus::OkData,
const TString& sid = TString());
TActorId TestWatch(const TPathId& pathId, const TActorId& watcher = {}, ui64 key = 0);
@@ -108,15 +108,15 @@ UNIT_TEST_SUITE_REGISTRATION(TCacheTest);
void TCacheTest::Navigate() {
TestNavigate("/Root", TNavigate::EStatus::Ok);
-
- ui64 txId = 100;
-
- TestMkDir(*Context, ++txId, "/Root", "DirA");
+
+ ui64 txId = 100;
+
+ TestMkDir(*Context, ++txId, "/Root", "DirA");
TestWaitNotification(*Context, {txId}, CreateNotificationSubscriber(*Context, TTestTxConfig::SchemeShard));
-
- auto entry = TestNavigate("/Root/DirA", TNavigate::EStatus::Ok);
-
- TestNavigateByTableId(entry.TableId, TNavigate::EStatus::Ok, "/Root/DirA");
+
+ auto entry = TestNavigate("/Root/DirA", TNavigate::EStatus::Ok);
+
+ TestNavigateByTableId(entry.TableId, TNavigate::EStatus::Ok, "/Root/DirA");
TestNavigateByTableId(TTableId(1ull << 56, 1), TNavigate::EStatus::RootUnknown, "");
}
@@ -166,9 +166,9 @@ void TCacheTest::Recreate() {
TestRmDir(*Context, ++txId, "/Root", "DirA");
TestWaitNotification(*Context, {txId}, CreateNotificationSubscriber(*Context, TTestTxConfig::SchemeShard));
auto ev = Context->GrabEdgeEvent<TSchemeBoardEvents::TEvNotifyDelete>(edge);
- auto pathId = ev->Get()->PathId;
- TTableId tableId(pathId.OwnerId, pathId.LocalPathId);
- TestResolve(tableId, TResolve::EStatus::PathErrorNotExist);
+ auto pathId = ev->Get()->PathId;
+ TTableId tableId(pathId.OwnerId, pathId.LocalPathId);
+ TestResolve(tableId, TResolve::EStatus::PathErrorNotExist);
TestMkDir(*Context, ++txId, "/Root", "DirA");
TestWaitNotification(*Context, {txId}, CreateNotificationSubscriber(*Context, TTestTxConfig::SchemeShard));
@@ -293,49 +293,49 @@ void TCacheTest::CheckAccess() {
TestNavigateByTableId(entry.TableId, TNavigate::EStatus::Ok, "/Root/DirA", "user0@builtin");
}
-void TCacheTest::CheckSystemViewAccess() {
- ui64 txId = 100;
- TestCreateSubDomain(*Context, ++txId, "/Root", "Name: \"SubDomainA\"");
+void TCacheTest::CheckSystemViewAccess() {
+ ui64 txId = 100;
+ TestCreateSubDomain(*Context, ++txId, "/Root", "Name: \"SubDomainA\"");
TestWaitNotification(*Context, {txId}, CreateNotificationSubscriber(*Context, TTestTxConfig::SchemeShard));
- TestModifyACL(*Context, ++txId, "/Root", "SubDomainA", TString(), "user0@builtin");
-
- auto entry = TestNavigate("/Root/SubDomainA/.sys/partition_stats",
- TNavigate::EStatus::Ok, TString(), TNavigate::OpTable);
-
- auto tableId = entry.TableId;
- UNIT_ASSERT_VALUES_EQUAL(tableId.SysViewInfo, "partition_stats");
-
- TestNavigate("/Root/SubDomainA/.sys/partition_stats",
- TNavigate::EStatus::Ok, "user0@builtin", TNavigate::OpTable);
-
- TestNavigate("/Root/SubDomainA/.sys/partition_stats",
- TNavigate::EStatus::PathErrorUnknown, "user1@builtin", TNavigate::OpTable);
-
- TestResolve(tableId, TResolve::EStatus::OkData);
- TestResolve(tableId, TResolve::EStatus::OkData, "user0@builtin");
- TestResolve(tableId, TResolve::EStatus::PathErrorNotExist, "user1@builtin");
-}
-
+ TestModifyACL(*Context, ++txId, "/Root", "SubDomainA", TString(), "user0@builtin");
+
+ auto entry = TestNavigate("/Root/SubDomainA/.sys/partition_stats",
+ TNavigate::EStatus::Ok, TString(), TNavigate::OpTable);
+
+ auto tableId = entry.TableId;
+ UNIT_ASSERT_VALUES_EQUAL(tableId.SysViewInfo, "partition_stats");
+
+ TestNavigate("/Root/SubDomainA/.sys/partition_stats",
+ TNavigate::EStatus::Ok, "user0@builtin", TNavigate::OpTable);
+
+ TestNavigate("/Root/SubDomainA/.sys/partition_stats",
+ TNavigate::EStatus::PathErrorUnknown, "user1@builtin", TNavigate::OpTable);
+
+ TestResolve(tableId, TResolve::EStatus::OkData);
+ TestResolve(tableId, TResolve::EStatus::OkData, "user0@builtin");
+ TestResolve(tableId, TResolve::EStatus::PathErrorNotExist, "user1@builtin");
+}
+
void TCacheTest::SystemView() {
- auto entry = TestNavigate("/Root/.sys/partition_stats", TNavigate::EStatus::Ok, TString(), TNavigate::OpTable);
-
- auto tableId = entry.TableId;
- UNIT_ASSERT_VALUES_EQUAL(tableId.SysViewInfo, "partition_stats");
-
- TestNavigateByTableId(tableId, TNavigate::EStatus::Ok, "/Root/.sys/partition_stats", TString(), TNavigate::OpTable);
-}
-
-void TCacheTest::SysLocks() {
- {
- auto entry = TestNavigate("/sys/locks", TNavigate::EStatus::Ok, TString(), TNavigate::OpTable, true);
- TestNavigateByTableId(entry.TableId, TNavigate::EStatus::Ok, "/sys/locks", TString(), TNavigate::OpTable, true);
- }
- {
- auto entry = TestNavigate("/sys/locks2", TNavigate::EStatus::Ok, TString(), TNavigate::OpTable, true);
- TestNavigateByTableId(entry.TableId, TNavigate::EStatus::Ok, "/sys/locks2", TString(), TNavigate::OpTable, true);
- }
+ auto entry = TestNavigate("/Root/.sys/partition_stats", TNavigate::EStatus::Ok, TString(), TNavigate::OpTable);
+
+ auto tableId = entry.TableId;
+ UNIT_ASSERT_VALUES_EQUAL(tableId.SysViewInfo, "partition_stats");
+
+ TestNavigateByTableId(tableId, TNavigate::EStatus::Ok, "/Root/.sys/partition_stats", TString(), TNavigate::OpTable);
}
+void TCacheTest::SysLocks() {
+ {
+ auto entry = TestNavigate("/sys/locks", TNavigate::EStatus::Ok, TString(), TNavigate::OpTable, true);
+ TestNavigateByTableId(entry.TableId, TNavigate::EStatus::Ok, "/sys/locks", TString(), TNavigate::OpTable, true);
+ }
+ {
+ auto entry = TestNavigate("/sys/locks2", TNavigate::EStatus::Ok, TString(), TNavigate::OpTable, true);
+ TestNavigateByTableId(entry.TableId, TNavigate::EStatus::Ok, "/sys/locks2", TString(), TNavigate::OpTable, true);
+ }
+}
+
void TCacheTest::TableSchemaVersion() {
ui64 txId = 100;
TestCreateTable(*Context, ++txId, "/Root", R"(
@@ -366,11 +366,11 @@ void TCacheTest::TableSchemaVersion() {
}
}
-TNavigate::TEntry TCacheTest::TestNavigateImpl(THolder<TNavigate> request, TNavigate::EStatus expectedStatus,
+TNavigate::TEntry TCacheTest::TestNavigateImpl(THolder<TNavigate> request, TNavigate::EStatus expectedStatus,
const TString& sid, TNavigate::EOp op, bool showPrivatePath, bool redirectRequired)
-{
+{
auto& entry = request->ResultSet.back();
- entry.Operation = op;
+ entry.Operation = op;
entry.ShowPrivatePath = showPrivatePath;
entry.RedirectRequired = redirectRequired;
@@ -390,40 +390,40 @@ TNavigate::TEntry TCacheTest::TestNavigateImpl(THolder<TNavigate> request, TNavi
return result;
}
-TNavigate::TEntry TCacheTest::TestNavigate(const TString& path, TNavigate::EStatus expectedStatus,
+TNavigate::TEntry TCacheTest::TestNavigate(const TString& path, TNavigate::EStatus expectedStatus,
const TString& sid, TNavigate::EOp op, bool showPrivatePath, bool redirectRequired, bool syncVersion)
-{
- auto request = MakeHolder<TNavigate>();
- request->ResultSet.push_back({});
- auto& entry = request->ResultSet.back();
-
- entry.Path = SplitPath(path);
+{
+ auto request = MakeHolder<TNavigate>();
+ request->ResultSet.push_back({});
+ auto& entry = request->ResultSet.back();
+
+ entry.Path = SplitPath(path);
entry.SyncVersion = syncVersion;
-
+
auto result = TestNavigateImpl(std::move(request), expectedStatus, sid, op, showPrivatePath, redirectRequired);
- return result;
-}
-
-TNavigate::TEntry TCacheTest::TestNavigateByTableId(const TTableId& tableId, TNavigate::EStatus expectedStatus,
+ return result;
+}
+
+TNavigate::TEntry TCacheTest::TestNavigateByTableId(const TTableId& tableId, TNavigate::EStatus expectedStatus,
const TString& expectedPath, const TString& sid, TNavigate::EOp op, bool showPrivatePath)
-{
- auto request = MakeHolder<TNavigate>();
- request->ResultSet.push_back({});
- auto& entry = request->ResultSet.back();
-
- entry.TableId = tableId;
- entry.RequestType = TNavigate::TEntry::ERequestType::ByTableId;
-
+{
+ auto request = MakeHolder<TNavigate>();
+ request->ResultSet.push_back({});
+ auto& entry = request->ResultSet.back();
+
+ entry.TableId = tableId;
+ entry.RequestType = TNavigate::TEntry::ERequestType::ByTableId;
+
auto result = TestNavigateImpl(std::move(request), expectedStatus, sid, op, showPrivatePath, true);
- UNIT_ASSERT_VALUES_EQUAL(CanonizePath(result.Path), expectedPath);
- return result;
-}
-
-TResolve::TEntry TCacheTest::TestResolve(const TTableId& tableId, TResolve::EStatus expectedStatus, const TString& sid) {
+ UNIT_ASSERT_VALUES_EQUAL(CanonizePath(result.Path), expectedPath);
+ return result;
+}
+
+TResolve::TEntry TCacheTest::TestResolve(const TTableId& tableId, TResolve::EStatus expectedStatus, const TString& sid) {
auto request = MakeHolder<TResolve>();
auto keyDesc = MakeHolder<TKeyDesc>(
- tableId,
+ tableId,
TTableRange({}),
TKeyDesc::ERowOperation::Unknown,
TVector<NScheme::TTypeId>(), TVector<TKeyDesc::TColumnOp>()
diff --git a/ydb/core/tx/scheme_board/replica.cpp b/ydb/core/tx/scheme_board/replica.cpp
index 638656e91db..a7aa4dd8f49 100644
--- a/ydb/core/tx/scheme_board/replica.cpp
+++ b/ydb/core/tx/scheme_board/replica.cpp
@@ -13,7 +13,7 @@
#include <library/cpp/actors/core/hfunc.h>
#include <library/cpp/actors/core/interconnect.h>
#include <library/cpp/actors/core/log.h>
-#include <library/cpp/actors/core/memory_track.h>
+#include <library/cpp/actors/core/memory_track.h>
#include <util/generic/hash.h>
#include <util/generic/map.h>
@@ -53,7 +53,7 @@ private:
, WaitForAck(false)
, LastVersionSent(0)
, NotifiedStrongly(true)
- , SyncRequestCookie(0)
+ , SyncRequestCookie(0)
, SyncResponseCookie(0)
{
}
@@ -145,8 +145,8 @@ private:
public:
class TDescription {
- static constexpr char MemoryLabelDescribeResult[] = "SchemeBoard/Replica/DescribeSchemeResult";
-
+ static constexpr char MemoryLabelDescribeResult[] = "SchemeBoard/Replica/DescribeSchemeResult";
+
void Notify() {
if (!Subscribers) {
return;
@@ -166,59 +166,59 @@ public:
MultiSend(subscribers, Owner->SelfId(), std::move(notify));
}
- void CalculateResultSize() {
- ResultSize = DescribeSchemeResult.ByteSizeLong();
- }
-
- size_t FullSize() const {
- size_t size = ResultSize;
- if (PreSerializedDescribeSchemeResult) {
- size += PreSerializedDescribeSchemeResult->size();
- }
- return size;
- }
-
- void TrackMemory() const {
- NActors::NMemory::TLabel<MemoryLabelDescribeResult>::Add(FullSize());
- }
-
- void UntrackMemory() const {
- NActors::NMemory::TLabel<MemoryLabelDescribeResult>::Sub(FullSize());
- }
-
- void Move(TDescription&& other) {
- UntrackMemory();
- other.UntrackMemory();
-
- Owner = other.Owner;
- Path = std::move(other.Path);
- PathId = std::move(other.PathId);
- DescribeSchemeResult = std::move(other.DescribeSchemeResult);
- PreSerializedDescribeSchemeResult = std::move(other.PreSerializedDescribeSchemeResult);
- ExplicitlyDeleted = other.ExplicitlyDeleted;
- Subscribers = std::move(other.Subscribers);
-
- ResultSize = other.ResultSize;
- other.ResultSize = 0;
- TrackNotify = other.TrackNotify;
-
- TrackMemory();
- other.TrackMemory();
- }
-
+ void CalculateResultSize() {
+ ResultSize = DescribeSchemeResult.ByteSizeLong();
+ }
+
+ size_t FullSize() const {
+ size_t size = ResultSize;
+ if (PreSerializedDescribeSchemeResult) {
+ size += PreSerializedDescribeSchemeResult->size();
+ }
+ return size;
+ }
+
+ void TrackMemory() const {
+ NActors::NMemory::TLabel<MemoryLabelDescribeResult>::Add(FullSize());
+ }
+
+ void UntrackMemory() const {
+ NActors::NMemory::TLabel<MemoryLabelDescribeResult>::Sub(FullSize());
+ }
+
+ void Move(TDescription&& other) {
+ UntrackMemory();
+ other.UntrackMemory();
+
+ Owner = other.Owner;
+ Path = std::move(other.Path);
+ PathId = std::move(other.PathId);
+ DescribeSchemeResult = std::move(other.DescribeSchemeResult);
+ PreSerializedDescribeSchemeResult = std::move(other.PreSerializedDescribeSchemeResult);
+ ExplicitlyDeleted = other.ExplicitlyDeleted;
+ Subscribers = std::move(other.Subscribers);
+
+ ResultSize = other.ResultSize;
+ other.ResultSize = 0;
+ TrackNotify = other.TrackNotify;
+
+ TrackMemory();
+ other.TrackMemory();
+ }
+
public:
explicit TDescription(TReplica* owner, const TString& path)
: Owner(owner)
, Path(path)
{
- TrackMemory();
+ TrackMemory();
}
explicit TDescription(TReplica* owner, const TPathId& pathId)
: Owner(owner)
, PathId(pathId)
{
- TrackMemory();
+ TrackMemory();
}
explicit TDescription(TReplica* owner, const TString& path, const TPathId& pathId)
@@ -226,7 +226,7 @@ public:
, Path(path)
, PathId(pathId)
{
- TrackMemory();
+ TrackMemory();
}
explicit TDescription(
@@ -237,8 +237,8 @@ public:
, PathId(pathId)
, DescribeSchemeResult(std::move(describeSchemeResult))
{
- CalculateResultSize();
- TrackMemory();
+ CalculateResultSize();
+ TrackMemory();
}
explicit TDescription(
@@ -251,28 +251,28 @@ public:
, PathId(pathId)
, DescribeSchemeResult(std::move(describeSchemeResult))
{
- CalculateResultSize();
- TrackMemory();
- }
-
- TDescription(TDescription&& other) {
- TrackMemory();
- Move(std::move(other));
- }
-
- TDescription& operator=(TDescription&& other) {
- Move(std::move(other));
- return *this;
- }
-
- TDescription(const TDescription& other) = delete;
- TDescription& operator=(const TDescription& other) = delete;
-
- ~TDescription()
- {
- UntrackMemory();
- }
-
+ CalculateResultSize();
+ TrackMemory();
+ }
+
+ TDescription(TDescription&& other) {
+ TrackMemory();
+ Move(std::move(other));
+ }
+
+ TDescription& operator=(TDescription&& other) {
+ Move(std::move(other));
+ return *this;
+ }
+
+ TDescription(const TDescription& other) = delete;
+ TDescription& operator=(const TDescription& other) = delete;
+
+ ~TDescription()
+ {
+ UntrackMemory();
+ }
+
bool operator<(const TDescription& other) const {
return GetVersion() < other.GetVersion();
}
@@ -309,11 +309,11 @@ public:
<< ", rigth pathId# " << other.PathId
<< ", rigth version# " << other.GetVersion());
- UntrackMemory();
- other.UntrackMemory();
- TrackNotify = false;
- other.TrackNotify = false;
-
+ UntrackMemory();
+ other.UntrackMemory();
+ TrackNotify = false;
+ other.TrackNotify = false;
+
if (*this > other) {
other.DescribeSchemeResult.Swap(&DescribeSchemeResult);
other.PreSerializedDescribeSchemeResult.Clear();
@@ -328,14 +328,14 @@ public:
Notify();
}
- CalculateResultSize();
- other.CalculateResultSize();
-
- TrackNotify = true;
- other.TrackNotify = true;
- TrackMemory();
- other.TrackMemory();
-
+ CalculateResultSize();
+ other.CalculateResultSize();
+
+ TrackNotify = true;
+ other.TrackNotify = true;
+ TrackMemory();
+ other.TrackMemory();
+
Subscribers.insert(other.Subscribers.begin(), other.Subscribers.end());
return *this;
@@ -388,11 +388,11 @@ public:
void Clear() {
ExplicitlyDeleted = true;
- UntrackMemory();
+ UntrackMemory();
TDescribeSchemeResult().Swap(&DescribeSchemeResult);
PreSerializedDescribeSchemeResult.Clear();
- ResultSize = 0;
- TrackMemory();
+ ResultSize = 0;
+ TrackMemory();
Notify();
}
@@ -415,13 +415,13 @@ public:
if (!PreSerializedDescribeSchemeResult) {
TString serialized;
Y_PROTOBUF_SUPPRESS_NODISCARD DescribeSchemeResult.SerializeToString(&serialized);
- if (TrackNotify) {
- UntrackMemory();
- }
+ if (TrackNotify) {
+ UntrackMemory();
+ }
PreSerializedDescribeSchemeResult = std::move(serialized);
- if (TrackNotify) {
- TrackMemory();
- }
+ if (TrackNotify) {
+ TrackMemory();
+ }
}
notify->SetDescribeSchemeResult(*PreSerializedDescribeSchemeResult);
@@ -479,10 +479,10 @@ public:
// subscribers
THashMap<TActorId, TSubscriberInfo> Subscribers;
- // memory tracking
- size_t ResultSize = 0;
- bool TrackNotify = true;
-
+ // memory tracking
+ size_t ResultSize = 0;
+ bool TrackNotify = true;
+
}; // TDescription
struct TMerger {
diff --git a/ydb/core/tx/scheme_cache/scheme_cache.h b/ydb/core/tx/scheme_cache/scheme_cache.h
index 26a07897a11..5c74daa5e8c 100644
--- a/ydb/core/tx/scheme_cache/scheme_cache.h
+++ b/ydb/core/tx/scheme_cache/scheme_cache.h
@@ -202,15 +202,15 @@ struct TSchemeCacheNavigate {
};
struct TEntry {
- enum class ERequestType : ui8 {
- ByPath,
- ByTableId
- };
+ enum class ERequestType : ui8 {
+ ByPath,
+ ByTableId
+ };
// in
TVector<TString> Path;
TTableId TableId;
- ERequestType RequestType = ERequestType::ByPath;
+ ERequestType RequestType = ERequestType::ByPath;
EOp Operation = OpUnknown;
bool RedirectRequired = true;
bool ShowPrivatePath = false;
@@ -523,8 +523,8 @@ public:
};
};
-inline TActorId MakeSchemeCacheID() {
+inline TActorId MakeSchemeCacheID() {
return TActorId(0, TStringBuf("SchmCcheSrv"));
}
-
+
} // NKikimr
diff --git a/ydb/core/tx/schemeshard/schemeshard.h b/ydb/core/tx/schemeshard/schemeshard.h
index d33173508ce..17d3f5947e0 100644
--- a/ydb/core/tx/schemeshard/schemeshard.h
+++ b/ydb/core/tx/schemeshard/schemeshard.h
@@ -235,8 +235,8 @@ struct TEvSchemeShard {
TEvDescribeScheme(TTableId tableId)
{
- Record.SetSchemeshardId(tableId.PathId.OwnerId);
- Record.SetPathId(tableId.PathId.LocalPathId);
+ Record.SetSchemeshardId(tableId.PathId.OwnerId);
+ Record.SetPathId(tableId.PathId.LocalPathId);
}
TEvDescribeScheme(NKikimr::TPathId pathId)
@@ -425,7 +425,7 @@ struct TEvSchemeShard {
ui64 effectiveACLVersion,
ui64 subdomainVersion,
ui64 userAttrsVersion,
- ui64 tenantHive,
+ ui64 tenantHive,
ui64 tenantSysViewProcessor,
const TString& rootACL)
{
@@ -440,7 +440,7 @@ struct TEvSchemeShard {
Record.SetUserAttributesVersion(userAttrsVersion);
Record.SetTenantHive(tenantHive);
- Record.SetTenantSysViewProcessor(tenantSysViewProcessor);
+ Record.SetTenantSysViewProcessor(tenantSysViewProcessor);
Record.SetTenantRootACL(rootACL);
}
@@ -483,10 +483,10 @@ struct TEvSchemeShard {
void SetTenantHive(ui64 hive) {
Record.SetTenantHive(hive);
}
-
- void SetTenantSysViewProcessor(ui64 svp) {
- Record.SetTenantSysViewProcessor(svp);
- }
+
+ void SetTenantSysViewProcessor(ui64 svp) {
+ Record.SetTenantSysViewProcessor(svp);
+ }
void SetUpdateTenantRootACL(const TString& acl) {
Record.SetUpdateTenantRootACL(acl);
diff --git a/ydb/core/tx/schemeshard/schemeshard__delete_tablet_reply.cpp b/ydb/core/tx/schemeshard/schemeshard__delete_tablet_reply.cpp
index 855553cddc8..20fe55ed246 100644
--- a/ydb/core/tx/schemeshard/schemeshard__delete_tablet_reply.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__delete_tablet_reply.cpp
@@ -89,9 +89,9 @@ struct TSchemeShard::TTxDeleteTabletReply : public TSchemeShard::TRwTxBase {
case ETabletType::PersQueueReadBalancer:
Self->TabletCounters->Simple()[COUNTER_PQ_RB_SHARD_COUNT].Sub(1);
break;
- case ETabletType::SysViewProcessor:
+ case ETabletType::SysViewProcessor:
Self->TabletCounters->Simple()[COUNTER_SYS_VIEW_PROCESSOR_COUNT].Sub(1);
- break;
+ break;
case ETabletType::ColumnShard:
Self->TabletCounters->Simple()[COUNTER_OLAP_COLUMN_SHARDS].Sub(-1);
break;
diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp
index 650506549e0..8a44d513c66 100644
--- a/ydb/core/tx/schemeshard/schemeshard__init.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp
@@ -2022,7 +2022,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> {
Y_VERIFY(!partitions.empty());
Y_VERIFY(Self->Tables.contains(prevTableId));
TTableInfo::TPtr tableInfo = Self->Tables.at(prevTableId);
- Self->SetPartitioning(prevTableId, tableInfo, std::move(partitions));
+ Self->SetPartitioning(prevTableId, tableInfo, std::move(partitions));
partitions.clear();
}
@@ -2055,7 +2055,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> {
Y_VERIFY(!partitions.empty());
Y_VERIFY(Self->Tables.contains(prevTableId));
TTableInfo::TPtr tableInfo = Self->Tables.at(prevTableId);
- Self->SetPartitioning(prevTableId, tableInfo, std::move(partitions));
+ Self->SetPartitioning(prevTableId, tableInfo, std::move(partitions));
}
}
@@ -3710,9 +3710,9 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> {
case ETabletType::Hive:
Self->TabletCounters->Simple()[COUNTER_SUB_DOMAIN_HIVE_COUNT].Add(1);
break;
- case ETabletType::SysViewProcessor:
- Self->TabletCounters->Simple()[COUNTER_SYS_VIEW_PROCESSOR_COUNT].Add(1);
- break;
+ case ETabletType::SysViewProcessor:
+ Self->TabletCounters->Simple()[COUNTER_SYS_VIEW_PROCESSOR_COUNT].Add(1);
+ break;
case ETabletType::ColumnShard:
Self->TabletCounters->Simple()[COUNTER_OLAP_COLUMN_SHARDS].Add(1);
break;
diff --git a/ydb/core/tx/schemeshard/schemeshard__init_root.cpp b/ydb/core/tx/schemeshard/schemeshard__init_root.cpp
index 74399680cda..a771d556123 100644
--- a/ydb/core/tx/schemeshard/schemeshard__init_root.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__init_root.cpp
@@ -328,9 +328,9 @@ struct TSchemeShard::TTxInitTenantSchemeShard : public TSchemeShard::TRwTxBase {
if (processingParams.HasHive()) {
RegisterShard(db, subdomain, TVector<ui64>{processingParams.GetHive()}, TTabletTypes::Hive);
}
- if (processingParams.HasSysViewProcessor()) {
- RegisterShard(db, subdomain, TVector<ui64>{processingParams.GetSysViewProcessor()}, TTabletTypes::SysViewProcessor);
- }
+ if (processingParams.HasSysViewProcessor()) {
+ RegisterShard(db, subdomain, TVector<ui64>{processingParams.GetSysViewProcessor()}, TTabletTypes::SysViewProcessor);
+ }
subdomain->Initialize(Self->ShardInfos);
diff --git a/ydb/core/tx/schemeshard/schemeshard__operation.cpp b/ydb/core/tx/schemeshard/schemeshard__operation.cpp
index daf1218bcf7..9d6c966229c 100644
--- a/ydb/core/tx/schemeshard/schemeshard__operation.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__operation.cpp
@@ -210,7 +210,7 @@ THolder<TProposeResponse> TSchemeShard::IgniteOperation(TProposeRequest& request
struct TSchemeShard::TTxOperationPropose: public NTabletFlatExecutor::TTransactionBase<TSchemeShard> {
using TBase = NTabletFlatExecutor::TTransactionBase<TSchemeShard>;
-
+
TProposeRequest::TPtr Request;
THolder<TProposeResponse> Response = nullptr;
@@ -538,14 +538,14 @@ TOperation::TConsumeQuotaResult TOperation::ConsumeQuota(const TTxTransaction& t
return result;
}
-TOperation::TSplitTransactionsResult TOperation::SplitIntoTransactions(const TTxTransaction& tx, const TOperationContext& context) {
- TSplitTransactionsResult result;
-
+TOperation::TSplitTransactionsResult TOperation::SplitIntoTransactions(const TTxTransaction& tx, const TOperationContext& context) {
+ TSplitTransactionsResult result;
+
const TPath parentPath = TPath::Resolve(tx.GetWorkingDir(), context.SS);
{
TPath::TChecker checks = parentPath.Check();
checks
- .NotUnderDomainUpgrade()
+ .NotUnderDomainUpgrade()
.IsAtLocalSchemeShard()
.IsResolved()
.NotDeleted()
@@ -554,8 +554,8 @@ TOperation::TSplitTransactionsResult TOperation::SplitIntoTransactions(const TTx
.IsLikeDirectory();
if (!checks) {
- result.Transactions.push_back(tx);
- return result;
+ result.Transactions.push_back(tx);
+ return result;
}
}
@@ -567,8 +567,8 @@ TOperation::TSplitTransactionsResult TOperation::SplitIntoTransactions(const TTx
break;
case NKikimrSchemeOp::EOperationType::ESchemeOpCreateTable:
if (tx.GetCreateTable().HasCopyFromTable()) {
- result.Transactions.push_back(tx);
- return result;
+ result.Transactions.push_back(tx);
+ return result;
}
targetName = tx.GetCreateTable().GetName();
break;
@@ -604,13 +604,13 @@ TOperation::TSplitTransactionsResult TOperation::SplitIntoTransactions(const TTx
targetName = tx.GetCreateColumnTable().GetName();
break;
default:
- result.Transactions.push_back(tx);
- return result;
+ result.Transactions.push_back(tx);
+ return result;
}
if (!targetName || targetName.StartsWith('/') || targetName.EndsWith('/')) {
- result.Transactions.push_back(tx);
- return result;
+ result.Transactions.push_back(tx);
+ return result;
}
TPath path = TPath::Resolve(JoinPath(tx.GetWorkingDir(), targetName), context.SS);
@@ -640,9 +640,9 @@ TOperation::TSplitTransactionsResult TOperation::SplitIntoTransactions(const TTx
}
if (!checks) {
- result.Status = checks.GetStatus(&result.Reason);
- result.Transactions.push_back(tx);
- return result;
+ result.Status = checks.GetStatus(&result.Reason);
+ result.Transactions.push_back(tx);
+ return result;
}
const TString name = path.LeafName();
@@ -694,7 +694,7 @@ TOperation::TSplitTransactionsResult TOperation::SplitIntoTransactions(const TTx
Y_UNREACHABLE();
}
- result.Transactions.push_back(create);
+ result.Transactions.push_back(create);
if (exists) {
return result;
@@ -703,9 +703,9 @@ TOperation::TSplitTransactionsResult TOperation::SplitIntoTransactions(const TTx
while (path != parentPath) {
TPath::TChecker checks = path.Check();
- checks
- .NotUnderDomainUpgrade()
- .IsAtLocalSchemeShard();
+ checks
+ .NotUnderDomainUpgrade()
+ .IsAtLocalSchemeShard();
if (path.IsResolved()) {
checks.IsResolved();
@@ -733,7 +733,7 @@ TOperation::TSplitTransactionsResult TOperation::SplitIntoTransactions(const TTx
checks
.IsValidLeafName()
.DepthLimit()
- .PathsLimit(result.Transactions.size() + 1);
+ .PathsLimit(result.Transactions.size() + 1);
}
if (checks && path.Parent().IsResolved()) {
@@ -741,10 +741,10 @@ TOperation::TSplitTransactionsResult TOperation::SplitIntoTransactions(const TTx
}
if (!checks) {
- result.Status = checks.GetStatus(&result.Reason);
- result.Transactions.clear();
- result.Transactions.push_back(tx);
- return result;
+ result.Status = checks.GetStatus(&result.Reason);
+ result.Transactions.clear();
+ result.Transactions.push_back(tx);
+ return result;
}
const TString name = path.LeafName();
@@ -755,10 +755,10 @@ TOperation::TSplitTransactionsResult TOperation::SplitIntoTransactions(const TTx
mkdir.SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpMkDir);
mkdir.SetWorkingDir(path.PathString());
mkdir.MutableMkDir()->SetName(name);
- result.Transactions.push_back(mkdir);
+ result.Transactions.push_back(mkdir);
}
- Reverse(result.Transactions.begin(), result.Transactions.end());
+ Reverse(result.Transactions.begin(), result.Transactions.end());
return result;
}
diff --git a/ydb/core/tx/schemeshard/schemeshard__operation.h b/ydb/core/tx/schemeshard/schemeshard__operation.h
index 52f1848aa12..b4fe170b5e4 100644
--- a/ydb/core/tx/schemeshard/schemeshard__operation.h
+++ b/ydb/core/tx/schemeshard/schemeshard__operation.h
@@ -51,12 +51,12 @@ struct TOperation: TSimpleRefCount<TOperation> {
TString Reason;
};
- struct TSplitTransactionsResult {
+ struct TSplitTransactionsResult {
NKikimrScheme::EStatus Status = NKikimrScheme::StatusSuccess;
- TString Reason;
- TVector<TTxTransaction> Transactions;
- };
-
+ TString Reason;
+ TVector<TTxTransaction> Transactions;
+ };
+
TOperation(TTxId txId)
: TxId(txId)
{}
@@ -66,7 +66,7 @@ struct TOperation: TSimpleRefCount<TOperation> {
static TConsumeQuotaResult ConsumeQuota(const TTxTransaction& tx, const TOperationContext& context);
- static TSplitTransactionsResult SplitIntoTransactions(const TTxTransaction& tx, const TOperationContext& context);
+ static TSplitTransactionsResult SplitIntoTransactions(const TTxTransaction& tx, const TOperationContext& context);
ISubOperationBase::TPtr RestorePart(TTxState::ETxType opType, TTxState::ETxState opState);
diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_alter_extsubdomain.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_alter_extsubdomain.cpp
index 463f3bb3cf9..af683144e30 100644
--- a/ydb/core/tx/schemeshard/schemeshard__operation_alter_extsubdomain.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__operation_alter_extsubdomain.cpp
@@ -254,8 +254,8 @@ public:
"Malformed subdomain request: unable to change ExternalHive, only set it up");
return result;
}
- }
-
+ }
+
if (addExternalHive) {
shardsToCreate += 1;
allowOverTheLimitShards += 1;
@@ -418,8 +418,8 @@ public:
if (addViewProcessors) {
DeclareShards(txState, OperationId.GetTxId(), subDomain->PathId, 1, TTabletTypes::SysViewProcessor, channelBindings, context.SS);
- }
-
+ }
+
for (auto& shard: txState.Shards) {
alterData->AddPrivateShard(shard.Idx);
}
diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_common.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_common.cpp
index 5a88636ffae..2f547ce4a4d 100644
--- a/ydb/core/tx/schemeshard/schemeshard__operation_common.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__operation_common.cpp
@@ -436,9 +436,9 @@ void NTableState::UpdatePartitioningForCopyTable(TOperationId operationId, TTxSt
TShardInfo datashardInfo = TShardInfo::DataShardInfo(operationId.GetTxId(), txState.TargetPathId);
datashardInfo.BindedChannels = channelsBinding;
- context.SS->SetPartitioning(txState.TargetPathId, dstTableInfo,
- ApplyPartitioningCopyTable(datashardInfo, srcTableInfo, txState, context.SS));
-
+ context.SS->SetPartitioning(txState.TargetPathId, dstTableInfo,
+ ApplyPartitioningCopyTable(datashardInfo, srcTableInfo, txState, context.SS));
+
ui32 newShardCout = dstTableInfo->GetPartitions().size();
dstPath->SetShardsInside(newShardCout);
diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_common.h b/ydb/core/tx/schemeshard/schemeshard__operation_common.h
index 5a3da32cade..f3bb8fca029 100644
--- a/ydb/core/tx/schemeshard/schemeshard__operation_common.h
+++ b/ydb/core/tx/schemeshard/schemeshard__operation_common.h
@@ -317,16 +317,16 @@ public:
context.OnComplete.BindMsgToPipe(OperationId, tabletID, idx, event);
break;
}
- case ETabletType::SysViewProcessor: {
- LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
- "Send configure request to sysview processor: " << tabletID <<
- " opId: " << OperationId <<
- " schemeshard: " << ssId);
- auto event = new NSysView::TEvSysView::TEvConfigureProcessor(path.PathString());
- shard.Operation = TTxState::ConfigureParts;
- context.OnComplete.BindMsgToPipe(OperationId, tabletID, idx, event);
- break;
- }
+ case ETabletType::SysViewProcessor: {
+ LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
+ "Send configure request to sysview processor: " << tabletID <<
+ " opId: " << OperationId <<
+ " schemeshard: " << ssId);
+ auto event = new NSysView::TEvSysView::TEvConfigureProcessor(path.PathString());
+ shard.Operation = TTxState::ConfigureParts;
+ context.OnComplete.BindMsgToPipe(OperationId, tabletID, idx, event);
+ break;
+ }
case ETabletType::SchemeShard: {
auto event = new TEvSchemeShard::TEvInitTenantSchemeShard(ui64(ssId),
pathId.LocalPathId, path.PathString(),
@@ -622,9 +622,9 @@ public:
case ETabletType::Hive:
context.SS->TabletCounters->Simple()[COUNTER_SUB_DOMAIN_HIVE_COUNT].Add(1);
break;
- case ETabletType::SysViewProcessor:
- context.SS->TabletCounters->Simple()[COUNTER_SYS_VIEW_PROCESSOR_COUNT].Add(1);
- break;
+ case ETabletType::SysViewProcessor:
+ context.SS->TabletCounters->Simple()[COUNTER_SYS_VIEW_PROCESSOR_COUNT].Add(1);
+ break;
default:
break;
}
diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp
index 50045225298..a7ab5a79e26 100644
--- a/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp
@@ -122,7 +122,7 @@ void ApplyPartitioning(TTxId txId,
partitions[i].ShardIdx = idx;
}
- ss->SetPartitioning(pathId, tableInfo, std::move(partitions));
+ ss->SetPartitioning(pathId, tableInfo, std::move(partitions));
}
diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_split_merge.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_split_merge.cpp
index 645d7f37e19..bbec44fc46e 100644
--- a/ydb/core/tx/schemeshard/schemeshard__operation_split_merge.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__operation_split_merge.cpp
@@ -272,7 +272,7 @@ public:
// Delete the whole old partitioning and persist the whole new partitionig as the indexes have changed
context.SS->DeleteTablePartitioning(db, tableId, tableInfo);
- context.SS->SetPartitioning(tableId, tableInfo, std::move(newPartitioning));
+ context.SS->SetPartitioning(tableId, tableInfo, std::move(newPartitioning));
context.SS->PersistTablePartitioning(db, tableId, tableInfo);
context.SS->PersistTablePartitionStats(db, tableId, tableInfo);
context.SS->TabletCounters->Simple()[COUNTER_TABLE_SHARD_ACTIVE_COUNT].Sub(allSrcShardIdxs.size());
diff --git a/ydb/core/tx/schemeshard/schemeshard__sync_update_tenants.cpp b/ydb/core/tx/schemeshard/schemeshard__sync_update_tenants.cpp
index d04be460803..a81218a2160 100644
--- a/ydb/core/tx/schemeshard/schemeshard__sync_update_tenants.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__sync_update_tenants.cpp
@@ -134,47 +134,47 @@ struct TSchemeShard::TTxUpdateTenant : public TSchemeShard::TRwTxBase {
MakeSync();
}
- auto addPrivateShard = [&] (TTabletId tabletId, TTabletTypes::EType tabletType) {
+ auto addPrivateShard = [&] (TTabletId tabletId, TTabletTypes::EType tabletType) {
const auto shardIdx = Self->RegisterShardInfo(
TShardInfo(InvalidTxId, Self->RootPathId(), tabletType)
.WithTabletID(tabletId));
- Self->PersistUpdateNextShardIdx(db);
+ Self->PersistUpdateNextShardIdx(db);
Self->PersistShardMapping(db, shardIdx, tabletId, Self->RootPathId(), InvalidTxId, tabletType);
- Y_VERIFY(record.GetSubdomainVersion() >= subdomain->GetVersion());
- if (record.GetSubdomainVersion() > subdomain->GetVersion()) {
- subdomain->SetVersion(record.GetSubdomainVersion());
- }
-
+ Y_VERIFY(record.GetSubdomainVersion() >= subdomain->GetVersion());
+ if (record.GetSubdomainVersion() > subdomain->GetVersion()) {
+ subdomain->SetVersion(record.GetSubdomainVersion());
+ }
+
subdomain->AddPrivateShard(shardIdx);
subdomain->AddInternalShard(shardIdx);
- subdomain->Initialize(Self->ShardInfos);
- Self->PersistSubDomain(db, Self->RootPathId(), *subdomain);
+ subdomain->Initialize(Self->ShardInfos);
+ Self->PersistSubDomain(db, Self->RootPathId(), *subdomain);
path->IncShardsInside(1);
- SideEffects.PublishToSchemeBoard(InvalidOperationId, Self->RootPathId());
- MakeSync();
- };
-
- if (record.HasTenantHive()) {
- TTabletId tenantHive = TTabletId(record.GetTenantHive());
- if (!subdomain->GetTenantHiveID()) {
- addPrivateShard(tenantHive, ETabletType::Hive);
- }
- Y_VERIFY(tenantHive == subdomain->GetTenantHiveID());
- }
-
- if (record.HasTenantSysViewProcessor()) {
- TTabletId tenantSVP = TTabletId(record.GetTenantSysViewProcessor());
- if (!subdomain->GetTenantSysViewProcessorID()) {
- addPrivateShard(tenantSVP, ETabletType::SysViewProcessor);
+ SideEffects.PublishToSchemeBoard(InvalidOperationId, Self->RootPathId());
+ MakeSync();
+ };
+
+ if (record.HasTenantHive()) {
+ TTabletId tenantHive = TTabletId(record.GetTenantHive());
+ if (!subdomain->GetTenantHiveID()) {
+ addPrivateShard(tenantHive, ETabletType::Hive);
}
- Y_VERIFY(tenantSVP == subdomain->GetTenantSysViewProcessorID());
+ Y_VERIFY(tenantHive == subdomain->GetTenantHiveID());
}
+ if (record.HasTenantSysViewProcessor()) {
+ TTabletId tenantSVP = TTabletId(record.GetTenantSysViewProcessor());
+ if (!subdomain->GetTenantSysViewProcessorID()) {
+ addPrivateShard(tenantSVP, ETabletType::SysViewProcessor);
+ }
+ Y_VERIFY(tenantSVP == subdomain->GetTenantSysViewProcessorID());
+ }
+
if (record.HasUpdateTenantRootACL()) {
// KIKIMR-10699: transfer tenants root ACL from GSS to the TSS
// here TSS sees the ACL from GSS
diff --git a/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp b/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp
index b2e7db88b79..68c53c94fc6 100644
--- a/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp
@@ -22,44 +22,44 @@ static ui64 GetIops(const T& c) {
}
void TSchemeShard::Handle(NSysView::TEvSysView::TEvGetPartitionStats::TPtr& ev, const TActorContext& ctx) {
- ctx.Send(ev->Forward(SysPartitionStatsCollector));
-}
-
+ ctx.Send(ev->Forward(SysPartitionStatsCollector));
+}
+
auto TSchemeShard::BuildStatsForCollector(TPathId pathId, TShardIdx shardIdx, TTabletId datashardId,
- TMaybe<ui32> nodeId, TMaybe<ui64> startTime, const TTableInfo::TPartitionStats& stats)
-{
- auto ev = MakeHolder<NSysView::TEvSysView::TEvSendPartitionStats>(
- GetDomainKey(pathId), pathId, std::make_pair(ui64(shardIdx.GetOwnerId()), ui64(shardIdx.GetLocalId())));
-
- auto& sysStats = ev->Stats;
- sysStats.SetDataSize(stats.DataSize);
- sysStats.SetRowCount(stats.RowCount);
- sysStats.SetIndexSize(stats.IndexSize);
+ TMaybe<ui32> nodeId, TMaybe<ui64> startTime, const TTableInfo::TPartitionStats& stats)
+{
+ auto ev = MakeHolder<NSysView::TEvSysView::TEvSendPartitionStats>(
+ GetDomainKey(pathId), pathId, std::make_pair(ui64(shardIdx.GetOwnerId()), ui64(shardIdx.GetLocalId())));
+
+ auto& sysStats = ev->Stats;
+ sysStats.SetDataSize(stats.DataSize);
+ sysStats.SetRowCount(stats.RowCount);
+ sysStats.SetIndexSize(stats.IndexSize);
sysStats.SetCPUCores(std::min(stats.GetCurrentRawCpuUsage() / 1000000., 1.0));
sysStats.SetTabletId(ui64(datashardId));
- sysStats.SetAccessTime(stats.LastAccessTime.MilliSeconds());
- sysStats.SetUpdateTime(stats.LastUpdateTime.MilliSeconds());
- sysStats.SetInFlightTxCount(stats.InFlightTxCount);
- sysStats.SetRowUpdates(stats.RowUpdates);
- sysStats.SetRowDeletes(stats.RowDeletes);
- sysStats.SetRowReads(stats.RowReads);
- sysStats.SetRangeReads(stats.RangeReads);
- sysStats.SetRangeReadRows(stats.RangeReadRows);
- sysStats.SetImmediateTxCompleted(stats.ImmediateTxCompleted);
- sysStats.SetPlannedTxCompleted(stats.PlannedTxCompleted);
- sysStats.SetTxRejectedByOverload(stats.TxRejectedByOverload);
- sysStats.SetTxRejectedBySpace(stats.TxRejectedBySpace);
-
- if (nodeId) {
- sysStats.SetNodeId(*nodeId);
- }
- if (startTime) {
- sysStats.SetStartTime(*startTime);
- }
-
+ sysStats.SetAccessTime(stats.LastAccessTime.MilliSeconds());
+ sysStats.SetUpdateTime(stats.LastUpdateTime.MilliSeconds());
+ sysStats.SetInFlightTxCount(stats.InFlightTxCount);
+ sysStats.SetRowUpdates(stats.RowUpdates);
+ sysStats.SetRowDeletes(stats.RowDeletes);
+ sysStats.SetRowReads(stats.RowReads);
+ sysStats.SetRangeReads(stats.RangeReads);
+ sysStats.SetRangeReadRows(stats.RangeReadRows);
+ sysStats.SetImmediateTxCompleted(stats.ImmediateTxCompleted);
+ sysStats.SetPlannedTxCompleted(stats.PlannedTxCompleted);
+ sysStats.SetTxRejectedByOverload(stats.TxRejectedByOverload);
+ sysStats.SetTxRejectedBySpace(stats.TxRejectedBySpace);
+
+ if (nodeId) {
+ sysStats.SetNodeId(*nodeId);
+ }
+ if (startTime) {
+ sysStats.SetStartTime(*startTime);
+ }
+
return ev;
-}
-
+}
+
class TTxStorePartitionStats: public NTabletFlatExecutor::TTransactionBase<TSchemeShard> {
TEvDataShard::TEvPeriodicTableStats::TPtr Ev;
@@ -219,17 +219,17 @@ bool TTxStorePartitionStats::Execute(TTransactionContext& txc, const TActorConte
Self->PersistTablePartitionStats(db, tableId, shardIdx, table);
if (AppData(ctx)->FeatureFlags.GetEnableSystemViews()) {
- TMaybe<ui32> nodeId;
- if (rec.HasNodeId()) {
- nodeId = rec.GetNodeId();
- }
- TMaybe<ui64> startTime;
- if (rec.HasStartTime()) {
- startTime = rec.GetStartTime();
- }
+ TMaybe<ui32> nodeId;
+ if (rec.HasNodeId()) {
+ nodeId = rec.GetNodeId();
+ }
+ TMaybe<ui64> startTime;
+ if (rec.HasStartTime()) {
+ startTime = rec.GetStartTime();
+ }
StatsCollectorEv = Self->BuildStatsForCollector(tableId, shardIdx, datashardId, nodeId, startTime, newStats);
- }
-
+ }
+
const auto& shardToPartition = table->GetShard2PartitionIdx();
if (table->IsTTLEnabled() && shardToPartition.contains(shardIdx)) {
const ui64 partitionIdx = shardToPartition.at(shardIdx);
diff --git a/ydb/core/tx/schemeshard/schemeshard_domain_links.cpp b/ydb/core/tx/schemeshard/schemeshard_domain_links.cpp
index b50aaa509a3..3011e042296 100644
--- a/ydb/core/tx/schemeshard/schemeshard_domain_links.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard_domain_links.cpp
@@ -26,7 +26,7 @@ THolder<TEvSchemeShard::TEvSyncTenantSchemeShard> TParentDomainLink::MakeSyncMsg
Self->ParentDomainEffectiveACLVersion,
rootSubdomain->GetVersion(),
rootPath->UserAttrs->AlterVersion,
- ui64(rootSubdomain->GetTenantHiveID()),
+ ui64(rootSubdomain->GetTenantHiveID()),
ui64(rootSubdomain->GetTenantSysViewProcessorID()),
rootPath->ACL);
}
@@ -106,7 +106,7 @@ void TSubDomainsLinks::TLink::Out(IOutputStream& stream) const {
<< ", SubdomainVersion: " << SubdomainVersion
<< ", UserAttributesVersion: " << UserAttributesVersion
<< ", TenantHive: " << TenantHive
- << ", TenantSysViewProcessor: " << TenantSysViewProcessor
+ << ", TenantSysViewProcessor: " << TenantSysViewProcessor
<< ", TenantRootACL: " << TenantRootACL
<< "}";
}
@@ -119,8 +119,8 @@ TSubDomainsLinks::TLink::TLink(const NKikimrScheme::TEvSyncTenantSchemeShard &re
, SubdomainVersion(record.GetSubdomainVersion())
, UserAttributesVersion(record.GetUserAttributesVersion())
, TenantHive(record.HasTenantHive() ? TTabletId(record.GetTenantHive()) : InvalidTabletId)
- , TenantSysViewProcessor(record.HasTenantSysViewProcessor() ?
- TTabletId(record.GetTenantSysViewProcessor()) : InvalidTabletId)
+ , TenantSysViewProcessor(record.HasTenantSysViewProcessor() ?
+ TTabletId(record.GetTenantSysViewProcessor()) : InvalidTabletId)
, TenantRootACL(record.GetTenantRootACL())
{}
diff --git a/ydb/core/tx/schemeshard/schemeshard_domain_links.h b/ydb/core/tx/schemeshard/schemeshard_domain_links.h
index d9cb6133c42..0acbc4c9d4d 100644
--- a/ydb/core/tx/schemeshard/schemeshard_domain_links.h
+++ b/ydb/core/tx/schemeshard/schemeshard_domain_links.h
@@ -40,7 +40,7 @@ public:
ui64 SubdomainVersion = 0;
ui64 UserAttributesVersion = 0;
TTabletId TenantHive = InvalidTabletId;
- TTabletId TenantSysViewProcessor = InvalidTabletId;
+ TTabletId TenantSysViewProcessor = InvalidTabletId;
TString TenantRootACL;
TLink() = default;
diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp
index 0715543a229..bf193ff06f9 100644
--- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp
@@ -990,12 +990,12 @@ TSubDomainInfo::TPtr TSchemeShard::ResolveDomainInfo(TPathElement::TPtr pathEl)
}
TPathId TSchemeShard::GetDomainKey(TPathId pathId) const {
- auto domainId = ResolveDomainId(pathId);
- TPathElement::TPtr domainElement = PathsById.at(domainId);
- Y_VERIFY(domainElement);
- return domainElement->IsRoot() ? ParentDomainId : domainId;
-}
-
+ auto domainId = ResolveDomainId(pathId);
+ TPathElement::TPtr domainElement = PathsById.at(domainId);
+ Y_VERIFY(domainElement);
+ return domainElement->IsRoot() ? ParentDomainId : domainId;
+}
+
const NKikimrSubDomains::TProcessingParams &TSchemeShard::SelectProcessingPrarams(TPathId id) const {
TPathElement::TPtr item = PathsById.at(id);
return SelectProcessingPrarams(item);
@@ -3248,11 +3248,11 @@ void TSchemeShard::PersistRemoveTable(NIceDb::TNiceDb& db, TPathId pathId, const
Tables.erase(pathId);
DecrementPathDbRefCount(pathId, "remove table");
-
+
if (AppData()->FeatureFlags.GetEnableSystemViews()) {
- auto ev = MakeHolder<NSysView::TEvSysView::TEvRemoveTable>(GetDomainKey(pathId), pathId);
- Send(SysPartitionStatsCollector, ev.Release());
- }
+ auto ev = MakeHolder<NSysView::TEvSysView::TEvRemoveTable>(GetDomainKey(pathId), pathId);
+ Send(SysPartitionStatsCollector, ev.Release());
+ }
}
void TSchemeShard::PersistRemoveTableIndex(NIceDb::TNiceDb &db, TPathId pathId)
@@ -3699,8 +3699,8 @@ bool TSchemeShard::IsShemeShardConfigured() const {
void TSchemeShard::Die(const TActorContext &ctx) {
ctx.Send(SchemeBoardPopulator, new TEvents::TEvPoisonPill());
ctx.Send(TxAllocatorClient, new TEvents::TEvPoisonPill());
- ctx.Send(SysPartitionStatsCollector, new TEvents::TEvPoisonPill());
-
+ ctx.Send(SysPartitionStatsCollector, new TEvents::TEvPoisonPill());
+
ShardDeleter.Shutdown(ctx);
ParentDomainLink.Shutdown(ctx);
@@ -3773,8 +3773,8 @@ void TSchemeShard::OnActivateExecutor(const TActorContext &ctx) {
TxAllocatorClient = RegisterWithSameMailbox(CreateTxAllocatorClient(CollectTxAllocators(appData)));
- SysPartitionStatsCollector = Register(NSysView::CreatePartitionStatsCollector().Release());
-
+ SysPartitionStatsCollector = Register(NSysView::CreatePartitionStatsCollector().Release());
+
SplitSettings.Register(appData->Icb);
Executor()->RegisterExternalTabletCounters(TabletCountersPtr);
@@ -3915,8 +3915,8 @@ void TSchemeShard::StateWork(STFUNC_SIG) {
HFuncTraced(TEvPrivate::TEvServerlessStorageBilling, Handle);
- HFuncTraced(NSysView::TEvSysView::TEvGetPartitionStats, Handle);
-
+ HFuncTraced(NSysView::TEvSysView::TEvGetPartitionStats, Handle);
+
HFuncTraced(TEvSubDomain::TEvConfigureStatus, Handle);
HFuncTraced(TEvSchemeShard::TEvInitTenantSchemeShard, Handle);
HFuncTraced(TEvSchemeShard::TEvInitTenantSchemeShardResult, Handle);
@@ -5766,19 +5766,19 @@ bool TSchemeShard::FillUniformPartitioning(TVector<TString>& rangeEnds, ui32 key
void TSchemeShard::SetPartitioning(TPathId pathId, TTableInfo::TPtr tableInfo, TVector<TTableShardInfo>&& newPartitioning) {
if (AppData()->FeatureFlags.GetEnableSystemViews()) {
- TVector<std::pair<ui64, ui64>> shardIndices;
- shardIndices.reserve(newPartitioning.size());
- for (auto& info : newPartitioning) {
- shardIndices.push_back(
- std::make_pair(ui64(info.ShardIdx.GetOwnerId()), ui64(info.ShardIdx.GetLocalId())));
- }
-
- auto path = TPath::Init(pathId, this);
- auto ev = MakeHolder<NSysView::TEvSysView::TEvSetPartitioning>(GetDomainKey(pathId), pathId, path.PathString());
- ev->ShardIndices.swap(shardIndices);
- Send(SysPartitionStatsCollector, ev.Release());
- }
-
+ TVector<std::pair<ui64, ui64>> shardIndices;
+ shardIndices.reserve(newPartitioning.size());
+ for (auto& info : newPartitioning) {
+ shardIndices.push_back(
+ std::make_pair(ui64(info.ShardIdx.GetOwnerId()), ui64(info.ShardIdx.GetLocalId())));
+ }
+
+ auto path = TPath::Init(pathId, this);
+ auto ev = MakeHolder<NSysView::TEvSysView::TEvSetPartitioning>(GetDomainKey(pathId), pathId, path.PathString());
+ ev->ShardIndices.swap(shardIndices);
+ Send(SysPartitionStatsCollector, ev.Release());
+ }
+
if (!tableInfo->IsBackup) {
for (const auto& p: newPartitioning) {
ui64 searchHeight = 0;
@@ -5793,9 +5793,9 @@ void TSchemeShard::SetPartitioning(TPathId pathId, TTableInfo::TPtr tableInfo, T
}
}
- tableInfo->SetPartitioning(std::move(newPartitioning));
-}
-
+ tableInfo->SetPartitioning(std::move(newPartitioning));
+}
+
void TSchemeShard::FillAsyncIndexInfo(const TPathId& tableId, NKikimrTxDataShard::TFlatSchemeTransaction& tx) {
Y_VERIFY(PathsById.contains(tableId));
diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.h b/ydb/core/tx/schemeshard/schemeshard_impl.h
index e1a1d08b9a6..fcbcffa2916 100644
--- a/ydb/core/tx/schemeshard/schemeshard_impl.h
+++ b/ydb/core/tx/schemeshard/schemeshard_impl.h
@@ -210,7 +210,7 @@ public:
TAutoPtr<TSelfPinger> SelfPinger;
TActorId SysPartitionStatsCollector;
-
+
TSet<TPathId> CleanDroppedPathsCandidates;
TSet<TPathId> CleanDroppedSubDomainsCandidates;
bool CleanDroppedPathsInFly = false;
@@ -368,8 +368,8 @@ public:
TSubDomainInfo::TPtr ResolveDomainInfo(TPathId pathId) const;
TSubDomainInfo::TPtr ResolveDomainInfo(TPathElement::TPtr pathEl) const;
- TPathId GetDomainKey(TPathId pathId) const;
-
+ TPathId GetDomainKey(TPathId pathId) const;
+
const NKikimrSubDomains::TProcessingParams& SelectProcessingPrarams(TPathId id) const;
const NKikimrSubDomains::TProcessingParams& SelectProcessingPrarams(TPathElement::TPtr pathEl) const;
@@ -412,9 +412,9 @@ public:
void DoShardsDeletion(const THashSet<TShardIdx>& shardIdx, const TActorContext& ctx);
- void SetPartitioning(TPathId pathId, TTableInfo::TPtr tableInfo, TVector<TTableShardInfo>&& newPartitioning);
+ void SetPartitioning(TPathId pathId, TTableInfo::TPtr tableInfo, TVector<TTableShardInfo>&& newPartitioning);
auto BuildStatsForCollector(TPathId tableId, TShardIdx shardIdx, TTabletId datashardId,
- TMaybe<ui32> nodeId, TMaybe<ui64> startTime, const TTableInfo::TPartitionStats& stats);
+ TMaybe<ui32> nodeId, TMaybe<ui64> startTime, const TTableInfo::TPartitionStats& stats);
bool ReadSysValue(NIceDb::TNiceDb& db, ui64 sysTag, TString& value, TString defValue = TString());
bool ReadSysValue(NIceDb::TNiceDb& db, ui64 sysTag, ui64& value, ui64 defVal = 0);
@@ -818,8 +818,8 @@ public:
void Handle(TEvPrivate::TEvRunConditionalErase::TPtr& ev, const TActorContext& ctx);
void Handle(TEvDataShard::TEvConditionalEraseRowsResponse::TPtr& ev, const TActorContext& ctx);
- void Handle(NSysView::TEvSysView::TEvGetPartitionStats::TPtr& ev, const TActorContext& ctx);
-
+ void Handle(NSysView::TEvSysView::TEvGetPartitionStats::TPtr& ev, const TActorContext& ctx);
+
void ScheduleServerlessStorageBilling(const TActorContext& ctx);
void Handle(TEvPrivate::TEvServerlessStorageBilling::TPtr& ev, const TActorContext& ctx);
diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h
index 2e5710c3d3e..4bb201c7c2e 100644
--- a/ydb/core/tx/schemeshard/schemeshard_info_types.h
+++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h
@@ -1264,13 +1264,13 @@ struct TSubDomainInfo: TSimpleRefCount<TSubDomainInfo> {
return TTabletId(ProcessingParams.GetHive());
}
- TTabletId GetTenantSysViewProcessorID() const {
- if (!ProcessingParams.HasSysViewProcessor()) {
- return InvalidTabletId;
- }
- return TTabletId(ProcessingParams.GetSysViewProcessor());
- }
-
+ TTabletId GetTenantSysViewProcessorID() const {
+ if (!ProcessingParams.HasSysViewProcessor()) {
+ return InvalidTabletId;
+ }
+ return TTabletId(ProcessingParams.GetSysViewProcessor());
+ }
+
ui64 GetPathsInside() const {
return PathsInsideCount;
}
@@ -1606,13 +1606,13 @@ struct TSubDomainInfo: TSimpleRefCount<TSubDomainInfo> {
ProcessingParams.SetHive(ui64(hives.front()));
SetSharedHive(InvalidTabletId); // set off shared hive when our own hive has found
}
-
- ProcessingParams.ClearSysViewProcessor();
- TVector<TTabletId> sysViewProcessors = FilterTablets(ETabletType::SysViewProcessor, allShards);
- Y_VERIFY_S(sysViewProcessors.size() <= 1, "size was: " << sysViewProcessors.size());
- if (sysViewProcessors.size()) {
- ProcessingParams.SetSysViewProcessor(ui64(sysViewProcessors.front()));
- }
+
+ ProcessingParams.ClearSysViewProcessor();
+ TVector<TTabletId> sysViewProcessors = FilterTablets(ETabletType::SysViewProcessor, allShards);
+ Y_VERIFY_S(sysViewProcessors.size() <= 1, "size was: " << sysViewProcessors.size());
+ if (sysViewProcessors.size()) {
+ ProcessingParams.SetSysViewProcessor(ui64(sysViewProcessors.front()));
+ }
}
void InitializeAsGlobal(NKikimrSubDomains::TProcessingParams&& processingParams) {
diff --git a/ydb/core/tx/schemeshard/schemeshard_path.cpp b/ydb/core/tx/schemeshard/schemeshard_path.cpp
index 7535ee673c8..74d299fbdee 100644
--- a/ydb/core/tx/schemeshard/schemeshard_path.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard_path.cpp
@@ -4,7 +4,7 @@
#include <ydb/core/base/path.h>
#include <ydb/core/sys_view/common/path.h>
-
+
namespace NKikimr {
namespace NSchemeShard {
@@ -1617,11 +1617,11 @@ bool TPath::IsValidLeafName(TString& explain) const {
}
if (AppData()->FeatureFlags.GetEnableSystemViews() && leaf == NSysView::SysPathName) {
- explain += TStringBuilder()
- << "path part '" << NSysView::SysPathName << "' is reserved by the system";
- return false;
- }
-
+ explain += TStringBuilder()
+ << "path part '" << NSysView::SysPathName << "' is reserved by the system";
+ return false;
+ }
+
auto brokenAt = PathPartBrokenAt(leaf, schemeLimits.ExtraPathSymbolsAllowed);
if (brokenAt != leaf.end()) {
explain += TStringBuilder()
diff --git a/ydb/core/tx/schemeshard/schemeshard_path_element.h b/ydb/core/tx/schemeshard/schemeshard_path_element.h
index 9912880c057..e9131da453b 100644
--- a/ydb/core/tx/schemeshard/schemeshard_path_element.h
+++ b/ydb/core/tx/schemeshard/schemeshard_path_element.h
@@ -2,7 +2,7 @@
#include "schemeshard_types.h"
#include "schemeshard_effective_acl.h"
-#include "schemeshard_user_attr_limits.h"
+#include "schemeshard_user_attr_limits.h"
#include <ydb/core/protos/flat_scheme_op.pb.h>
#include <ydb/library/aclib/aclib.h>
@@ -170,7 +170,7 @@ struct TUserAttributes: TSimpleRefCount<TUserAttributes> {
bool CheckLimits(TString& errStr) const {
const ui64 bytes = Bytes();
- if (bytes > TUserAttributesLimits::MaxBytes) {
+ if (bytes > TUserAttributesLimits::MaxBytes) {
errStr = Sprintf("UserArttibutes::CheckLimits: user attributes too big: %" PRIu64, bytes);
return false;
}
@@ -184,13 +184,13 @@ struct TUserAttributes: TSimpleRefCount<TUserAttributes> {
return true;
}
- if (name.size() > TUserAttributesLimits::MaxNameLen) {
+ if (name.size() > TUserAttributesLimits::MaxNameLen) {
errStr = Sprintf("UserArttibutes: name too long, name# '%s' value# '%s'"
, name.c_str(), value.c_str());
return false;
}
- if (value.size() > TUserAttributesLimits::MaxValueLen) {
+ if (value.size() > TUserAttributesLimits::MaxValueLen) {
errStr = Sprintf("UserArttibutes: value too long, name# '%s' value# '%s'"
, name.c_str(), value.c_str());
return false;
diff --git a/ydb/core/tx/schemeshard/schemeshard_user_attr_limits.h b/ydb/core/tx/schemeshard/schemeshard_user_attr_limits.h
index aeddaa22be6..b28102fff21 100644
--- a/ydb/core/tx/schemeshard/schemeshard_user_attr_limits.h
+++ b/ydb/core/tx/schemeshard/schemeshard_user_attr_limits.h
@@ -1,15 +1,15 @@
-#pragma once
-
-#include <util/system/types.h>
-
-namespace NKikimr {
+#pragma once
+
+#include <util/system/types.h>
+
+namespace NKikimr {
namespace NSchemeShard {
-
-struct TUserAttributesLimits {
- static constexpr ui32 MaxNameLen = 100;
- static constexpr ui32 MaxValueLen = 4 * 1024;
- static constexpr ui32 MaxBytes = 10 * 1024;
-};
-
-}
-}
+
+struct TUserAttributesLimits {
+ static constexpr ui32 MaxNameLen = 100;
+ static constexpr ui32 MaxValueLen = 4 * 1024;
+ static constexpr ui32 MaxBytes = 10 * 1024;
+};
+
+}
+}
diff --git a/ydb/core/tx/schemeshard/ut_base.cpp b/ydb/core/tx/schemeshard/ut_base.cpp
index a76dcf9d5bb..2df6f0b712b 100644
--- a/ydb/core/tx/schemeshard/ut_base.cpp
+++ b/ydb/core/tx/schemeshard/ut_base.cpp
@@ -9303,22 +9303,22 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) {
env.TestWaitNotification(runtime, txId);
env.TestWaitTabletDeletion(runtime, xrange(TTestTxConfig::FakeHiveTablets+2, TTestTxConfig::FakeHiveTablets+10));
}
-
- Y_UNIT_TEST(RejectSystemViewPath) {
- TTestBasicRuntime runtime;
- TTestEnv env(runtime, 4, true, &CreateFlatTxSchemeShard, true);
- ui64 txId = 100;
-
- TestCreateTable(runtime, ++txId, "/MyRoot",
- "Name: \".sys\""
- "Columns { Name: \"key\" Type: \"Uint32\"}"
- "KeyColumnNames: [\"key\"]",
+
+ Y_UNIT_TEST(RejectSystemViewPath) {
+ TTestBasicRuntime runtime;
+ TTestEnv env(runtime, 4, true, &CreateFlatTxSchemeShard, true);
+ ui64 txId = 100;
+
+ TestCreateTable(runtime, ++txId, "/MyRoot",
+ "Name: \".sys\""
+ "Columns { Name: \"key\" Type: \"Uint32\"}"
+ "KeyColumnNames: [\"key\"]",
{NKikimrScheme::StatusSchemeError});
-
+
TestMkDir(runtime, ++txId, "/MyRoot", ".sys", {NKikimrScheme::StatusSchemeError});
TestMkDir(runtime, ++txId, "/MyRoot", ".sys/partition_stats", {NKikimrScheme::StatusSchemeError});
TestMkDir(runtime, ++txId, "/MyRoot", "DirA/.sys/partition_stats", {NKikimrScheme::StatusSchemeError});
- }
+ }
Y_UNIT_TEST(DocumentApiVersion) {
TTestBasicRuntime runtime;
diff --git a/ydb/core/tx/schemeshard/ut_extsubdomain.cpp b/ydb/core/tx/schemeshard/ut_extsubdomain.cpp
index 0fa6a27bce0..c895f6bacfe 100644
--- a/ydb/core/tx/schemeshard/ut_extsubdomain.cpp
+++ b/ydb/core/tx/schemeshard/ut_extsubdomain.cpp
@@ -674,23 +674,23 @@ Y_UNIT_TEST_SUITE(TSchemeShardExtSubDomainTest) {
UNIT_ASSERT(!CheckLocalRowExists(runtime, TTestTxConfig::SchemeShard, "SubDomains", "PathId", 2));
UNIT_ASSERT(!CheckLocalRowExists(runtime, TTestTxConfig::SchemeShard, "Paths", "Id", 2));
}
-
- Y_UNIT_TEST(SysViewProcessorSync) {
- TTestBasicRuntime runtime;
- TTestEnv env(runtime);
- ui64 txId = 100;
-
+
+ Y_UNIT_TEST(SysViewProcessorSync) {
+ TTestBasicRuntime runtime;
+ TTestEnv env(runtime);
+ ui64 txId = 100;
+
NSchemeShard::TSchemeLimits lowLimits;
lowLimits.MaxShardsInPath = 3;
SetSchemeshardSchemaLimits(runtime, lowLimits);
- TestCreateExtSubDomain(runtime, ++txId, "/MyRoot",
- "Name: \"USER_0\"");
-
+ TestCreateExtSubDomain(runtime, ++txId, "/MyRoot",
+ "Name: \"USER_0\"");
+
// check that limits have a power, try create 4 shards
- TestAlterExtSubDomain(runtime, ++txId, "/MyRoot",
- "Name: \"USER_0\" "
- "PlanResolution: 50 "
+ TestAlterExtSubDomain(runtime, ++txId, "/MyRoot",
+ "Name: \"USER_0\" "
+ "PlanResolution: 50 "
"Coordinators: 2 "
"Mediators: 1 "
"TimeCastBucketsPerMediator: 2 "
@@ -701,68 +701,68 @@ Y_UNIT_TEST_SUITE(TSchemeShardExtSubDomainTest) {
TestAlterExtSubDomain(runtime, ++txId, "/MyRoot",
"Name: \"USER_0\" "
"PlanResolution: 50 "
- "Coordinators: 1 "
- "Mediators: 1 "
- "TimeCastBucketsPerMediator: 2 "
+ "Coordinators: 1 "
+ "Mediators: 1 "
+ "TimeCastBucketsPerMediator: 2 "
"ExternalSchemeShard: true "
"StoragePools { "
" Name: \"/dc-1/users/tenant-1:hdd\" "
" Kind: \"hdd\" "
"} ");
- env.TestWaitNotification(runtime, {txId, txId - 1});
-
+ env.TestWaitNotification(runtime, {txId, txId - 1});
+
lowLimits.MaxShardsInPath = 2;
SetSchemeshardSchemaLimits(runtime, lowLimits);
// one more, but for free
- TestAlterExtSubDomain(runtime, ++txId, "/MyRoot",
- "Name: \"USER_0\" "
- "ExternalSysViewProcessor: true ");
-
- env.TestWaitNotification(runtime, txId);
-
- ui64 tenantSchemeShard = 0;
- ui64 tenantSVP = 0;
- TestDescribeResult(DescribePath(runtime, "/MyRoot/USER_0"),
- {NLs::PathExist,
- NLs::IsExternalSubDomain("USER_0"),
- NLs::ExtractTenantSchemeshard(&tenantSchemeShard),
- NLs::ExtractTenantSysViewProcessor(&tenantSVP)});
-
- UNIT_ASSERT(tenantSchemeShard != 0
- && tenantSchemeShard != (ui64)-1
+ TestAlterExtSubDomain(runtime, ++txId, "/MyRoot",
+ "Name: \"USER_0\" "
+ "ExternalSysViewProcessor: true ");
+
+ env.TestWaitNotification(runtime, txId);
+
+ ui64 tenantSchemeShard = 0;
+ ui64 tenantSVP = 0;
+ TestDescribeResult(DescribePath(runtime, "/MyRoot/USER_0"),
+ {NLs::PathExist,
+ NLs::IsExternalSubDomain("USER_0"),
+ NLs::ExtractTenantSchemeshard(&tenantSchemeShard),
+ NLs::ExtractTenantSysViewProcessor(&tenantSVP)});
+
+ UNIT_ASSERT(tenantSchemeShard != 0
+ && tenantSchemeShard != (ui64)-1
&& tenantSchemeShard != TTestTxConfig::SchemeShard);
-
- UNIT_ASSERT(tenantSVP != 0 && tenantSVP != (ui64)-1);
-
- ui64 tenantSVPOnTSS = 0;
- TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/USER_0"),
- {NLs::PathExist,
- NLs::ExtractTenantSysViewProcessor(&tenantSVPOnTSS)});
-
- UNIT_ASSERT_EQUAL(tenantSVP, tenantSVPOnTSS);
-
+
+ UNIT_ASSERT(tenantSVP != 0 && tenantSVP != (ui64)-1);
+
+ ui64 tenantSVPOnTSS = 0;
+ TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/USER_0"),
+ {NLs::PathExist,
+ NLs::ExtractTenantSysViewProcessor(&tenantSVPOnTSS)});
+
+ UNIT_ASSERT_EQUAL(tenantSVP, tenantSVPOnTSS);
+
RebootTablet(runtime, tenantSchemeShard, runtime.AllocateEdgeActor());
-
- TestCreateTable(runtime, tenantSchemeShard, ++txId, "/MyRoot/USER_0",
- "Name: \"table\""
- "Columns { Name: \"RowId\" Type: \"Uint64\"}"
- "Columns { Name: \"Value\" Type: \"Utf8\"}"
- "KeyColumnNames: [\"RowId\"]");
-
- env.TestWaitNotification(runtime, txId, tenantSchemeShard);
-
- TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/USER_0/table"),
- {NLs::PathExist});
-
+
+ TestCreateTable(runtime, tenantSchemeShard, ++txId, "/MyRoot/USER_0",
+ "Name: \"table\""
+ "Columns { Name: \"RowId\" Type: \"Uint64\"}"
+ "Columns { Name: \"Value\" Type: \"Utf8\"}"
+ "KeyColumnNames: [\"RowId\"]");
+
+ env.TestWaitNotification(runtime, txId, tenantSchemeShard);
+
+ TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/USER_0/table"),
+ {NLs::PathExist});
+
RebootTablet(runtime, tenantSchemeShard, runtime.AllocateEdgeActor());
-
- TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/USER_0"),
- {NLs::PathExist,
- NLs::ExtractTenantSysViewProcessor(&tenantSVPOnTSS)});
-
- UNIT_ASSERT_EQUAL(tenantSVP, tenantSVPOnTSS);
- }
+
+ TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/USER_0"),
+ {NLs::PathExist,
+ NLs::ExtractTenantSysViewProcessor(&tenantSVPOnTSS)});
+
+ UNIT_ASSERT_EQUAL(tenantSVP, tenantSVPOnTSS);
+ }
Y_UNIT_TEST(SchemeQuotas) {
TTestBasicRuntime runtime;
diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp
index c5d6c5a973a..8dd69c9c68b 100644
--- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp
+++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp
@@ -99,15 +99,15 @@ TCheckFunc ExtractTenantSchemeshard(ui64* tenantSchemeShardId) {
TCheckFunc ExtractTenantSysViewProcessor(ui64* tenantSVPId) {
return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) {
UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrScheme::StatusSuccess);
- const auto& pathDescr = record.GetPathDescription();
- UNIT_ASSERT(pathDescr.HasDomainDescription());
- const auto& domainDesc = pathDescr.GetDomainDescription();
- UNIT_ASSERT(domainDesc.HasProcessingParams());
- const auto& procParams = domainDesc.GetProcessingParams();
- *tenantSVPId = procParams.GetSysViewProcessor();
- };
-}
-
+ const auto& pathDescr = record.GetPathDescription();
+ UNIT_ASSERT(pathDescr.HasDomainDescription());
+ const auto& domainDesc = pathDescr.GetDomainDescription();
+ UNIT_ASSERT(domainDesc.HasProcessingParams());
+ const auto& procParams = domainDesc.GetProcessingParams();
+ *tenantSVPId = procParams.GetSysViewProcessor();
+ };
+}
+
void InExternalSubdomain(const NKikimrScheme::TEvDescribeSchemeResult& record) {
PathRedirected(record);
diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h
index 33022e42576..6c8c9eedfb9 100644
--- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h
+++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h
@@ -60,7 +60,7 @@ namespace NLs {
TCheckFunc IsExternalSubDomain(const TString& name);
void InExternalSubdomain(const NKikimrScheme::TEvDescribeSchemeResult& record);
TCheckFunc ExtractTenantSchemeshard(ui64* tenantSchemeShard);
- TCheckFunc ExtractTenantSysViewProcessor(ui64* tenantSVP);
+ TCheckFunc ExtractTenantSysViewProcessor(ui64* tenantSVP);
void NotFinished(const NKikimrScheme::TEvDescribeSchemeResult& record);
void Finished(const NKikimrScheme::TEvDescribeSchemeResult& record);
diff --git a/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp b/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp
index 9cfd8037b7e..430c5fabcbb 100644
--- a/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp
+++ b/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp
@@ -506,7 +506,7 @@ NSchemeShardUT_Private::TTestEnv::TTestEnv(TTestActorRuntime& runtime, const TTe
app.SetEnableProtoSourceIdInfo(opts.EnableProtoSourceIdInfo_);
app.SetEnableBackgroundCompaction(opts.EnableBackgroundCompaction_);
app.FeatureFlags.SetEnablePublicApiExternalBlobs(true);
-
+
for (const auto& sid : opts.SystemBackupSIDs_) {
app.AddSystemBackupSID(sid);
}
diff --git a/ydb/core/tx/schemeshard/ut_helpers/test_env.h b/ydb/core/tx/schemeshard/ut_helpers/test_env.h
index 72b934c570d..39128d54ea8 100644
--- a/ydb/core/tx/schemeshard/ut_helpers/test_env.h
+++ b/ydb/core/tx/schemeshard/ut_helpers/test_env.h
@@ -62,8 +62,8 @@ namespace NSchemeShardUT_Private {
TActorId MeteringFake;
public:
- TTestEnv(TTestActorRuntime& runtime, ui32 nchannels = 4, bool enablePipeRetries = true,
- TSchemeShardFactory ssFactory = &CreateFlatTxSchemeShard, bool enableSystemViews = false);
+ TTestEnv(TTestActorRuntime& runtime, ui32 nchannels = 4, bool enablePipeRetries = true,
+ TSchemeShardFactory ssFactory = &CreateFlatTxSchemeShard, bool enableSystemViews = false);
TTestEnv(TTestActorRuntime& runtime, const TTestEnvOptions& opts,
TSchemeShardFactory ssFactory = &CreateFlatTxSchemeShard, std::shared_ptr<NKikimr::NDataShard::IExportFactory> dsExportFactory = {});
diff --git a/ydb/core/tx/schemeshard/ya.make b/ydb/core/tx/schemeshard/ya.make
index ceb87ce3883..3fcaa3d02c1 100644
--- a/ydb/core/tx/schemeshard/ya.make
+++ b/ydb/core/tx/schemeshard/ya.make
@@ -179,7 +179,7 @@ SRCS(
schemeshard_types.cpp
schemeshard_types.h
schemeshard_ui64id.cpp
- schemeshard_user_attr_limits.h
+ schemeshard_user_attr_limits.h
schemeshard_utils.cpp
schemeshard_utils.h
schemeshard_export__cancel.cpp
diff --git a/ydb/core/tx/tx_proxy/datareq.cpp b/ydb/core/tx/tx_proxy/datareq.cpp
index af48b42d60f..0433ff7ff90 100644
--- a/ydb/core/tx/tx_proxy/datareq.cpp
+++ b/ydb/core/tx/tx_proxy/datareq.cpp
@@ -788,7 +788,7 @@ void TDataReq::ReportStatus(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus
const auto* e = PerTablet.FindPtr(shard);
if (!e)
return "";
- return Sprintf("%" PRIu64 "/%" PRIu64, e->TableId.PathId.OwnerId, e->TableId.PathId.LocalPathId);
+ return Sprintf("%" PRIu64 "/%" PRIu64, e->TableId.PathId.OwnerId, e->TableId.PathId.LocalPathId);
};
if (FlatMKQLRequest && FlatMKQLRequest->CollectStats) {
@@ -906,12 +906,12 @@ void TDataReq::BuildTxStats(NKikimrQueryStats::TTxStats& stats) {
TTableId tableId(table.GetTableInfo().GetSchemeshardId(), table.GetTableInfo().GetPathId());
auto& tableStats = byTable[tableId];
if (!tableStats.HasTableInfo()) {
- tableStats.MutableTableInfo()->SetSchemeshardId(tableId.PathId.OwnerId);
- tableStats.MutableTableInfo()->SetPathId(tableId.PathId.LocalPathId);
+ tableStats.MutableTableInfo()->SetSchemeshardId(tableId.PathId.OwnerId);
+ tableStats.MutableTableInfo()->SetPathId(tableId.PathId.LocalPathId);
tableStats.MutableTableInfo()->SetName(table.GetTableInfo().GetName());
}
- Aggregate(tableStats, table);
- tableStats.SetShardCount(tableStats.GetShardCount() + 1);
+ Aggregate(tableStats, table);
+ tableStats.SetShardCount(tableStats.GetShardCount() + 1);
}
if (shard.second.Stats->PerShardStatsSize() == 1) {
auto shardStats = stats.AddPerShardStats();
@@ -1418,14 +1418,14 @@ void TDataReq::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr &ev,
auto &res = resp->ResultSet[0];
ReadTableRequest->TableId = res.TableId;
- if (res.TableId.IsSystemView()) {
- IssueManager.RaiseIssue(MakeIssue(NKikimrIssues::TIssuesIds::GENERIC_RESOLVE_ERROR,
- Sprintf("Table '%s' is a system view. Read table is not supported", ReadTableRequest->TablePath.data())));
- ReportStatus(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ResolveError, NKikimrIssues::TStatusIds::SCHEME_ERROR, true, ctx);
- TxProxyMon->ResolveKeySetWrongRequest->Inc();
- return Die(ctx);
- }
-
+ if (res.TableId.IsSystemView()) {
+ IssueManager.RaiseIssue(MakeIssue(NKikimrIssues::TIssuesIds::GENERIC_RESOLVE_ERROR,
+ Sprintf("Table '%s' is a system view. Read table is not supported", ReadTableRequest->TablePath.data())));
+ ReportStatus(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ResolveError, NKikimrIssues::TStatusIds::SCHEME_ERROR, true, ctx);
+ TxProxyMon->ResolveKeySetWrongRequest->Inc();
+ return Die(ctx);
+ }
+
TVector<NScheme::TTypeId> keyColumnTypes(res.Columns.size());
TVector<TKeyDesc::TColumnOp> columns(res.Columns.size());
size_t keySize = 0;
@@ -1544,14 +1544,14 @@ void TDataReq::Handle(TEvTxProxySchemeCache::TEvResolveKeySetResult::TPtr &ev, c
switch (x.Status) {
case NSchemeCache::TSchemeCacheRequest::EStatus::PathErrorNotExist:
gotHardResolveError = true;
- ss << "table not exists: " << x.KeyDescription->TableId;
+ ss << "table not exists: " << x.KeyDescription->TableId;
break;
case NSchemeCache::TSchemeCacheRequest::EStatus::TypeCheckError:
gotHardResolveError = true;
- ss << "type check error: " << x.KeyDescription->TableId;
+ ss << "type check error: " << x.KeyDescription->TableId;
break;
default:
- ss << "unresolved table: " << x.KeyDescription->TableId << ". Status: " << x.Status;
+ ss << "unresolved table: " << x.KeyDescription->TableId << ". Status: " << x.Status;
break;
}
@@ -1599,7 +1599,7 @@ void TDataReq::Handle(TEvTxProxySchemeCache::TEvResolveKeySetResult::TPtr &ev, c
TStringStream explanation;
explanation << "Access denied for " << UserToken->GetUserSID()
<< " with access " << NACLib::AccessRightsToString(access)
- << " to tableId# " << entry.KeyDescription->TableId;
+ << " to tableId# " << entry.KeyDescription->TableId;
LOG_ERROR_S(ctx, NKikimrServices::TX_PROXY, explanation.Str());
IssueManager.RaiseIssue(MakeIssue(NKikimrIssues::TIssuesIds::ACCESS_DENIED, explanation.Str()));
diff --git a/ydb/core/tx/tx_proxy/describe.cpp b/ydb/core/tx/tx_proxy/describe.cpp
index deb38ca2b7d..4065ebb8b10 100644
--- a/ydb/core/tx/tx_proxy/describe.cpp
+++ b/ydb/core/tx/tx_proxy/describe.cpp
@@ -67,115 +67,115 @@ class TDescribeReq : public TActor<TDescribeReq> {
}
void FillSystemViewDescr(NKikimrSchemeOp::TDirEntry* descr, ui64 schemeShardId) {
- descr->SetSchemeshardId(schemeShardId);
- descr->SetPathId(InvalidLocalPathId);
- descr->SetParentPathId(InvalidLocalPathId);
- descr->SetCreateFinished(true);
- descr->SetCreateTxId(0);
- descr->SetCreateStep(0);
- }
-
- void SendSystemViewResult(const NSchemeCache::TSchemeCacheNavigate::TEntry& entry, const TString& path,
- const TActorContext& ctx)
- {
- auto schemeShardId = entry.DomainInfo->DomainKey.OwnerId;
-
+ descr->SetSchemeshardId(schemeShardId);
+ descr->SetPathId(InvalidLocalPathId);
+ descr->SetParentPathId(InvalidLocalPathId);
+ descr->SetCreateFinished(true);
+ descr->SetCreateTxId(0);
+ descr->SetCreateStep(0);
+ }
+
+ void SendSystemViewResult(const NSchemeCache::TSchemeCacheNavigate::TEntry& entry, const TString& path,
+ const TActorContext& ctx)
+ {
+ auto schemeShardId = entry.DomainInfo->DomainKey.OwnerId;
+
auto result = MakeHolder<NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResultBuilder>(
- path, schemeShardId, TPathId());
-
- auto* pathDescription = result->Record.MutablePathDescription();
- auto* self = pathDescription->MutableSelf();
-
- Y_VERIFY(!entry.Path.empty());
- self->SetName(entry.Path.back());
+ path, schemeShardId, TPathId());
+
+ auto* pathDescription = result->Record.MutablePathDescription();
+ auto* self = pathDescription->MutableSelf();
+
+ Y_VERIFY(!entry.Path.empty());
+ self->SetName(entry.Path.back());
self->SetPathType(NKikimrSchemeOp::EPathTypeTable);
- FillSystemViewDescr(self, schemeShardId);
-
- auto* table = pathDescription->MutableTable();
-
- TVector<ui32> keyColumnIds(entry.Columns.size());
- size_t keySize = 0;
-
- table->SetName(entry.Path.back());
- table->MutableColumns()->Reserve(entry.Columns.size());
-
- for (const auto& [id, column] : entry.Columns) {
- auto* col = table->AddColumns();
- col->SetName(column.Name);
- col->SetType(AppData(ctx)->TypeRegistry->GetTypeName(column.PType));
- col->SetTypeId(column.PType);
- col->SetId(id);
- if (column.KeyOrder >= 0) {
- Y_VERIFY((size_t)column.KeyOrder < keyColumnIds.size());
- keyColumnIds[column.KeyOrder] = id;
- ++keySize;
- }
- }
-
- table->MutableKeyColumnNames()->Reserve(keySize);
- table->MutableKeyColumnIds()->Reserve(keySize);
-
- for (size_t i = 0; i < keySize; ++i) {
- auto columnId = keyColumnIds[i];
- auto columnIt = entry.Columns.find(columnId);
- Y_VERIFY(columnIt != entry.Columns.end());
- table->AddKeyColumnIds(columnId);
- table->AddKeyColumnNames(columnIt->second.Name);
- }
-
- auto* stats = pathDescription->MutableTableStats();
- stats->SetPartCount(0);
-
- LOG_DEBUG_S(ctx, NKikimrServices::TX_PROXY,
- "Actor# " << ctx.SelfID.ToString() <<
- " Send sysview TEvDescribeSchemeResult to# " << Source.ToString() <<
- " Cookie: " << SourceCookie <<
- " TEvDescribeSchemeResult: " << result->ToString());
-
- TxProxyMon->NavigateLatency->Collect((ctx.Now() - WallClockStarted).MilliSeconds());
-
+ FillSystemViewDescr(self, schemeShardId);
+
+ auto* table = pathDescription->MutableTable();
+
+ TVector<ui32> keyColumnIds(entry.Columns.size());
+ size_t keySize = 0;
+
+ table->SetName(entry.Path.back());
+ table->MutableColumns()->Reserve(entry.Columns.size());
+
+ for (const auto& [id, column] : entry.Columns) {
+ auto* col = table->AddColumns();
+ col->SetName(column.Name);
+ col->SetType(AppData(ctx)->TypeRegistry->GetTypeName(column.PType));
+ col->SetTypeId(column.PType);
+ col->SetId(id);
+ if (column.KeyOrder >= 0) {
+ Y_VERIFY((size_t)column.KeyOrder < keyColumnIds.size());
+ keyColumnIds[column.KeyOrder] = id;
+ ++keySize;
+ }
+ }
+
+ table->MutableKeyColumnNames()->Reserve(keySize);
+ table->MutableKeyColumnIds()->Reserve(keySize);
+
+ for (size_t i = 0; i < keySize; ++i) {
+ auto columnId = keyColumnIds[i];
+ auto columnIt = entry.Columns.find(columnId);
+ Y_VERIFY(columnIt != entry.Columns.end());
+ table->AddKeyColumnIds(columnId);
+ table->AddKeyColumnNames(columnIt->second.Name);
+ }
+
+ auto* stats = pathDescription->MutableTableStats();
+ stats->SetPartCount(0);
+
+ LOG_DEBUG_S(ctx, NKikimrServices::TX_PROXY,
+ "Actor# " << ctx.SelfID.ToString() <<
+ " Send sysview TEvDescribeSchemeResult to# " << Source.ToString() <<
+ " Cookie: " << SourceCookie <<
+ " TEvDescribeSchemeResult: " << result->ToString());
+
+ TxProxyMon->NavigateLatency->Collect((ctx.Now() - WallClockStarted).MilliSeconds());
+
result->Record.SetStatus(NKikimrScheme::StatusSuccess);
- ctx.Send(Source, result.Release(), 0, SourceCookie);
- return Die(ctx);
- }
-
- void SendSystemViewFolderResult(const NSchemeCache::TSchemeCacheNavigate::TEntry& entry, const TString& path,
- const TActorContext& ctx)
- {
- auto schemeShardId = entry.DomainInfo->DomainKey.OwnerId;
-
+ ctx.Send(Source, result.Release(), 0, SourceCookie);
+ return Die(ctx);
+ }
+
+ void SendSystemViewFolderResult(const NSchemeCache::TSchemeCacheNavigate::TEntry& entry, const TString& path,
+ const TActorContext& ctx)
+ {
+ auto schemeShardId = entry.DomainInfo->DomainKey.OwnerId;
+
auto result = MakeHolder<NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResultBuilder>(
- path, schemeShardId, TPathId());
-
- auto* pathDescription = result->Record.MutablePathDescription();
- auto* self = pathDescription->MutableSelf();
-
- self->SetName(TString(NSysView::SysPathName));
+ path, schemeShardId, TPathId());
+
+ auto* pathDescription = result->Record.MutablePathDescription();
+ auto* self = pathDescription->MutableSelf();
+
+ self->SetName(TString(NSysView::SysPathName));
self->SetPathType(NKikimrSchemeOp::EPathTypeDir);
- FillSystemViewDescr(self, schemeShardId);
-
- if (entry.ListNodeEntry) {
- for (const auto& child : entry.ListNodeEntry->Children) {
- auto descr = pathDescription->AddChildren();
- descr->SetName(child.Name);
+ FillSystemViewDescr(self, schemeShardId);
+
+ if (entry.ListNodeEntry) {
+ for (const auto& child : entry.ListNodeEntry->Children) {
+ auto descr = pathDescription->AddChildren();
+ descr->SetName(child.Name);
descr->SetPathType(NKikimrSchemeOp::EPathTypeTable);
- FillSystemViewDescr(descr, schemeShardId);
- }
- };
-
- LOG_DEBUG_S(ctx, NKikimrServices::TX_PROXY,
- "Actor# " << ctx.SelfID.ToString() <<
- " Send sysview TEvDescribeSchemeResult to# " << Source.ToString() <<
- " Cookie: " << SourceCookie <<
- " TEvDescribeSchemeResult: " << result->ToString());
-
- TxProxyMon->NavigateLatency->Collect((ctx.Now() - WallClockStarted).MilliSeconds());
-
+ FillSystemViewDescr(descr, schemeShardId);
+ }
+ };
+
+ LOG_DEBUG_S(ctx, NKikimrServices::TX_PROXY,
+ "Actor# " << ctx.SelfID.ToString() <<
+ " Send sysview TEvDescribeSchemeResult to# " << Source.ToString() <<
+ " Cookie: " << SourceCookie <<
+ " TEvDescribeSchemeResult: " << result->ToString());
+
+ TxProxyMon->NavigateLatency->Collect((ctx.Now() - WallClockStarted).MilliSeconds());
+
result->Record.SetStatus(NKikimrScheme::StatusSuccess);
- ctx.Send(Source, result.Release(), 0, SourceCookie);
- return Die(ctx);
- }
-
+ ctx.Send(Source, result.Release(), 0, SourceCookie);
+ return Die(ctx);
+ }
+
void Handle(TEvTxProxyReq::TEvNavigateScheme::TPtr &ev, const TActorContext &ctx);
void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr &ev, const TActorContext &ctx);
void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr &ev, const TActorContext &ctx);
@@ -283,9 +283,9 @@ void TDescribeReq::Handle(TEvTxProxyReq::TEvNavigateScheme::TPtr &ev, const TAct
}
request->ResultSet.emplace_back(entry);
-
+
ctx.Send(Services.SchemeCache, new TEvTxProxySchemeCache::TEvNavigateKeySet(request), 0, SourceCookie);
-
+
SchemeRequest = ev->Release();
Become(&TThis::StateWaitResolve);
}
@@ -350,24 +350,24 @@ void TDescribeReq::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr &
}
}
- const auto& describePath = SchemeRequest->Ev->Get()->Record.GetDescribePath();
-
- if (entry.TableId.IsSystemView()) {
- // don't go to schemeshard
- const auto& path = describePath.GetPath();
-
- if (entry.TableId.SysViewInfo == NSysView::SysPathName) {
- return SendSystemViewFolderResult(entry, path, ctx);
- }
-
- return SendSystemViewResult(entry, path, ctx);
- }
-
+ const auto& describePath = SchemeRequest->Ev->Get()->Record.GetDescribePath();
+
+ if (entry.TableId.IsSystemView()) {
+ // don't go to schemeshard
+ const auto& path = describePath.GetPath();
+
+ if (entry.TableId.SysViewInfo == NSysView::SysPathName) {
+ return SendSystemViewFolderResult(entry, path, ctx);
+ }
+
+ return SendSystemViewResult(entry, path, ctx);
+ }
+
const ui64 shardToRequest = entry.DomainInfo->ExtractSchemeShard();
-
+
TAutoPtr<NSchemeShard::TEvSchemeShard::TEvDescribeScheme> req(
new NSchemeShard::TEvSchemeShard::TEvDescribeScheme(describePath));
-
+
LOG_DEBUG_S(ctx, NKikimrServices::TX_PROXY, "Actor# " << ctx.SelfID.ToString()
<< " SEND to# " << shardToRequest << " shardToRequest " << req->ToString());
@@ -387,48 +387,48 @@ void TDescribeReq::Handle(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult:
TxProxyMon->NavigateLatency->Collect((ctx.Now() - WallClockStarted).MilliSeconds());
if (AppData()->FeatureFlags.GetEnableSystemViews()) {
- const auto& pathDescription = ev->Get()->GetRecord().GetPathDescription();
- const auto& self = pathDescription.GetSelf();
-
- TDomainsInfo *domainsInfo = AppData()->DomainsInfo.Get();
- Y_VERIFY(!domainsInfo->Domains.empty());
-
- bool needSysFolder = false;
+ const auto& pathDescription = ev->Get()->GetRecord().GetPathDescription();
+ const auto& self = pathDescription.GetSelf();
+
+ TDomainsInfo *domainsInfo = AppData()->DomainsInfo.Get();
+ Y_VERIFY(!domainsInfo->Domains.empty());
+
+ bool needSysFolder = false;
if (self.GetPathType() == NKikimrSchemeOp::EPathType::EPathTypeSubDomain ||
self.GetPathType() == NKikimrSchemeOp::EPathType::EPathTypeColumnStore ||
self.GetPathType() == NKikimrSchemeOp::EPathType::EPathTypeColumnTable)
{
- needSysFolder = true;
+ needSysFolder = true;
} else if (self.GetPathId() == NSchemeShard::RootPathId) {
- for (const auto& [_, domain] : domainsInfo->Domains) {
- if (domain->SchemeRoot == self.GetSchemeshardId()) {
- needSysFolder = true;
- break;
- }
- }
- }
-
- if (needSysFolder) {
- bool hasSysFolder = false;
-
- const auto& children = pathDescription.GetChildren();
- if (!children.empty()) {
- auto size = children.size();
- if (children[size - 1].GetName() == NSysView::SysPathName) {
- hasSysFolder = true;
- }
- }
-
- if (!hasSysFolder) {
- auto* record = ev->Get()->MutableRecord();
- auto* descr = record->MutablePathDescription()->AddChildren();
- descr->SetName(TString(NSysView::SysPathName));
+ for (const auto& [_, domain] : domainsInfo->Domains) {
+ if (domain->SchemeRoot == self.GetSchemeshardId()) {
+ needSysFolder = true;
+ break;
+ }
+ }
+ }
+
+ if (needSysFolder) {
+ bool hasSysFolder = false;
+
+ const auto& children = pathDescription.GetChildren();
+ if (!children.empty()) {
+ auto size = children.size();
+ if (children[size - 1].GetName() == NSysView::SysPathName) {
+ hasSysFolder = true;
+ }
+ }
+
+ if (!hasSysFolder) {
+ auto* record = ev->Get()->MutableRecord();
+ auto* descr = record->MutablePathDescription()->AddChildren();
+ descr->SetName(TString(NSysView::SysPathName));
descr->SetPathType(NKikimrSchemeOp::EPathTypeDir);
- FillSystemViewDescr(descr, self.GetSchemeshardId());
- }
- }
- }
-
+ FillSystemViewDescr(descr, self.GetSchemeshardId());
+ }
+ }
+ }
+
ctx.ExecutorThread.Send(ev->Forward(Source));
return Die(ctx);
}
diff --git a/ydb/core/tx/tx_proxy/mon.cpp b/ydb/core/tx/tx_proxy/mon.cpp
index 14f5d47064e..d18b6d5edb4 100644
--- a/ydb/core/tx/tx_proxy/mon.cpp
+++ b/ydb/core/tx/tx_proxy/mon.cpp
@@ -34,29 +34,29 @@ TTxProxyMon::TTxProxyMon(const TIntrusivePtr<NMonitoring::TDynamicCounters>& cou
ReportStatusNotOK = DataReqGroup->GetCounter("ReportStatus/NotOK", true);
ReportStatusStreamData = DataReqGroup->GetCounter("ReportStatus/StreamData", true);
- TxPrepareTimeHgram = DataReqGroup->GetHistogram("TxPrepareTimesMs",
- NMonitoring::ExponentialHistogram(20, 2, 1));
- TxExecuteTimeHgram = DataReqGroup->GetHistogram("TxExecuteTimesMs",
- NMonitoring::ExponentialHistogram(20, 2, 1));
- TxTotalTimeHgram = DataReqGroup->GetHistogram("TxTotalTimesMs",
- NMonitoring::ExponentialHistogram(20, 2, 1));
- TxPrepareSpreadHgram = DataReqGroup->GetHistogram("TxPrepareSpreadMs",
- NMonitoring::ExponentialHistogram(10, 2, 1));
- TxPrepareArriveSpreadHgram = DataReqGroup->GetHistogram("TxPrepareArriveSpreadMs",
- NMonitoring::ExponentialHistogram(10, 2, 1));
- TxPrepareCompleteSpreadHgram = DataReqGroup->GetHistogram("TxPrepareCompleteSpreadMs",
- NMonitoring::ExponentialHistogram(10, 2, 1));
- TxExecSpreadHgram = DataReqGroup->GetHistogram("TxExecSpreadMs",
- NMonitoring::ExponentialHistogram(10, 2, 1));
-
- TxPrepareSetProgramHgram = DataReqGroup->GetHistogram("TxPrepareSetProgramUs",
- NMonitoring::ExponentialHistogram(10, 2, 100));
- TxPrepareResolveHgram = DataReqGroup->GetHistogram("TxPrepareResolveUs",
- NMonitoring::ExponentialHistogram(10, 2, 100));
- TxPrepareBuildShardProgramsHgram = DataReqGroup->GetHistogram("TxPrepareBuildShardProgramsUs",
- NMonitoring::ExponentialHistogram(10, 2, 100));
- TxPrepareSendShardProgramsHgram = DataReqGroup->GetHistogram("TxPrepareSendShardProgramsUs",
- NMonitoring::ExponentialHistogram(10, 2, 100));
+ TxPrepareTimeHgram = DataReqGroup->GetHistogram("TxPrepareTimesMs",
+ NMonitoring::ExponentialHistogram(20, 2, 1));
+ TxExecuteTimeHgram = DataReqGroup->GetHistogram("TxExecuteTimesMs",
+ NMonitoring::ExponentialHistogram(20, 2, 1));
+ TxTotalTimeHgram = DataReqGroup->GetHistogram("TxTotalTimesMs",
+ NMonitoring::ExponentialHistogram(20, 2, 1));
+ TxPrepareSpreadHgram = DataReqGroup->GetHistogram("TxPrepareSpreadMs",
+ NMonitoring::ExponentialHistogram(10, 2, 1));
+ TxPrepareArriveSpreadHgram = DataReqGroup->GetHistogram("TxPrepareArriveSpreadMs",
+ NMonitoring::ExponentialHistogram(10, 2, 1));
+ TxPrepareCompleteSpreadHgram = DataReqGroup->GetHistogram("TxPrepareCompleteSpreadMs",
+ NMonitoring::ExponentialHistogram(10, 2, 1));
+ TxExecSpreadHgram = DataReqGroup->GetHistogram("TxExecSpreadMs",
+ NMonitoring::ExponentialHistogram(10, 2, 1));
+
+ TxPrepareSetProgramHgram = DataReqGroup->GetHistogram("TxPrepareSetProgramUs",
+ NMonitoring::ExponentialHistogram(10, 2, 100));
+ TxPrepareResolveHgram = DataReqGroup->GetHistogram("TxPrepareResolveUs",
+ NMonitoring::ExponentialHistogram(10, 2, 100));
+ TxPrepareBuildShardProgramsHgram = DataReqGroup->GetHistogram("TxPrepareBuildShardProgramsUs",
+ NMonitoring::ExponentialHistogram(10, 2, 100));
+ TxPrepareSendShardProgramsHgram = DataReqGroup->GetHistogram("TxPrepareSendShardProgramsUs",
+ NMonitoring::ExponentialHistogram(10, 2, 100));
MiniKQLResolveSentToShard = DataReqGroup->GetCounter("MiniKQLResolve/SentToShard", true);
MiniKQLWrongRequest = DataReqGroup->GetCounter("MiniKQLResolve/WrongRequest", true);
diff --git a/ydb/core/tx/tx_proxy/proxy.h b/ydb/core/tx/tx_proxy/proxy.h
index 86d4f1ac79e..45a2f236e0a 100644
--- a/ydb/core/tx/tx_proxy/proxy.h
+++ b/ydb/core/tx/tx_proxy/proxy.h
@@ -164,8 +164,8 @@ struct TEvTxUserProxy {
struct TEvInvalidateTable : public TEventPB<TEvInvalidateTable, NKikimrTxUserProxy::TEvInvalidateTable, EvInvalidateTable> {
TEvInvalidateTable() = default;
TEvInvalidateTable(const TTableId& tableId) {
- Record.SetSchemeShardId(tableId.PathId.OwnerId);
- Record.SetTableId(tableId.PathId.LocalPathId);
+ Record.SetSchemeShardId(tableId.PathId.OwnerId);
+ Record.SetTableId(tableId.PathId.LocalPathId);
}
};
diff --git a/ydb/core/tx/tx_proxy/proxy_ut_helpers.h b/ydb/core/tx/tx_proxy/proxy_ut_helpers.h
index 846b0cf568c..9afd28473e1 100644
--- a/ydb/core/tx/tx_proxy/proxy_ut_helpers.h
+++ b/ydb/core/tx/tx_proxy/proxy_ut_helpers.h
@@ -195,7 +195,7 @@ public:
GetSettings().SetNodeCount(staticNodes);
GetSettings().SetDynamicNodeCount(dynamicNodes);
- GetSettings().SetEnableSystemViews(false);
+ GetSettings().SetEnableSystemViews(false);
for (ui32 poolNum = 1; poolNum <= poolsCount; ++poolNum) {
GetSettings().AddStoragePoolType("storage-pool-number-" + ToString(poolNum));
diff --git a/ydb/core/tx/tx_proxy/upload_rows_common_impl.h b/ydb/core/tx/tx_proxy/upload_rows_common_impl.h
index 0aaee7e7f59..fb859c90a26 100644
--- a/ydb/core/tx/tx_proxy/upload_rows_common_impl.h
+++ b/ydb/core/tx/tx_proxy/upload_rows_common_impl.h
@@ -475,11 +475,11 @@ private:
TableKind = request.ResultSet.front().Kind;
bool isOlapTable = (TableKind == NSchemeCache::TSchemeCacheNavigate::KindOlapTable);
- if (request.ResultSet.front().TableId.IsSystemView()) {
- return ReplyWithError(Ydb::StatusIds::SCHEME_ERROR,
+ if (request.ResultSet.front().TableId.IsSystemView()) {
+ return ReplyWithError(Ydb::StatusIds::SCHEME_ERROR,
Sprintf("Table '%s' is a system view. Bulk upsert is not supported.", GetTable().c_str()), ctx);
- }
-
+ }
+
ResolveNamesResult = ev->Get()->Request;
bool makeYdbSchema = isOlapTable || (GetSourceType() != EUploadSource::ProtoValues);
diff --git a/ydb/core/util/concurrent_rw_hash.cpp b/ydb/core/util/concurrent_rw_hash.cpp
index 8d86320f021..f3aabfc056b 100644
--- a/ydb/core/util/concurrent_rw_hash.cpp
+++ b/ydb/core/util/concurrent_rw_hash.cpp
@@ -1 +1 @@
-#include "concurrent_rw_hash.h"
+#include "concurrent_rw_hash.h"
diff --git a/ydb/core/util/concurrent_rw_hash.h b/ydb/core/util/concurrent_rw_hash.h
index 4a471bc5675..46ca5b55f41 100644
--- a/ydb/core/util/concurrent_rw_hash.h
+++ b/ydb/core/util/concurrent_rw_hash.h
@@ -1,117 +1,117 @@
-#pragma once
-
-#include <util/generic/hash.h>
-#include <util/system/rwlock.h>
-
-#include <array>
-
-template <typename K, typename V, size_t BucketCount = 64>
-class TConcurrentRWHashMap {
-public:
- using TActualMap = THashMap<K, V>;
-
- struct TBucket {
- friend class TConcurrentRWHashMap;
-
- private:
- TActualMap Map;
- mutable TRWMutex RWLock;
-
- public:
- TRWMutex& GetLock() const {
- return RWLock;
- }
-
- TActualMap& GetMap() {
- return Map;
- }
-
- const TActualMap& GetMap() const {
- return Map;
- }
- };
-
- std::array<TBucket, BucketCount> Buckets;
-
-public:
- Y_FORCE_INLINE TBucket& GetBucketForKey(const K& key) {
- return Buckets[THash<K>()(key) % BucketCount];
- }
-
- Y_FORCE_INLINE const TBucket& GetBucketForKey(const K& key) const {
- return Buckets[THash<K>()(key) % BucketCount];
- }
-
- void Insert(const K& key, const V& value) {
- TBucket& bucket = GetBucketForKey(key);
- TWriteGuard guard(bucket.RWLock);
-
- bucket.Map[key] = value;
- }
-
- V& InsertIfAbsent(const K& key, const V& value) {
- TBucket& bucket = GetBucketForKey(key);
- TWriteGuard guard(bucket.RWLock);
-
- return bucket.Map.insert(std::make_pair(key, value)).first->second;
- }
-
- template <typename Callable>
- V& InsertIfAbsentWithInit(const K& key, Callable initFunc) {
- TBucket& bucket = GetBucketForKey(key);
- TWriteGuard guard(bucket.RWLock);
-
- typename TActualMap::iterator it = bucket.Map.find(key);
- if (it != bucket.Map.end()) {
- return it->second;
- }
- return bucket.Map.insert(std::make_pair(key, initFunc())).first->second;
- }
-
- V Get(const K& key) const {
- const TBucket& bucket = GetBucketForKey(key);
- TReadGuard guard(bucket.RWLock);
-
- typename TActualMap::const_iterator it = bucket.Map.find(key);
- Y_VERIFY(it != bucket.Map.end(), "not found by key");
- return it->second;
- }
-
- bool Get(const K& key, V& result) const {
- const TBucket& bucket = GetBucketForKey(key);
- TReadGuard guard(bucket.RWLock);
-
- typename TActualMap::const_iterator it = bucket.Map.find(key);
- if (it != bucket.Map.end()) {
- result = it->second;
- return true;
- }
- return false;
- }
-
- V Remove(const K& key) {
- TBucket& bucket = GetBucketForKey(key);
- TWriteGuard guard(bucket.RWLock);
-
- typename TActualMap::iterator it = bucket.Map.find(key);
- Y_VERIFY(it != bucket.Map.end(), "removing non-existent key");
- V r = it->second;
- bucket.Map.erase(it);
- return r;
- }
-
- bool Has(const K& key) const {
- const TBucket& bucket = GetBucketForKey(key);
- TReadGuard guard(bucket.RWLock);
-
- typename TActualMap::const_iterator it = bucket.Map.find(key);
- return (it != bucket.Map.end());
- }
-
- bool Erase(const K& key) {
- TBucket& bucket = GetBucketForKey(key);
- TWriteGuard guard(bucket.RWLock);
-
- return bucket.Map.erase(key);
- }
-};
+#pragma once
+
+#include <util/generic/hash.h>
+#include <util/system/rwlock.h>
+
+#include <array>
+
+template <typename K, typename V, size_t BucketCount = 64>
+class TConcurrentRWHashMap {
+public:
+ using TActualMap = THashMap<K, V>;
+
+ struct TBucket {
+ friend class TConcurrentRWHashMap;
+
+ private:
+ TActualMap Map;
+ mutable TRWMutex RWLock;
+
+ public:
+ TRWMutex& GetLock() const {
+ return RWLock;
+ }
+
+ TActualMap& GetMap() {
+ return Map;
+ }
+
+ const TActualMap& GetMap() const {
+ return Map;
+ }
+ };
+
+ std::array<TBucket, BucketCount> Buckets;
+
+public:
+ Y_FORCE_INLINE TBucket& GetBucketForKey(const K& key) {
+ return Buckets[THash<K>()(key) % BucketCount];
+ }
+
+ Y_FORCE_INLINE const TBucket& GetBucketForKey(const K& key) const {
+ return Buckets[THash<K>()(key) % BucketCount];
+ }
+
+ void Insert(const K& key, const V& value) {
+ TBucket& bucket = GetBucketForKey(key);
+ TWriteGuard guard(bucket.RWLock);
+
+ bucket.Map[key] = value;
+ }
+
+ V& InsertIfAbsent(const K& key, const V& value) {
+ TBucket& bucket = GetBucketForKey(key);
+ TWriteGuard guard(bucket.RWLock);
+
+ return bucket.Map.insert(std::make_pair(key, value)).first->second;
+ }
+
+ template <typename Callable>
+ V& InsertIfAbsentWithInit(const K& key, Callable initFunc) {
+ TBucket& bucket = GetBucketForKey(key);
+ TWriteGuard guard(bucket.RWLock);
+
+ typename TActualMap::iterator it = bucket.Map.find(key);
+ if (it != bucket.Map.end()) {
+ return it->second;
+ }
+ return bucket.Map.insert(std::make_pair(key, initFunc())).first->second;
+ }
+
+ V Get(const K& key) const {
+ const TBucket& bucket = GetBucketForKey(key);
+ TReadGuard guard(bucket.RWLock);
+
+ typename TActualMap::const_iterator it = bucket.Map.find(key);
+ Y_VERIFY(it != bucket.Map.end(), "not found by key");
+ return it->second;
+ }
+
+ bool Get(const K& key, V& result) const {
+ const TBucket& bucket = GetBucketForKey(key);
+ TReadGuard guard(bucket.RWLock);
+
+ typename TActualMap::const_iterator it = bucket.Map.find(key);
+ if (it != bucket.Map.end()) {
+ result = it->second;
+ return true;
+ }
+ return false;
+ }
+
+ V Remove(const K& key) {
+ TBucket& bucket = GetBucketForKey(key);
+ TWriteGuard guard(bucket.RWLock);
+
+ typename TActualMap::iterator it = bucket.Map.find(key);
+ Y_VERIFY(it != bucket.Map.end(), "removing non-existent key");
+ V r = it->second;
+ bucket.Map.erase(it);
+ return r;
+ }
+
+ bool Has(const K& key) const {
+ const TBucket& bucket = GetBucketForKey(key);
+ TReadGuard guard(bucket.RWLock);
+
+ typename TActualMap::const_iterator it = bucket.Map.find(key);
+ return (it != bucket.Map.end());
+ }
+
+ bool Erase(const K& key) {
+ TBucket& bucket = GetBucketForKey(key);
+ TWriteGuard guard(bucket.RWLock);
+
+ return bucket.Map.erase(key);
+ }
+};
diff --git a/ydb/core/util/concurrent_rw_hash_ut.cpp b/ydb/core/util/concurrent_rw_hash_ut.cpp
index 35d175a5449..e761b69ecef 100644
--- a/ydb/core/util/concurrent_rw_hash_ut.cpp
+++ b/ydb/core/util/concurrent_rw_hash_ut.cpp
@@ -1,154 +1,154 @@
-#include <concurrent_rw_hash.h>
-
-#include <library/cpp/testing/unittest/gtest.h>
-
-#include <util/generic/string.h>
-#include <util/generic/vector.h>
-
-TEST(TConcurrentRWHashTest, TEmptyGetTest) {
- TConcurrentRWHashMap<TString, ui32> h;
-
- EXPECT_FALSE(h.Has("key"));
-
- ui32 res = 100;
- EXPECT_FALSE(h.Get("key", res));
- EXPECT_EQ(res, 100);
-
- // Can't check h.Get("key") here because it has Y_VERIFY inside o_O
-}
-
-TEST(TConcurrentRWHashTest, TInsertTest) {
- TConcurrentRWHashMap<TString, ui32> h;
-
- h.Insert("key1", 1);
- h.Insert("key2", 2);
-
- EXPECT_EQ(h.Get("key1"), 1);
- EXPECT_EQ(h.Get("key2"), 2);
-
- ui32 res = 100;
- EXPECT_TRUE(h.Has("key1"));
- EXPECT_TRUE(h.Get("key1", res));
- EXPECT_EQ(res, 1);
-
- EXPECT_TRUE(h.Has("key2"));
- EXPECT_TRUE(h.Get("key2", res));
- EXPECT_EQ(res, 2);
-
- EXPECT_FALSE(h.Has("key3"));
- EXPECT_FALSE(h.Get("key3", res));
- EXPECT_EQ(res, 2);
-}
-
-TEST(TConcurrentRWHashTest, TInsertIfAbsentTest) {
- TConcurrentRWHashMap<TString, ui32> h;
-
- ui32 res;
- EXPECT_FALSE(h.Has("key"));
- EXPECT_FALSE(h.Get("key", res));
-
- EXPECT_EQ(h.InsertIfAbsent("key", 1), static_cast<ui32>(1));
- EXPECT_EQ(h.Get("key"), 1);
-
- EXPECT_EQ(h.InsertIfAbsent("key", 2), static_cast<ui32>(1));
- EXPECT_EQ(h.Get("key"), 1);
-}
-
-TEST(TConcurrentRWHashTest, TInsertIfAbsentTestFunc) {
- TConcurrentRWHashMap<TString, ui32> h;
-
- bool initialized = false;
- auto f = [&initialized]() {
- initialized = true;
- return static_cast<ui32>(1);
- };
-
- ui32 res = 0;
- EXPECT_FALSE(h.Get("key", res));
- EXPECT_EQ(res, 0);
-
- EXPECT_EQ(h.InsertIfAbsentWithInit("key", f), 1);
- EXPECT_EQ(h.Get("key"), 1);
- EXPECT_TRUE(initialized);
-
- initialized = false;
- EXPECT_EQ(h.InsertIfAbsentWithInit("key", f), 1);
- EXPECT_EQ(h.Get("key"), 1);
- EXPECT_FALSE(initialized);
-
- h.Insert("key", 2);
- EXPECT_EQ(h.InsertIfAbsentWithInit("key", f), 2);
- EXPECT_EQ(h.Get("key"), 2);
- EXPECT_FALSE(initialized);
-}
-
-TEST(TConcurrentRWHashTest, TRemoveTest) {
- TConcurrentRWHashMap<TString, ui32> h;
-
- EXPECT_FALSE(h.Has("key1"));
- EXPECT_FALSE(h.Has("key2"));
- EXPECT_FALSE(h.Has("key3"));
-
- h.Insert("key1", 1);
- EXPECT_TRUE(h.Has("key1"));
- EXPECT_FALSE(h.Has("key2"));
- EXPECT_FALSE(h.Has("key3"));
- EXPECT_EQ(h.Get("key1"), 1);
-
- EXPECT_EQ(h.Remove("key1"), 1);
- EXPECT_FALSE(h.Has("key1"));
- EXPECT_FALSE(h.Has("key2"));
- EXPECT_FALSE(h.Has("key3"));
-
- h.Insert("key1", 1);
- h.Insert("key2", 2);
-
- EXPECT_TRUE(h.Has("key1"));
- EXPECT_TRUE(h.Has("key2"));
- EXPECT_FALSE(h.Has("key3"));
- EXPECT_EQ(h.Get("key1"), 1);
- EXPECT_EQ(h.Get("key2"), 2);
-
- EXPECT_EQ(h.Remove("key2"), 2);
- EXPECT_TRUE(h.Has("key1"));
- EXPECT_FALSE(h.Has("key2"));
- EXPECT_FALSE(h.Has("key3"));
- EXPECT_EQ(h.Get("key1"), 1);
-}
-
-TEST(TConcurrentRWHashTest, TEraseTest) {
- TConcurrentRWHashMap<TString, ui32> h;
-
- EXPECT_FALSE(h.Has("key1"));
- EXPECT_FALSE(h.Has("key2"));
- EXPECT_FALSE(h.Has("key3"));
-
- h.Insert("key1", 1);
- EXPECT_TRUE(h.Has("key1"));
- EXPECT_FALSE(h.Has("key2"));
- EXPECT_FALSE(h.Has("key3"));
- EXPECT_EQ(h.Get("key1"), 1);
-
- EXPECT_TRUE(h.Erase("key1"));
- EXPECT_FALSE(h.Has("key1"));
- EXPECT_FALSE(h.Has("key2"));
- EXPECT_FALSE(h.Has("key3"));
-
- h.Insert("key1", 1);
- h.Insert("key2", 2);
-
- EXPECT_TRUE(h.Has("key1"));
- EXPECT_TRUE(h.Has("key2"));
- EXPECT_FALSE(h.Has("key3"));
- EXPECT_EQ(h.Get("key1"), 1);
- EXPECT_EQ(h.Get("key2"), 2);
-
- EXPECT_TRUE(h.Erase("key2"));
- EXPECT_TRUE(h.Has("key1"));
- EXPECT_FALSE(h.Has("key2"));
- EXPECT_FALSE(h.Has("key3"));
- EXPECT_EQ(h.Get("key1"), 1);
-
- EXPECT_FALSE(h.Erase("key2"));
-}
-
+#include <concurrent_rw_hash.h>
+
+#include <library/cpp/testing/unittest/gtest.h>
+
+#include <util/generic/string.h>
+#include <util/generic/vector.h>
+
+TEST(TConcurrentRWHashTest, TEmptyGetTest) {
+ TConcurrentRWHashMap<TString, ui32> h;
+
+ EXPECT_FALSE(h.Has("key"));
+
+ ui32 res = 100;
+ EXPECT_FALSE(h.Get("key", res));
+ EXPECT_EQ(res, 100);
+
+ // Can't check h.Get("key") here because it has Y_VERIFY inside o_O
+}
+
+TEST(TConcurrentRWHashTest, TInsertTest) {
+ TConcurrentRWHashMap<TString, ui32> h;
+
+ h.Insert("key1", 1);
+ h.Insert("key2", 2);
+
+ EXPECT_EQ(h.Get("key1"), 1);
+ EXPECT_EQ(h.Get("key2"), 2);
+
+ ui32 res = 100;
+ EXPECT_TRUE(h.Has("key1"));
+ EXPECT_TRUE(h.Get("key1", res));
+ EXPECT_EQ(res, 1);
+
+ EXPECT_TRUE(h.Has("key2"));
+ EXPECT_TRUE(h.Get("key2", res));
+ EXPECT_EQ(res, 2);
+
+ EXPECT_FALSE(h.Has("key3"));
+ EXPECT_FALSE(h.Get("key3", res));
+ EXPECT_EQ(res, 2);
+}
+
+TEST(TConcurrentRWHashTest, TInsertIfAbsentTest) {
+ TConcurrentRWHashMap<TString, ui32> h;
+
+ ui32 res;
+ EXPECT_FALSE(h.Has("key"));
+ EXPECT_FALSE(h.Get("key", res));
+
+ EXPECT_EQ(h.InsertIfAbsent("key", 1), static_cast<ui32>(1));
+ EXPECT_EQ(h.Get("key"), 1);
+
+ EXPECT_EQ(h.InsertIfAbsent("key", 2), static_cast<ui32>(1));
+ EXPECT_EQ(h.Get("key"), 1);
+}
+
+TEST(TConcurrentRWHashTest, TInsertIfAbsentTestFunc) {
+ TConcurrentRWHashMap<TString, ui32> h;
+
+ bool initialized = false;
+ auto f = [&initialized]() {
+ initialized = true;
+ return static_cast<ui32>(1);
+ };
+
+ ui32 res = 0;
+ EXPECT_FALSE(h.Get("key", res));
+ EXPECT_EQ(res, 0);
+
+ EXPECT_EQ(h.InsertIfAbsentWithInit("key", f), 1);
+ EXPECT_EQ(h.Get("key"), 1);
+ EXPECT_TRUE(initialized);
+
+ initialized = false;
+ EXPECT_EQ(h.InsertIfAbsentWithInit("key", f), 1);
+ EXPECT_EQ(h.Get("key"), 1);
+ EXPECT_FALSE(initialized);
+
+ h.Insert("key", 2);
+ EXPECT_EQ(h.InsertIfAbsentWithInit("key", f), 2);
+ EXPECT_EQ(h.Get("key"), 2);
+ EXPECT_FALSE(initialized);
+}
+
+TEST(TConcurrentRWHashTest, TRemoveTest) {
+ TConcurrentRWHashMap<TString, ui32> h;
+
+ EXPECT_FALSE(h.Has("key1"));
+ EXPECT_FALSE(h.Has("key2"));
+ EXPECT_FALSE(h.Has("key3"));
+
+ h.Insert("key1", 1);
+ EXPECT_TRUE(h.Has("key1"));
+ EXPECT_FALSE(h.Has("key2"));
+ EXPECT_FALSE(h.Has("key3"));
+ EXPECT_EQ(h.Get("key1"), 1);
+
+ EXPECT_EQ(h.Remove("key1"), 1);
+ EXPECT_FALSE(h.Has("key1"));
+ EXPECT_FALSE(h.Has("key2"));
+ EXPECT_FALSE(h.Has("key3"));
+
+ h.Insert("key1", 1);
+ h.Insert("key2", 2);
+
+ EXPECT_TRUE(h.Has("key1"));
+ EXPECT_TRUE(h.Has("key2"));
+ EXPECT_FALSE(h.Has("key3"));
+ EXPECT_EQ(h.Get("key1"), 1);
+ EXPECT_EQ(h.Get("key2"), 2);
+
+ EXPECT_EQ(h.Remove("key2"), 2);
+ EXPECT_TRUE(h.Has("key1"));
+ EXPECT_FALSE(h.Has("key2"));
+ EXPECT_FALSE(h.Has("key3"));
+ EXPECT_EQ(h.Get("key1"), 1);
+}
+
+TEST(TConcurrentRWHashTest, TEraseTest) {
+ TConcurrentRWHashMap<TString, ui32> h;
+
+ EXPECT_FALSE(h.Has("key1"));
+ EXPECT_FALSE(h.Has("key2"));
+ EXPECT_FALSE(h.Has("key3"));
+
+ h.Insert("key1", 1);
+ EXPECT_TRUE(h.Has("key1"));
+ EXPECT_FALSE(h.Has("key2"));
+ EXPECT_FALSE(h.Has("key3"));
+ EXPECT_EQ(h.Get("key1"), 1);
+
+ EXPECT_TRUE(h.Erase("key1"));
+ EXPECT_FALSE(h.Has("key1"));
+ EXPECT_FALSE(h.Has("key2"));
+ EXPECT_FALSE(h.Has("key3"));
+
+ h.Insert("key1", 1);
+ h.Insert("key2", 2);
+
+ EXPECT_TRUE(h.Has("key1"));
+ EXPECT_TRUE(h.Has("key2"));
+ EXPECT_FALSE(h.Has("key3"));
+ EXPECT_EQ(h.Get("key1"), 1);
+ EXPECT_EQ(h.Get("key2"), 2);
+
+ EXPECT_TRUE(h.Erase("key2"));
+ EXPECT_TRUE(h.Has("key1"));
+ EXPECT_FALSE(h.Has("key2"));
+ EXPECT_FALSE(h.Has("key3"));
+ EXPECT_EQ(h.Get("key1"), 1);
+
+ EXPECT_FALSE(h.Erase("key2"));
+}
+
diff --git a/ydb/core/util/memory_tracker.cpp b/ydb/core/util/memory_tracker.cpp
index 848b36e2761..fab42824201 100644
--- a/ydb/core/util/memory_tracker.cpp
+++ b/ydb/core/util/memory_tracker.cpp
@@ -1,307 +1,307 @@
-#include "memory_tracker.h"
-
+#include "memory_tracker.h"
+
#include <ydb/core/base/appdata.h>
#include <ydb/core/base/counters.h>
#include <ydb/core/mon/mon.h>
-
-#include <library/cpp/actors/core/actor_bootstrapped.h>
-#include <library/cpp/actors/core/memory_tracker.h>
-#include <library/cpp/html/escape/escape.h>
-#include <library/cpp/monlib/service/pages/templates.h>
-
-#include <util/generic/xrange.h>
-
-using namespace NMonitoring;
-using namespace NActors::NMemory::NPrivate;
-
-namespace NKikimr {
-namespace NMemory {
-
-class TMemoryTrackerActor : public TActorBootstrapped<TMemoryTrackerActor> {
-public:
- TMemoryTrackerActor(TDuration updateInterval, TDynamicCounterPtr appCounters)
- : UpdateInterval(updateInterval)
- , AppCounters(appCounters)
- {
- auto utilsCounters = GetServiceCounters(AppCounters, "utils");
- Counters = utilsCounters->GetSubgroup("component", "memory_tracker");
- }
-
- static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
- return NKikimrServices::TActivity::MEMORY_TRACKER;
- }
-
- void Bootstrap(const TActorContext&) {
-#if defined(ENABLE_MEMORY_TRACKING)
- TMemoryTracker::Instance()->Initialize();
- RegisterCounters(Counters);
-
- TMon* mon = AppData()->Mon;
- if (mon) {
- auto *actorsMonPage = mon->RegisterIndexPage("actors", "Actors");
- mon->RegisterActorPage(actorsMonPage, "memory_tracker", "Memory tracker",
- false, TlsActivationContext->ExecutorThread.ActorSystem, SelfId());
- }
-
- Collect();
-#endif
- Become(&TMemoryTrackerActor::StateWork);
- }
-
- STRICT_STFUNC(StateWork,
- cFunc(TEvents::TEvWakeup::EventType, Wakeup)
- hFunc(NMon::TEvHttpInfo, HttpInfo)
- cFunc(TEvents::TEvPoison::EventType, PassAway)
- )
-
-private:
- void Wakeup() {
- Collect();
- }
-
- void Collect() {
- Schedule(UpdateInterval, new TEvents::TEvWakeup());
-
- std::vector<TMetric> metrics;
- TMemoryTracker::Instance()->GatherMetrics(metrics);
-
- auto count = metrics.size();
- if (Static.size() != count) {
- return;
- }
-
- const auto& sensors = TMemoryTracker::Instance()->GetSensors();
-
- for (auto i : xrange(count)) {
- auto& metric = Static[i];
- metric.Current = metrics[i];
- metric.Peak.CalculatePeak(metric.Current);
-
- if (sensors.find(i) != sensors.end()) {
- *metric.CurrentMemCounter = metric.Current.GetMemory();
- *metric.PeakMemCounter = metric.Peak.GetMemory();
- *metric.CurrentCountCounter = metric.Current.GetCount();
- *metric.PeakCountCounter = metric.Peak.GetCount();
- }
- }
-
- GatherDynamicMetrics();
- }
-
- void RegisterCounters(NMonitoring::TDynamicCounterPtr counters) {
- auto componentGroup = counters->GetSubgroup("component", "memory_tracker");
-
- auto count = TMemoryTracker::Instance()->GetCount();
- Static.resize(count);
-
- const auto& sensors = TMemoryTracker::Instance()->GetSensors();
- for (auto index : sensors) {
- auto name = TMemoryTracker::Instance()->GetName(index);
- auto& metric = Static[index];
- auto group = componentGroup->GetSubgroup("label", name);
- metric.CurrentMemCounter = group->GetCounter("MT/Memory");
- metric.PeakMemCounter = group->GetCounter("MT/PeakMemory");
- metric.CurrentCountCounter = group->GetCounter("MT/Count");
- metric.PeakCountCounter = group->GetCounter("MT/PeakCount");
- }
- }
-
- void UpdateDynamicCounters() {
- auto utils = GetServiceCounters(AppCounters, "utils");
-
- if (Dynamic.find(AnonRssSize) == Dynamic.end()) {
- Dynamic.emplace(AnonRssSize, utils->GetCounter("Process/AnonRssSize"));
- }
-
- if (Dynamic.find(LFMmapped) == Dynamic.end()) {
- auto lfAllocGroup = utils->FindSubgroup("component", "lfalloc");
- if (lfAllocGroup) {
- Dynamic.emplace(LFMmapped, lfAllocGroup->GetCounter("ActiveBytesMmapped"));
- Dynamic.emplace(LFSmall, lfAllocGroup->GetCounter("ActiveBytesSmall"));
- Dynamic.emplace(LFLarge, lfAllocGroup->GetCounter("ActiveBytesLarge"));
- Dynamic.emplace(LFSystem, lfAllocGroup->GetCounter("ActiveBytesSystem"));
- }
- }
-
- if (Dynamic.find(MkqlGlobalPoolTotalBytes) == Dynamic.end()) {
- auto mkqlAllocGroup = utils->FindSubgroup("subsystem", "mkqlalloc");
- if (mkqlAllocGroup) {
- Dynamic.emplace(MkqlGlobalPoolTotalBytes, mkqlAllocGroup->GetCounter("GlobalPoolTotalBytes"));
-
- auto getNamedPagePoolCounter = [&mkqlAllocGroup](const char* name) {
- return mkqlAllocGroup->GetCounter(TString(name) + "/TotalBytesAllocated");
- };
- Dynamic.emplace(MkqlKqp, getNamedPagePoolCounter("kqp"));
- Dynamic.emplace(MkqlCompile, getNamedPagePoolCounter("compile"));
- Dynamic.emplace(MkqlDatashard, getNamedPagePoolCounter("datashard"));
- Dynamic.emplace(MkqlLocalTx, getNamedPagePoolCounter("local_tx"));
- Dynamic.emplace(MkqlTxProxy, getNamedPagePoolCounter("tx_proxy"));
- }
- }
- }
-
- void GatherDynamicMetrics() {
- UpdateDynamicCounters();
-
- for (auto& [_, metric] : Dynamic) {
- if (metric.Counter) {
- metric.Current.SetMemory(metric.Counter->Val());
- metric.Peak.CalculatePeak(metric.Current);
- }
- }
- }
-
- static TString PrettyNumber(ssize_t val) {
- if (val == 0) {
- return "0";
- }
- TStringStream str;
- if (val < 0) {
- str << "- ";
- }
- ui64 value = val < 0 ? -val : val;
- TSmallVec<ui64> parts;
- while (value > 0) {
- auto head = value / 1000;
- parts.push_back(value - head * 1000);
- value = head;
- }
- str << parts.back();
- parts.pop_back();
- while (!parts.empty()) {
- str << Sprintf(" %03" PRIu64, parts.back());
- parts.pop_back();
- }
- return str.Str();
- }
-
- void HttpInfo(NMon::TEvHttpInfo::TPtr& ev) {
- TStringStream str;
-
- HTML(str) {
-#if defined(ENABLE_MEMORY_TRACKING)
- TABLE_CLASS("table") {
- TABLEHEAD() {
- TABLER() {
- TABLEH() { str << "Metric"; }
- TABLEH() { str << "Memory"; }
- TABLEH() { str << "Peak memory"; }
- }
- }
- TABLEBODY() {
- auto outputDynamic = [&str, &__stream] (const char* name, const TDynamicMetric& metric) {
- TABLED() { str << name; }
- TAG_CLASS_STYLE(TTableD, "", "text-align:right") {
- str << PrettyNumber(metric.Current.GetMemory()); }
- TAG_CLASS_STYLE(TTableD, "", "text-align:right") {
- str << PrettyNumber(metric.Peak.GetMemory()); }
- };
-
- if (Dynamic.find(AnonRssSize) != Dynamic.end()) {
- TABLER() { outputDynamic("AnonRssSize", Dynamic[AnonRssSize]); }
- }
- if (Dynamic.find(LFMmapped) != Dynamic.end()) {
- TABLER() {
- TABLED() { str << "LFAlloc"; }
- }
- TABLER() { outputDynamic("Mmapped", Dynamic[LFMmapped]); }
- TABLER() { outputDynamic("Small", Dynamic[LFSmall]); }
- TABLER() { outputDynamic("Large", Dynamic[LFLarge]); }
- TABLER() { outputDynamic("System", Dynamic[LFSystem]); }
- }
- if (Dynamic.find(MkqlGlobalPoolTotalBytes) != Dynamic.end()) {
- TABLER() {
- TABLED() { str << "MKQLAlloc"; }
- }
- TABLER() { outputDynamic("GlobalPoolTotalBytes", Dynamic[MkqlGlobalPoolTotalBytes]); }
- TABLER() { outputDynamic("Kqp", Dynamic[MkqlKqp]); }
- TABLER() { outputDynamic("Compile", Dynamic[MkqlCompile]); }
- TABLER() { outputDynamic("Datashard", Dynamic[MkqlDatashard]); }
- TABLER() { outputDynamic("LocalTx", Dynamic[MkqlLocalTx]); }
- TABLER() { outputDynamic("TxProxy", Dynamic[MkqlTxProxy]); }
- }
- }
- }
- TABLE_CLASS("table") {
- TABLEHEAD() {
- TABLER() {
- TABLEH() { str << "Metric"; }
- TABLEH() { str << "Count"; }
- TABLEH() { str << "Peak count"; }
- TABLEH() { str << "Memory"; }
- TABLEH() { str << "Peak memory"; }
- }
- }
- TABLEBODY() {
- const auto& indices = TMemoryTracker::Instance()->GetMetricIndices();
- for (const auto& [name, index] : indices) {
- TABLER() {
- auto& metric = Static[index];
- TABLED() { str << NHtml::EscapeText(name); }
- TAG_CLASS_STYLE(TTableD, "", "text-align:right") {
- str << PrettyNumber(metric.Current.GetCount()); }
- TAG_CLASS_STYLE(TTableD, "", "text-align:right") {
- str << PrettyNumber(metric.Peak.GetCount()); }
- TAG_CLASS_STYLE(TTableD, "", "text-align:right") {
- str << PrettyNumber(metric.Current.GetMemory()); }
- TAG_CLASS_STYLE(TTableD, "", "text-align:right") {
- str << PrettyNumber(metric.Peak.GetMemory()); }
- }
- }
- }
- }
-#endif
- }
-
- Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str()));
- }
-
-private:
- const TDuration UpdateInterval;
-
- enum EDynamicMetric {
- AnonRssSize,
- LFMmapped,
- LFSmall,
- LFLarge,
- LFSystem,
- MkqlGlobalPoolTotalBytes,
- MkqlKqp,
- MkqlCompile,
- MkqlDatashard,
- MkqlLocalTx,
- MkqlTxProxy
- };
-
- TDynamicCounterPtr AppCounters;
- TDynamicCounterPtr Counters;
-
- struct TStaticMetric {
- TMetric Current;
- TMetric Peak;
- TDynamicCounters::TCounterPtr CurrentMemCounter;
- TDynamicCounters::TCounterPtr PeakMemCounter;
- TDynamicCounters::TCounterPtr CurrentCountCounter;
- TDynamicCounters::TCounterPtr PeakCountCounter;
- };
-
- struct TDynamicMetric {
- TMetric Current;
- TMetric Peak;
- TDynamicCounters::TCounterPtr Counter;
-
- TDynamicMetric(TDynamicCounters::TCounterPtr counter = {})
- : Counter(counter)
- {}
- };
-
- std::vector<TStaticMetric> Static;
- std::unordered_map<EDynamicMetric, TDynamicMetric> Dynamic;
-};
-
-NActors::IActor* CreateMemoryTrackerActor(TDuration updateInterval, TDynamicCounterPtr counters) {
- return new TMemoryTrackerActor(updateInterval, counters);
-}
-
-}
-}
+
+#include <library/cpp/actors/core/actor_bootstrapped.h>
+#include <library/cpp/actors/core/memory_tracker.h>
+#include <library/cpp/html/escape/escape.h>
+#include <library/cpp/monlib/service/pages/templates.h>
+
+#include <util/generic/xrange.h>
+
+using namespace NMonitoring;
+using namespace NActors::NMemory::NPrivate;
+
+namespace NKikimr {
+namespace NMemory {
+
+class TMemoryTrackerActor : public TActorBootstrapped<TMemoryTrackerActor> {
+public:
+ TMemoryTrackerActor(TDuration updateInterval, TDynamicCounterPtr appCounters)
+ : UpdateInterval(updateInterval)
+ , AppCounters(appCounters)
+ {
+ auto utilsCounters = GetServiceCounters(AppCounters, "utils");
+ Counters = utilsCounters->GetSubgroup("component", "memory_tracker");
+ }
+
+ static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
+ return NKikimrServices::TActivity::MEMORY_TRACKER;
+ }
+
+ void Bootstrap(const TActorContext&) {
+#if defined(ENABLE_MEMORY_TRACKING)
+ TMemoryTracker::Instance()->Initialize();
+ RegisterCounters(Counters);
+
+ TMon* mon = AppData()->Mon;
+ if (mon) {
+ auto *actorsMonPage = mon->RegisterIndexPage("actors", "Actors");
+ mon->RegisterActorPage(actorsMonPage, "memory_tracker", "Memory tracker",
+ false, TlsActivationContext->ExecutorThread.ActorSystem, SelfId());
+ }
+
+ Collect();
+#endif
+ Become(&TMemoryTrackerActor::StateWork);
+ }
+
+ STRICT_STFUNC(StateWork,
+ cFunc(TEvents::TEvWakeup::EventType, Wakeup)
+ hFunc(NMon::TEvHttpInfo, HttpInfo)
+ cFunc(TEvents::TEvPoison::EventType, PassAway)
+ )
+
+private:
+ void Wakeup() {
+ Collect();
+ }
+
+ void Collect() {
+ Schedule(UpdateInterval, new TEvents::TEvWakeup());
+
+ std::vector<TMetric> metrics;
+ TMemoryTracker::Instance()->GatherMetrics(metrics);
+
+ auto count = metrics.size();
+ if (Static.size() != count) {
+ return;
+ }
+
+ const auto& sensors = TMemoryTracker::Instance()->GetSensors();
+
+ for (auto i : xrange(count)) {
+ auto& metric = Static[i];
+ metric.Current = metrics[i];
+ metric.Peak.CalculatePeak(metric.Current);
+
+ if (sensors.find(i) != sensors.end()) {
+ *metric.CurrentMemCounter = metric.Current.GetMemory();
+ *metric.PeakMemCounter = metric.Peak.GetMemory();
+ *metric.CurrentCountCounter = metric.Current.GetCount();
+ *metric.PeakCountCounter = metric.Peak.GetCount();
+ }
+ }
+
+ GatherDynamicMetrics();
+ }
+
+ void RegisterCounters(NMonitoring::TDynamicCounterPtr counters) {
+ auto componentGroup = counters->GetSubgroup("component", "memory_tracker");
+
+ auto count = TMemoryTracker::Instance()->GetCount();
+ Static.resize(count);
+
+ const auto& sensors = TMemoryTracker::Instance()->GetSensors();
+ for (auto index : sensors) {
+ auto name = TMemoryTracker::Instance()->GetName(index);
+ auto& metric = Static[index];
+ auto group = componentGroup->GetSubgroup("label", name);
+ metric.CurrentMemCounter = group->GetCounter("MT/Memory");
+ metric.PeakMemCounter = group->GetCounter("MT/PeakMemory");
+ metric.CurrentCountCounter = group->GetCounter("MT/Count");
+ metric.PeakCountCounter = group->GetCounter("MT/PeakCount");
+ }
+ }
+
+ void UpdateDynamicCounters() {
+ auto utils = GetServiceCounters(AppCounters, "utils");
+
+ if (Dynamic.find(AnonRssSize) == Dynamic.end()) {
+ Dynamic.emplace(AnonRssSize, utils->GetCounter("Process/AnonRssSize"));
+ }
+
+ if (Dynamic.find(LFMmapped) == Dynamic.end()) {
+ auto lfAllocGroup = utils->FindSubgroup("component", "lfalloc");
+ if (lfAllocGroup) {
+ Dynamic.emplace(LFMmapped, lfAllocGroup->GetCounter("ActiveBytesMmapped"));
+ Dynamic.emplace(LFSmall, lfAllocGroup->GetCounter("ActiveBytesSmall"));
+ Dynamic.emplace(LFLarge, lfAllocGroup->GetCounter("ActiveBytesLarge"));
+ Dynamic.emplace(LFSystem, lfAllocGroup->GetCounter("ActiveBytesSystem"));
+ }
+ }
+
+ if (Dynamic.find(MkqlGlobalPoolTotalBytes) == Dynamic.end()) {
+ auto mkqlAllocGroup = utils->FindSubgroup("subsystem", "mkqlalloc");
+ if (mkqlAllocGroup) {
+ Dynamic.emplace(MkqlGlobalPoolTotalBytes, mkqlAllocGroup->GetCounter("GlobalPoolTotalBytes"));
+
+ auto getNamedPagePoolCounter = [&mkqlAllocGroup](const char* name) {
+ return mkqlAllocGroup->GetCounter(TString(name) + "/TotalBytesAllocated");
+ };
+ Dynamic.emplace(MkqlKqp, getNamedPagePoolCounter("kqp"));
+ Dynamic.emplace(MkqlCompile, getNamedPagePoolCounter("compile"));
+ Dynamic.emplace(MkqlDatashard, getNamedPagePoolCounter("datashard"));
+ Dynamic.emplace(MkqlLocalTx, getNamedPagePoolCounter("local_tx"));
+ Dynamic.emplace(MkqlTxProxy, getNamedPagePoolCounter("tx_proxy"));
+ }
+ }
+ }
+
+ void GatherDynamicMetrics() {
+ UpdateDynamicCounters();
+
+ for (auto& [_, metric] : Dynamic) {
+ if (metric.Counter) {
+ metric.Current.SetMemory(metric.Counter->Val());
+ metric.Peak.CalculatePeak(metric.Current);
+ }
+ }
+ }
+
+ static TString PrettyNumber(ssize_t val) {
+ if (val == 0) {
+ return "0";
+ }
+ TStringStream str;
+ if (val < 0) {
+ str << "- ";
+ }
+ ui64 value = val < 0 ? -val : val;
+ TSmallVec<ui64> parts;
+ while (value > 0) {
+ auto head = value / 1000;
+ parts.push_back(value - head * 1000);
+ value = head;
+ }
+ str << parts.back();
+ parts.pop_back();
+ while (!parts.empty()) {
+ str << Sprintf(" %03" PRIu64, parts.back());
+ parts.pop_back();
+ }
+ return str.Str();
+ }
+
+ void HttpInfo(NMon::TEvHttpInfo::TPtr& ev) {
+ TStringStream str;
+
+ HTML(str) {
+#if defined(ENABLE_MEMORY_TRACKING)
+ TABLE_CLASS("table") {
+ TABLEHEAD() {
+ TABLER() {
+ TABLEH() { str << "Metric"; }
+ TABLEH() { str << "Memory"; }
+ TABLEH() { str << "Peak memory"; }
+ }
+ }
+ TABLEBODY() {
+ auto outputDynamic = [&str, &__stream] (const char* name, const TDynamicMetric& metric) {
+ TABLED() { str << name; }
+ TAG_CLASS_STYLE(TTableD, "", "text-align:right") {
+ str << PrettyNumber(metric.Current.GetMemory()); }
+ TAG_CLASS_STYLE(TTableD, "", "text-align:right") {
+ str << PrettyNumber(metric.Peak.GetMemory()); }
+ };
+
+ if (Dynamic.find(AnonRssSize) != Dynamic.end()) {
+ TABLER() { outputDynamic("AnonRssSize", Dynamic[AnonRssSize]); }
+ }
+ if (Dynamic.find(LFMmapped) != Dynamic.end()) {
+ TABLER() {
+ TABLED() { str << "LFAlloc"; }
+ }
+ TABLER() { outputDynamic("Mmapped", Dynamic[LFMmapped]); }
+ TABLER() { outputDynamic("Small", Dynamic[LFSmall]); }
+ TABLER() { outputDynamic("Large", Dynamic[LFLarge]); }
+ TABLER() { outputDynamic("System", Dynamic[LFSystem]); }
+ }
+ if (Dynamic.find(MkqlGlobalPoolTotalBytes) != Dynamic.end()) {
+ TABLER() {
+ TABLED() { str << "MKQLAlloc"; }
+ }
+ TABLER() { outputDynamic("GlobalPoolTotalBytes", Dynamic[MkqlGlobalPoolTotalBytes]); }
+ TABLER() { outputDynamic("Kqp", Dynamic[MkqlKqp]); }
+ TABLER() { outputDynamic("Compile", Dynamic[MkqlCompile]); }
+ TABLER() { outputDynamic("Datashard", Dynamic[MkqlDatashard]); }
+ TABLER() { outputDynamic("LocalTx", Dynamic[MkqlLocalTx]); }
+ TABLER() { outputDynamic("TxProxy", Dynamic[MkqlTxProxy]); }
+ }
+ }
+ }
+ TABLE_CLASS("table") {
+ TABLEHEAD() {
+ TABLER() {
+ TABLEH() { str << "Metric"; }
+ TABLEH() { str << "Count"; }
+ TABLEH() { str << "Peak count"; }
+ TABLEH() { str << "Memory"; }
+ TABLEH() { str << "Peak memory"; }
+ }
+ }
+ TABLEBODY() {
+ const auto& indices = TMemoryTracker::Instance()->GetMetricIndices();
+ for (const auto& [name, index] : indices) {
+ TABLER() {
+ auto& metric = Static[index];
+ TABLED() { str << NHtml::EscapeText(name); }
+ TAG_CLASS_STYLE(TTableD, "", "text-align:right") {
+ str << PrettyNumber(metric.Current.GetCount()); }
+ TAG_CLASS_STYLE(TTableD, "", "text-align:right") {
+ str << PrettyNumber(metric.Peak.GetCount()); }
+ TAG_CLASS_STYLE(TTableD, "", "text-align:right") {
+ str << PrettyNumber(metric.Current.GetMemory()); }
+ TAG_CLASS_STYLE(TTableD, "", "text-align:right") {
+ str << PrettyNumber(metric.Peak.GetMemory()); }
+ }
+ }
+ }
+ }
+#endif
+ }
+
+ Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str()));
+ }
+
+private:
+ const TDuration UpdateInterval;
+
+ enum EDynamicMetric {
+ AnonRssSize,
+ LFMmapped,
+ LFSmall,
+ LFLarge,
+ LFSystem,
+ MkqlGlobalPoolTotalBytes,
+ MkqlKqp,
+ MkqlCompile,
+ MkqlDatashard,
+ MkqlLocalTx,
+ MkqlTxProxy
+ };
+
+ TDynamicCounterPtr AppCounters;
+ TDynamicCounterPtr Counters;
+
+ struct TStaticMetric {
+ TMetric Current;
+ TMetric Peak;
+ TDynamicCounters::TCounterPtr CurrentMemCounter;
+ TDynamicCounters::TCounterPtr PeakMemCounter;
+ TDynamicCounters::TCounterPtr CurrentCountCounter;
+ TDynamicCounters::TCounterPtr PeakCountCounter;
+ };
+
+ struct TDynamicMetric {
+ TMetric Current;
+ TMetric Peak;
+ TDynamicCounters::TCounterPtr Counter;
+
+ TDynamicMetric(TDynamicCounters::TCounterPtr counter = {})
+ : Counter(counter)
+ {}
+ };
+
+ std::vector<TStaticMetric> Static;
+ std::unordered_map<EDynamicMetric, TDynamicMetric> Dynamic;
+};
+
+NActors::IActor* CreateMemoryTrackerActor(TDuration updateInterval, TDynamicCounterPtr counters) {
+ return new TMemoryTrackerActor(updateInterval, counters);
+}
+
+}
+}
diff --git a/ydb/core/util/memory_tracker.h b/ydb/core/util/memory_tracker.h
index 1db4ee3e9e7..af99e84424b 100644
--- a/ydb/core/util/memory_tracker.h
+++ b/ydb/core/util/memory_tracker.h
@@ -1,17 +1,17 @@
-#pragma once
-
-#include <library/cpp/monlib/dynamic_counters/counters.h>
-
-namespace NActors {
- class IActor;
-}
-
-namespace NKikimr {
-namespace NMemory {
-
-NActors::IActor* CreateMemoryTrackerActor(
- TDuration updateInterval,
- NMonitoring::TDynamicCounterPtr counters);
-
-}
-}
+#pragma once
+
+#include <library/cpp/monlib/dynamic_counters/counters.h>
+
+namespace NActors {
+ class IActor;
+}
+
+namespace NKikimr {
+namespace NMemory {
+
+NActors::IActor* CreateMemoryTrackerActor(
+ TDuration updateInterval,
+ NMonitoring::TDynamicCounterPtr counters);
+
+}
+}
diff --git a/ydb/core/util/ut/ya.make b/ydb/core/util/ut/ya.make
index 315713cad86..295b6895f0a 100644
--- a/ydb/core/util/ut/ya.make
+++ b/ydb/core/util/ut/ya.make
@@ -26,7 +26,7 @@ SRCS(
btree_ut.cpp
cache_ut.cpp
circular_queue_ut.cpp
- concurrent_rw_hash_ut.cpp
+ concurrent_rw_hash_ut.cpp
fast_tls_ut.cpp
fragmented_buffer_ut.cpp
hazard_ut.cpp
diff --git a/ydb/core/util/ya.make b/ydb/core/util/ya.make
index 4f325d8fcbd..34b6df67aa4 100644
--- a/ydb/core/util/ya.make
+++ b/ydb/core/util/ya.make
@@ -11,8 +11,8 @@ SRCS(
cache.h
cache_cache.h
circular_queue.h
- concurrent_rw_hash.cpp
- concurrent_rw_hash.h
+ concurrent_rw_hash.cpp
+ concurrent_rw_hash.h
console.cpp
console.h
counted_leaky_bucket.h
@@ -33,8 +33,8 @@ SRCS(
intrusive_heap.h
intrusive_stack.h
log_priority_mute_checker.h
- memory_tracker.cpp
- memory_tracker.h
+ memory_tracker.cpp
+ memory_tracker.h
operation_queue.h
page_map.cpp
pb.h
@@ -65,7 +65,7 @@ PEERDIR(
library/cpp/actors/interconnect/mock
library/cpp/actors/util
library/cpp/containers/stack_vector
- library/cpp/html/escape
+ library/cpp/html/escape
library/cpp/ipmath
library/cpp/lwtrace
library/cpp/monlib/dynamic_counters
diff --git a/ydb/library/yql/core/common_opt/yql_co_flow1.cpp b/ydb/library/yql/core/common_opt/yql_co_flow1.cpp
index 728838dc8eb..37a89608f3b 100644
--- a/ydb/library/yql/core/common_opt/yql_co_flow1.cpp
+++ b/ydb/library/yql/core/common_opt/yql_co_flow1.cpp
@@ -1291,8 +1291,8 @@ TExprNode::TPtr CountAggregateRewrite(const TCoAggregate& node, TExprContext& ct
auto hoppingSetting = GetSetting(settings.Ref(), "hopping");
if (hoppingSetting) {
return node.Ptr();
- }
-
+ }
+
if (GetSetting(settings.Ref(), "session")) {
// TODO: support
return node.Ptr();
diff --git a/ydb/library/yql/core/common_opt/yql_co_simple2.cpp b/ydb/library/yql/core/common_opt/yql_co_simple2.cpp
index 6d59989b6a4..0666f8c545c 100644
--- a/ydb/library/yql/core/common_opt/yql_co_simple2.cpp
+++ b/ydb/library/yql/core/common_opt/yql_co_simple2.cpp
@@ -2,7 +2,7 @@
#include <ydb/library/yql/core/yql_opt_utils.h>
#include <ydb/library/yql/core/yql_expr_csee.h>
-
+
#include <ydb/library/yql/utils/log/log.h>
#include <ydb/library/yql/utils/yql_panic.h>
@@ -198,8 +198,8 @@ TExprNode::TPtr DeduplicateAggregateSameTraits(const TExprNode::TPtr& node, TExp
.Atom(1, "_yql_time", TNodeFlags::Default)
.Seal()
.Seal();
- }
-
+ }
+
return structObj.Seal();
})
.Seal()
diff --git a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.h b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.h
index 01cc7771185..af5680a3f5c 100644
--- a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.h
+++ b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.h
@@ -180,19 +180,19 @@ public:
}
TNodeBuilder<TParent, TCoLambda>& Args(const std::vector<TCoArgument>& list)
- {
- Y_VERIFY_DEBUG(!this->ArgsHolder.IsValid());
-
+ {
+ Y_VERIFY_DEBUG(!this->ArgsHolder.IsValid());
+
TExprNode::TListType argNodes;
- for (auto arg : list) {
- argNodes.push_back(arg.Ptr());
- }
-
- auto argsNode = this->Ctx.NewArguments(this->Pos, std::move(argNodes));
+ for (auto arg : list) {
+ argNodes.push_back(arg.Ptr());
+ }
+
+ auto argsNode = this->Ctx.NewArguments(this->Pos, std::move(argNodes));
this->ArgsHolder = TCoArguments(argsNode);
- return *this;
- }
-
+ return *this;
+ }
+
TNodeBuilder<TParent, TCoLambda>& Args(const TExprNode::TListType& list)
{
Y_VERIFY_DEBUG(!this->ArgsHolder.IsValid());
diff --git a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json
index edfcaa15302..b4b7a979185 100644
--- a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json
+++ b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json
@@ -549,18 +549,18 @@
},
{
"Name": "TCoHoppingTraits",
- "Base": "TCallable",
- "Match": {"Type": "Callable", "Name": "HoppingTraits"},
- "Children": [
+ "Base": "TCallable",
+ "Match": {"Type": "Callable", "Name": "HoppingTraits"},
+ "Children": [
{"Index": 0, "Name": "ItemType", "Type": "TExprBase"},
{"Index": 1, "Name": "TimeExtractor", "Type": "TCoLambda"},
{"Index": 2, "Name": "Hop", "Type": "TExprBase"},
{"Index": 3, "Name": "Interval", "Type": "TExprBase"},
{"Index": 4, "Name": "Delay", "Type": "TExprBase"},
{"Index": 5, "Name": "DataWatermarks", "Type": "TExprBase"}
- ]
- },
- {
+ ]
+ },
+ {
"Name": "TCoSessionWindowTraits",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "SessionWindowTraits"},
@@ -583,8 +583,8 @@
{"Index": 3, "Name": "SaveHandler", "Type": "TCoLambda"},
{"Index": 4, "Name": "LoadHandler", "Type": "TCoLambda"},
{"Index": 5, "Name": "MergeHandler", "Type": "TCoLambda"},
- {"Index": 6, "Name": "FinishHandler", "Type": "TCoLambda"},
- {"Index": 7, "Name": "DefVal", "Type": "TExprBase"}
+ {"Index": 6, "Name": "FinishHandler", "Type": "TCoLambda"},
+ {"Index": 7, "Name": "DefVal", "Type": "TExprBase"}
]
},
{
@@ -1579,28 +1579,28 @@
{"Index": 2, "Name": "MatchLambda", "Type": "TCoLambda"},
{"Index": 3, "Name": "NotMatchLambda", "Type": "TCoLambda"}
]
- },
- {
+ },
+ {
"Name": "TCoIterator",
- "Base": "TCallable",
- "Match": {"Type": "Callable", "Name": "Iterator"},
- "Children": [
- {"Index": 0, "Name": "List", "Type": "TExprBase"}
- ]
- },
- {
+ "Base": "TCallable",
+ "Match": {"Type": "Callable", "Name": "Iterator"},
+ "Children": [
+ {"Index": 0, "Name": "List", "Type": "TExprBase"}
+ ]
+ },
+ {
"Name": "TCoForwardList",
- "Base": "TCallable",
- "Match": {"Type": "Callable", "Name": "ForwardList"},
- "Children": [
- {"Index": 0, "Name": "Stream", "Type": "TExprBase"}
- ]
- },
- {
+ "Base": "TCallable",
+ "Match": {"Type": "Callable", "Name": "ForwardList"},
+ "Children": [
+ {"Index": 0, "Name": "Stream", "Type": "TExprBase"}
+ ]
+ },
+ {
"Name": "TCoHoppingCore",
"Base": "TCoInputBase",
- "Match": {"Type": "Callable", "Name": "HoppingCore"},
- "Children": [
+ "Match": {"Type": "Callable", "Name": "HoppingCore"},
+ "Children": [
{"Index": 1, "Name": "TimeExtractor", "Type": "TCoLambda"},
{"Index": 2, "Name": "Hop", "Type": "TExprBase"},
{"Index": 3, "Name": "Interval", "Type": "TExprBase"},
@@ -1611,7 +1611,7 @@
{"Index": 8, "Name": "LoadHandler", "Type": "TExprBase"},
{"Index": 9, "Name": "MergeHandler", "Type": "TCoLambda"},
{"Index": 10, "Name": "FinishHandler", "Type": "TCoLambda"}
- ]
+ ]
},
{
"Name": "TCoMultiHoppingCore",
@@ -1673,8 +1673,8 @@
"Children": [
{"Index": 1, "Name": "Type", "Type": "TCoAtom"}
]
- },
- {
+ },
+ {
"Name": "TCoCondense",
"Base": "TCoInputBase",
"Match": {"Type": "Callable", "Name": "Condense"},
diff --git a/ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.jnj b/ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.jnj
index 4b5733693f4..9f2327dbf2c 100644
--- a/ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.jnj
+++ b/ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.jnj
@@ -203,9 +203,9 @@ public:
{%- for child in node.aux.allChildren %}
TBuilder<TParent, TDerived>& {{child.Name}}(const TStringBuf& argName) { {{child.aux.holderName}} = {{child.Type}}(this->GetArgFunc(argName)); return *static_cast<TBuilder<TParent, TDerived>*>(this); }
- {% if child.Optional %}
- TBuilder<TParent, TDerived>& {{child.Name}}(const TMaybe<{{child.Type}}>& node) { {{child.aux.holderName}} = node; return *static_cast<TBuilder<TParent, TDerived>*>(this); }
- {% endif %}
+ {% if child.Optional %}
+ TBuilder<TParent, TDerived>& {{child.Name}}(const TMaybe<{{child.Type}}>& node) { {{child.aux.holderName}} = node; return *static_cast<TBuilder<TParent, TDerived>*>(this); }
+ {% endif %}
TBuilder<TParent, TDerived>& {{child.Name}}(const {{child.Type}}& node) { {{child.aux.holderName}} = node; return *static_cast<TBuilder<TParent, TDerived>*>(this); }
TBuilder<TParent, TDerived>& {{child.Name}}(const TExprNode::TPtr& node) { {{child.aux.holderName}} = {{child.Type}}(node); return *static_cast<TBuilder<TParent, TDerived>*>(this); }
diff --git a/ydb/library/yql/core/facade/yql_facade.cpp b/ydb/library/yql/core/facade/yql_facade.cpp
index b74abc1c34c..4b5a7558eee 100644
--- a/ydb/library/yql/core/facade/yql_facade.cpp
+++ b/ydb/library/yql/core/facade/yql_facade.cpp
@@ -1085,40 +1085,40 @@ IGraphTransformer::TStatistics TProgram::GetRawDiagnostics() {
return Transformer_ ? Transformer_->GetStatistics() : IGraphTransformer::TStatistics::NotPresent();
}
-TMaybe<TString> TProgram::GetTasksInfo() {
- if (!TypeCtx_) {
- return Nothing();
- }
-
- bool hasTasks = false;
-
- TStringStream out;
+TMaybe<TString> TProgram::GetTasksInfo() {
+ if (!TypeCtx_) {
+ return Nothing();
+ }
+
+ bool hasTasks = false;
+
+ TStringStream out;
NYson::TYsonWriter writer(&out, ResultFormat_);
-
- writer.OnBeginMap();
- writer.OnKeyedItem("Write");
- writer.OnBeginList();
- writer.OnListItem();
- writer.OnBeginMap();
- writer.OnKeyedItem("Tasks");
- writer.OnBeginList();
-
- for (auto& datasink : TypeCtx_->DataSinks) {
+
+ writer.OnBeginMap();
+ writer.OnKeyedItem("Write");
+ writer.OnBeginList();
+ writer.OnListItem();
+ writer.OnBeginMap();
+ writer.OnKeyedItem("Tasks");
+ writer.OnBeginList();
+
+ for (auto& datasink : TypeCtx_->DataSinks) {
hasTasks = hasTasks || datasink->GetTasksInfo(writer);
- }
-
- writer.OnEndList();
- writer.OnEndMap();
- writer.OnEndList();
- writer.OnEndMap();
-
- if (hasTasks) {
- return out.Str();
- } else {
- return Nothing();
- }
-}
-
+ }
+
+ writer.OnEndList();
+ writer.OnEndMap();
+ writer.OnEndList();
+ writer.OnEndMap();
+
+ if (hasTasks) {
+ return out.Str();
+ } else {
+ return Nothing();
+ }
+}
+
TMaybe<TString> TProgram::GetStatistics(bool totalOnly) {
if (!TypeCtx_) {
return Nothing();
@@ -1311,9 +1311,9 @@ TTypeAnnotationContextPtr TProgram::BuildTypeAnnotationContext(const TString& us
}
if (providerNames.contains(RtmrProviderName)) {
- resultProviderDataSources.push_back(TString(RtmrProviderName));
- }
-
+ resultProviderDataSources.push_back(TString(RtmrProviderName));
+ }
+
if (providerNames.contains(PqProviderName)) {
resultProviderDataSources.push_back(TString(PqProviderName));
}
@@ -1384,13 +1384,13 @@ void TProgram::Print(IOutputStream* exprOut, IOutputStream* planOut, bool cleanP
InstantTransform(*compositeTransformer, ExprRoot_, *ExprCtx_);
}
-bool TProgram::HasActiveProcesses() {
- for (const auto& dp : DataProviders_) {
- if (dp.HasActiveProcesses && dp.HasActiveProcesses()) {
- return true;
- }
- }
- return false;
-}
-
+bool TProgram::HasActiveProcesses() {
+ for (const auto& dp : DataProviders_) {
+ if (dp.HasActiveProcesses && dp.HasActiveProcesses()) {
+ return true;
+ }
+ }
+ return false;
+}
+
} // namespace NYql
diff --git a/ydb/library/yql/core/facade/yql_facade.h b/ydb/library/yql/core/facade/yql_facade.h
index 1f8f47681da..7e3308e1c2e 100644
--- a/ydb/library/yql/core/facade/yql_facade.h
+++ b/ydb/library/yql/core/facade/yql_facade.h
@@ -156,8 +156,8 @@ public:
TFutureStatus ContinueAsync();
- bool HasActiveProcesses();
-
+ bool HasActiveProcesses();
+
void Abort();
inline TIssues Issues() {
@@ -215,8 +215,8 @@ public:
TMaybe<TString> GetDiagnostics();
IGraphTransformer::TStatistics GetRawDiagnostics();
- TMaybe<TString> GetTasksInfo();
-
+ TMaybe<TString> GetTasksInfo();
+
TMaybe<TString> GetStatistics(bool totalOnly = false);
TMaybe<TString> GetDiscoveredData();
diff --git a/ydb/library/yql/core/services/yql_plan.cpp b/ydb/library/yql/core/services/yql_plan.cpp
index 4d7cfffaf5a..c3f3534511d 100644
--- a/ydb/library/yql/core/services/yql_plan.cpp
+++ b/ydb/library/yql/core/services/yql_plan.cpp
@@ -374,9 +374,9 @@ public:
info.IsVisible = (*provider)->GetPlanFormatter().GetDependencies(*node, dependencies, true);
} else {
info.IsVisible = false;
- for (auto& child : node->Children()) {
- dependencies.push_back(child.Get());
- }
+ for (auto& child : node->Children()) {
+ dependencies.push_back(child.Get());
+ }
}
for (const auto& child : dependencies) {
diff --git a/ydb/library/yql/core/services/yql_plan.h b/ydb/library/yql/core/services/yql_plan.h
index 608c54fb8e4..f13a84d33e8 100644
--- a/ydb/library/yql/core/services/yql_plan.h
+++ b/ydb/library/yql/core/services/yql_plan.h
@@ -3,7 +3,7 @@
#include <ydb/library/yql/core/type_ann/type_ann_core.h>
#include <library/cpp/yson/writer.h>
-
+
namespace NYql {
class IPlanBuilder {
diff --git a/ydb/library/yql/core/type_ann/type_ann_core.cpp b/ydb/library/yql/core/type_ann/type_ann_core.cpp
index 5846e6cb108..1388aa77cc4 100644
--- a/ydb/library/yql/core/type_ann/type_ann_core.cpp
+++ b/ydb/library/yql/core/type_ann/type_ann_core.cpp
@@ -2043,7 +2043,7 @@ namespace NTypeAnnImpl {
}
template <bool Equal, bool Order>
- IGraphTransformer::TStatus AggrCompareWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
+ IGraphTransformer::TStatus AggrCompareWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
if (!EnsureArgsCount(*input, 2, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -2067,18 +2067,18 @@ namespace NTypeAnnImpl {
if (!Order) {
input->SetUnorderedChildren();
}
- return IGraphTransformer::TStatus::Ok;
- }
-
- IGraphTransformer::TStatus AggrMinMaxWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
- if (!EnsureArgsCount(*input, 2, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
+ return IGraphTransformer::TStatus::Ok;
+ }
+
+ IGraphTransformer::TStatus AggrMinMaxWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
+ if (!EnsureArgsCount(*input, 2, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
if (!EnsureComparableType(input->Pos(), *input->Head().GetTypeAnn(), ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
+ return IGraphTransformer::TStatus::Error;
+ }
+
if (!IsSameAnnotation(*input->Head().GetTypeAnn(), *input->Tail().GetTypeAnn())) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Type mismatch, left: "
<< *input->Head().GetTypeAnn() << ", right:" << *input->Tail().GetTypeAnn()));
@@ -12994,8 +12994,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
Functions["CommonJoinCore"] = &CommonJoinCoreWrapper;
Functions["CombineCore"] = &CombineCoreWrapper;
Functions["GroupingCore"] = &GroupingCoreWrapper;
- Functions["HoppingTraits"] = &HoppingTraitsWrapper;
- Functions["HoppingCore"] = &HoppingCoreWrapper;
+ Functions["HoppingTraits"] = &HoppingTraitsWrapper;
+ Functions["HoppingCore"] = &HoppingCoreWrapper;
Functions["MultiHoppingCore"] = &MultiHoppingCoreWrapper;
Functions["EquiJoin"] = &EquiJoinWrapper;
Functions["OptionalReduce"] = &OptionalReduceWrapper;
diff --git a/ydb/library/yql/core/type_ann/type_ann_list.cpp b/ydb/library/yql/core/type_ann/type_ann_list.cpp
index b145cde3893..a1790c89575 100644
--- a/ydb/library/yql/core/type_ann/type_ann_list.cpp
+++ b/ydb/library/yql/core/type_ann/type_ann_list.cpp
@@ -3456,7 +3456,7 @@ namespace {
IGraphTransformer::TStatus SqueezeWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
Y_UNUSED(output);
- if (!EnsureArgsCount(*input, 5, ctx.Expr)) {
+ if (!EnsureArgsCount(*input, 5, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -3468,16 +3468,16 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- auto& lambda = input->ChildRef(2);
-
- auto status = ConvertToLambda(lambda, ctx.Expr, 2);
+ auto& lambda = input->ChildRef(2);
+
+ auto status = ConvertToLambda(lambda, ctx.Expr, 2);
if (status.Level != IGraphTransformer::TStatus::Ok) {
return status;
}
auto stateType = input->Child(1)->GetTypeAnn();
auto itemType = input->Head().GetTypeAnn()->Cast<TStreamExprType>()->GetItemType();
- if (!UpdateLambdaAllArgumentsTypes(lambda, { itemType, stateType }, ctx.Expr)) {
+ if (!UpdateLambdaAllArgumentsTypes(lambda, { itemType, stateType }, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
if (!lambda->GetTypeAnn()) {
@@ -3490,48 +3490,48 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- auto& saveLambda = input->ChildRef(3);
- auto& loadLambda = input->ChildRef(4);
-
- if (saveLambda->IsCallable("Void") != loadLambda->IsCallable("Void")) {
+ auto& saveLambda = input->ChildRef(3);
+ auto& loadLambda = input->ChildRef(4);
+
+ if (saveLambda->IsCallable("Void") != loadLambda->IsCallable("Void")) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(saveLambda->Pos()), TStringBuilder() <<
- "Save and load lambdas must be specified at the same time"));
- return IGraphTransformer::TStatus::Error;
- }
-
- if (!saveLambda->IsCallable("Void")) {
- auto status = ConvertToLambda(saveLambda, ctx.Expr, 1);
- status = status.Combine(ConvertToLambda(loadLambda, ctx.Expr, 1));
- if (status.Level != IGraphTransformer::TStatus::Ok) {
- return status;
- }
-
- if (!UpdateLambdaAllArgumentsTypes(saveLambda, {stateType}, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
+ "Save and load lambdas must be specified at the same time"));
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ if (!saveLambda->IsCallable("Void")) {
+ auto status = ConvertToLambda(saveLambda, ctx.Expr, 1);
+ status = status.Combine(ConvertToLambda(loadLambda, ctx.Expr, 1));
+ if (status.Level != IGraphTransformer::TStatus::Ok) {
+ return status;
+ }
+
+ if (!UpdateLambdaAllArgumentsTypes(saveLambda, {stateType}, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
if (!saveLambda->GetTypeAnn()) {
- return IGraphTransformer::TStatus::Repeat;
- }
-
+ return IGraphTransformer::TStatus::Repeat;
+ }
+
auto savedStateType = saveLambda->GetTypeAnn();
- if (!EnsurePersistableType(saveLambda->Pos(), *savedStateType, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- if (!UpdateLambdaAllArgumentsTypes(loadLambda, {savedStateType}, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
+ if (!EnsurePersistableType(saveLambda->Pos(), *savedStateType, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ if (!UpdateLambdaAllArgumentsTypes(loadLambda, {savedStateType}, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
if (!loadLambda->GetTypeAnn()) {
- return IGraphTransformer::TStatus::Repeat;
- }
-
+ return IGraphTransformer::TStatus::Repeat;
+ }
+
if (!IsSameAnnotation(*loadLambda->GetTypeAnn(), *stateType)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(loadLambda->Pos()), TStringBuilder() << "Mismatch of load lambda return type and state type, "
<< *loadLambda->GetTypeAnn() << " != " << *stateType));
- return IGraphTransformer::TStatus::Error;
- }
- }
-
+ return IGraphTransformer::TStatus::Error;
+ }
+ }
+
auto issue = TIssue(ctx.Expr.GetPosition(input->Head().Pos()), "Deprecated Squeeze, use Condense instead.");
SetIssueCode(EYqlIssueCode::TIssuesIds_EIssueCode_YQL_DEPRECATED_FUNCTION_OR_SIGNATURE, issue);
if (!ctx.Expr.AddWarning(issue)) {
@@ -3544,7 +3544,7 @@ namespace {
IGraphTransformer::TStatus Squeeze1Wrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
Y_UNUSED(output);
- if (!EnsureArgsCount(*input, 5, ctx.Expr)) {
+ if (!EnsureArgsCount(*input, 5, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -3552,11 +3552,11 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- auto& initLambda = input->ChildRef(1);
- auto& updateLambda = input->ChildRef(2);
-
- auto status = ConvertToLambda(initLambda, ctx.Expr, 1);
- status = status.Combine(ConvertToLambda(updateLambda, ctx.Expr, 2));
+ auto& initLambda = input->ChildRef(1);
+ auto& updateLambda = input->ChildRef(2);
+
+ auto status = ConvertToLambda(initLambda, ctx.Expr, 1);
+ status = status.Combine(ConvertToLambda(updateLambda, ctx.Expr, 2));
if (status.Level != IGraphTransformer::TStatus::Ok) {
return status;
}
@@ -3585,48 +3585,48 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- auto& saveLambda = input->ChildRef(3);
- auto& loadLambda = input->ChildRef(4);
-
- if (saveLambda->IsCallable("Void") != loadLambda->IsCallable("Void")) {
+ auto& saveLambda = input->ChildRef(3);
+ auto& loadLambda = input->ChildRef(4);
+
+ if (saveLambda->IsCallable("Void") != loadLambda->IsCallable("Void")) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(saveLambda->Pos()), TStringBuilder() <<
- "Save and load lambdas must be specified at the same time"));
- return IGraphTransformer::TStatus::Error;
- }
-
- if (!saveLambda->IsCallable("Void")) {
- auto status = ConvertToLambda(saveLambda, ctx.Expr, 1);
- status = status.Combine(ConvertToLambda(loadLambda, ctx.Expr, 1));
- if (status.Level != IGraphTransformer::TStatus::Ok) {
- return status;
- }
-
- if (!UpdateLambdaAllArgumentsTypes(saveLambda, {stateType}, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
+ "Save and load lambdas must be specified at the same time"));
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ if (!saveLambda->IsCallable("Void")) {
+ auto status = ConvertToLambda(saveLambda, ctx.Expr, 1);
+ status = status.Combine(ConvertToLambda(loadLambda, ctx.Expr, 1));
+ if (status.Level != IGraphTransformer::TStatus::Ok) {
+ return status;
+ }
+
+ if (!UpdateLambdaAllArgumentsTypes(saveLambda, {stateType}, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
if (!saveLambda->GetTypeAnn()) {
- return IGraphTransformer::TStatus::Repeat;
- }
-
+ return IGraphTransformer::TStatus::Repeat;
+ }
+
auto savedStateType = saveLambda->GetTypeAnn();
- if (!EnsurePersistableType(saveLambda->Pos(), *savedStateType, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- if (!UpdateLambdaAllArgumentsTypes(loadLambda, {savedStateType}, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
+ if (!EnsurePersistableType(saveLambda->Pos(), *savedStateType, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ if (!UpdateLambdaAllArgumentsTypes(loadLambda, {savedStateType}, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
if (!loadLambda->GetTypeAnn()) {
- return IGraphTransformer::TStatus::Repeat;
- }
-
+ return IGraphTransformer::TStatus::Repeat;
+ }
+
if (!IsSameAnnotation(*loadLambda->GetTypeAnn(), *stateType)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(loadLambda->Pos()), TStringBuilder() << "Mismatch of load lambda return type and state type, "
<< *loadLambda->GetTypeAnn() << " != " << *stateType));
- return IGraphTransformer::TStatus::Error;
- }
- }
-
+ return IGraphTransformer::TStatus::Error;
+ }
+ }
+
auto issue = TIssue(ctx.Expr.GetPosition(input->Head().Pos()), "Deprecated Squeeze1, use Condense1 instead.");
SetIssueCode(EYqlIssueCode::TIssuesIds_EIssueCode_YQL_DEPRECATED_FUNCTION_OR_SIGNATURE, issue);
if (!ctx.Expr.AddWarning(issue)) {
@@ -4307,12 +4307,12 @@ namespace {
}
IGraphTransformer::TStatus AggregateWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
- if (!EnsureMinArgsCount(*input, 3, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- if (!EnsureMaxArgsCount(*input, 4, ctx.Expr)) {
+ if (!EnsureMinArgsCount(*input, 3, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
+ if (!EnsureMaxArgsCount(*input, 4, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
if (IsEmptyList(input->Head())) {
output = input->HeadPtr();
@@ -4551,7 +4551,7 @@ namespace {
}
} else {
auto defVal = child->Child(1)->Child(7);
- if (defVal->IsCallable("Null") && !isOptional && !isHopping && input->Child(1)->ChildrenSize() == 0) {
+ if (defVal->IsCallable("Null") && !isOptional && !isHopping && input->Child(1)->ChildrenSize() == 0) {
if (finishType->GetKind() != ETypeAnnotationKind::Null) {
finishType = ctx.Expr.MakeType<TOptionalExprType>(finishType);
}
@@ -4566,11 +4566,11 @@ namespace {
}
}
- auto rowType = ctx.Expr.MakeType<TStructExprType>(rowColumns);
- if (!rowType->Validate(input->Pos(), ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
+ auto rowType = ctx.Expr.MakeType<TStructExprType>(rowColumns);
+ if (!rowType->Validate(input->Pos(), ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
input->SetTypeAnn(isStream
? (const TTypeAnnotationNode*)ctx.Expr.MakeType<TStreamExprType>(rowType)
: (const TTypeAnnotationNode*)ctx.Expr.MakeType<TListExprType>(rowType));
@@ -5397,79 +5397,79 @@ namespace {
return IGraphTransformer::TStatus::Ok;
}
- IGraphTransformer::TStatus HoppingTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
- Y_UNUSED(output);
+ IGraphTransformer::TStatus HoppingTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
+ Y_UNUSED(output);
if (!EnsureMinArgsCount(*input, 5, ctx.Expr) || !EnsureMaxArgsCount(*input, 6, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
+ return IGraphTransformer::TStatus::Error;
+ }
if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) {
return status;
- }
-
+ }
+
auto itemType = input->Head().GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (!EnsureComputableType(input->Head().Pos(), *itemType, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- auto& lambdaTimeExtractor = input->ChildRef(1);
- auto status = ConvertToLambda(lambdaTimeExtractor, ctx.Expr, 1);
- if (status.Level != IGraphTransformer::TStatus::Ok) {
- return status;
- }
-
- if (!UpdateLambdaAllArgumentsTypes(lambdaTimeExtractor, {itemType}, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ auto& lambdaTimeExtractor = input->ChildRef(1);
+ auto status = ConvertToLambda(lambdaTimeExtractor, ctx.Expr, 1);
+ if (status.Level != IGraphTransformer::TStatus::Ok) {
+ return status;
+ }
+
+ if (!UpdateLambdaAllArgumentsTypes(lambdaTimeExtractor, {itemType}, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
if (!lambdaTimeExtractor->GetTypeAnn()) {
- return IGraphTransformer::TStatus::Repeat;
- }
-
+ return IGraphTransformer::TStatus::Repeat;
+ }
+
const TTypeAnnotationNode* timeType = ctx.Expr.MakeType<TOptionalExprType>(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Timestamp));
-
+
if (!IsSameAnnotation(*lambdaTimeExtractor->GetTypeAnn(), *timeType)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaTimeExtractor->Pos()), TStringBuilder()
- << "Mismatch hopping window time extractor lambda output type, expected: "
+ << "Mismatch hopping window time extractor lambda output type, expected: "
<< *timeType << ", but got: " << *lambdaTimeExtractor->GetTypeAnn()));
- return IGraphTransformer::TStatus::Error;
- }
-
+ return IGraphTransformer::TStatus::Error;
+ }
+
const TTypeAnnotationNode* intervalType = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Interval);
-
- auto checkWindowParam = [&] (TExprNode::TPtr& param) -> IGraphTransformer::TStatus {
+
+ auto checkWindowParam = [&] (TExprNode::TPtr& param) -> IGraphTransformer::TStatus {
auto type = param->GetTypeAnn();
- if (type->GetKind() == ETypeAnnotationKind::Optional) {
- if (param->IsCallable("Nothing")) {
+ if (type->GetKind() == ETypeAnnotationKind::Optional) {
+ if (param->IsCallable("Nothing")) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(param->Pos()), "Hopping window parameter value cannot be evaluated"));
- return IGraphTransformer::TStatus::Error;
- }
- type = type->Cast<TOptionalExprType>()->GetItemType();
- }
- if (!IsSameAnnotation(*type, *intervalType)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ type = type->Cast<TOptionalExprType>()->GetItemType();
+ }
+ if (!IsSameAnnotation(*type, *intervalType)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(param->Pos()), TStringBuilder()
- << "Mismatch hopping window parameter type, expected: "
- << *intervalType << ", but got: " << *type));
- return IGraphTransformer::TStatus::Error;
- }
- if (!IsPureIsolatedLambda(*param)) {
+ << "Mismatch hopping window parameter type, expected: "
+ << *intervalType << ", but got: " << *type));
+ return IGraphTransformer::TStatus::Error;
+ }
+ if (!IsPureIsolatedLambda(*param)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(param->Pos()), "Parameter is not a pure expression"));
- return IGraphTransformer::TStatus::Error;
- }
- return IGraphTransformer::TStatus::Ok;
- };
-
- auto convertStatus = checkWindowParam(input->ChildRef(2));
- if (convertStatus.Level != IGraphTransformer::TStatus::Ok) {
- return convertStatus;
- }
- convertStatus = checkWindowParam(input->ChildRef(3));
- if (convertStatus.Level != IGraphTransformer::TStatus::Ok) {
- return convertStatus;
- }
- convertStatus = checkWindowParam(input->ChildRef(4));
- if (convertStatus.Level != IGraphTransformer::TStatus::Ok) {
- return convertStatus;
- }
-
+ return IGraphTransformer::TStatus::Error;
+ }
+ return IGraphTransformer::TStatus::Ok;
+ };
+
+ auto convertStatus = checkWindowParam(input->ChildRef(2));
+ if (convertStatus.Level != IGraphTransformer::TStatus::Ok) {
+ return convertStatus;
+ }
+ convertStatus = checkWindowParam(input->ChildRef(3));
+ if (convertStatus.Level != IGraphTransformer::TStatus::Ok) {
+ return convertStatus;
+ }
+ convertStatus = checkWindowParam(input->ChildRef(4));
+ if (convertStatus.Level != IGraphTransformer::TStatus::Ok) {
+ return convertStatus;
+ }
+
if (input->ChildrenSize() == 6) {
const auto& dataWatermarksNodePtr = input->ChildRef(5);
if (dataWatermarksNodePtr->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Unit) {
@@ -5481,17 +5481,17 @@ namespace {
}
input->SetTypeAnn(ctx.Expr.MakeType<TUnitExprType>());
- return IGraphTransformer::TStatus::Ok;
- }
-
- IGraphTransformer::TStatus HoppingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
- Y_UNUSED(output);
+ return IGraphTransformer::TStatus::Ok;
+ }
+
+ IGraphTransformer::TStatus HoppingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
+ Y_UNUSED(output);
if (!EnsureArgsCount(*input, 11, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- auto& item = input->ChildRef(0);
+ return IGraphTransformer::TStatus::Error;
+ }
+ auto& item = input->ChildRef(0);
+
auto& lambdaTimeExtractor = input->ChildRef(1);
auto* hop = input->Child(2);
auto* interval = input->Child(3);
@@ -5504,26 +5504,26 @@ namespace {
auto& lambdaMerge = input->ChildRef(9);
auto& lambdaFinish = input->ChildRef(10);
- if (!EnsureStreamType(*item, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
+ if (!EnsureStreamType(*item, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
auto status = ConvertToLambda(lambdaTimeExtractor, ctx.Expr, 1);
status = status.Combine(ConvertToLambda(lambdaInit, ctx.Expr, 1));
- status = status.Combine(ConvertToLambda(lambdaUpdate, ctx.Expr, 2));
- status = status.Combine(ConvertToLambda(lambdaMerge, ctx.Expr, 2));
- status = status.Combine(ConvertToLambda(lambdaFinish, ctx.Expr, 2));
- if (status.Level != IGraphTransformer::TStatus::Ok) {
- return status;
- }
-
+ status = status.Combine(ConvertToLambda(lambdaUpdate, ctx.Expr, 2));
+ status = status.Combine(ConvertToLambda(lambdaMerge, ctx.Expr, 2));
+ status = status.Combine(ConvertToLambda(lambdaFinish, ctx.Expr, 2));
+ if (status.Level != IGraphTransformer::TStatus::Ok) {
+ return status;
+ }
+
auto timeType = ctx.Expr.MakeType<TOptionalExprType>(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Timestamp));
auto itemType = item->GetTypeAnn()->Cast<TStreamExprType>()->GetItemType();
-
- if (!EnsureStructType(input->Pos(), *itemType, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
+
+ if (!EnsureStructType(input->Pos(), *itemType, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
if (!UpdateLambdaAllArgumentsTypes(lambdaTimeExtractor, {itemType}, ctx.Expr))
{
return IGraphTransformer::TStatus::Error;
@@ -5542,99 +5542,99 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- if (!UpdateLambdaAllArgumentsTypes(lambdaInit, {itemType}, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
+ if (!UpdateLambdaAllArgumentsTypes(lambdaInit, {itemType}, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
if (!lambdaInit->GetTypeAnn()) {
- return IGraphTransformer::TStatus::Repeat;
- }
-
+ return IGraphTransformer::TStatus::Repeat;
+ }
+
auto stateType = lambdaInit->GetTypeAnn();
- if (!EnsureComputableType(lambdaInit->Pos(), *stateType, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- if (!UpdateLambdaAllArgumentsTypes(lambdaUpdate, {itemType, stateType}, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
+ if (!EnsureComputableType(lambdaInit->Pos(), *stateType, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ if (!UpdateLambdaAllArgumentsTypes(lambdaUpdate, {itemType, stateType}, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
if (!lambdaUpdate->GetTypeAnn()) {
- return IGraphTransformer::TStatus::Repeat;
- }
-
+ return IGraphTransformer::TStatus::Repeat;
+ }
+
if (!IsSameAnnotation(*lambdaUpdate->GetTypeAnn(), *stateType)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaUpdate->Pos()), TStringBuilder() << "Mismatch update lambda result type, expected: "
<< *stateType << ", but got: " << *lambdaUpdate->GetTypeAnn()));
- return IGraphTransformer::TStatus::Error;
- }
-
- if (!UpdateLambdaAllArgumentsTypes(lambdaMerge, {stateType, stateType}, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ if (!UpdateLambdaAllArgumentsTypes(lambdaMerge, {stateType, stateType}, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
if (!lambdaMerge->GetTypeAnn()) {
- return IGraphTransformer::TStatus::Repeat;
- }
-
+ return IGraphTransformer::TStatus::Repeat;
+ }
+
if (!IsSameAnnotation(*lambdaMerge->GetTypeAnn(), *stateType)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaMerge->Pos()), TStringBuilder() << "Mismatch merge lambda result type, expected: "
<< *stateType << ", but got: " << *lambdaMerge->GetTypeAnn()));
- return IGraphTransformer::TStatus::Error;
- }
-
- if (!UpdateLambdaAllArgumentsTypes(lambdaFinish, {stateType, timeType}, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ if (!UpdateLambdaAllArgumentsTypes(lambdaFinish, {stateType, timeType}, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
if (!lambdaFinish->GetTypeAnn()) {
- return IGraphTransformer::TStatus::Repeat;
- }
+ return IGraphTransformer::TStatus::Repeat;
+ }
if (lambdaFinish->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Struct) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaFinish->Pos()), TStringBuilder() <<
"Expected struct type as finish lambda result type, but got: " << *lambdaMerge->GetTypeAnn()));
- return IGraphTransformer::TStatus::Error;
- }
-
- if (saveLambda->IsCallable("Void") != loadLambda->IsCallable("Void")) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ if (saveLambda->IsCallable("Void") != loadLambda->IsCallable("Void")) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(saveLambda->Pos()), TStringBuilder() <<
- "Save and load lambdas must be specified at the same time"));
- return IGraphTransformer::TStatus::Error;
- }
-
- if (!saveLambda->IsCallable("Void")) {
- auto status = ConvertToLambda(saveLambda, ctx.Expr, 1);
- status = status.Combine(ConvertToLambda(loadLambda, ctx.Expr, 1));
- if (status.Level != IGraphTransformer::TStatus::Ok) {
- return status;
- }
-
- if (!UpdateLambdaAllArgumentsTypes(saveLambda, {stateType}, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
+ "Save and load lambdas must be specified at the same time"));
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ if (!saveLambda->IsCallable("Void")) {
+ auto status = ConvertToLambda(saveLambda, ctx.Expr, 1);
+ status = status.Combine(ConvertToLambda(loadLambda, ctx.Expr, 1));
+ if (status.Level != IGraphTransformer::TStatus::Ok) {
+ return status;
+ }
+
+ if (!UpdateLambdaAllArgumentsTypes(saveLambda, {stateType}, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
if (!saveLambda->GetTypeAnn()) {
- return IGraphTransformer::TStatus::Repeat;
- }
-
+ return IGraphTransformer::TStatus::Repeat;
+ }
+
auto savedStateType = saveLambda->GetTypeAnn();
- if (!EnsurePersistableType(saveLambda->Pos(), *savedStateType, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- if (!UpdateLambdaAllArgumentsTypes(loadLambda, {savedStateType}, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
+ if (!EnsurePersistableType(saveLambda->Pos(), *savedStateType, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ if (!UpdateLambdaAllArgumentsTypes(loadLambda, {savedStateType}, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
if (!loadLambda->GetTypeAnn()) {
- return IGraphTransformer::TStatus::Repeat;
- }
-
+ return IGraphTransformer::TStatus::Repeat;
+ }
+
if (!IsSameAnnotation(*loadLambda->GetTypeAnn(), *stateType)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(loadLambda->Pos()), TStringBuilder() << "Mismatch of load lambda return type and state type, "
<< *loadLambda->GetTypeAnn() << " != " << *stateType));
- return IGraphTransformer::TStatus::Error;
- }
- }
-
+ return IGraphTransformer::TStatus::Error;
+ }
+ }
+
input->SetTypeAnn(ctx.Expr.MakeType<TStreamExprType>(lambdaFinish->GetTypeAnn()));
- return IGraphTransformer::TStatus::Ok;
- }
-
+ return IGraphTransformer::TStatus::Ok;
+ }
+
IGraphTransformer::TStatus MultiHoppingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
Y_UNUSED(output);
if (!EnsureArgsCount(*input, 13, ctx.Expr)) {
diff --git a/ydb/library/yql/core/type_ann/type_ann_list.h b/ydb/library/yql/core/type_ann/type_ann_list.h
index 608b813fe8b..ca433faa5f7 100644
--- a/ydb/library/yql/core/type_ann/type_ann_list.h
+++ b/ydb/library/yql/core/type_ann/type_ann_list.h
@@ -104,9 +104,9 @@ namespace NTypeAnnImpl {
IGraphTransformer::TStatus WinLeadLagWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
IGraphTransformer::TStatus WinRowNumberWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
IGraphTransformer::TStatus WinRankWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
- IGraphTransformer::TStatus HoppingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
+ IGraphTransformer::TStatus HoppingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
IGraphTransformer::TStatus MultiHoppingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
- IGraphTransformer::TStatus HoppingTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
+ IGraphTransformer::TStatus HoppingTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
IGraphTransformer::TStatus ListMinWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
IGraphTransformer::TStatus ListMaxWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
IGraphTransformer::TStatus ListSumWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
diff --git a/ydb/library/yql/core/yql_data_provider.h b/ydb/library/yql/core/yql_data_provider.h
index 3ffbdc041da..2c2b6ce3af8 100644
--- a/ydb/library/yql/core/yql_data_provider.h
+++ b/ydb/library/yql/core/yql_data_provider.h
@@ -102,7 +102,7 @@ public:
//-- configuration
virtual IGraphTransformer& GetConfigurationTransformer() = 0;
- virtual TExprNode::TPtr GetClusterInfo(const TString& cluster, TExprContext& ctx) = 0;
+ virtual TExprNode::TPtr GetClusterInfo(const TString& cluster, TExprContext& ctx) = 0;
virtual const THashMap<TString, TString>* GetClusterTokens() = 0;
//-- discovery & rewrite
@@ -152,13 +152,13 @@ public:
virtual void GetRequiredChildren(const TExprNode& node, TExprNode::TListType& children) = 0;
virtual IGraphTransformer& GetCallableExecutionTransformer() = 0;
- //-- finalizing
- virtual IGraphTransformer& GetFinalizingTransformer() = 0;
+ //-- finalizing
+ virtual IGraphTransformer& GetFinalizingTransformer() = 0;
virtual bool CollectDiagnostics(NYson::TYsonWriter& writer) = 0;
virtual bool GetTasksInfo(NYson::TYsonWriter& writer) = 0;
virtual bool CollectStatistics(NYson::TYsonWriter& writer, bool totalOnly) = 0;
virtual bool CollectDiscoveredData(NYson::TYsonWriter& writer) = 0;
-
+
//-- plan
virtual IGraphTransformer& GetPlanInfoTransformer() = 0;
virtual IPlanFormatter& GetPlanFormatter() = 0;
@@ -220,8 +220,8 @@ struct TDataProviderInfo {
const TOperationProgressWriter& progressWriter, const TYqlOperationOptions& operationOptions,
TIntrusivePtr<IRandomProvider> randomProvider, TIntrusivePtr<ITimeProvider> timeProvider)> OpenSession;
- std::function<bool()> HasActiveProcesses;
-
+ std::function<bool()> HasActiveProcesses;
+
std::function<void(const TString& sessionId)> CloseSession;
std::function<void(const TString& sessionId)> CleanupSession;
diff --git a/ydb/library/yql/core/yql_execution.cpp b/ydb/library/yql/core/yql_execution.cpp
index eb138b1b4f3..9a0a645327b 100644
--- a/ydb/library/yql/core/yql_execution.cpp
+++ b/ydb/library/yql/core/yql_execution.cpp
@@ -1,6 +1,6 @@
#include "yql_execution.h"
#include "yql_expr_optimize.h"
-#include "yql_opt_proposed_by_data.h"
+#include "yql_opt_proposed_by_data.h"
#include <ydb/library/yql/utils/log/log.h>
#include <ydb/library/yql/utils/yql_panic.h>
@@ -45,13 +45,13 @@ public:
}
TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final {
- if (FinalizingTransformer) {
+ if (FinalizingTransformer) {
YQL_CLOG(INFO, CoreExecution) << "FinalizingTransformer, root #" << input->UniqueId();
auto status = FinalizingTransformer->Transform(input, output, ctx);
YQL_CLOG(INFO, CoreExecution) << "FinalizingTransformer done, output #" << output->UniqueId() << ", status: " << status;
return status;
- }
-
+ }
+
YQL_CLOG(INFO, CoreExecution) << "Begin, root #" << input->UniqueId();
output = input;
if (RewriteSanityCheck) {
@@ -82,26 +82,26 @@ public:
YQL_CLOG(INFO, CoreExecution) << "Finish, output #" << output->UniqueId() << ", status: " << status;
if (status != TStatus::Ok || !WithFinalize) {
- return status;
- }
-
- FinalizingTransformer = CreateCompositeFinalizingTransformer(Types);
- return FinalizingTransformer->Transform(input, output, ctx);
+ return status;
+ }
+
+ FinalizingTransformer = CreateCompositeFinalizingTransformer(Types);
+ return FinalizingTransformer->Transform(input, output, ctx);
}
NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode& input) final {
- return FinalizingTransformer ?
- FinalizingTransformer->GetAsyncFuture(input) :
- State->Promise.GetFuture();
+ return FinalizingTransformer ?
+ FinalizingTransformer->GetAsyncFuture(input) :
+ State->Promise.GetFuture();
}
TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final {
- if (FinalizingTransformer) {
- return FinalizingTransformer->ApplyAsyncChanges(input, output, ctx);
- }
-
+ if (FinalizingTransformer) {
+ return FinalizingTransformer->ApplyAsyncChanges(input, output, ctx);
+ }
+
output = input;
-
+
TStatus combinedStatus = TStatus::Ok;
TState::TQueueType completed;
@@ -630,7 +630,7 @@ private:
const bool WithFinalize;
TStatePtr State;
TNodeOnNodeOwnedMap NewNodes;
- TAutoPtr<IGraphTransformer> FinalizingTransformer;
+ TAutoPtr<IGraphTransformer> FinalizingTransformer;
THashMap<ui32, TOperationProgress> Progresses;
struct TTrackableNodeInfo
diff --git a/ydb/library/yql/core/yql_expr_type_annotation.cpp b/ydb/library/yql/core/yql_expr_type_annotation.cpp
index f2b793af8d3..0de311687ea 100644
--- a/ydb/library/yql/core/yql_expr_type_annotation.cpp
+++ b/ydb/library/yql/core/yql_expr_type_annotation.cpp
@@ -180,7 +180,7 @@ IGraphTransformer::TStatus TryConvertToImpl(TExprContext& ctx, TExprNode::TPtr&
.Add(1, std::move(node))
.Seal()
.Build();
-
+
return IGraphTransformer::TStatus::Repeat;
} else if ((fromSlot == EDataSlot::Yson || fromSlot == EDataSlot::Json) && to == "Yson.Node") {
node = ctx.Builder(node->Pos())
@@ -206,31 +206,31 @@ IGraphTransformer::TStatus TryConvertToImpl(TExprContext& ctx, TExprNode::TPtr&
.Build();
return IGraphTransformer::TStatus::Repeat;
-
+
} else if ((GetDataTypeInfo(fromSlot).Features & (NUdf::EDataTypeFeatures::DateType | NUdf::EDataTypeFeatures::TzDateType)) && to == "DateTime2.TM") {
- node = ctx.Builder(node->Pos())
- .Callable("Apply")
- .Callable(0, "Udf")
+ node = ctx.Builder(node->Pos())
+ .Callable("Apply")
+ .Callable(0, "Udf")
.Atom(0, "DateTime2.Split", TNodeFlags::Default)
- .Callable(1, "Void")
- .Seal()
- .Callable(2, "TupleType")
- .Callable(0, "TupleType")
- .Callable(0, "DataType")
+ .Callable(1, "Void")
+ .Seal()
+ .Callable(2, "TupleType")
+ .Callable(0, "TupleType")
+ .Callable(0, "DataType")
.Atom(0, sourceType.Cast<TDataExprType>()->GetName(), TNodeFlags::Default)
- .Seal()
- .Seal()
- .Callable(1, "StructType")
- .Seal()
- .Callable(2, "TupleType")
- .Seal()
- .Seal()
- .Seal()
+ .Seal()
+ .Seal()
+ .Callable(1, "StructType")
+ .Seal()
+ .Callable(2, "TupleType")
+ .Seal()
+ .Seal()
+ .Seal()
.Add(1, std::move(node))
- .Seal()
- .Build();
-
- return IGraphTransformer::TStatus::Repeat;
+ .Seal()
+ .Build();
+
+ return IGraphTransformer::TStatus::Repeat;
} else if (fromSlot == EDataSlot::Json && to == "JsonNode") {
node = ctx.Builder(node->Pos())
.Callable("Apply")
@@ -2778,11 +2778,11 @@ bool EnsureComparableType(TPositionHandle position, const TTypeAnnotationNode& t
if (!type.IsComparable()) {
ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder()
<< "Expected comparable type, i.e. combination of Data, Optional, List or Tuple, but got:" << type));
- return false;
- }
+ return false;
+ }
return true;
-}
-
+}
+
bool EnsureEquatableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) {
if (HasError(&type, ctx)) {
return false;
diff --git a/ydb/library/yql/core/yql_opt_proposed_by_data.cpp b/ydb/library/yql/core/yql_opt_proposed_by_data.cpp
index 23a912303fb..4dfaca5db48 100644
--- a/ydb/library/yql/core/yql_opt_proposed_by_data.cpp
+++ b/ydb/library/yql/core/yql_opt_proposed_by_data.cpp
@@ -367,15 +367,15 @@ TAutoPtr<IGraphTransformer> CreateTableMetadataLoader(const TTypeAnnotationConte
);
}
-TAutoPtr<IGraphTransformer> CreateCompositeFinalizingTransformer(const TTypeAnnotationContext& types) {
- return CreateDataProposalsInspector<ESource::DataSink>(
- types,
- [](IDataProvider* provider) -> IGraphTransformer& {
- return provider->GetFinalizingTransformer();
- }
- );
-}
-
+TAutoPtr<IGraphTransformer> CreateCompositeFinalizingTransformer(const TTypeAnnotationContext& types) {
+ return CreateDataProposalsInspector<ESource::DataSink>(
+ types,
+ [](IDataProvider* provider) -> IGraphTransformer& {
+ return provider->GetFinalizingTransformer();
+ }
+ );
+}
+
TAutoPtr<IGraphTransformer> CreatePlanInfoTransformer(const TTypeAnnotationContext& types) {
return CreateDataProposalsInspector<ESource::DataSink>(
types,
diff --git a/ydb/library/yql/core/yql_opt_proposed_by_data.h b/ydb/library/yql/core/yql_opt_proposed_by_data.h
index 0ec0bcab43f..2781fa7d098 100644
--- a/ydb/library/yql/core/yql_opt_proposed_by_data.h
+++ b/ydb/library/yql/core/yql_opt_proposed_by_data.h
@@ -18,7 +18,7 @@ TAutoPtr<IGraphTransformer> CreateLogicalDataProposalsInspector(const TTypeAnnot
TAutoPtr<IGraphTransformer> CreatePhysicalDataProposalsInspector(const TTypeAnnotationContext& types);
TAutoPtr<IGraphTransformer> CreatePhysicalFinalizers(const TTypeAnnotationContext& types);
TAutoPtr<IGraphTransformer> CreateTableMetadataLoader(const TTypeAnnotationContext& types);
-TAutoPtr<IGraphTransformer> CreateCompositeFinalizingTransformer(const TTypeAnnotationContext& types);
+TAutoPtr<IGraphTransformer> CreateCompositeFinalizingTransformer(const TTypeAnnotationContext& types);
TAutoPtr<IGraphTransformer> CreatePlanInfoTransformer(const TTypeAnnotationContext& types);
}
diff --git a/ydb/library/yql/core/yql_opt_utils.cpp b/ydb/library/yql/core/yql_opt_utils.cpp
index 69aa84a500f..9c09520e0fe 100644
--- a/ydb/library/yql/core/yql_opt_utils.cpp
+++ b/ydb/library/yql/core/yql_opt_utils.cpp
@@ -110,7 +110,7 @@ bool IsRenameFlatMap(const NNodes::TCoFlatMapBase& node, TExprNode::TPtr& struct
return true;
}
-bool IsPassthroughFlatMap(const TCoFlatMapBase& flatmap, TMaybe<THashSet<TStringBuf>>* passthroughFields, bool analyzeJustMember) {
+bool IsPassthroughFlatMap(const TCoFlatMapBase& flatmap, TMaybe<THashSet<TStringBuf>>* passthroughFields, bool analyzeJustMember) {
return IsPassthroughLambda(flatmap.Lambda(), passthroughFields, analyzeJustMember);
}
@@ -143,7 +143,7 @@ bool IsPassthroughLambda(const TCoLambda& lambda, TMaybe<THashSet<TStringBuf>>*
for (auto child : maybeStruct.Cast()) {
auto tuple = child.Cast<TCoNameValueTuple>();
auto value = tuple.Value();
- if (analyzeJustMember && value.Maybe<TCoJust>()) {
+ if (analyzeJustMember && value.Maybe<TCoJust>()) {
value = value.Cast<TCoJust>().Input();
}
@@ -341,15 +341,15 @@ const TTypeAnnotationNode* GetSeqItemType(const TTypeAnnotationNode* type) {
THROW yexception() << "Impossible to get item type from " << *type;
}
-TExprNode::TPtr GetSetting(const TExprNode& settings, const TStringBuf& name) {
- for (auto& setting : settings.Children()) {
- if (setting->ChildrenSize() != 0 && setting->Child(0)->Content() == name) {
- return setting;
- }
- }
- return nullptr;
-}
-
+TExprNode::TPtr GetSetting(const TExprNode& settings, const TStringBuf& name) {
+ for (auto& setting : settings.Children()) {
+ if (setting->ChildrenSize() != 0 && setting->Child(0)->Content() == name) {
+ return setting;
+ }
+ }
+ return nullptr;
+}
+
bool HasSetting(const TExprNode& settings, const TStringBuf& name) {
return GetSetting(settings, name) != nullptr;
}
diff --git a/ydb/library/yql/core/yql_opt_utils.h b/ydb/library/yql/core/yql_opt_utils.h
index 00635fea81e..b192fcc0d74 100644
--- a/ydb/library/yql/core/yql_opt_utils.h
+++ b/ydb/library/yql/core/yql_opt_utils.h
@@ -18,7 +18,7 @@ bool IsPredicateFlatMap(const TExprNode& node);
bool IsFilterFlatMap(const NNodes::TCoLambda& lambda);
bool IsListReorder(const TExprNode& node);
bool IsRenameFlatMap(const NNodes::TCoFlatMapBase& node, TExprNode::TPtr& structNode);
-bool IsPassthroughFlatMap(const NNodes::TCoFlatMapBase& flatmap, TMaybe<THashSet<TStringBuf>>* passthroughFields, bool analyzeJustMember = false);
+bool IsPassthroughFlatMap(const NNodes::TCoFlatMapBase& flatmap, TMaybe<THashSet<TStringBuf>>* passthroughFields, bool analyzeJustMember = false);
bool IsPassthroughLambda(const NNodes::TCoLambda& lambda, TMaybe<THashSet<TStringBuf>>* passthroughFields, bool analyzeJustMember = false);
bool IsTablePropsDependent(const TExprNode& node);
@@ -36,12 +36,12 @@ bool IsDepended(const TExprNode& from, const TExprNode& to);
bool IsEmpty(const TExprNode& node, const TTypeAnnotationContext& typeCtx);
bool IsEmptyOptional(const TExprNode& node);
bool IsEmptyContainer(const TExprNode& node);
-
+
const TTypeAnnotationNode* RemoveOptionalType(const TTypeAnnotationNode* type);
const TTypeAnnotationNode* RemoveAllOptionals(const TTypeAnnotationNode* type);
const TTypeAnnotationNode* GetSeqItemType(const TTypeAnnotationNode* seq);
-TExprNode::TPtr GetSetting(const TExprNode& settings, const TStringBuf& name);
+TExprNode::TPtr GetSetting(const TExprNode& settings, const TStringBuf& name);
bool HasSetting(const TExprNode& settings, const TStringBuf& name);
bool HasAnySetting(const TExprNode& settings, const THashSet<TString>& names);
TExprNode::TPtr RemoveSetting(const TExprNode& settings, const TStringBuf& name, TExprContext& ctx);
diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor.h
index 84eeda2edbf..95ef857c25b 100644
--- a/ydb/library/yql/dq/actors/compute/dq_compute_actor.h
+++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor.h
@@ -27,7 +27,7 @@ struct TEvDqCompute {
struct TEvChannelData : public NActors::TEventPB<TEvChannelData, NDqProto::TEvComputeChannelData,
TDqComputeEvents::EvChannelData> {};
-
+
struct TEvChannelDataAck : public NActors::TEventPB<TEvChannelDataAck, NDqProto::TEvComputeChannelDataAck,
TDqComputeEvents::EvChannelDataAck> {};
diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h
index deaadefb42c..99033b83b12 100644
--- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h
+++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h
@@ -17,7 +17,7 @@
#include <ydb/library/yql/core/issue/yql_issue.h>
#include <ydb/library/yql/minikql/comp_nodes/mkql_saveload.h>
#include <ydb/library/yql/public/issue/yql_issue_message.h>
-
+
#include <library/cpp/actors/core/interconnect.h>
#include <util/generic/size_literals.h>
@@ -469,13 +469,13 @@ protected:
auto& record = execEv->Record;
record.SetState(State);
- record.SetStatus(status);
+ record.SetStatus(status);
record.SetTaskId(Task.GetId());
if (RuntimeSettings.StatsMode >= NDqProto::DQ_STATS_MODE_BASIC) {
FillStats(record.MutableStats(), /* last */ true);
}
IssuesToMessage(issues, record.MutableIssues());
-
+
this->Send(ExecuterId, execEv.Release());
if (Checkpoints && State == NDqProto::COMPUTE_STATE_FINISHED) {
diff --git a/ydb/library/yql/dq/actors/dq_events_ids.h b/ydb/library/yql/dq/actors/dq_events_ids.h
index 698377a393a..622fd20da3d 100644
--- a/ydb/library/yql/dq/actors/dq_events_ids.h
+++ b/ydb/library/yql/dq/actors/dq_events_ids.h
@@ -19,13 +19,13 @@ struct TDqEvents {
struct TDqComputeEvents {
enum EDqComputeEvents {
Unused0 = EventSpaceBegin(TDqEvents::ES_DQ_COMPUTE_KQP_COMPATIBLE) + 200,
- EvState,
- EvResumeExecution,
- EvChannelData,
+ EvState,
+ EvResumeExecution,
+ EvChannelData,
ReservedKqp_EvScanData,
ReservedKqp_EvScanDataAck,
EvChannelsInfo,
- EvChannelDataAck,
+ EvChannelDataAck,
ReservedKqp_EvScanError,
Unused1,
EvRetryChannelData,
diff --git a/ydb/library/yql/dq/actors/protos/dq_events.proto b/ydb/library/yql/dq/actors/protos/dq_events.proto
index 0e38ab3bbc1..479d57c0125 100644
--- a/ydb/library/yql/dq/actors/protos/dq_events.proto
+++ b/ydb/library/yql/dq/actors/protos/dq_events.proto
@@ -47,11 +47,11 @@ message TComputeActorStats {
reserved 2; //optional NKqpProto.TKqpStatsRun LegacyStats = 2;
};
-message TEvComputeActorState {
+message TEvComputeActorState {
optional uint32 State = 1; // == EComputeState
optional TComputeActorStats LegacyStats = 2;
- optional Ydb.StatusIds.StatusCode Status = 3;
- repeated Ydb.Issue.IssueMessage Issues = 4;
+ optional Ydb.StatusIds.StatusCode Status = 3;
+ repeated Ydb.Issue.IssueMessage Issues = 4;
optional uint64 TaskId = 5;
optional TDqComputeActorStats Stats = 6;
};
@@ -71,7 +71,7 @@ message TEvComputeChannelDataAck {
optional uint64 SeqNo = 2;
optional int64 FreeSpace = 3;
optional bool Finish = 4;
-};
+};
message TEvAbortExecution {
optional Ydb.StatusIds.StatusCode StatusCode = 1;
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_condense.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_condense.cpp
index c179abab492..c856baa929c 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_condense.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_condense.cpp
@@ -1,6 +1,6 @@
#include "mkql_condense.h"
#include "mkql_squeeze_state.h"
-
+
#include <ydb/library/yql/minikql/mkql_node_cast.h>
#include <ydb/library/yql/minikql/mkql_node_builder.h>
#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
@@ -175,35 +175,35 @@ public:
public:
using TBase = TComputationValue<TValue>;
- TValue(
- TMemoryUsageInfo* memInfo,
- NUdf::TUnboxedValue&& stream,
+ TValue(
+ TMemoryUsageInfo* memInfo,
+ NUdf::TUnboxedValue&& stream,
const TSqueezeState& state,
- TComputationContext& ctx)
+ TComputationContext& ctx)
: TBase(memInfo)
, Stream(std::move(stream))
- , Ctx(ctx)
+ , Ctx(ctx)
, State(state)
{}
private:
ui32 GetTraverseCount() const final {
- return 1;
- }
-
+ return 1;
+ }
+
NUdf::TUnboxedValue GetTraverseItem(ui32 index) const final {
- Y_UNUSED(index);
- return Stream;
- }
-
+ Y_UNUSED(index);
+ return Stream;
+ }
+
NUdf::TUnboxedValue Save() const final {
return State.Save(Ctx);
- }
-
+ }
+
void Load(const NUdf::TStringRef& state) final {
State.Load(Ctx, state);
- }
-
+ }
+
NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) override {
if (ESqueezeState::Finished == State.Stage)
return NUdf::EFetchStatus::Finish;
@@ -246,13 +246,13 @@ public:
}
const NUdf::TUnboxedValue Stream;
- TComputationContext& Ctx;
+ TComputationContext& Ctx;
TSqueezeState State;
};
TCondenseWrapper(
- TComputationMutables& mutables,
- IComputationNode* stream,
+ TComputationMutables& mutables,
+ IComputationNode* stream,
IComputationExternalNode* item,
IComputationExternalNode* state,
IComputationNode* outSwitch,
@@ -266,9 +266,9 @@ public:
: TBaseComputation(mutables)
, Stream(stream)
, State(item, state, outSwitch, initState, updateState, inSave, outSave, inLoad, outLoad, stateType)
- {
+ {
this->Stateless = false;
- }
+ }
NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const {
#ifndef MKQL_DISABLE_CODEGEN
@@ -298,13 +298,13 @@ private:
FetchFunc = GenerateFetch(codegen);
codegen->ExportSymbol(FetchFunc);
}
-
+
void FinalizeFunctions(const NYql::NCodegen::ICodegen::TPtr& codegen) final {
if (FetchFunc) {
Fetch = reinterpret_cast<TFetchPtr>(codegen->GetPointerToFunction(FetchFunc));
}
}
-
+
Function* GenerateFetch(const NYql::NCodegen::ICodegen::TPtr& codegen) const {
auto& module = codegen->GetModule();
auto& context = codegen->GetContext();
@@ -473,28 +473,28 @@ IComputationNode* WrapCondense(TCallable& callable, const TComputationNodeFactor
}
IComputationNode* WrapSqueeze(TCallable& callable, const TComputationNodeFactoryContext& ctx) {
- MKQL_ENSURE(callable.GetInputsCount() == 9, "Expected 9 args");
-
+ MKQL_ENSURE(callable.GetInputsCount() == 9, "Expected 9 args");
+
const auto stream = LocateNode(ctx.NodeLocator, callable, 0);
const auto initState = LocateNode(ctx.NodeLocator, callable, 1);
const auto updateState = LocateNode(ctx.NodeLocator, callable, 4);
const auto item = LocateExternalNode(ctx.NodeLocator, callable, 2);
const auto state = LocateExternalNode(ctx.NodeLocator, callable, 3);
-
+
IComputationExternalNode* inSave = nullptr;
- IComputationNode* outSave = nullptr;
+ IComputationNode* outSave = nullptr;
IComputationExternalNode* inLoad = nullptr;
- IComputationNode* outLoad = nullptr;
-
+ IComputationNode* outLoad = nullptr;
+
const auto hasSaveLoad = !callable.GetInput(6).GetStaticType()->IsVoid();
- if (hasSaveLoad) {
- outSave = LocateNode(ctx.NodeLocator, callable, 6);
- outLoad = LocateNode(ctx.NodeLocator, callable, 8);
+ if (hasSaveLoad) {
+ outSave = LocateNode(ctx.NodeLocator, callable, 6);
+ outLoad = LocateNode(ctx.NodeLocator, callable, 8);
inSave = LocateExternalNode(ctx.NodeLocator, callable, 5);
inLoad = LocateExternalNode(ctx.NodeLocator, callable, 7);
- }
+ }
const auto stateType = hasSaveLoad ? callable.GetInput(6).GetStaticType() : nullptr;
-
+
return new TCondenseWrapper<false>(ctx.Mutables, stream, item, state, nullptr, initState, updateState, inSave, outSave, inLoad, outLoad, stateType);
}
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_condense1.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_condense1.cpp
index 2f5ac894325..345cb6bd3a1 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_condense1.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_condense1.cpp
@@ -1,6 +1,6 @@
#include "mkql_condense1.h"
#include "mkql_squeeze_state.h"
-
+
#include <ydb/library/yql/minikql/mkql_node_cast.h>
#include <ydb/library/yql/minikql/mkql_node_builder.h>
#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
@@ -188,35 +188,35 @@ public:
public:
using TBase = TComputationValue<TValue>;
- TValue(
- TMemoryUsageInfo* memInfo,
- NUdf::TUnboxedValue&& stream,
+ TValue(
+ TMemoryUsageInfo* memInfo,
+ NUdf::TUnboxedValue&& stream,
const TSqueezeState& state,
- TComputationContext& ctx)
+ TComputationContext& ctx)
: TBase(memInfo)
, Stream(std::move(stream))
- , Ctx(ctx)
+ , Ctx(ctx)
, State(state)
{}
private:
ui32 GetTraverseCount() const final {
- return 1;
- }
-
+ return 1;
+ }
+
NUdf::TUnboxedValue GetTraverseItem(ui32 index) const final {
- Y_UNUSED(index);
- return Stream;
- }
-
+ Y_UNUSED(index);
+ return Stream;
+ }
+
NUdf::TUnboxedValue Save() const final {
return State.Save(Ctx);
- }
-
+ }
+
void Load(const NUdf::TStringRef& state) final {
State.Load(Ctx, state);
- }
-
+ }
+
NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) final {
if (ESqueezeState::Finished == State.Stage)
return NUdf::EFetchStatus::Finish;
@@ -263,13 +263,13 @@ public:
}
const NUdf::TUnboxedValue Stream;
- TComputationContext& Ctx;
+ TComputationContext& Ctx;
TSqueezeState State;
};
TCondense1Wrapper(
- TComputationMutables& mutables,
- IComputationNode* stream,
+ TComputationMutables& mutables,
+ IComputationNode* stream,
IComputationExternalNode* item,
IComputationExternalNode* state,
IComputationNode* outSwitch,
@@ -283,9 +283,9 @@ public:
: TBaseComputation(mutables)
, Stream(stream)
, State(item, state, outSwitch, initState, updateState, inSave, outSave, inLoad, outLoad, stateType)
- {
+ {
this->Stateless = false;
- }
+ }
NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const {
#ifndef MKQL_DISABLE_CODEGEN
@@ -315,13 +315,13 @@ private:
FetchFunc = GenerateFetch(codegen);
codegen->ExportSymbol(FetchFunc);
}
-
+
void FinalizeFunctions(const NYql::NCodegen::ICodegen::TPtr& codegen) final {
if (FetchFunc) {
Fetch = reinterpret_cast<TFetchPtr>(codegen->GetPointerToFunction(FetchFunc));
}
}
-
+
Function* GenerateFetch(const NYql::NCodegen::ICodegen::TPtr& codegen) const {
auto& module = codegen->GetModule();
auto& context = codegen->GetContext();
@@ -504,28 +504,28 @@ IComputationNode* WrapCondense1(TCallable& callable, const TComputationNodeFacto
}
IComputationNode* WrapSqueeze1(TCallable& callable, const TComputationNodeFactoryContext& ctx) {
- MKQL_ENSURE(callable.GetInputsCount() == 9, "Expected 9 args");
-
+ MKQL_ENSURE(callable.GetInputsCount() == 9, "Expected 9 args");
+
const auto stream = LocateNode(ctx.NodeLocator, callable, 0);
const auto initState = LocateNode(ctx.NodeLocator, callable, 2);
const auto updateState = LocateNode(ctx.NodeLocator, callable, 4);
const auto item = LocateExternalNode(ctx.NodeLocator, callable, 1);
const auto state = LocateExternalNode(ctx.NodeLocator, callable, 3);
-
+
IComputationExternalNode* inSave = nullptr;
- IComputationNode* outSave = nullptr;
+ IComputationNode* outSave = nullptr;
IComputationExternalNode* inLoad = nullptr;
- IComputationNode* outLoad = nullptr;
-
+ IComputationNode* outLoad = nullptr;
+
const auto hasSaveLoad = !callable.GetInput(6).GetStaticType()->IsVoid();
- if (hasSaveLoad) {
- outSave = LocateNode(ctx.NodeLocator, callable, 6);
- outLoad = LocateNode(ctx.NodeLocator, callable, 8);
+ if (hasSaveLoad) {
+ outSave = LocateNode(ctx.NodeLocator, callable, 6);
+ outLoad = LocateNode(ctx.NodeLocator, callable, 8);
inSave = LocateExternalNode(ctx.NodeLocator, callable, 5);
inLoad = LocateExternalNode(ctx.NodeLocator, callable, 7);
- }
+ }
const auto stateType = hasSaveLoad ? callable.GetInput(6).GetStaticType() : nullptr;
-
+
return new TCondense1Wrapper<false>(ctx.Mutables, stream, item, state, nullptr, initState, updateState, inSave, outSave, inLoad, outLoad, stateType);
}
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp
index 57599acb641..b4b80c4c5f8 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp
@@ -36,7 +36,7 @@
#include "mkql_group.h"
#include "mkql_heap.h"
#include "mkql_hasitems.h"
-#include "mkql_hopping.h"
+#include "mkql_hopping.h"
#include "mkql_if.h"
#include "mkql_ifpresent.h"
#include "mkql_invoke.h"
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_filter.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_filter.cpp
index 7de0ebd8305..6649dbeda44 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_filter.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_filter.cpp
@@ -226,14 +226,14 @@ protected:
private:
ui32 GetTraverseCount() const final {
- return 1;
- }
-
+ return 1;
+ }
+
NUdf::TUnboxedValue GetTraverseItem(ui32 index) const final {
- Y_UNUSED(index);
- return Stream;
- }
-
+ Y_UNUSED(index);
+ return Stream;
+ }
+
NUdf::TUnboxedValue Save() const final {
return NUdf::TUnboxedValue::Zero();
}
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_flatmap.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_flatmap.cpp
index a9d49e109d2..d1967d78eef 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_flatmap.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_flatmap.cpp
@@ -956,15 +956,15 @@ public:
{}
private:
- ui32 GetTraverseCount() const override {
- return 1;
- }
-
- NUdf::TUnboxedValue GetTraverseItem(ui32 index) const override {
- Y_UNUSED(index);
- return Stream;
- }
-
+ ui32 GetTraverseCount() const override {
+ return 1;
+ }
+
+ NUdf::TUnboxedValue GetTraverseItem(ui32 index) const override {
+ Y_UNUSED(index);
+ return Stream;
+ }
+
NUdf::TUnboxedValue Save() const override {
return NUdf::TUnboxedValue::Zero();
}
@@ -1077,15 +1077,15 @@ public:
{}
private:
- ui32 GetTraverseCount() const override {
- return 1;
- }
-
- NUdf::TUnboxedValue GetTraverseItem(ui32 index) const override {
- Y_UNUSED(index);
- return Stream;
- }
-
+ ui32 GetTraverseCount() const override {
+ return 1;
+ }
+
+ NUdf::TUnboxedValue GetTraverseItem(ui32 index) const override {
+ Y_UNUSED(index);
+ return Stream;
+ }
+
NUdf::TUnboxedValue Save() const override {
return NUdf::TUnboxedValue::Zero();
}
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_hopping.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_hopping.cpp
index 27d1e000ac8..8c98e2aa306 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_hopping.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_hopping.cpp
@@ -1,111 +1,111 @@
-#include "mkql_hopping.h"
-#include "mkql_saveload.h"
-
+#include "mkql_hopping.h"
+#include "mkql_saveload.h"
+
#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
#include <ydb/library/yql/minikql/mkql_node_cast.h>
#include <ydb/library/yql/minikql/mkql_stats_registry.h>
#include <ydb/library/yql/minikql/mkql_string_util.h>
-
+
#include <util/generic/scope.h>
-namespace NKikimr {
-namespace NMiniKQL {
-
+namespace NKikimr {
+namespace NMiniKQL {
+
namespace {
const TStatKey Hop_NewHopsCount("Hop_NewHopsCount", true);
const TStatKey Hop_ThrownEventsCount("Hop_ThrownEventsCount", true);
-class THoppingCoreWrapper : public TMutableComputationNode<THoppingCoreWrapper> {
+class THoppingCoreWrapper : public TMutableComputationNode<THoppingCoreWrapper> {
typedef TMutableComputationNode<THoppingCoreWrapper> TBaseComputation;
-public:
- using TSelf = THoppingCoreWrapper;
-
- class TStreamValue : public TComputationValue<TStreamValue> {
- public:
- using TBase = TComputationValue<TStreamValue>;
-
- TStreamValue(
- TMemoryUsageInfo* memInfo,
- NUdf::TUnboxedValue&& stream,
+public:
+ using TSelf = THoppingCoreWrapper;
+
+ class TStreamValue : public TComputationValue<TStreamValue> {
+ public:
+ using TBase = TComputationValue<TStreamValue>;
+
+ TStreamValue(
+ TMemoryUsageInfo* memInfo,
+ NUdf::TUnboxedValue&& stream,
const TSelf* self,
ui64 hopTime,
ui64 intervalHopCount,
- ui64 delayHopCount,
- TComputationContext& ctx)
- : TBase(memInfo)
- , Stream(std::move(stream))
- , Self(self)
+ ui64 delayHopCount,
+ TComputationContext& ctx)
+ : TBase(memInfo)
+ , Stream(std::move(stream))
+ , Self(self)
, HopTime(hopTime)
, IntervalHopCount(intervalHopCount)
, DelayHopCount(delayHopCount)
, Buckets(IntervalHopCount + DelayHopCount)
- , Ctx(ctx)
- {}
-
- private:
- ui32 GetTraverseCount() const override {
- return 1;
- }
-
- NUdf::TUnboxedValue GetTraverseItem(ui32 index) const override {
- Y_UNUSED(index);
- return Stream;
- }
-
- NUdf::TUnboxedValue Save() const override {
- MKQL_ENSURE(Ready.empty(), "Inconsistent state to save, not all elements are fetched");
-
- TString out;
- WriteUi32(out, Buckets.size());
- for (const auto& bucket : Buckets) {
- WriteBool(out, bucket.HasValue);
- if (bucket.HasValue) {
- Self->InSave->SetValue(Ctx, NUdf::TUnboxedValue(bucket.Value));
+ , Ctx(ctx)
+ {}
+
+ private:
+ ui32 GetTraverseCount() const override {
+ return 1;
+ }
+
+ NUdf::TUnboxedValue GetTraverseItem(ui32 index) const override {
+ Y_UNUSED(index);
+ return Stream;
+ }
+
+ NUdf::TUnboxedValue Save() const override {
+ MKQL_ENSURE(Ready.empty(), "Inconsistent state to save, not all elements are fetched");
+
+ TString out;
+ WriteUi32(out, Buckets.size());
+ for (const auto& bucket : Buckets) {
+ WriteBool(out, bucket.HasValue);
+ if (bucket.HasValue) {
+ Self->InSave->SetValue(Ctx, NUdf::TUnboxedValue(bucket.Value));
if (Self->StateType) {
WriteUnboxedValue(out, Self->Packer.RefMutableObject(Ctx, false, Self->StateType), Self->OutSave->GetValue(Ctx));
}
- }
- }
-
- WriteUi64(out, HopIndex);
- WriteBool(out, Started);
- WriteBool(out, Finished);
-
+ }
+ }
+
+ WriteUi64(out, HopIndex);
+ WriteBool(out, Started);
+ WriteBool(out, Finished);
+
auto strRef = NUdf::TStringRef(out.data(), out.size());
return MakeString(strRef);
- }
-
- void Load(const NUdf::TStringRef& state) override {
- TStringBuf in(state.Data(), state.Size());
-
- auto size = ReadUi32(in);
- Buckets.resize(size);
- for (auto& bucket : Buckets) {
- bucket.HasValue = ReadBool(in);
- if (bucket.HasValue) {
+ }
+
+ void Load(const NUdf::TStringRef& state) override {
+ TStringBuf in(state.Data(), state.Size());
+
+ auto size = ReadUi32(in);
+ Buckets.resize(size);
+ for (auto& bucket : Buckets) {
+ bucket.HasValue = ReadBool(in);
+ if (bucket.HasValue) {
if (Self->StateType) {
Self->InLoad->SetValue(Ctx, ReadUnboxedValue(in, Self->Packer.RefMutableObject(Ctx, false, Self->StateType), Ctx));
}
bucket.Value = Self->OutLoad->GetValue(Ctx);
- }
- }
-
- HopIndex = ReadUi64(in);
- Started = ReadBool(in);
- Finished = ReadBool(in);
- }
-
- NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) override {
+ }
+ }
+
+ HopIndex = ReadUi64(in);
+ Started = ReadBool(in);
+ Finished = ReadBool(in);
+ }
+
+ NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) override {
if (!Ready.empty()) {
result = std::move(Ready.front());
Ready.pop_front();
- return NUdf::EFetchStatus::Ok;
- }
+ return NUdf::EFetchStatus::Ok;
+ }
if (Finished) {
- return NUdf::EFetchStatus::Finish;
- }
-
+ return NUdf::EFetchStatus::Finish;
+ }
+
i64 thrownEvents = 0;
i64 newHops = 0;
Y_DEFER {
@@ -117,95 +117,95 @@ public:
}
};
- for (NUdf::TUnboxedValue item;;) {
- if (!Ready.empty()) {
- result = std::move(Ready.front());
- Ready.pop_front();
- return NUdf::EFetchStatus::Ok;
- }
-
- const auto status = Stream.Fetch(item);
- if (status != NUdf::EFetchStatus::Ok) {
- if (status == NUdf::EFetchStatus::Finish) {
+ for (NUdf::TUnboxedValue item;;) {
+ if (!Ready.empty()) {
+ result = std::move(Ready.front());
+ Ready.pop_front();
+ return NUdf::EFetchStatus::Ok;
+ }
+
+ const auto status = Stream.Fetch(item);
+ if (status != NUdf::EFetchStatus::Ok) {
+ if (status == NUdf::EFetchStatus::Finish) {
Finished = true;
- }
- return status;
- }
-
+ }
+ return status;
+ }
+
Self->Item->SetValue(Ctx, std::move(item));
auto time = Self->OutTime->GetValue(Ctx);
- if (!time) {
- continue;
- }
-
- auto hopIndex = time.Get<ui64>() / HopTime;
-
+ if (!time) {
+ continue;
+ }
+
+ auto hopIndex = time.Get<ui64>() / HopTime;
+
if (!Started) {
HopIndex = hopIndex + 1;
Started = true;
- }
-
+ }
+
while (hopIndex >= HopIndex) {
auto firstBucketIndex = HopIndex % Buckets.size();
-
- auto bucketIndex = firstBucketIndex;
- TMaybe<NUdf::TUnboxedValue> aggregated;
-
- for (ui64 i = 0; i < IntervalHopCount; ++i) {
- const auto& bucket = Buckets[bucketIndex];
- if (bucket.HasValue) {
- if (!aggregated) { // todo: clone
- Self->InSave->SetValue(Ctx, NUdf::TUnboxedValue(bucket.Value));
- Self->InLoad->SetValue(Ctx, Self->OutSave->GetValue(Ctx));
- aggregated = Self->OutLoad->GetValue(Ctx);
- } else {
- Self->State->SetValue(Ctx, NUdf::TUnboxedValue(bucket.Value));
- Self->State2->SetValue(Ctx, NUdf::TUnboxedValue(*aggregated));
+
+ auto bucketIndex = firstBucketIndex;
+ TMaybe<NUdf::TUnboxedValue> aggregated;
+
+ for (ui64 i = 0; i < IntervalHopCount; ++i) {
+ const auto& bucket = Buckets[bucketIndex];
+ if (bucket.HasValue) {
+ if (!aggregated) { // todo: clone
+ Self->InSave->SetValue(Ctx, NUdf::TUnboxedValue(bucket.Value));
+ Self->InLoad->SetValue(Ctx, Self->OutSave->GetValue(Ctx));
+ aggregated = Self->OutLoad->GetValue(Ctx);
+ } else {
+ Self->State->SetValue(Ctx, NUdf::TUnboxedValue(bucket.Value));
+ Self->State2->SetValue(Ctx, NUdf::TUnboxedValue(*aggregated));
aggregated = Self->OutMerge->GetValue(Ctx);
- }
- }
+ }
+ }
if (++bucketIndex == Buckets.size()) {
- bucketIndex = 0;
- }
- }
-
+ bucketIndex = 0;
+ }
+ }
+
auto& clearBucket = Buckets[firstBucketIndex];
- clearBucket.Value = NUdf::TUnboxedValue();
- clearBucket.HasValue = false;
-
- if (aggregated) {
- Self->State->SetValue(Ctx, NUdf::TUnboxedValue(*aggregated));
+ clearBucket.Value = NUdf::TUnboxedValue();
+ clearBucket.HasValue = false;
+
+ if (aggregated) {
+ Self->State->SetValue(Ctx, NUdf::TUnboxedValue(*aggregated));
Self->Time->SetValue(Ctx, NUdf::TUnboxedValuePod((HopIndex - DelayHopCount) * HopTime));
Ready.emplace_back(Self->OutFinish->GetValue(Ctx));
- }
-
+ }
+
++newHops;
++HopIndex;
- }
-
- if (hopIndex + DelayHopCount + 1 >= HopIndex) {
+ }
+
+ if (hopIndex + DelayHopCount + 1 >= HopIndex) {
auto& bucket = Buckets[hopIndex % Buckets.size()];
- if (!bucket.HasValue) {
+ if (!bucket.HasValue) {
bucket.Value = Self->OutInit->GetValue(Ctx);
- bucket.HasValue = true;
- } else {
- Self->State->SetValue(Ctx, NUdf::TUnboxedValue(bucket.Value));
+ bucket.HasValue = true;
+ } else {
+ Self->State->SetValue(Ctx, NUdf::TUnboxedValue(bucket.Value));
bucket.Value = Self->OutUpdate->GetValue(Ctx);
- }
+ }
} else {
++thrownEvents;
- }
- }
- }
-
+ }
+ }
+ }
+
const NUdf::TUnboxedValue Stream;
const TSelf *const Self;
- const ui64 HopTime;
- const ui64 IntervalHopCount;
- const ui64 DelayHopCount;
-
+ const ui64 HopTime;
+ const ui64 IntervalHopCount;
+ const ui64 DelayHopCount;
+
struct TBucket {
NUdf::TUnboxedValue Value;
bool HasValue = false;
@@ -216,73 +216,73 @@ public:
ui64 HopIndex = 0;
bool Started = false;
bool Finished = false;
-
- TComputationContext& Ctx;
- };
-
- THoppingCoreWrapper(
+
+ TComputationContext& Ctx;
+ };
+
+ THoppingCoreWrapper(
TComputationMutables& mutables,
- IComputationNode* stream,
+ IComputationNode* stream,
IComputationExternalNode* item,
IComputationExternalNode* state,
IComputationExternalNode* state2,
IComputationExternalNode* time,
IComputationExternalNode* inSave,
IComputationExternalNode* inLoad,
- IComputationNode* outTime,
- IComputationNode* outInit,
- IComputationNode* outUpdate,
- IComputationNode* outSave,
- IComputationNode* outLoad,
- IComputationNode* outMerge,
- IComputationNode* outFinish,
- IComputationNode* hop,
- IComputationNode* interval,
- IComputationNode* delay,
- TType* stateType)
+ IComputationNode* outTime,
+ IComputationNode* outInit,
+ IComputationNode* outUpdate,
+ IComputationNode* outSave,
+ IComputationNode* outLoad,
+ IComputationNode* outMerge,
+ IComputationNode* outFinish,
+ IComputationNode* hop,
+ IComputationNode* interval,
+ IComputationNode* delay,
+ TType* stateType)
: TBaseComputation(mutables)
, Stream(stream)
- , Item(item)
- , State(state)
- , State2(state2)
- , Time(time)
- , InSave(inSave)
- , InLoad(inLoad)
- , OutTime(outTime)
- , OutInit(outInit)
- , OutUpdate(outUpdate)
- , OutSave(outSave)
- , OutLoad(outLoad)
- , OutMerge(outMerge)
- , OutFinish(outFinish)
- , Hop(hop)
- , Interval(interval)
- , Delay(delay)
+ , Item(item)
+ , State(state)
+ , State2(state2)
+ , Time(time)
+ , InSave(inSave)
+ , InLoad(inLoad)
+ , OutTime(outTime)
+ , OutInit(outInit)
+ , OutUpdate(outUpdate)
+ , OutSave(outSave)
+ , OutLoad(outLoad)
+ , OutMerge(outMerge)
+ , OutFinish(outFinish)
+ , Hop(hop)
+ , Interval(interval)
+ , Delay(delay)
, StateType(stateType)
, Packer(mutables)
- {
- Stateless = false;
- }
-
+ {
+ Stateless = false;
+ }
+
NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const {
const auto hopTime = Hop->GetValue(ctx).Get<i64>();
const auto interval = Interval->GetValue(ctx).Get<i64>();
const auto delay = Delay->GetValue(ctx).Get<i64>();
-
- // TODO: move checks from here
- MKQL_ENSURE(hopTime > 0, "hop must be positive");
+
+ // TODO: move checks from here
+ MKQL_ENSURE(hopTime > 0, "hop must be positive");
MKQL_ENSURE(interval >= hopTime, "interval should be greater or equal to hop");
MKQL_ENSURE(delay >= hopTime, "delay should be greater or equal to hop");
-
+
const auto intervalHopCount = interval / hopTime;
const auto delayHopCount = delay / hopTime;
-
+
MKQL_ENSURE(intervalHopCount <= 100000, "too many hops in interval");
MKQL_ENSURE(delayHopCount <= 100000, "too many hops in delay");
-
+
return ctx.HolderFactory.Create<TStreamValue>(Stream->GetValue(ctx), this, (ui64)hopTime, (ui64)intervalHopCount, (ui64)delayHopCount, ctx);
- }
-
+ }
+
private:
void RegisterDependencies() const final {
DependsOn(Stream);
@@ -302,80 +302,80 @@ private:
DependsOn(Hop);
DependsOn(Interval);
DependsOn(Delay);
- }
-
- IComputationNode* const Stream;
-
+ }
+
+ IComputationNode* const Stream;
+
IComputationExternalNode* const Item;
IComputationExternalNode* const State;
IComputationExternalNode* const State2;
IComputationExternalNode* const Time;
IComputationExternalNode* const InSave;
IComputationExternalNode* const InLoad;
-
- IComputationNode* const OutTime;
- IComputationNode* const OutInit;
- IComputationNode* const OutUpdate;
- IComputationNode* const OutSave;
- IComputationNode* const OutLoad;
- IComputationNode* const OutMerge;
- IComputationNode* const OutFinish;
-
- IComputationNode* const Hop;
- IComputationNode* const Interval;
- IComputationNode* const Delay;
-
+
+ IComputationNode* const OutTime;
+ IComputationNode* const OutInit;
+ IComputationNode* const OutUpdate;
+ IComputationNode* const OutSave;
+ IComputationNode* const OutLoad;
+ IComputationNode* const OutMerge;
+ IComputationNode* const OutFinish;
+
+ IComputationNode* const Hop;
+ IComputationNode* const Interval;
+ IComputationNode* const Delay;
+
TType* const StateType;
TMutableObjectOverBoxedValue<TValuePackerBoxed> Packer;
-};
-
+};
+
}
-IComputationNode* WrapHoppingCore(TCallable& callable, const TComputationNodeFactoryContext& ctx) {
- MKQL_ENSURE(callable.GetInputsCount() == 17, "Expected 17 args");
-
- auto hasSaveLoad = !callable.GetInput(10).GetStaticType()->IsVoid();
-
+IComputationNode* WrapHoppingCore(TCallable& callable, const TComputationNodeFactoryContext& ctx) {
+ MKQL_ENSURE(callable.GetInputsCount() == 17, "Expected 17 args");
+
+ auto hasSaveLoad = !callable.GetInput(10).GetStaticType()->IsVoid();
+
IComputationExternalNode* inSave = nullptr;
- IComputationNode* outSave = nullptr;
+ IComputationNode* outSave = nullptr;
IComputationExternalNode* inLoad = nullptr;
- IComputationNode* outLoad = nullptr;
-
- auto streamType = callable.GetInput(0).GetStaticType();
- MKQL_ENSURE(streamType->IsStream(), "Expected stream");
-
- auto stream = LocateNode(ctx.NodeLocator, callable, 0);
-
- auto outTime = LocateNode(ctx.NodeLocator, callable, 7);
- auto outInit = LocateNode(ctx.NodeLocator, callable, 8);
- auto outUpdate = LocateNode(ctx.NodeLocator, callable, 9);
- if (hasSaveLoad) {
- outSave = LocateNode(ctx.NodeLocator, callable, 10);
- outLoad = LocateNode(ctx.NodeLocator, callable, 11);
- }
- auto outMerge = LocateNode(ctx.NodeLocator, callable, 12);
- auto outFinish = LocateNode(ctx.NodeLocator, callable, 13);
-
- auto hop = LocateNode(ctx.NodeLocator, callable, 14);
- auto interval = LocateNode(ctx.NodeLocator, callable, 15);
- auto delay = LocateNode(ctx.NodeLocator, callable, 16);
-
+ IComputationNode* outLoad = nullptr;
+
+ auto streamType = callable.GetInput(0).GetStaticType();
+ MKQL_ENSURE(streamType->IsStream(), "Expected stream");
+
+ auto stream = LocateNode(ctx.NodeLocator, callable, 0);
+
+ auto outTime = LocateNode(ctx.NodeLocator, callable, 7);
+ auto outInit = LocateNode(ctx.NodeLocator, callable, 8);
+ auto outUpdate = LocateNode(ctx.NodeLocator, callable, 9);
+ if (hasSaveLoad) {
+ outSave = LocateNode(ctx.NodeLocator, callable, 10);
+ outLoad = LocateNode(ctx.NodeLocator, callable, 11);
+ }
+ auto outMerge = LocateNode(ctx.NodeLocator, callable, 12);
+ auto outFinish = LocateNode(ctx.NodeLocator, callable, 13);
+
+ auto hop = LocateNode(ctx.NodeLocator, callable, 14);
+ auto interval = LocateNode(ctx.NodeLocator, callable, 15);
+ auto delay = LocateNode(ctx.NodeLocator, callable, 16);
+
auto item = LocateExternalNode(ctx.NodeLocator, callable, 1);
auto state = LocateExternalNode(ctx.NodeLocator, callable, 2);
auto state2 = LocateExternalNode(ctx.NodeLocator, callable, 3);
auto time = LocateExternalNode(ctx.NodeLocator, callable, 4);
- if (hasSaveLoad) {
+ if (hasSaveLoad) {
inSave = LocateExternalNode(ctx.NodeLocator, callable, 5);
inLoad = LocateExternalNode(ctx.NodeLocator, callable, 6);
- }
-
- auto stateType = hasSaveLoad ? callable.GetInput(10).GetStaticType() : nullptr;
-
+ }
+
+ auto stateType = hasSaveLoad ? callable.GetInput(10).GetStaticType() : nullptr;
+
return new THoppingCoreWrapper(ctx.Mutables,
- stream, item, state, state2, time, inSave, inLoad,
- outTime, outInit, outUpdate, outSave, outLoad, outMerge, outFinish,
- hop, interval, delay, stateType);
-}
-
-}
-}
+ stream, item, state, state2, time, inSave, inLoad,
+ outTime, outInit, outUpdate, outSave, outLoad, outMerge, outFinish,
+ hop, interval, delay, stateType);
+}
+
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_hopping.h b/ydb/library/yql/minikql/comp_nodes/mkql_hopping.h
index 6f5950ed49d..57e69d22350 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_hopping.h
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_hopping.h
@@ -1,10 +1,10 @@
-#pragma once
+#pragma once
#include <ydb/library/yql/minikql/computation/mkql_computation_node_impl.h>
-
-namespace NKikimr {
-namespace NMiniKQL {
-
-IComputationNode* WrapHoppingCore(TCallable& callable, const TComputationNodeFactoryContext& ctx);
-
-}
-}
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+IComputationNode* WrapHoppingCore(TCallable& callable, const TComputationNodeFactoryContext& ctx);
+
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_map.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_map.cpp
index 04997bd1f1d..938609b22e3 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_map.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_map.cpp
@@ -152,12 +152,12 @@ protected:
private:
ui32 GetTraverseCount() const final {
return 1U;
- }
-
+ }
+
NUdf::TUnboxedValue GetTraverseItem(ui32) const final {
- return Stream;
- }
-
+ return Stream;
+ }
+
NUdf::TUnboxedValue Save() const final {
return NUdf::TUnboxedValuePod::Zero();
}
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_saveload.h b/ydb/library/yql/minikql/comp_nodes/mkql_saveload.h
index 0440994f8d2..6af5b211afa 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_saveload.h
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_saveload.h
@@ -1,41 +1,41 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/minikql/defs.h>
#include <ydb/library/yql/minikql/pack_num.h>
#include <ydb/library/yql/minikql/computation/mkql_computation_node_pack.h>
-
-#include <util/generic/strbuf.h>
-
+
+#include <util/generic/strbuf.h>
+
#include <string_view>
-namespace NKikimr {
-namespace NMiniKQL {
-
+namespace NKikimr {
+namespace NMiniKQL {
+
Y_FORCE_INLINE void WriteByte(TString& out, ui8 value) {
out.append((char)value);
}
-Y_FORCE_INLINE void WriteBool(TString& out, bool value) {
- out.append((char)value);
-}
-
-Y_FORCE_INLINE void WriteUi32(TString& out, ui32 value) {
- char buf[MAX_PACKED32_SIZE];
- out.AppendNoAlias(buf, Pack32(value, buf));
-}
-
-Y_FORCE_INLINE void WriteUi64(TString& out, ui64 value) {
- char buf[MAX_PACKED64_SIZE];
- out.AppendNoAlias(buf, Pack64(value, buf));
-}
-
-Y_FORCE_INLINE bool ReadBool(TStringBuf& in) {
+Y_FORCE_INLINE void WriteBool(TString& out, bool value) {
+ out.append((char)value);
+}
+
+Y_FORCE_INLINE void WriteUi32(TString& out, ui32 value) {
+ char buf[MAX_PACKED32_SIZE];
+ out.AppendNoAlias(buf, Pack32(value, buf));
+}
+
+Y_FORCE_INLINE void WriteUi64(TString& out, ui64 value) {
+ char buf[MAX_PACKED64_SIZE];
+ out.AppendNoAlias(buf, Pack64(value, buf));
+}
+
+Y_FORCE_INLINE bool ReadBool(TStringBuf& in) {
MKQL_ENSURE(in.size(), "Serialized state is corrupted");
bool result = (bool)*in.data();
- in.Skip(1);
- return result;
-}
-
+ in.Skip(1);
+ return result;
+}
+
Y_FORCE_INLINE ui8 ReadByte(TStringBuf& in) {
MKQL_ENSURE(in.size(), "Serialized state is corrupted");
ui8 result = *in.data();
@@ -43,22 +43,22 @@ Y_FORCE_INLINE ui8 ReadByte(TStringBuf& in) {
return result;
}
-Y_FORCE_INLINE ui32 ReadUi32(TStringBuf& in) {
- ui32 result;
+Y_FORCE_INLINE ui32 ReadUi32(TStringBuf& in) {
+ ui32 result;
auto count = Unpack32(in.data(), in.size(), result);
- MKQL_ENSURE(count, "Serialized state is corrupted");
- in.Skip(count);
- return result;
-}
-
-Y_FORCE_INLINE ui64 ReadUi64(TStringBuf& in) {
- ui64 result;
+ MKQL_ENSURE(count, "Serialized state is corrupted");
+ in.Skip(count);
+ return result;
+}
+
+Y_FORCE_INLINE ui64 ReadUi64(TStringBuf& in) {
+ ui64 result;
auto count = Unpack64(in.data(), in.size(), result);
- MKQL_ENSURE(count, "Serialized state is corrupted");
- in.Skip(count);
- return result;
-}
-
+ MKQL_ENSURE(count, "Serialized state is corrupted");
+ in.Skip(count);
+ return result;
+}
+
Y_FORCE_INLINE void WriteString(TString& out, std::string_view str) {
WriteUi32(out, str.size());
out.AppendNoAlias(str.data(), str.size());
@@ -72,19 +72,19 @@ Y_FORCE_INLINE std::string_view ReadString(TStringBuf& in) {
return head;
}
-Y_FORCE_INLINE void WriteUnboxedValue(TString& out, const TValuePacker& packer, const NUdf::TUnboxedValue& value) {
- auto state = packer.Pack(value);
+Y_FORCE_INLINE void WriteUnboxedValue(TString& out, const TValuePacker& packer, const NUdf::TUnboxedValue& value) {
+ auto state = packer.Pack(value);
WriteUi32(out, state.size());
out.AppendNoAlias(state.data(), state.size());
-}
-
-Y_FORCE_INLINE NUdf::TUnboxedValue ReadUnboxedValue(TStringBuf& in, const TValuePacker& packer, TComputationContext& ctx) {
- auto size = ReadUi32(in);
+}
+
+Y_FORCE_INLINE NUdf::TUnboxedValue ReadUnboxedValue(TStringBuf& in, const TValuePacker& packer, TComputationContext& ctx) {
+ auto size = ReadUi32(in);
MKQL_ENSURE(size <= in.size(), "Serialized state is corrupted");
auto value = packer.Unpack(TStringBuf(in.data(), in.data() + size), ctx.HolderFactory);
- in.Skip(size);
- return value;
-}
-
-} // namespace NMiniKQL
-} // namespace NKikimr
+ in.Skip(size);
+ return value;
+}
+
+} // namespace NMiniKQL
+} // namespace NKikimr
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_while.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_while.cpp
index a8650d57540..caf6dc06ac0 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_while.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_while.cpp
@@ -205,15 +205,15 @@ protected:
}
private:
- ui32 GetTraverseCount() const override {
- return 1;
- }
-
- NUdf::TUnboxedValue GetTraverseItem(ui32 index) const override {
- Y_UNUSED(index);
- return Stream;
- }
-
+ ui32 GetTraverseCount() const override {
+ return 1;
+ }
+
+ NUdf::TUnboxedValue GetTraverseItem(ui32 index) const override {
+ Y_UNUSED(index);
+ return Stream;
+ }
+
NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) override {
if (FilterWorkFinished) {
return SkipOrTake ? Stream.Fetch(result) : NUdf::EFetchStatus::Finish;
diff --git a/ydb/library/yql/minikql/comp_nodes/ya.make b/ydb/library/yql/minikql/comp_nodes/ya.make
index 55838c40123..23211965bdf 100644
--- a/ydb/library/yql/minikql/comp_nodes/ya.make
+++ b/ydb/library/yql/minikql/comp_nodes/ya.make
@@ -82,8 +82,8 @@ SRCS(
mkql_hasitems.h
mkql_heap.cpp
mkql_heap.h
- mkql_hopping.cpp
- mkql_hopping.h
+ mkql_hopping.cpp
+ mkql_hopping.h
mkql_if.cpp
mkql_if.h
mkql_ifpresent.cpp
@@ -144,7 +144,7 @@ SRCS(
mkql_round.h
mkql_safe_circular_buffer.cpp
mkql_safe_circular_buffer.h
- mkql_saveload.h
+ mkql_saveload.h
mkql_seq.cpp
mkql_seq.h
mkql_size.cpp
diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_codegen.h b/ydb/library/yql/minikql/computation/mkql_computation_node_codegen.h
index d9feede204e..bcd7c82578c 100644
--- a/ydb/library/yql/minikql/computation/mkql_computation_node_codegen.h
+++ b/ydb/library/yql/minikql/computation/mkql_computation_node_codegen.h
@@ -641,7 +641,7 @@ protected:
}
Value* CreateGetValue(const TCodegenContext& ctx, BasicBlock*& block) const final {
- if (*this->Stateless) {
+ if (*this->Stateless) {
const auto newValue = static_cast<const TDerived*>(this)->DoGenerateGetValue(ctx, block);
if (newValue->getType()->isPointerTy()) {
ValueRelease(this->GetRepresentation(), newValue, ctx, block);
diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.cpp
index c481a5815e9..e05b19acb9c 100644
--- a/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.cpp
+++ b/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.cpp
@@ -1,93 +1,93 @@
-#include "mkql_computation_node_graph_saveload.h"
-#include "mkql_computation_node_holders.h"
-
+#include "mkql_computation_node_graph_saveload.h"
+#include "mkql_computation_node_holders.h"
+
#include <ydb/library/yql/minikql/pack_num.h>
#include <ydb/library/yql/minikql/comp_nodes/mkql_saveload.h>
-
-namespace NKikimr {
-namespace NMiniKQL {
-
-namespace {
-
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+namespace {
+
void TraverseGraph(const NUdf::TUnboxedValue* roots, ui32 rootCount, TVector<NUdf::TUnboxedValue>& values) {
THashSet<NUdf::IBoxedValue*> dedup;
-
- for (ui32 i = 0; i < rootCount; ++i) {
- const auto& value = roots[i];
- if (!value.IsBoxed()) {
- continue;
- }
- auto* ptr = value.AsBoxed().Get();
+
+ for (ui32 i = 0; i < rootCount; ++i) {
+ const auto& value = roots[i];
+ if (!value.IsBoxed()) {
+ continue;
+ }
+ auto* ptr = value.AsBoxed().Get();
if (dedup.contains(ptr)) {
- continue;
- }
- dedup.insert(ptr);
- values.push_back(value);
- }
-
- for (ui32 from = 0, to = values.size(); from != to; ++from) {
+ continue;
+ }
+ dedup.insert(ptr);
+ values.push_back(value);
+ }
+
+ for (ui32 from = 0, to = values.size(); from != to; ++from) {
auto current = values[from];
- auto count = current.GetTraverseCount();
-
- for (ui32 i = 0; i < count; ++i) {
- auto value = current.GetTraverseItem(i);
- if (!value.IsBoxed()) {
- continue;
- }
- auto* ptr = value.AsBoxed().Get();
+ auto count = current.GetTraverseCount();
+
+ for (ui32 i = 0; i < count; ++i) {
+ auto value = current.GetTraverseItem(i);
+ if (!value.IsBoxed()) {
+ continue;
+ }
+ auto* ptr = value.AsBoxed().Get();
if (dedup.contains(ptr)) {
- continue;
- }
- dedup.insert(ptr);
- values.push_back(value);
- ++to;
- }
- }
-}
-
-}
-
-void SaveGraphState(const NUdf::TUnboxedValue* roots, ui32 rootCount, ui64 hash, TString& out) {
- out.clear();
- out.AppendNoAlias((const char*)&hash, sizeof(hash));
-
+ continue;
+ }
+ dedup.insert(ptr);
+ values.push_back(value);
+ ++to;
+ }
+ }
+}
+
+}
+
+void SaveGraphState(const NUdf::TUnboxedValue* roots, ui32 rootCount, ui64 hash, TString& out) {
+ out.clear();
+ out.AppendNoAlias((const char*)&hash, sizeof(hash));
+
TVector<NUdf::TUnboxedValue> values;
- TraverseGraph(roots, rootCount, values);
-
- for (ui32 i = 0; i < values.size(); ++i) {
- auto state = values[i].Save();
- auto strRef = state.AsStringRef();
- auto size = strRef.Size();
- WriteUi32(out, size);
- if (size) {
- out.AppendNoAlias(strRef.Data(), size);
- }
- }
-}
-
-void LoadGraphState(const NUdf::TUnboxedValue* roots, ui32 rootCount, ui64 hash, const TStringBuf& in) {
- TStringBuf state(in);
-
+ TraverseGraph(roots, rootCount, values);
+
+ for (ui32 i = 0; i < values.size(); ++i) {
+ auto state = values[i].Save();
+ auto strRef = state.AsStringRef();
+ auto size = strRef.Size();
+ WriteUi32(out, size);
+ if (size) {
+ out.AppendNoAlias(strRef.Data(), size);
+ }
+ }
+}
+
+void LoadGraphState(const NUdf::TUnboxedValue* roots, ui32 rootCount, ui64 hash, const TStringBuf& in) {
+ TStringBuf state(in);
+
MKQL_ENSURE(state.size() >= sizeof(ui64), "Serialized state is corrupted - no hash");
ui64 storedHash = *(ui64*)state.data();
- state.Skip(sizeof(storedHash));
-
- MKQL_ENSURE(hash == storedHash, "Unable to load graph state, different hashes");
-
+ state.Skip(sizeof(storedHash));
+
+ MKQL_ENSURE(hash == storedHash, "Unable to load graph state, different hashes");
+
TVector<NUdf::TUnboxedValue> values;
- TraverseGraph(roots, rootCount, values);
-
- for (ui32 i = 0; i < values.size(); ++i) {
- auto size = ReadUi32(state);
- if (size) {
+ TraverseGraph(roots, rootCount, values);
+
+ for (ui32 i = 0; i < values.size(); ++i) {
+ auto size = ReadUi32(state);
+ if (size) {
MKQL_ENSURE(size <= state.size(), "Serialized state is corrupted");
values[i].Load(NUdf::TStringRef(state.data(), size));
- state.Skip(size);
- }
- }
-
+ state.Skip(size);
+ }
+ }
+
MKQL_ENSURE(state.size() == 0, "State was not loaded correctly");
-}
-
-} // namespace NMiniKQL
-} // namespace NKikimr
+}
+
+} // namespace NMiniKQL
+} // namespace NKikimr
diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.h b/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.h
index 55564400f2d..171a83fa691 100644
--- a/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.h
+++ b/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.h
@@ -1,13 +1,13 @@
-#pragma once
-
-#include "mkql_computation_node.h"
-
-namespace NKikimr {
-namespace NMiniKQL {
-
-void SaveGraphState(const NUdf::TUnboxedValue* roots, ui32 rootCount, ui64 hash, TString& out);
-
-void LoadGraphState(const NUdf::TUnboxedValue* roots, ui32 rootCount, ui64 hash, const TStringBuf& in);
-
-} // namespace NMiniKQL
-} // namespace NKikimr
+#pragma once
+
+#include "mkql_computation_node.h"
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+void SaveGraphState(const NUdf::TUnboxedValue* roots, ui32 rootCount, ui64 hash, TString& out);
+
+void LoadGraphState(const NUdf::TUnboxedValue* roots, ui32 rootCount, ui64 hash, const TStringBuf& in);
+
+} // namespace NMiniKQL
+} // namespace NKikimr
diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload_ut.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload_ut.cpp
index cc734e11d60..48848db3264 100644
--- a/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload_ut.cpp
+++ b/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload_ut.cpp
@@ -7,77 +7,77 @@
#include <ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.h>
#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h>
#include <ydb/library/yql/minikql/comp_nodes/mkql_factories.h>
-
+
#include <library/cpp/testing/unittest/registar.h>
-
-namespace NKikimr {
-namespace NMiniKQL {
-
-namespace {
- TIntrusivePtr<IRandomProvider> CreateRandomProvider() {
- return CreateDeterministicRandomProvider(1);
- }
-
- TIntrusivePtr<ITimeProvider> CreateTimeProvider() {
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+namespace {
+ TIntrusivePtr<IRandomProvider> CreateRandomProvider() {
+ return CreateDeterministicRandomProvider(1);
+ }
+
+ TIntrusivePtr<ITimeProvider> CreateTimeProvider() {
return CreateDeterministicTimeProvider(10000000);
- }
-
- TComputationNodeFactory GetAuxCallableFactory() {
- return [](TCallable& callable, const TComputationNodeFactoryContext& ctx) -> IComputationNode* {
- if (callable.GetType()->GetName() == "OneYieldStream") {
- return new TExternalComputationNode(ctx.Mutables);
- }
-
- return GetBuiltinFactory()(callable, ctx);
- };
- }
-
- struct TSetup {
- TSetup(TScopedAlloc& alloc)
- : Alloc(alloc)
- {
- FunctionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry());
- RandomProvider = CreateRandomProvider();
- TimeProvider = CreateTimeProvider();
-
- Env.Reset(new TTypeEnvironment(Alloc));
- PgmBuilder.Reset(new TProgramBuilder(*Env, *FunctionRegistry));
- }
-
+ }
+
+ TComputationNodeFactory GetAuxCallableFactory() {
+ return [](TCallable& callable, const TComputationNodeFactoryContext& ctx) -> IComputationNode* {
+ if (callable.GetType()->GetName() == "OneYieldStream") {
+ return new TExternalComputationNode(ctx.Mutables);
+ }
+
+ return GetBuiltinFactory()(callable, ctx);
+ };
+ }
+
+ struct TSetup {
+ TSetup(TScopedAlloc& alloc)
+ : Alloc(alloc)
+ {
+ FunctionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry());
+ RandomProvider = CreateRandomProvider();
+ TimeProvider = CreateTimeProvider();
+
+ Env.Reset(new TTypeEnvironment(Alloc));
+ PgmBuilder.Reset(new TProgramBuilder(*Env, *FunctionRegistry));
+ }
+
THolder<IComputationGraph> BuildGraph(TRuntimeNode pgm, const std::vector<TNode*>& entryPoints = std::vector<TNode*>()) {
- Explorer.Walk(pgm.GetNode(), *Env);
- TComputationPatternOpts opts(Alloc.Ref(), *Env, GetAuxCallableFactory(),
+ Explorer.Walk(pgm.GetNode(), *Env);
+ TComputationPatternOpts opts(Alloc.Ref(), *Env, GetAuxCallableFactory(),
FunctionRegistry.Get(),
NUdf::EValidateMode::None, NUdf::EValidatePolicy::Fail, "OFF", EGraphPerProcess::Multi);
- Pattern = MakeComputationPattern(Explorer, pgm, entryPoints, opts);
+ Pattern = MakeComputationPattern(Explorer, pgm, entryPoints, opts);
TComputationOptsFull compOpts = opts.ToComputationOptions(*RandomProvider, *TimeProvider);
- return Pattern->Clone(compOpts);
- }
-
- TIntrusivePtr<IFunctionRegistry> FunctionRegistry;
- TIntrusivePtr<IRandomProvider> RandomProvider;
- TIntrusivePtr<ITimeProvider> TimeProvider;
-
- TScopedAlloc& Alloc;
- THolder<TTypeEnvironment> Env;
- THolder<TProgramBuilder> PgmBuilder;
-
- TExploringNodeVisitor Explorer;
+ return Pattern->Clone(compOpts);
+ }
+
+ TIntrusivePtr<IFunctionRegistry> FunctionRegistry;
+ TIntrusivePtr<IRandomProvider> RandomProvider;
+ TIntrusivePtr<ITimeProvider> TimeProvider;
+
+ TScopedAlloc& Alloc;
+ THolder<TTypeEnvironment> Env;
+ THolder<TProgramBuilder> PgmBuilder;
+
+ TExploringNodeVisitor Explorer;
IComputationPattern::TPtr Pattern;
- };
-
- struct TStreamWithYield : public NUdf::TBoxedValue {
+ };
+
+ struct TStreamWithYield : public NUdf::TBoxedValue {
TStreamWithYield(const TUnboxedValueVector& items, ui32 yieldPos, ui32 index)
- : Items(items)
- , YieldPos(yieldPos)
- , Index(index)
- {}
-
- private:
+ : Items(items)
+ , YieldPos(yieldPos)
+ , Index(index)
+ {}
+
+ private:
TUnboxedValueVector Items;
- ui32 YieldPos;
- ui32 Index;
-
+ ui32 YieldPos;
+ ui32 Index;
+
ui32 GetTraverseCount() const override {
return 0;
}
@@ -91,290 +91,290 @@ namespace {
}
NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) final {
- if (Index >= Items.size()) {
- return NUdf::EFetchStatus::Finish;
- }
- if (Index == YieldPos) {
- return NUdf::EFetchStatus::Yield;
- }
- result = Items[Index++];
- return NUdf::EFetchStatus::Ok;
- }
- };
-}
-
+ if (Index >= Items.size()) {
+ return NUdf::EFetchStatus::Finish;
+ }
+ if (Index == YieldPos) {
+ return NUdf::EFetchStatus::Yield;
+ }
+ result = Items[Index++];
+ return NUdf::EFetchStatus::Ok;
+ }
+ };
+}
+
Y_UNIT_TEST_SUITE(TMiniKQLSaveLoadTest) {
Y_UNIT_TEST(TestSqueezeSaveLoad) {
- TScopedAlloc alloc;
-
+ TScopedAlloc alloc;
+
const std::vector<ui32> items = {2, 3, 4, 5, 6, 7, 8};
-
- auto buildGraph = [&items] (TSetup& setup, ui32 yieldPos, ui32 startIndex) -> THolder<IComputationGraph> {
- TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
-
- auto dataType = pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id);
- auto streamType = pgmBuilder.NewStreamType(dataType);
-
- TCallableBuilder inStream(pgmBuilder.GetTypeEnvironment(), "OneYieldStream", streamType);
- auto streamNode = inStream.Build();
-
- auto pgmReturn = pgmBuilder.Squeeze(
- TRuntimeNode(streamNode, false),
- pgmBuilder.NewDataLiteral<ui32>(1),
- [&](TRuntimeNode item, TRuntimeNode state) {
- return pgmBuilder.Add(item, state);
- },
- [](TRuntimeNode state) {
- return state;
- },
- [](TRuntimeNode state) {
- return state;
- });
-
+
+ auto buildGraph = [&items] (TSetup& setup, ui32 yieldPos, ui32 startIndex) -> THolder<IComputationGraph> {
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ auto dataType = pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id);
+ auto streamType = pgmBuilder.NewStreamType(dataType);
+
+ TCallableBuilder inStream(pgmBuilder.GetTypeEnvironment(), "OneYieldStream", streamType);
+ auto streamNode = inStream.Build();
+
+ auto pgmReturn = pgmBuilder.Squeeze(
+ TRuntimeNode(streamNode, false),
+ pgmBuilder.NewDataLiteral<ui32>(1),
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pgmBuilder.Add(item, state);
+ },
+ [](TRuntimeNode state) {
+ return state;
+ },
+ [](TRuntimeNode state) {
+ return state;
+ });
+
TUnboxedValueVector streamItems;
- for (auto item : items) {
- streamItems.push_back(NUdf::TUnboxedValuePod(item));
- }
-
- auto graph = setup.BuildGraph(pgmReturn, {streamNode});
- auto streamValue = NUdf::TUnboxedValuePod(new TStreamWithYield(streamItems, yieldPos, startIndex));
+ for (auto item : items) {
+ streamItems.push_back(NUdf::TUnboxedValuePod(item));
+ }
+
+ auto graph = setup.BuildGraph(pgmReturn, {streamNode});
+ auto streamValue = NUdf::TUnboxedValuePod(new TStreamWithYield(streamItems, yieldPos, startIndex));
graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), std::move(streamValue));
- return graph;
- };
-
- for (ui32 yieldPos = 0; yieldPos < items.size(); ++yieldPos) {
- TSetup setup1(alloc);
- auto graph1 = buildGraph(setup1, yieldPos, 0);
-
+ return graph;
+ };
+
+ for (ui32 yieldPos = 0; yieldPos < items.size(); ++yieldPos) {
+ TSetup setup1(alloc);
+ auto graph1 = buildGraph(setup1, yieldPos, 0);
+
auto root1 = graph1->GetValue();
- NUdf::TUnboxedValue res;
+ NUdf::TUnboxedValue res;
auto status = root1.Fetch(res);
- UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Yield);
-
- TString graphState;
- SaveGraphState(&root1, 1, 0ULL, graphState);
-
- TSetup setup2(alloc);
- auto graph2 = buildGraph(setup2, -1, yieldPos);
-
- auto root2 = graph2->GetValue();
+ UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Yield);
+
+ TString graphState;
+ SaveGraphState(&root1, 1, 0ULL, graphState);
+
+ TSetup setup2(alloc);
+ auto graph2 = buildGraph(setup2, -1, yieldPos);
+
+ auto root2 = graph2->GetValue();
LoadGraphState(&root2, 1, 0ULL, graphState);
-
+
status = root2.Fetch(res);
- UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Ok);
- UNIT_ASSERT_EQUAL(res.Get<ui32>(), 36);
-
+ UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Ok);
+ UNIT_ASSERT_EQUAL(res.Get<ui32>(), 36);
+
status = root2.Fetch(res);
- UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Finish);
- }
- }
-
+ UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Finish);
+ }
+ }
+
Y_UNIT_TEST(TestSqueeze1SaveLoad) {
- TScopedAlloc alloc;
-
+ TScopedAlloc alloc;
+
const std::vector<ui32> items = {1, 2, 3, 4, 5, 6, 7, 8};
-
- auto buildGraph = [&items] (TSetup& setup, ui32 yieldPos, ui32 startIndex) -> THolder<IComputationGraph> {
- TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
-
- auto dataType = pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id);
- auto streamType = pgmBuilder.NewStreamType(dataType);
-
- TCallableBuilder inStream(pgmBuilder.GetTypeEnvironment(), "OneYieldStream", streamType);
- auto streamNode = inStream.Build();
-
- auto pgmReturn = pgmBuilder.Squeeze1(
- TRuntimeNode(streamNode, false),
- [](TRuntimeNode item) {
- return item;
- },
- [&](TRuntimeNode item, TRuntimeNode state) {
- return pgmBuilder.Add(item, state);
- },
- [](TRuntimeNode state) {
- return state;
- },
- [](TRuntimeNode state) {
- return state;
- });
-
+
+ auto buildGraph = [&items] (TSetup& setup, ui32 yieldPos, ui32 startIndex) -> THolder<IComputationGraph> {
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ auto dataType = pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id);
+ auto streamType = pgmBuilder.NewStreamType(dataType);
+
+ TCallableBuilder inStream(pgmBuilder.GetTypeEnvironment(), "OneYieldStream", streamType);
+ auto streamNode = inStream.Build();
+
+ auto pgmReturn = pgmBuilder.Squeeze1(
+ TRuntimeNode(streamNode, false),
+ [](TRuntimeNode item) {
+ return item;
+ },
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pgmBuilder.Add(item, state);
+ },
+ [](TRuntimeNode state) {
+ return state;
+ },
+ [](TRuntimeNode state) {
+ return state;
+ });
+
TUnboxedValueVector streamItems;
- for (auto item : items) {
- streamItems.push_back(NUdf::TUnboxedValuePod(item));
- }
-
- auto graph = setup.BuildGraph(pgmReturn, {streamNode});
- auto streamValue = NUdf::TUnboxedValuePod(new TStreamWithYield(streamItems, yieldPos, startIndex));
+ for (auto item : items) {
+ streamItems.push_back(NUdf::TUnboxedValuePod(item));
+ }
+
+ auto graph = setup.BuildGraph(pgmReturn, {streamNode});
+ auto streamValue = NUdf::TUnboxedValuePod(new TStreamWithYield(streamItems, yieldPos, startIndex));
graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), std::move(streamValue));
- return graph;
- };
-
- for (ui32 yieldPos = 0; yieldPos < items.size(); ++yieldPos) {
- TSetup setup1(alloc);
- auto graph1 = buildGraph(setup1, yieldPos, 0);
-
+ return graph;
+ };
+
+ for (ui32 yieldPos = 0; yieldPos < items.size(); ++yieldPos) {
+ TSetup setup1(alloc);
+ auto graph1 = buildGraph(setup1, yieldPos, 0);
+
auto root1 = graph1->GetValue();
- NUdf::TUnboxedValue res;
+ NUdf::TUnboxedValue res;
auto status = root1.Fetch(res);
- UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Yield);
-
- TString graphState;
- SaveGraphState(&root1, 1, 0ULL, graphState);
-
- TSetup setup2(alloc);
- auto graph2 = buildGraph(setup2, -1, yieldPos);
-
- auto root2 = graph2->GetValue();
+ UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Yield);
+
+ TString graphState;
+ SaveGraphState(&root1, 1, 0ULL, graphState);
+
+ TSetup setup2(alloc);
+ auto graph2 = buildGraph(setup2, -1, yieldPos);
+
+ auto root2 = graph2->GetValue();
LoadGraphState(&root2, 1, 0ULL, graphState);
-
+
status = root2.Fetch(res);
- UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Ok);
- UNIT_ASSERT_EQUAL(res.Get<ui32>(), 36);
-
+ UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Ok);
+ UNIT_ASSERT_EQUAL(res.Get<ui32>(), 36);
+
status = root2.Fetch(res);
- UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Finish);
- }
- }
-
+ UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Finish);
+ }
+ }
+
Y_UNIT_TEST(TestHoppingSaveLoad) {
- TScopedAlloc alloc;
-
+ TScopedAlloc alloc;
+
const std::vector<std::pair<i64, ui32>> items = {
- {1, 2},
- {2, 3},
- {15, 4},
- {23, 6},
- {24, 5},
- {25, 7},
- {40, 2},
- {47, 1},
- {51, 6},
- {59, 2},
- {85, 8},
- {55, 1000},
- {200, 0}
- };
-
- auto buildGraph = [&items] (TSetup& setup, ui32 yieldPos, ui32 startIndex) -> THolder<IComputationGraph> {
- TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
-
- auto structType = pgmBuilder.NewEmptyStructType();
+ {1, 2},
+ {2, 3},
+ {15, 4},
+ {23, 6},
+ {24, 5},
+ {25, 7},
+ {40, 2},
+ {47, 1},
+ {51, 6},
+ {59, 2},
+ {85, 8},
+ {55, 1000},
+ {200, 0}
+ };
+
+ auto buildGraph = [&items] (TSetup& setup, ui32 yieldPos, ui32 startIndex) -> THolder<IComputationGraph> {
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ auto structType = pgmBuilder.NewEmptyStructType();
structType = pgmBuilder.NewStructType(structType, "time",
- pgmBuilder.NewDataType(NUdf::TDataType<NUdf::TTimestamp>::Id));
+ pgmBuilder.NewDataType(NUdf::TDataType<NUdf::TTimestamp>::Id));
structType = pgmBuilder.NewStructType(structType, "sum",
- pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id));
- auto timeIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("time");
- auto sumIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("sum");
-
- auto inStreamType = pgmBuilder.NewStreamType(structType);
-
- TCallableBuilder inStream(pgmBuilder.GetTypeEnvironment(), "OneYieldStream", inStreamType);
- auto streamNode = inStream.Build();
-
- ui64 hop = 10, interval = 30, delay = 20;
-
- auto pgmReturn = pgmBuilder.HoppingCore(
- TRuntimeNode(streamNode, false),
- [&](TRuntimeNode item) { // timeExtractor
+ pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id));
+ auto timeIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("time");
+ auto sumIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("sum");
+
+ auto inStreamType = pgmBuilder.NewStreamType(structType);
+
+ TCallableBuilder inStream(pgmBuilder.GetTypeEnvironment(), "OneYieldStream", inStreamType);
+ auto streamNode = inStream.Build();
+
+ ui64 hop = 10, interval = 30, delay = 20;
+
+ auto pgmReturn = pgmBuilder.HoppingCore(
+ TRuntimeNode(streamNode, false),
+ [&](TRuntimeNode item) { // timeExtractor
return pgmBuilder.Member(item, "time");
- },
- [&](TRuntimeNode item) { // init
+ },
+ [&](TRuntimeNode item) { // init
std::vector<std::pair<std::string_view, TRuntimeNode>> members;
members.emplace_back("sum", pgmBuilder.Member(item, "sum"));
- return pgmBuilder.NewStruct(members);
- },
- [&](TRuntimeNode item, TRuntimeNode state) { // update
- auto add = pgmBuilder.AggrAdd(
+ return pgmBuilder.NewStruct(members);
+ },
+ [&](TRuntimeNode item, TRuntimeNode state) { // update
+ auto add = pgmBuilder.AggrAdd(
pgmBuilder.Member(item, "sum"),
pgmBuilder.Member(state, "sum"));
std::vector<std::pair<std::string_view, TRuntimeNode>> members;
members.emplace_back("sum", add);
- return pgmBuilder.NewStruct(members);
- },
- [&](TRuntimeNode state) { // save
+ return pgmBuilder.NewStruct(members);
+ },
+ [&](TRuntimeNode state) { // save
return pgmBuilder.Member(state, "sum");
- },
- [&](TRuntimeNode savedState) { // load
+ },
+ [&](TRuntimeNode savedState) { // load
std::vector<std::pair<std::string_view, TRuntimeNode>> members;
members.emplace_back("sum", savedState);
- return pgmBuilder.NewStruct(members);
- },
- [&](TRuntimeNode state1, TRuntimeNode state2) { // merge
- auto add = pgmBuilder.AggrAdd(
+ return pgmBuilder.NewStruct(members);
+ },
+ [&](TRuntimeNode state1, TRuntimeNode state2) { // merge
+ auto add = pgmBuilder.AggrAdd(
pgmBuilder.Member(state1, "sum"),
pgmBuilder.Member(state2, "sum"));
std::vector<std::pair<std::string_view, TRuntimeNode>> members;
members.emplace_back("sum", add);
- return pgmBuilder.NewStruct(members);
- },
- [&](TRuntimeNode state, TRuntimeNode time) { // finish
- Y_UNUSED(time);
+ return pgmBuilder.NewStruct(members);
+ },
+ [&](TRuntimeNode state, TRuntimeNode time) { // finish
+ Y_UNUSED(time);
std::vector<std::pair<std::string_view, TRuntimeNode>> members;
members.emplace_back("sum", pgmBuilder.Member(state, "sum"));
- return pgmBuilder.NewStruct(members);
- },
- pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&hop, sizeof(hop))), // hop
- pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&interval, sizeof(interval))), // interval
- pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&delay, sizeof(delay))) // delay
- );
-
- auto graph = setup.BuildGraph(pgmReturn, {streamNode});
-
+ return pgmBuilder.NewStruct(members);
+ },
+ pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&hop, sizeof(hop))), // hop
+ pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&interval, sizeof(interval))), // interval
+ pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&delay, sizeof(delay))) // delay
+ );
+
+ auto graph = setup.BuildGraph(pgmReturn, {streamNode});
+
TUnboxedValueVector streamItems;
- for (size_t i = 0; i < items.size(); ++i) {
- NUdf::TUnboxedValue* itemsPtr;
- auto structValues = graph->GetHolderFactory().CreateDirectArrayHolder(2, itemsPtr);
- itemsPtr[timeIndex] = NUdf::TUnboxedValuePod(items[i].first);
- itemsPtr[sumIndex] = NUdf::TUnboxedValuePod(items[i].second);
- streamItems.push_back(std::move(structValues));
- }
-
- auto streamValue = NUdf::TUnboxedValuePod(new TStreamWithYield(streamItems, yieldPos, startIndex));
+ for (size_t i = 0; i < items.size(); ++i) {
+ NUdf::TUnboxedValue* itemsPtr;
+ auto structValues = graph->GetHolderFactory().CreateDirectArrayHolder(2, itemsPtr);
+ itemsPtr[timeIndex] = NUdf::TUnboxedValuePod(items[i].first);
+ itemsPtr[sumIndex] = NUdf::TUnboxedValuePod(items[i].second);
+ streamItems.push_back(std::move(structValues));
+ }
+
+ auto streamValue = NUdf::TUnboxedValuePod(new TStreamWithYield(streamItems, yieldPos, startIndex));
graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), std::move(streamValue));
- return graph;
- };
-
- for (ui32 yieldPos = 0; yieldPos < items.size(); ++yieldPos) {
+ return graph;
+ };
+
+ for (ui32 yieldPos = 0; yieldPos < items.size(); ++yieldPos) {
std::vector<ui32> result;
-
- TSetup setup1(alloc);
- auto graph1 = buildGraph(setup1, yieldPos, 0);
- auto root1 = graph1->GetValue();
-
- NUdf::EFetchStatus status = NUdf::EFetchStatus::Ok;
- while (status == NUdf::EFetchStatus::Ok) {
- NUdf::TUnboxedValue val;
- status = root1.Fetch(val);
- if (status == NUdf::EFetchStatus::Ok) {
- result.push_back(val.GetElement(0).Get<ui32>());
- }
- }
- UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Yield);
-
- TString graphState;
- SaveGraphState(&root1, 1, 0ULL, graphState);
-
- TSetup setup2(alloc);
- auto graph2 = buildGraph(setup2, -1, yieldPos);
- auto root2 = graph2->GetValue();
+
+ TSetup setup1(alloc);
+ auto graph1 = buildGraph(setup1, yieldPos, 0);
+ auto root1 = graph1->GetValue();
+
+ NUdf::EFetchStatus status = NUdf::EFetchStatus::Ok;
+ while (status == NUdf::EFetchStatus::Ok) {
+ NUdf::TUnboxedValue val;
+ status = root1.Fetch(val);
+ if (status == NUdf::EFetchStatus::Ok) {
+ result.push_back(val.GetElement(0).Get<ui32>());
+ }
+ }
+ UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Yield);
+
+ TString graphState;
+ SaveGraphState(&root1, 1, 0ULL, graphState);
+
+ TSetup setup2(alloc);
+ auto graph2 = buildGraph(setup2, -1, yieldPos);
+ auto root2 = graph2->GetValue();
LoadGraphState(&root2, 1, 0ULL, graphState);
-
- status = NUdf::EFetchStatus::Ok;
- while (status == NUdf::EFetchStatus::Ok) {
- NUdf::TUnboxedValue val;
- status = root2.Fetch(val);
- if (status == NUdf::EFetchStatus::Ok) {
- result.push_back(val.GetElement(0).Get<ui32>());
- }
- }
- UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Finish);
-
+
+ status = NUdf::EFetchStatus::Ok;
+ while (status == NUdf::EFetchStatus::Ok) {
+ NUdf::TUnboxedValue val;
+ status = root2.Fetch(val);
+ if (status == NUdf::EFetchStatus::Ok) {
+ result.push_back(val.GetElement(0).Get<ui32>());
+ }
+ }
+ UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Finish);
+
const std::vector<ui32> resultCompare = {5, 9, 27, 22, 21, 11, 11, 8, 8, 8, 8};
- UNIT_ASSERT_EQUAL(result, resultCompare);
- }
- }
-}
-
-} // namespace NMiniKQL
-} // namespace NKikimr
+ UNIT_ASSERT_EQUAL(result, resultCompare);
+ }
+ }
+}
+
+} // namespace NMiniKQL
+} // namespace NKikimr
diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_holders.h b/ydb/library/yql/minikql/computation/mkql_computation_node_holders.h
index 4486d6f43a1..1fd7a9bde83 100644
--- a/ydb/library/yql/minikql/computation/mkql_computation_node_holders.h
+++ b/ydb/library/yql/minikql/computation/mkql_computation_node_holders.h
@@ -56,11 +56,11 @@ inline int CompareValues(NUdf::EDataSlot type,
cmp = 1;
}
else {
- cmp = NUdf::CompareValues(type, lhs, rhs);
+ cmp = NUdf::CompareValues(type, lhs, rhs);
}
}
else {
- cmp = NUdf::CompareValues(type, lhs, rhs);
+ cmp = NUdf::CompareValues(type, lhs, rhs);
}
if (!asc) {
diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_impl.h b/ydb/library/yql/minikql/computation/mkql_computation_node_impl.h
index 0a63ca9d349..843d8388175 100644
--- a/ydb/library/yql/minikql/computation/mkql_computation_node_impl.h
+++ b/ydb/library/yql/minikql/computation/mkql_computation_node_impl.h
@@ -155,10 +155,10 @@ private:
}
void PrepareStageOne() final {
- if (!Stateless) {
+ if (!Stateless) {
Stateless = std::accumulate(this->Dependencies.cbegin(), this->Dependencies.cend(), 0,
std::bind(std::plus<i32>(), std::placeholders::_1, std::bind(&IComputationNode::GetDependencyWeight, std::placeholders::_2))) <= 1;
- }
+ }
}
void PrepareStageTwo() final {}
@@ -960,27 +960,27 @@ private:
return NUdf::EFetchStatus::Finish;
}
- ui32 GetTraverseCount() const override {
+ ui32 GetTraverseCount() const override {
ThrowNotSupported(__func__);
- return 0;
- }
-
- NUdf::TUnboxedValue GetTraverseItem(ui32 index) const override {
- Y_UNUSED(index);
+ return 0;
+ }
+
+ NUdf::TUnboxedValue GetTraverseItem(ui32 index) const override {
+ Y_UNUSED(index);
ThrowNotSupported(__func__);
- return {};
- }
-
- NUdf::TUnboxedValue Save() const override {
+ return {};
+ }
+
+ NUdf::TUnboxedValue Save() const override {
ThrowNotSupported(__func__);
- return NUdf::TUnboxedValue::Zero();
- }
-
- void Load(const NUdf::TStringRef& state) override {
- Y_UNUSED(state);
+ return NUdf::TUnboxedValue::Zero();
+ }
+
+ void Load(const NUdf::TStringRef& state) override {
+ Y_UNUSED(state);
ThrowNotSupported(__func__);
- }
-
+ }
+
void Push(const NUdf::TUnboxedValuePod& value) override {
Y_UNUSED(value);
ThrowNotSupported(__func__);
diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp
index 625a269eaf6..60fab952394 100644
--- a/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp
+++ b/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp
@@ -389,7 +389,7 @@ std::pair<ui32, bool> TValuePacker::SkipEmbeddedLength(TStringBuf& buf) {
return {length, emptySingleOptional};
}
-NUdf::TUnboxedValue TValuePacker::Unpack(TStringBuf buf, const THolderFactory& holderFactory) const {
+NUdf::TUnboxedValue TValuePacker::Unpack(TStringBuf buf, const THolderFactory& holderFactory) const {
auto pair = SkipEmbeddedLength(buf);
ui32 length = pair.first;
bool emptySingleOptional = pair.second;
@@ -417,7 +417,7 @@ NUdf::TUnboxedValue TValuePacker::Unpack(TStringBuf buf, const THolderFactory& h
}
NUdf::TUnboxedValue TValuePacker::UnpackImpl(const TType* type, TStringBuf& buf, ui32 topLength,
- const THolderFactory& holderFactory) const
+ const THolderFactory& holderFactory) const
{
switch (type->GetKind()) {
case TType::EKind::Void:
@@ -592,7 +592,7 @@ NUdf::TUnboxedValue TValuePacker::UnpackImpl(const TType* type, TStringBuf& buf,
}
}
-TStringBuf TValuePacker::Pack(const NUdf::TUnboxedValuePod& value) const {
+TStringBuf TValuePacker::Pack(const NUdf::TUnboxedValuePod& value) const {
OptionalUsageMask.Reset();
const size_t lengthReserve = sizeof(ui32);
Buffer.Proceed(lengthReserve + OptionalMaskReserve);
@@ -643,7 +643,7 @@ TStringBuf TValuePacker::Pack(const NUdf::TUnboxedValuePod& value) const {
return TStringBuf(Buffer.Data() + delta, len - delta);
}
-void TValuePacker::PackImpl(const TType* type, const NUdf::TUnboxedValuePod& value) const {
+void TValuePacker::PackImpl(const TType* type, const NUdf::TUnboxedValuePod& value) const {
switch (type->GetKind()) {
case TType::EKind::Void:
break;
diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_pack.h b/ydb/library/yql/minikql/computation/mkql_computation_node_pack.h
index c69f4467a3a..f9473ebff15 100644
--- a/ydb/library/yql/minikql/computation/mkql_computation_node_pack.h
+++ b/ydb/library/yql/minikql/computation/mkql_computation_node_pack.h
@@ -36,12 +36,12 @@ public:
TValuePacker(const TValuePacker& other);
// Returned buffer is temporary and should be copied before next Pack() call
- TStringBuf Pack(const NUdf::TUnboxedValuePod& value) const;
- NUdf::TUnboxedValue Unpack(TStringBuf buf, const THolderFactory& holderFactory) const;
+ TStringBuf Pack(const NUdf::TUnboxedValuePod& value) const;
+ NUdf::TUnboxedValue Unpack(TStringBuf buf, const THolderFactory& holderFactory) const;
private:
- void PackImpl(const TType* type, const NUdf::TUnboxedValuePod& value) const;
- NUdf::TUnboxedValue UnpackImpl(const TType* type, TStringBuf& buf, ui32 topLength, const THolderFactory& holderFactory) const;
+ void PackImpl(const TType* type, const NUdf::TUnboxedValuePod& value) const;
+ NUdf::TUnboxedValue UnpackImpl(const TType* type, TStringBuf& buf, ui32 topLength, const THolderFactory& holderFactory) const;
static TProperties ScanTypeProperties(const TType* type);
static bool HasOptionalFields(const TType* type);
// Returns length and empty single optional flag
@@ -54,12 +54,12 @@ private:
#endif
const bool Stable;
const TType* Type;
- // TODO: real thread safety with external state
- mutable TBuffer Buffer;
+ // TODO: real thread safety with external state
+ mutable TBuffer Buffer;
TProperties Properties;
- mutable size_t OptionalMaskReserve;
- mutable NDetails::TOptionalUsageMask OptionalUsageMask;
- mutable TPlainContainerCache TopStruct;
+ mutable size_t OptionalMaskReserve;
+ mutable NDetails::TOptionalUsageMask OptionalUsageMask;
+ mutable TPlainContainerCache TopStruct;
mutable TVector<TVector<std::pair<NUdf::TUnboxedValue, NUdf::TUnboxedValue>>> DictBuffers;
mutable TVector<TVector<std::tuple<NUdf::TUnboxedValue, NUdf::TUnboxedValue, NUdf::TUnboxedValue>>> EncodedDictBuffers;
TPackFunction PackFunc = nullptr;
diff --git a/ydb/library/yql/minikql/computation/mkql_value_builder.cpp b/ydb/library/yql/minikql/computation/mkql_value_builder.cpp
index ad476617158..c27a241428e 100644
--- a/ydb/library/yql/minikql/computation/mkql_value_builder.cpp
+++ b/ydb/library/yql/minikql/computation/mkql_value_builder.cpp
@@ -131,24 +131,24 @@ NUdf::IDictValueBuilder::TPtr TDefaultValueBuilder::NewDict(const NUdf::TType* d
return HolderFactory_.NewDict(dictType, flags);
}
-bool TDefaultValueBuilder::MakeDate(ui32 year, ui32 month, ui32 day, ui16& value) const {
- return ::NKikimr::NMiniKQL::MakeDate(year, month, day, value);
-}
-
-bool TDefaultValueBuilder::SplitDate(ui16 value, ui32& year, ui32& month, ui32& day) const {
- return ::NKikimr::NMiniKQL::SplitDate(value, year, month, day);
-}
-
-bool TDefaultValueBuilder::MakeDatetime(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 minute, ui32 second, ui32& value, ui16 tzId) const
-{
- return ::NKikimr::NMiniKQL::MakeTzDatetime(year, month, day, hour, minute, second, value, tzId);
-}
-
-bool TDefaultValueBuilder::SplitDatetime(ui32 value, ui32& year, ui32& month, ui32& day, ui32& hour, ui32& minute, ui32& second, ui16 tzId) const
-{
- return ::NKikimr::NMiniKQL::SplitTzDatetime(value, year, month, day, hour, minute, second, tzId);
-}
-
+bool TDefaultValueBuilder::MakeDate(ui32 year, ui32 month, ui32 day, ui16& value) const {
+ return ::NKikimr::NMiniKQL::MakeDate(year, month, day, value);
+}
+
+bool TDefaultValueBuilder::SplitDate(ui16 value, ui32& year, ui32& month, ui32& day) const {
+ return ::NKikimr::NMiniKQL::SplitDate(value, year, month, day);
+}
+
+bool TDefaultValueBuilder::MakeDatetime(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 minute, ui32 second, ui32& value, ui16 tzId) const
+{
+ return ::NKikimr::NMiniKQL::MakeTzDatetime(year, month, day, hour, minute, second, value, tzId);
+}
+
+bool TDefaultValueBuilder::SplitDatetime(ui32 value, ui32& year, ui32& month, ui32& day, ui32& hour, ui32& minute, ui32& second, ui16 tzId) const
+{
+ return ::NKikimr::NMiniKQL::SplitTzDatetime(value, year, month, day, hour, minute, second, tzId);
+}
+
bool TDefaultValueBuilder::FullSplitDate(ui16 value, ui32& year, ui32& month, ui32& day,
ui32& dayOfYear, ui32& weekOfYear, ui32& dayOfWeek, ui16 tzId) const {
ui32 unusedWeekOfYearIso8601 = 0;
@@ -171,20 +171,20 @@ bool TDefaultValueBuilder::FullSplitDatetime2(ui32 value, ui32& year, ui32& mont
return ::NKikimr::NMiniKQL::SplitTzDatetime(value, year, month, day, hour, minute, second, dayOfYear, weekOfYear, weekOfYearIso8601, dayOfWeek, tzId);
}
-bool TDefaultValueBuilder::EnrichDate(ui16 date, ui32& dayOfYear, ui32& weekOfYear, ui32& dayOfWeek) const {
+bool TDefaultValueBuilder::EnrichDate(ui16 date, ui32& dayOfYear, ui32& weekOfYear, ui32& dayOfWeek) const {
ui32 unusedWeekOfYearIso8601 = 0;
return ::NKikimr::NMiniKQL::EnrichDate(date, dayOfYear, weekOfYear, unusedWeekOfYearIso8601, dayOfWeek);
-}
-
+}
+
bool TDefaultValueBuilder::EnrichDate2(ui16 date, ui32& dayOfYear, ui32& weekOfYear, ui32& weekOfYearIso8601, ui32& dayOfWeek) const {
return ::NKikimr::NMiniKQL::EnrichDate(date, dayOfYear, weekOfYear, weekOfYearIso8601, dayOfWeek);
}
-bool TDefaultValueBuilder::GetTimezoneShift(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 minute, ui32 second, ui16 tzId, i32& value) const
-{
- return ::NKikimr::NMiniKQL::GetTimezoneShift(year, month, day, hour, minute, second, tzId, value);
-}
-
+bool TDefaultValueBuilder::GetTimezoneShift(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 minute, ui32 second, ui16 tzId, i32& value) const
+{
+ return ::NKikimr::NMiniKQL::GetTimezoneShift(year, month, day, hour, minute, second, tzId, value);
+}
+
const NUdf::TSourcePosition* TDefaultValueBuilder::CalleePosition() const {
return *CalleePositionPtr_;
}
diff --git a/ydb/library/yql/minikql/computation/mkql_value_builder.h b/ydb/library/yql/minikql/computation/mkql_value_builder.h
index b293b263a3d..eea453b6b17 100644
--- a/ydb/library/yql/minikql/computation/mkql_value_builder.h
+++ b/ydb/library/yql/minikql/computation/mkql_value_builder.h
@@ -61,17 +61,17 @@ public:
NUdf::TFlatArrayBlockPtr NewFlatArrayBlock(ui32 count) const;
NUdf::TSingleBlockPtr NewSingleBlock(const NUdf::TUnboxedValue& value) const;
- bool MakeDate(ui32 year, ui32 month, ui32 day, ui16& value) const final;
- bool SplitDate(ui16 value, ui32& year, ui32& month, ui32& day) const final;
+ bool MakeDate(ui32 year, ui32 month, ui32 day, ui16& value) const final;
+ bool SplitDate(ui16 value, ui32& year, ui32& month, ui32& day) const final;
- bool MakeDatetime(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 minute, ui32 second, ui32& value, ui16 tzId = 0) const final;
- bool SplitDatetime(ui32 value, ui32& year, ui32& month, ui32& day, ui32& hour, ui32& minute, ui32& second, ui16 tzId = 0) const final;
-
- bool EnrichDate(ui16 date, ui32& dayOfYear, ui32& weekOfYear, ui32& dayOfWeek) const final;
-
- // in minutes
- bool GetTimezoneShift(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 minute, ui32 second, ui16 tzId, i32& value) const final;
+ bool MakeDatetime(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 minute, ui32 second, ui32& value, ui16 tzId = 0) const final;
+ bool SplitDatetime(ui32 value, ui32& year, ui32& month, ui32& day, ui32& hour, ui32& minute, ui32& second, ui16 tzId = 0) const final;
+ bool EnrichDate(ui16 date, ui32& dayOfYear, ui32& weekOfYear, ui32& dayOfWeek) const final;
+
+ // in minutes
+ bool GetTimezoneShift(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 minute, ui32 second, ui16 tzId, i32& value) const final;
+
bool FullSplitDate(ui16 value, ui32& year, ui32& month, ui32& day,
ui32& dayOfYear, ui32& weekOfYear, ui32& dayOfWeek, ui16 timezoneId = 0) const final;
bool FullSplitDatetime(ui32 value, ui32& year, ui32& month, ui32& day, ui32& hour, ui32& minute, ui32& second,
diff --git a/ydb/library/yql/minikql/computation/presort.cpp b/ydb/library/yql/minikql/computation/presort.cpp
index 26b96a60aaa..25f0a4806ce 100644
--- a/ydb/library/yql/minikql/computation/presort.cpp
+++ b/ydb/library/yql/minikql/computation/presort.cpp
@@ -1,510 +1,510 @@
-#include "presort.h"
+#include "presort.h"
#include "mkql_computation_node_holders.h"
#include <ydb/library/yql/minikql/defs.h>
#include <ydb/library/yql/minikql/mkql_string_util.h>
-
+
#include <ydb/library/yql/utils/swap_bytes.h>
#include <ydb/library/yql/public/decimal/yql_decimal_serialize.h>
-
-#include <util/system/unaligned_mem.h>
+
+#include <util/system/unaligned_mem.h>
#include <util/string/builder.h>
-
-namespace NKikimr {
-namespace NMiniKQL {
-
-namespace NDetail {
-
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+namespace NDetail {
+
using NYql::SwapBytes;
-Y_FORCE_INLINE
-void EnsureInputSize(TStringBuf& input, size_t size) {
- MKQL_ENSURE(input.size() >= size, "premature end of input");
-}
-
-template <bool Desc>
-Y_FORCE_INLINE
-void EncodeBool(TVector<ui8>& output, bool value) {
- output.push_back(Desc ? 0xFF ^ ui8(value) : ui8(value));
-}
-
-template <bool Desc>
-Y_FORCE_INLINE
-bool DecodeBool(TStringBuf& input) {
- EnsureInputSize(input, 1);
- auto result = Desc ? bool(0xFF ^ ui8(input[0])) : bool(input[0]);
- input.Skip(1);
- return result;
-}
-
-template <typename TUnsigned, bool Desc>
-Y_FORCE_INLINE
-void EncodeUnsigned(TVector<ui8>& output, TUnsigned value) {
- constexpr size_t size = sizeof(TUnsigned);
-
- if (Desc) {
- value = ~value;
- }
-
- output.resize(output.size() + size);
- WriteUnaligned<TUnsigned>(output.end() - size, SwapBytes(value));
-}
-
-template <typename TUnsigned, bool Desc>
-Y_FORCE_INLINE
-TUnsigned DecodeUnsigned(TStringBuf& input) {
- constexpr size_t size = sizeof(TUnsigned);
-
- EnsureInputSize(input, size);
+Y_FORCE_INLINE
+void EnsureInputSize(TStringBuf& input, size_t size) {
+ MKQL_ENSURE(input.size() >= size, "premature end of input");
+}
+
+template <bool Desc>
+Y_FORCE_INLINE
+void EncodeBool(TVector<ui8>& output, bool value) {
+ output.push_back(Desc ? 0xFF ^ ui8(value) : ui8(value));
+}
+
+template <bool Desc>
+Y_FORCE_INLINE
+bool DecodeBool(TStringBuf& input) {
+ EnsureInputSize(input, 1);
+ auto result = Desc ? bool(0xFF ^ ui8(input[0])) : bool(input[0]);
+ input.Skip(1);
+ return result;
+}
+
+template <typename TUnsigned, bool Desc>
+Y_FORCE_INLINE
+void EncodeUnsigned(TVector<ui8>& output, TUnsigned value) {
+ constexpr size_t size = sizeof(TUnsigned);
+
+ if (Desc) {
+ value = ~value;
+ }
+
+ output.resize(output.size() + size);
+ WriteUnaligned<TUnsigned>(output.end() - size, SwapBytes(value));
+}
+
+template <typename TUnsigned, bool Desc>
+Y_FORCE_INLINE
+TUnsigned DecodeUnsigned(TStringBuf& input) {
+ constexpr size_t size = sizeof(TUnsigned);
+
+ EnsureInputSize(input, size);
auto value = ReadUnaligned<TUnsigned>(input.data());
- input.Skip(size);
-
- value = SwapBytes(value);
- if (Desc) {
- value = ~value;
- }
- return value;
-}
-
-template <typename TSigned, bool Desc>
-Y_FORCE_INLINE
-void EncodeSigned(TVector<ui8>& output, TSigned value) {
+ input.Skip(size);
+
+ value = SwapBytes(value);
+ if (Desc) {
+ value = ~value;
+ }
+ return value;
+}
+
+template <typename TSigned, bool Desc>
+Y_FORCE_INLINE
+void EncodeSigned(TVector<ui8>& output, TSigned value) {
using TUnsigned = std::make_unsigned_t<TSigned>;
- constexpr size_t size = sizeof(TUnsigned);
- constexpr TUnsigned shift = TUnsigned(1) << (size * 8 - 1);
-
- EncodeUnsigned<TUnsigned, Desc>(output, TUnsigned(value) + shift);
-}
-
-template <typename TSigned, bool Desc>
-Y_FORCE_INLINE
-TSigned DecodeSigned(TStringBuf& input) {
+ constexpr size_t size = sizeof(TUnsigned);
+ constexpr TUnsigned shift = TUnsigned(1) << (size * 8 - 1);
+
+ EncodeUnsigned<TUnsigned, Desc>(output, TUnsigned(value) + shift);
+}
+
+template <typename TSigned, bool Desc>
+Y_FORCE_INLINE
+TSigned DecodeSigned(TStringBuf& input) {
using TUnsigned = std::make_unsigned_t<TSigned>;
- constexpr size_t size = sizeof(TUnsigned);
- constexpr TUnsigned shift = TUnsigned(1) << (size * 8 - 1);
-
- return TSigned(DecodeUnsigned<TUnsigned, Desc>(input) - shift);
-}
-
-enum class EFPCode : ui8 {
- NegInf = 0,
- Neg = 1,
- Zero = 2,
- Pos = 3,
- PosInf = 4,
- Nan = 5
-};
-
-template <typename TFloat>
-struct TFloatToInteger {};
-
-template <>
-struct TFloatToInteger<float> {
- using TType = ui32;
-};
-
-template <>
-struct TFloatToInteger<double> {
- using TType = ui64;
-};
-
-static_assert(std::numeric_limits<float>::is_iec559, "float type is not iec559(ieee754)");
-static_assert(std::numeric_limits<double>::is_iec559, "double type is not iec559(ieee754)");
-
-template <typename TFloat, bool Desc>
-Y_FORCE_INLINE
-void EncodeFloating(TVector<ui8>& output, TFloat value) {
- using TInteger = typename TFloatToInteger<TFloat>::TType;
- EFPCode code;
-
- switch (std::fpclassify(value)) {
- case FP_NORMAL:
- case FP_SUBNORMAL: {
- auto integer = ReadUnaligned<TInteger>(&value);
- if (value < 0) {
- integer = ~integer;
- code = EFPCode::Neg;
- } else {
- code = EFPCode::Pos;
- }
- output.push_back(Desc ? 0xFF ^ ui8(code) : ui8(code));
- EncodeUnsigned<TInteger, Desc>(output, integer);
- return;
- }
- case FP_ZERO:
- code = EFPCode::Zero;
- break;
- case FP_INFINITE:
- code = value < 0 ? EFPCode::NegInf : EFPCode::PosInf;
- break;
- default:
- code = EFPCode::Nan;
- break;
- }
- output.push_back(Desc ? 0xFF ^ ui8(code) : ui8(code));
-}
-
-template <typename TFloat, bool Desc>
-Y_FORCE_INLINE
-TFloat DecodeFloating(TStringBuf& input) {
- using TInteger = typename TFloatToInteger<TFloat>::TType;
-
- EnsureInputSize(input, 1);
- auto code = EFPCode(Desc ? 0xFF ^ input[0] : input[0]);
- input.Skip(1);
-
- bool negative;
- switch (code) {
- case EFPCode::Zero:
- return 0;
- case EFPCode::NegInf:
- return -std::numeric_limits<TFloat>::infinity();
- case EFPCode::PosInf:
- return std::numeric_limits<TFloat>::infinity();
- case EFPCode::Nan:
- return std::numeric_limits<TFloat>::quiet_NaN();
- case EFPCode::Neg:
- negative = true;
- break;
- case EFPCode::Pos:
- negative = false;
- break;
- default:
- MKQL_ENSURE(false, "floating point data is corrupted");
- }
-
- auto integer = DecodeUnsigned<TInteger, Desc>(input);
- if (negative) {
- integer = ~integer;
- }
-
- return ReadUnaligned<TFloat>(&integer);
-}
-
-constexpr ui8 BlockCode = 0x1F;
-constexpr size_t BlockSize = 15;
-constexpr size_t BlockSizeUi64 = BlockSize / 8 + 1;
-
-template <bool Desc>
-Y_FORCE_INLINE
-void EncodeString(TVector<ui8>& output, TStringBuf value) {
- size_t part = 0;
-
- while (!value.empty()) {
- union {
- ui8 buffer[BlockSize + 1];
- ui64 buffer64[BlockSizeUi64];
- };
-
- part = std::min(value.size(), BlockSize);
- if (part == BlockSize) {
- std::memcpy(buffer + 1, value.data(), BlockSize);
- } else {
- for (size_t i = 0; i < BlockSizeUi64; ++i) {
- buffer64[i] = 0;
- }
- std::memcpy(buffer + 1, value.data(), part);
- }
- value.Skip(part);
-
- buffer[0] = BlockCode;
-
- if (Desc) {
- for (size_t i = 0; i < BlockSizeUi64; ++i) {
- buffer64[i] ^= std::numeric_limits<ui64>::max();
- }
- }
-
- output.insert(output.end(), buffer, buffer + BlockSize + 1);
- }
-
- auto lastLength = ui8(part);
- output.push_back(Desc ? 0xFF ^ lastLength : lastLength);
-}
-
-template <bool Desc>
-Y_FORCE_INLINE
-TStringBuf DecodeString(TStringBuf& input, TVector<ui8>& value) {
- EnsureInputSize(input, 1);
- ui8 code = Desc ? 0xFF ^ input[0] : input[0];
- input.Skip(1);
-
- if (code != BlockCode) {
- MKQL_ENSURE(code == 0, TStringBuilder() << "unknown string block code: " << code);
- return TStringBuf();
- }
-
- while (code == BlockCode) {
- union {
- ui8 buffer[BlockSize + 1];
- ui64 buffer64[BlockSizeUi64];
- };
-
- EnsureInputSize(input, BlockSize + 1);
- std::memcpy(buffer, input.data(), BlockSize + 1);
- input.Skip(BlockSize + 1);
-
- if (Desc) {
- for (size_t i = 0; i < BlockSizeUi64; ++i) {
- buffer64[i] ^= std::numeric_limits<ui64>::max();
- }
- }
-
- value.insert(value.end(), buffer, buffer + BlockSize);
- code = buffer[BlockSize];
- }
-
- auto begin = (const char*)value.begin();
- auto end = (const char*)value.end() - BlockSize + code;
- return TStringBuf(begin, end - begin);
-}
-
-constexpr size_t UuidSize = 16;
-
-template <bool Desc>
-Y_FORCE_INLINE
-void EncodeUuid(TVector<ui8>& output, const char* data) {
- output.resize(output.size() + UuidSize);
- auto ptr = output.end() - UuidSize;
-
- if (Desc) {
- for (size_t i = 0; i < UuidSize; ++i) {
- *ptr++ = ui8(*data++) ^ 0xFF;
- }
- } else {
- std::memcpy(ptr, data, UuidSize);
- }
-}
-
-template <bool Desc>
-Y_FORCE_INLINE
-TStringBuf DecodeUuid(TStringBuf& input, TVector<ui8>& value) {
- EnsureInputSize(input, UuidSize);
+ constexpr size_t size = sizeof(TUnsigned);
+ constexpr TUnsigned shift = TUnsigned(1) << (size * 8 - 1);
+
+ return TSigned(DecodeUnsigned<TUnsigned, Desc>(input) - shift);
+}
+
+enum class EFPCode : ui8 {
+ NegInf = 0,
+ Neg = 1,
+ Zero = 2,
+ Pos = 3,
+ PosInf = 4,
+ Nan = 5
+};
+
+template <typename TFloat>
+struct TFloatToInteger {};
+
+template <>
+struct TFloatToInteger<float> {
+ using TType = ui32;
+};
+
+template <>
+struct TFloatToInteger<double> {
+ using TType = ui64;
+};
+
+static_assert(std::numeric_limits<float>::is_iec559, "float type is not iec559(ieee754)");
+static_assert(std::numeric_limits<double>::is_iec559, "double type is not iec559(ieee754)");
+
+template <typename TFloat, bool Desc>
+Y_FORCE_INLINE
+void EncodeFloating(TVector<ui8>& output, TFloat value) {
+ using TInteger = typename TFloatToInteger<TFloat>::TType;
+ EFPCode code;
+
+ switch (std::fpclassify(value)) {
+ case FP_NORMAL:
+ case FP_SUBNORMAL: {
+ auto integer = ReadUnaligned<TInteger>(&value);
+ if (value < 0) {
+ integer = ~integer;
+ code = EFPCode::Neg;
+ } else {
+ code = EFPCode::Pos;
+ }
+ output.push_back(Desc ? 0xFF ^ ui8(code) : ui8(code));
+ EncodeUnsigned<TInteger, Desc>(output, integer);
+ return;
+ }
+ case FP_ZERO:
+ code = EFPCode::Zero;
+ break;
+ case FP_INFINITE:
+ code = value < 0 ? EFPCode::NegInf : EFPCode::PosInf;
+ break;
+ default:
+ code = EFPCode::Nan;
+ break;
+ }
+ output.push_back(Desc ? 0xFF ^ ui8(code) : ui8(code));
+}
+
+template <typename TFloat, bool Desc>
+Y_FORCE_INLINE
+TFloat DecodeFloating(TStringBuf& input) {
+ using TInteger = typename TFloatToInteger<TFloat>::TType;
+
+ EnsureInputSize(input, 1);
+ auto code = EFPCode(Desc ? 0xFF ^ input[0] : input[0]);
+ input.Skip(1);
+
+ bool negative;
+ switch (code) {
+ case EFPCode::Zero:
+ return 0;
+ case EFPCode::NegInf:
+ return -std::numeric_limits<TFloat>::infinity();
+ case EFPCode::PosInf:
+ return std::numeric_limits<TFloat>::infinity();
+ case EFPCode::Nan:
+ return std::numeric_limits<TFloat>::quiet_NaN();
+ case EFPCode::Neg:
+ negative = true;
+ break;
+ case EFPCode::Pos:
+ negative = false;
+ break;
+ default:
+ MKQL_ENSURE(false, "floating point data is corrupted");
+ }
+
+ auto integer = DecodeUnsigned<TInteger, Desc>(input);
+ if (negative) {
+ integer = ~integer;
+ }
+
+ return ReadUnaligned<TFloat>(&integer);
+}
+
+constexpr ui8 BlockCode = 0x1F;
+constexpr size_t BlockSize = 15;
+constexpr size_t BlockSizeUi64 = BlockSize / 8 + 1;
+
+template <bool Desc>
+Y_FORCE_INLINE
+void EncodeString(TVector<ui8>& output, TStringBuf value) {
+ size_t part = 0;
+
+ while (!value.empty()) {
+ union {
+ ui8 buffer[BlockSize + 1];
+ ui64 buffer64[BlockSizeUi64];
+ };
+
+ part = std::min(value.size(), BlockSize);
+ if (part == BlockSize) {
+ std::memcpy(buffer + 1, value.data(), BlockSize);
+ } else {
+ for (size_t i = 0; i < BlockSizeUi64; ++i) {
+ buffer64[i] = 0;
+ }
+ std::memcpy(buffer + 1, value.data(), part);
+ }
+ value.Skip(part);
+
+ buffer[0] = BlockCode;
+
+ if (Desc) {
+ for (size_t i = 0; i < BlockSizeUi64; ++i) {
+ buffer64[i] ^= std::numeric_limits<ui64>::max();
+ }
+ }
+
+ output.insert(output.end(), buffer, buffer + BlockSize + 1);
+ }
+
+ auto lastLength = ui8(part);
+ output.push_back(Desc ? 0xFF ^ lastLength : lastLength);
+}
+
+template <bool Desc>
+Y_FORCE_INLINE
+TStringBuf DecodeString(TStringBuf& input, TVector<ui8>& value) {
+ EnsureInputSize(input, 1);
+ ui8 code = Desc ? 0xFF ^ input[0] : input[0];
+ input.Skip(1);
+
+ if (code != BlockCode) {
+ MKQL_ENSURE(code == 0, TStringBuilder() << "unknown string block code: " << code);
+ return TStringBuf();
+ }
+
+ while (code == BlockCode) {
+ union {
+ ui8 buffer[BlockSize + 1];
+ ui64 buffer64[BlockSizeUi64];
+ };
+
+ EnsureInputSize(input, BlockSize + 1);
+ std::memcpy(buffer, input.data(), BlockSize + 1);
+ input.Skip(BlockSize + 1);
+
+ if (Desc) {
+ for (size_t i = 0; i < BlockSizeUi64; ++i) {
+ buffer64[i] ^= std::numeric_limits<ui64>::max();
+ }
+ }
+
+ value.insert(value.end(), buffer, buffer + BlockSize);
+ code = buffer[BlockSize];
+ }
+
+ auto begin = (const char*)value.begin();
+ auto end = (const char*)value.end() - BlockSize + code;
+ return TStringBuf(begin, end - begin);
+}
+
+constexpr size_t UuidSize = 16;
+
+template <bool Desc>
+Y_FORCE_INLINE
+void EncodeUuid(TVector<ui8>& output, const char* data) {
+ output.resize(output.size() + UuidSize);
+ auto ptr = output.end() - UuidSize;
+
+ if (Desc) {
+ for (size_t i = 0; i < UuidSize; ++i) {
+ *ptr++ = ui8(*data++) ^ 0xFF;
+ }
+ } else {
+ std::memcpy(ptr, data, UuidSize);
+ }
+}
+
+template <bool Desc>
+Y_FORCE_INLINE
+TStringBuf DecodeUuid(TStringBuf& input, TVector<ui8>& value) {
+ EnsureInputSize(input, UuidSize);
auto data = input.data();
- input.Skip(UuidSize);
-
- value.resize(UuidSize);
- auto ptr = value.begin();
-
- if (Desc) {
- for (size_t i = 0; i < UuidSize; ++i) {
- *ptr++ = ui8(*data++) ^ 0xFF;
- }
- } else {
- std::memcpy(ptr, data, UuidSize);
- }
-
- return TStringBuf((const char*)value.begin(), (const char*)value.end());
-}
-
-template <typename TUnsigned, bool Desc>
-Y_FORCE_INLINE
-void EncodeTzUnsigned(TVector<ui8>& output, TUnsigned value, ui16 tzId) {
- constexpr size_t size = sizeof(TUnsigned);
-
- if (Desc) {
- value = ~value;
- tzId = ~tzId;
- }
-
- output.resize(output.size() + size + sizeof(ui16));
- WriteUnaligned<TUnsigned>(output.end() - size - sizeof(ui16), SwapBytes(value));
- WriteUnaligned<ui16>(output.end() - sizeof(ui16), SwapBytes(tzId));
-}
-
-template <typename TUnsigned, bool Desc>
-Y_FORCE_INLINE
-void DecodeTzUnsigned(TStringBuf& input, TUnsigned& value, ui16& tzId) {
- constexpr size_t size = sizeof(TUnsigned);
-
- EnsureInputSize(input, size + sizeof(ui16));
+ input.Skip(UuidSize);
+
+ value.resize(UuidSize);
+ auto ptr = value.begin();
+
+ if (Desc) {
+ for (size_t i = 0; i < UuidSize; ++i) {
+ *ptr++ = ui8(*data++) ^ 0xFF;
+ }
+ } else {
+ std::memcpy(ptr, data, UuidSize);
+ }
+
+ return TStringBuf((const char*)value.begin(), (const char*)value.end());
+}
+
+template <typename TUnsigned, bool Desc>
+Y_FORCE_INLINE
+void EncodeTzUnsigned(TVector<ui8>& output, TUnsigned value, ui16 tzId) {
+ constexpr size_t size = sizeof(TUnsigned);
+
+ if (Desc) {
+ value = ~value;
+ tzId = ~tzId;
+ }
+
+ output.resize(output.size() + size + sizeof(ui16));
+ WriteUnaligned<TUnsigned>(output.end() - size - sizeof(ui16), SwapBytes(value));
+ WriteUnaligned<ui16>(output.end() - sizeof(ui16), SwapBytes(tzId));
+}
+
+template <typename TUnsigned, bool Desc>
+Y_FORCE_INLINE
+void DecodeTzUnsigned(TStringBuf& input, TUnsigned& value, ui16& tzId) {
+ constexpr size_t size = sizeof(TUnsigned);
+
+ EnsureInputSize(input, size + sizeof(ui16));
auto v = ReadUnaligned<TUnsigned>(input.data());
auto t = ReadUnaligned<ui16>(input.data() + size);
- input.Skip(size + sizeof(ui16));
-
- if (Desc) {
- value = ~SwapBytes(v);
- tzId = ~SwapBytes(t);
- } else {
- value = SwapBytes(v);
- tzId = SwapBytes(t);
- }
-}
-
+ input.Skip(size + sizeof(ui16));
+
+ if (Desc) {
+ value = ~SwapBytes(v);
+ tzId = ~SwapBytes(t);
+ } else {
+ value = SwapBytes(v);
+ tzId = SwapBytes(t);
+ }
+}
+
constexpr size_t DecimalSize = sizeof(NYql::NDecimal::TInt128);
-
-template <bool Desc>
-Y_FORCE_INLINE
+
+template <bool Desc>
+Y_FORCE_INLINE
void EncodeDecimal(TVector<ui8>& output, NYql::NDecimal::TInt128 value) {
- output.resize(output.size() + DecimalSize);
- auto ptr = reinterpret_cast<char*>(output.end() - DecimalSize);
+ output.resize(output.size() + DecimalSize);
+ auto ptr = reinterpret_cast<char*>(output.end() - DecimalSize);
output.resize(output.size() + NYql::NDecimal::Serialize(Desc ? -value : value, ptr) - DecimalSize);
-}
-
-template <bool Desc>
-Y_FORCE_INLINE
+}
+
+template <bool Desc>
+Y_FORCE_INLINE
NYql::NDecimal::TInt128 DecodeDecimal(TStringBuf& input) {
MKQL_ENSURE(input.size() > 0U && input.size() <= DecimalSize, "premature end of input");
const auto des = NYql::NDecimal::Deserialize(input.data());
input.Skip(des.second);
return Desc ? -des.first : des.first;
-}
-
-
-template <bool Desc>
-Y_FORCE_INLINE
-void Encode(TVector<ui8>& output, NUdf::EDataSlot slot, const NUdf::TUnboxedValuePod& value) {
- switch (slot) {
-
- case NUdf::EDataSlot::Bool:
- EncodeBool<Desc>(output, value.Get<bool>());
- break;
- case NUdf::EDataSlot::Int8:
- EncodeSigned<i8, Desc>(output, value.Get<i8>());
- break;
- case NUdf::EDataSlot::Uint8:
- EncodeUnsigned<ui8, Desc>(output, value.Get<ui8>());
- break;
- case NUdf::EDataSlot::Int16:
- EncodeSigned<i16, Desc>(output, value.Get<i16>());
- break;
- case NUdf::EDataSlot::Uint16:
- case NUdf::EDataSlot::Date:
- EncodeUnsigned<ui16, Desc>(output, value.Get<ui16>());
- break;
- case NUdf::EDataSlot::Int32:
- EncodeSigned<i32, Desc>(output, value.Get<i32>());
- break;
- case NUdf::EDataSlot::Uint32:
- case NUdf::EDataSlot::Datetime:
- EncodeUnsigned<ui32, Desc>(output, value.Get<ui32>());
- break;
- case NUdf::EDataSlot::Int64:
- case NUdf::EDataSlot::Interval:
- EncodeSigned<i64, Desc>(output, value.Get<i64>());
- break;
- case NUdf::EDataSlot::Uint64:
- case NUdf::EDataSlot::Timestamp:
- EncodeUnsigned<ui64, Desc>(output, value.Get<ui64>());
- break;
- case NUdf::EDataSlot::Double:
- EncodeFloating<double, Desc>(output, value.Get<double>());
- break;
- case NUdf::EDataSlot::Float:
- EncodeFloating<float, Desc>(output, value.Get<float>());
- break;
+}
+
+
+template <bool Desc>
+Y_FORCE_INLINE
+void Encode(TVector<ui8>& output, NUdf::EDataSlot slot, const NUdf::TUnboxedValuePod& value) {
+ switch (slot) {
+
+ case NUdf::EDataSlot::Bool:
+ EncodeBool<Desc>(output, value.Get<bool>());
+ break;
+ case NUdf::EDataSlot::Int8:
+ EncodeSigned<i8, Desc>(output, value.Get<i8>());
+ break;
+ case NUdf::EDataSlot::Uint8:
+ EncodeUnsigned<ui8, Desc>(output, value.Get<ui8>());
+ break;
+ case NUdf::EDataSlot::Int16:
+ EncodeSigned<i16, Desc>(output, value.Get<i16>());
+ break;
+ case NUdf::EDataSlot::Uint16:
+ case NUdf::EDataSlot::Date:
+ EncodeUnsigned<ui16, Desc>(output, value.Get<ui16>());
+ break;
+ case NUdf::EDataSlot::Int32:
+ EncodeSigned<i32, Desc>(output, value.Get<i32>());
+ break;
+ case NUdf::EDataSlot::Uint32:
+ case NUdf::EDataSlot::Datetime:
+ EncodeUnsigned<ui32, Desc>(output, value.Get<ui32>());
+ break;
+ case NUdf::EDataSlot::Int64:
+ case NUdf::EDataSlot::Interval:
+ EncodeSigned<i64, Desc>(output, value.Get<i64>());
+ break;
+ case NUdf::EDataSlot::Uint64:
+ case NUdf::EDataSlot::Timestamp:
+ EncodeUnsigned<ui64, Desc>(output, value.Get<ui64>());
+ break;
+ case NUdf::EDataSlot::Double:
+ EncodeFloating<double, Desc>(output, value.Get<double>());
+ break;
+ case NUdf::EDataSlot::Float:
+ EncodeFloating<float, Desc>(output, value.Get<float>());
+ break;
case NUdf::EDataSlot::DyNumber:
- case NUdf::EDataSlot::String:
- case NUdf::EDataSlot::Utf8: {
- auto stringRef = value.AsStringRef();
- EncodeString<Desc>(output, TStringBuf(stringRef.Data(), stringRef.Size()));
- break;
- }
- case NUdf::EDataSlot::Uuid:
- EncodeUuid<Desc>(output, value.AsStringRef().Data());
- break;
- case NUdf::EDataSlot::TzDate:
- EncodeTzUnsigned<ui16, Desc>(output, value.Get<ui16>(), value.GetTimezoneId());
- break;
- case NUdf::EDataSlot::TzDatetime:
- EncodeTzUnsigned<ui32, Desc>(output, value.Get<ui32>(), value.GetTimezoneId());
- break;
- case NUdf::EDataSlot::TzTimestamp:
- EncodeTzUnsigned<ui64, Desc>(output, value.Get<ui64>(), value.GetTimezoneId());
- break;
- case NUdf::EDataSlot::Decimal:
- EncodeDecimal<Desc>(output, value.GetInt128());
- break;
-
- default:
- MKQL_ENSURE(false, TStringBuilder() << "unknown data slot for presort encoding: " << slot);
- }
-}
-
-template <bool Desc>
-Y_FORCE_INLINE
+ case NUdf::EDataSlot::String:
+ case NUdf::EDataSlot::Utf8: {
+ auto stringRef = value.AsStringRef();
+ EncodeString<Desc>(output, TStringBuf(stringRef.Data(), stringRef.Size()));
+ break;
+ }
+ case NUdf::EDataSlot::Uuid:
+ EncodeUuid<Desc>(output, value.AsStringRef().Data());
+ break;
+ case NUdf::EDataSlot::TzDate:
+ EncodeTzUnsigned<ui16, Desc>(output, value.Get<ui16>(), value.GetTimezoneId());
+ break;
+ case NUdf::EDataSlot::TzDatetime:
+ EncodeTzUnsigned<ui32, Desc>(output, value.Get<ui32>(), value.GetTimezoneId());
+ break;
+ case NUdf::EDataSlot::TzTimestamp:
+ EncodeTzUnsigned<ui64, Desc>(output, value.Get<ui64>(), value.GetTimezoneId());
+ break;
+ case NUdf::EDataSlot::Decimal:
+ EncodeDecimal<Desc>(output, value.GetInt128());
+ break;
+
+ default:
+ MKQL_ENSURE(false, TStringBuilder() << "unknown data slot for presort encoding: " << slot);
+ }
+}
+
+template <bool Desc>
+Y_FORCE_INLINE
NUdf::TUnboxedValue Decode(TStringBuf& input, NUdf::EDataSlot slot, TVector<ui8>& buffer)
-{
- switch (slot) {
-
- case NUdf::EDataSlot::Bool:
- return NUdf::TUnboxedValuePod(DecodeBool<Desc>(input));
-
- case NUdf::EDataSlot::Int8:
- return NUdf::TUnboxedValuePod(DecodeSigned<i8, Desc>(input));
-
- case NUdf::EDataSlot::Uint8:
- return NUdf::TUnboxedValuePod(DecodeUnsigned<ui8, Desc>(input));
-
- case NUdf::EDataSlot::Int16:
- return NUdf::TUnboxedValuePod(DecodeSigned<i16, Desc>(input));
-
- case NUdf::EDataSlot::Uint16:
- case NUdf::EDataSlot::Date:
- return NUdf::TUnboxedValuePod(DecodeUnsigned<ui16, Desc>(input));
-
- case NUdf::EDataSlot::Int32:
- return NUdf::TUnboxedValuePod(DecodeSigned<i32, Desc>(input));
-
- case NUdf::EDataSlot::Uint32:
- case NUdf::EDataSlot::Datetime:
- return NUdf::TUnboxedValuePod(DecodeUnsigned<ui32, Desc>(input));
-
- case NUdf::EDataSlot::Int64:
- case NUdf::EDataSlot::Interval:
- return NUdf::TUnboxedValuePod(DecodeSigned<i64, Desc>(input));
-
- case NUdf::EDataSlot::Uint64:
- case NUdf::EDataSlot::Timestamp:
- return NUdf::TUnboxedValuePod(DecodeUnsigned<ui64, Desc>(input));
-
- case NUdf::EDataSlot::Double:
- return NUdf::TUnboxedValuePod(DecodeFloating<double, Desc>(input));
-
- case NUdf::EDataSlot::Float:
- return NUdf::TUnboxedValuePod(DecodeFloating<float, Desc>(input));
-
+{
+ switch (slot) {
+
+ case NUdf::EDataSlot::Bool:
+ return NUdf::TUnboxedValuePod(DecodeBool<Desc>(input));
+
+ case NUdf::EDataSlot::Int8:
+ return NUdf::TUnboxedValuePod(DecodeSigned<i8, Desc>(input));
+
+ case NUdf::EDataSlot::Uint8:
+ return NUdf::TUnboxedValuePod(DecodeUnsigned<ui8, Desc>(input));
+
+ case NUdf::EDataSlot::Int16:
+ return NUdf::TUnboxedValuePod(DecodeSigned<i16, Desc>(input));
+
+ case NUdf::EDataSlot::Uint16:
+ case NUdf::EDataSlot::Date:
+ return NUdf::TUnboxedValuePod(DecodeUnsigned<ui16, Desc>(input));
+
+ case NUdf::EDataSlot::Int32:
+ return NUdf::TUnboxedValuePod(DecodeSigned<i32, Desc>(input));
+
+ case NUdf::EDataSlot::Uint32:
+ case NUdf::EDataSlot::Datetime:
+ return NUdf::TUnboxedValuePod(DecodeUnsigned<ui32, Desc>(input));
+
+ case NUdf::EDataSlot::Int64:
+ case NUdf::EDataSlot::Interval:
+ return NUdf::TUnboxedValuePod(DecodeSigned<i64, Desc>(input));
+
+ case NUdf::EDataSlot::Uint64:
+ case NUdf::EDataSlot::Timestamp:
+ return NUdf::TUnboxedValuePod(DecodeUnsigned<ui64, Desc>(input));
+
+ case NUdf::EDataSlot::Double:
+ return NUdf::TUnboxedValuePod(DecodeFloating<double, Desc>(input));
+
+ case NUdf::EDataSlot::Float:
+ return NUdf::TUnboxedValuePod(DecodeFloating<float, Desc>(input));
+
case NUdf::EDataSlot::DyNumber:
- case NUdf::EDataSlot::String:
- case NUdf::EDataSlot::Utf8:
- buffer.clear();
+ case NUdf::EDataSlot::String:
+ case NUdf::EDataSlot::Utf8:
+ buffer.clear();
return MakeString(NUdf::TStringRef(DecodeString<Desc>(input, buffer)));
-
- case NUdf::EDataSlot::Uuid:
- buffer.clear();
- return MakeString(NUdf::TStringRef(DecodeUuid<Desc>(input, buffer)));
-
- case NUdf::EDataSlot::TzDate: {
- ui16 date;
- ui16 tzId;
- DecodeTzUnsigned<ui16, Desc>(input, date, tzId);
- NUdf::TUnboxedValuePod value(date);
- value.SetTimezoneId(tzId);
- return value;
- }
- case NUdf::EDataSlot::TzDatetime: {
- ui32 datetime;
- ui16 tzId;
- DecodeTzUnsigned<ui32, Desc>(input, datetime, tzId);
- NUdf::TUnboxedValuePod value(datetime);
- value.SetTimezoneId(tzId);
- return value;
- }
- case NUdf::EDataSlot::TzTimestamp: {
- ui64 timestamp;
- ui16 tzId;
- DecodeTzUnsigned<ui64, Desc>(input, timestamp, tzId);
- NUdf::TUnboxedValuePod value(timestamp);
- value.SetTimezoneId(tzId);
- return value;
- }
- case NUdf::EDataSlot::Decimal:
- return NUdf::TUnboxedValuePod(DecodeDecimal<Desc>(input));
-
- default:
- MKQL_ENSURE(false, TStringBuilder() << "unknown data slot for presort decoding: " << slot);
- }
-}
-
+
+ case NUdf::EDataSlot::Uuid:
+ buffer.clear();
+ return MakeString(NUdf::TStringRef(DecodeUuid<Desc>(input, buffer)));
+
+ case NUdf::EDataSlot::TzDate: {
+ ui16 date;
+ ui16 tzId;
+ DecodeTzUnsigned<ui16, Desc>(input, date, tzId);
+ NUdf::TUnboxedValuePod value(date);
+ value.SetTimezoneId(tzId);
+ return value;
+ }
+ case NUdf::EDataSlot::TzDatetime: {
+ ui32 datetime;
+ ui16 tzId;
+ DecodeTzUnsigned<ui32, Desc>(input, datetime, tzId);
+ NUdf::TUnboxedValuePod value(datetime);
+ value.SetTimezoneId(tzId);
+ return value;
+ }
+ case NUdf::EDataSlot::TzTimestamp: {
+ ui64 timestamp;
+ ui16 tzId;
+ DecodeTzUnsigned<ui64, Desc>(input, timestamp, tzId);
+ NUdf::TUnboxedValuePod value(timestamp);
+ value.SetTimezoneId(tzId);
+ return value;
+ }
+ case NUdf::EDataSlot::Decimal:
+ return NUdf::TUnboxedValuePod(DecodeDecimal<Desc>(input));
+
+ default:
+ MKQL_ENSURE(false, TStringBuilder() << "unknown data slot for presort decoding: " << slot);
+ }
+}
+
struct TDictItem {
TString KeyBuffer;
NUdf::TUnboxedValue Payload;
@@ -716,80 +716,80 @@ NUdf::TUnboxedValue DecodeImpl(TType* type, TStringBuf& input, const THolderFact
}
}
-} // NDetail
-
-void TPresortCodec::AddType(NUdf::EDataSlot slot, bool isOptional, bool isDesc) {
- Types.push_back({slot, isOptional, isDesc});
-}
-
-void TPresortEncoder::Start() {
- Output.clear();
- Current = 0;
-}
-
-void TPresortEncoder::Start(TStringBuf prefix) {
- Output.clear();
+} // NDetail
+
+void TPresortCodec::AddType(NUdf::EDataSlot slot, bool isOptional, bool isDesc) {
+ Types.push_back({slot, isOptional, isDesc});
+}
+
+void TPresortEncoder::Start() {
+ Output.clear();
+ Current = 0;
+}
+
+void TPresortEncoder::Start(TStringBuf prefix) {
+ Output.clear();
auto data = reinterpret_cast<const ui8*>(prefix.data());
Output.insert(Output.begin(), data, data + prefix.size());
- Current = 0;
-}
-
-void TPresortEncoder::Encode(const NUdf::TUnboxedValuePod& value) {
- auto& type = Types[Current++];
-
- if (type.IsDesc) {
- if (type.IsOptional) {
- auto hasValue = (bool)value;
- NDetail::EncodeBool<true>(Output, hasValue);
- if (!hasValue) {
- return;
- }
- }
- NDetail::Encode<true>(Output, type.Slot, value);
- } else {
- if (type.IsOptional) {
- auto hasValue = (bool)value;
- NDetail::EncodeBool<false>(Output, hasValue);
- if (!hasValue) {
- return;
- }
- }
- NDetail::Encode<false>(Output, type.Slot, value);
- }
-}
-
-TStringBuf TPresortEncoder::Finish() {
- MKQL_ENSURE(Current == Types.size(), "not all fields were encoded");
- return TStringBuf((const char*)Output.data(), Output.size());
-}
-
-
-void TPresortDecoder::Start(TStringBuf input) {
- Input = input;
- Current = 0;
-}
-
-NUdf::TUnboxedValue TPresortDecoder::Decode() {
- auto& type = Types[Current++];
-
- if (type.IsDesc) {
- if (type.IsOptional && !NDetail::DecodeBool<true>(Input)) {
- return NUdf::TUnboxedValuePod();
- }
+ Current = 0;
+}
+
+void TPresortEncoder::Encode(const NUdf::TUnboxedValuePod& value) {
+ auto& type = Types[Current++];
+
+ if (type.IsDesc) {
+ if (type.IsOptional) {
+ auto hasValue = (bool)value;
+ NDetail::EncodeBool<true>(Output, hasValue);
+ if (!hasValue) {
+ return;
+ }
+ }
+ NDetail::Encode<true>(Output, type.Slot, value);
+ } else {
+ if (type.IsOptional) {
+ auto hasValue = (bool)value;
+ NDetail::EncodeBool<false>(Output, hasValue);
+ if (!hasValue) {
+ return;
+ }
+ }
+ NDetail::Encode<false>(Output, type.Slot, value);
+ }
+}
+
+TStringBuf TPresortEncoder::Finish() {
+ MKQL_ENSURE(Current == Types.size(), "not all fields were encoded");
+ return TStringBuf((const char*)Output.data(), Output.size());
+}
+
+
+void TPresortDecoder::Start(TStringBuf input) {
+ Input = input;
+ Current = 0;
+}
+
+NUdf::TUnboxedValue TPresortDecoder::Decode() {
+ auto& type = Types[Current++];
+
+ if (type.IsDesc) {
+ if (type.IsOptional && !NDetail::DecodeBool<true>(Input)) {
+ return NUdf::TUnboxedValuePod();
+ }
return NDetail::Decode<true>(Input, type.Slot, Buffer);
- } else {
- if (type.IsOptional && !NDetail::DecodeBool<false>(Input)) {
- return NUdf::TUnboxedValuePod();
- }
+ } else {
+ if (type.IsOptional && !NDetail::DecodeBool<false>(Input)) {
+ return NUdf::TUnboxedValuePod();
+ }
return NDetail::Decode<false>(Input, type.Slot, Buffer);
- }
-}
-
-void TPresortDecoder::Finish() {
- MKQL_ENSURE(Current == Types.size(), "not all fields were decoded");
- MKQL_ENSURE(Input.empty(), "buffer is not empty");
-}
-
+ }
+}
+
+void TPresortDecoder::Finish() {
+ MKQL_ENSURE(Current == Types.size(), "not all fields were decoded");
+ MKQL_ENSURE(Input.empty(), "buffer is not empty");
+}
+
TGenericPresortEncoder::TGenericPresortEncoder(TType* type)
: Type(type)
{}
@@ -825,5 +825,5 @@ NUdf::TUnboxedValue TGenericPresortEncoder::Decode(TStringBuf buf, bool desc, co
}
}
-} // NMiniKQL
-} // NKikimr
+} // NMiniKQL
+} // NKikimr
diff --git a/ydb/library/yql/minikql/computation/presort.h b/ydb/library/yql/minikql/computation/presort.h
index b90f54343b8..8bce45e5202 100644
--- a/ydb/library/yql/minikql/computation/presort.h
+++ b/ydb/library/yql/minikql/computation/presort.h
@@ -1,57 +1,57 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/minikql/mkql_node.h>
#include <ydb/library/yql/public/udf/udf_value.h>
-
-#include <util/generic/vector.h>
-
-namespace NKikimr {
-namespace NMiniKQL {
-
-class TPresortCodec {
-public:
- TPresortCodec() = default;
-
- struct TTypeInfo {
- NUdf::EDataSlot Slot;
- bool IsOptional;
- bool IsDesc;
- };
-
- void AddType(NUdf::EDataSlot slot, bool isOptional = false, bool isDesc = false);
-
-protected:
- size_t Current = 0;
- TVector<TTypeInfo> Types;
-};
-
-class TPresortEncoder : public TPresortCodec {
-public:
- TPresortEncoder() = default;
-
- void Start();
- void Start(TStringBuf prefix);
- void Encode(const NUdf::TUnboxedValuePod& value);
- TStringBuf Finish(); // user must copy
-
-private:
- TVector<ui8> Output;
-};
-
-class TPresortDecoder : public TPresortCodec {
-public:
+
+#include <util/generic/vector.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+class TPresortCodec {
+public:
+ TPresortCodec() = default;
+
+ struct TTypeInfo {
+ NUdf::EDataSlot Slot;
+ bool IsOptional;
+ bool IsDesc;
+ };
+
+ void AddType(NUdf::EDataSlot slot, bool isOptional = false, bool isDesc = false);
+
+protected:
+ size_t Current = 0;
+ TVector<TTypeInfo> Types;
+};
+
+class TPresortEncoder : public TPresortCodec {
+public:
+ TPresortEncoder() = default;
+
+ void Start();
+ void Start(TStringBuf prefix);
+ void Encode(const NUdf::TUnboxedValuePod& value);
+ TStringBuf Finish(); // user must copy
+
+private:
+ TVector<ui8> Output;
+};
+
+class TPresortDecoder : public TPresortCodec {
+public:
TPresortDecoder() = default;
-
- void Start(TStringBuf input);
- NUdf::TUnboxedValue Decode();
- void Finish();
-
-private:
- TVector<ui8> Buffer;
- TStringBuf Input;
-};
-
+
+ void Start(TStringBuf input);
+ NUdf::TUnboxedValue Decode();
+ void Finish();
+
+private:
+ TVector<ui8> Buffer;
+ TStringBuf Input;
+};
+
class THolderFactory;
class TGenericPresortEncoder {
@@ -65,5 +65,5 @@ private:
TVector<ui8> Buffer;
};
-} // NMiniKQL
-} // NKikimr
+} // NMiniKQL
+} // NKikimr
diff --git a/ydb/library/yql/minikql/computation/presort_ut.cpp b/ydb/library/yql/minikql/computation/presort_ut.cpp
index 4adf957ca1b..b95af1d50e5 100644
--- a/ydb/library/yql/minikql/computation/presort_ut.cpp
+++ b/ydb/library/yql/minikql/computation/presort_ut.cpp
@@ -1,411 +1,411 @@
-#include "presort.h"
-
+#include "presort.h"
+
#include <ydb/library/yql/minikql/mkql_alloc.h>
#include <ydb/library/yql/minikql/mkql_string_util.h>
#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h>
#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
-
+
#include <library/cpp/testing/unittest/registar.h>
-
-#include <util/string/hex.h>
-
+
+#include <util/string/hex.h>
+
using namespace std::literals::string_view_literals;
-namespace NKikimr {
-namespace NMiniKQL {
-
-namespace {
-
-#define TYPE_MAP(XX) \
- XX(bool, Bool) \
- XX(ui8, Uint8) \
- XX(ui16, Uint16) \
- XX(ui32, Uint32) \
- XX(ui64, Uint64) \
- XX(i8, Int8) \
- XX(i16, Int16) \
- XX(i32, Int32) \
- XX(i64, Int64) \
- XX(float, Float) \
- XX(double, Double)
-
-#define ALL_VALUES(xType, xName) \
- xType xName;
-
-struct TSimpleTypes {
- TYPE_MAP(ALL_VALUES)
-};
-#undef ALL_VALUES
-
-#define ADD_TYPE(xType, xName) \
- codec.AddType(NUdf::EDataSlot::xName, isOptional, isDesc);
-
-void AddTypes(TPresortCodec& codec, bool isOptional, bool isDesc) {
- TYPE_MAP(ADD_TYPE)
-}
-#undef ADD_TYPE
-
-#define ENCODE(xType, xName) \
- encoder.Encode(NUdf::TUnboxedValuePod(values.xName));
-
-TStringBuf Encode(NKikimr::NMiniKQL::TPresortEncoder& encoder, const TSimpleTypes& values) {
- encoder.Start();
- TYPE_MAP(ENCODE)
- return encoder.Finish();
-}
-#undef ENCODE
-
-#define DECODE(xType, xName) \
- UNIT_ASSERT_EQUAL(decoder.Decode().Get<xType>(), values.xName);
-
-void Decode(TPresortDecoder& decoder, TStringBuf input, const TSimpleTypes& values) {
- decoder.Start(input);
- TYPE_MAP(DECODE)
- decoder.Finish();
-}
-#undef DECODE
-
-#undef TYPE_MAP
-
-struct TPresortTest {
- TScopedAlloc Alloc;
- TMemoryUsageInfo MemInfo;
-
- TPresortTest()
- : MemInfo("Memory")
- {}
-
- template <typename T>
- void ValidateEncoding(bool isDesc, T value, const TString& hex) {
- TPresortEncoder encoder;
- encoder.AddType(NUdf::TDataType<T>::Slot, false, isDesc);
-
+namespace NKikimr {
+namespace NMiniKQL {
+
+namespace {
+
+#define TYPE_MAP(XX) \
+ XX(bool, Bool) \
+ XX(ui8, Uint8) \
+ XX(ui16, Uint16) \
+ XX(ui32, Uint32) \
+ XX(ui64, Uint64) \
+ XX(i8, Int8) \
+ XX(i16, Int16) \
+ XX(i32, Int32) \
+ XX(i64, Int64) \
+ XX(float, Float) \
+ XX(double, Double)
+
+#define ALL_VALUES(xType, xName) \
+ xType xName;
+
+struct TSimpleTypes {
+ TYPE_MAP(ALL_VALUES)
+};
+#undef ALL_VALUES
+
+#define ADD_TYPE(xType, xName) \
+ codec.AddType(NUdf::EDataSlot::xName, isOptional, isDesc);
+
+void AddTypes(TPresortCodec& codec, bool isOptional, bool isDesc) {
+ TYPE_MAP(ADD_TYPE)
+}
+#undef ADD_TYPE
+
+#define ENCODE(xType, xName) \
+ encoder.Encode(NUdf::TUnboxedValuePod(values.xName));
+
+TStringBuf Encode(NKikimr::NMiniKQL::TPresortEncoder& encoder, const TSimpleTypes& values) {
+ encoder.Start();
+ TYPE_MAP(ENCODE)
+ return encoder.Finish();
+}
+#undef ENCODE
+
+#define DECODE(xType, xName) \
+ UNIT_ASSERT_EQUAL(decoder.Decode().Get<xType>(), values.xName);
+
+void Decode(TPresortDecoder& decoder, TStringBuf input, const TSimpleTypes& values) {
+ decoder.Start(input);
+ TYPE_MAP(DECODE)
+ decoder.Finish();
+}
+#undef DECODE
+
+#undef TYPE_MAP
+
+struct TPresortTest {
+ TScopedAlloc Alloc;
+ TMemoryUsageInfo MemInfo;
+
+ TPresortTest()
+ : MemInfo("Memory")
+ {}
+
+ template <typename T>
+ void ValidateEncoding(bool isDesc, T value, const TString& hex) {
+ TPresortEncoder encoder;
+ encoder.AddType(NUdf::TDataType<T>::Slot, false, isDesc);
+
TPresortDecoder decoder;
- decoder.AddType(NUdf::TDataType<T>::Slot, false, isDesc);
-
- encoder.Start();
- encoder.Encode(NUdf::TUnboxedValuePod(value));
- auto bytes = encoder.Finish();
-
- UNIT_ASSERT_EQUAL(HexEncode(bytes.data(), bytes.size()), hex);
-
- decoder.Start(bytes);
- auto decoded = decoder.Decode().Get<T>();
- decoder.Finish();
-
- UNIT_ASSERT_EQUAL(decoded, value);
- };
-
- template <NUdf::EDataSlot Slot>
- void ValidateEncoding(bool isDesc, TStringBuf value, const TString& hex) {
- TPresortEncoder encoder;
- encoder.AddType(Slot, false, isDesc);
-
+ decoder.AddType(NUdf::TDataType<T>::Slot, false, isDesc);
+
+ encoder.Start();
+ encoder.Encode(NUdf::TUnboxedValuePod(value));
+ auto bytes = encoder.Finish();
+
+ UNIT_ASSERT_EQUAL(HexEncode(bytes.data(), bytes.size()), hex);
+
+ decoder.Start(bytes);
+ auto decoded = decoder.Decode().Get<T>();
+ decoder.Finish();
+
+ UNIT_ASSERT_EQUAL(decoded, value);
+ };
+
+ template <NUdf::EDataSlot Slot>
+ void ValidateEncoding(bool isDesc, TStringBuf value, const TString& hex) {
+ TPresortEncoder encoder;
+ encoder.AddType(Slot, false, isDesc);
+
TPresortDecoder decoder;
- decoder.AddType(Slot, false, isDesc);
-
- encoder.Start();
+ decoder.AddType(Slot, false, isDesc);
+
+ encoder.Start();
encoder.Encode(NUdf::TUnboxedValue(MakeString(NUdf::TStringRef(value))));
- auto bytes = encoder.Finish();
-
- UNIT_ASSERT_EQUAL(HexEncode(bytes.data(), bytes.size()), hex);
-
- decoder.Start(bytes);
- auto uv = decoder.Decode();
- decoder.Finish();
-
- auto stringRef = uv.AsStringRef();
- auto decoded = TStringBuf(stringRef.Data(), stringRef.Size());
-
- UNIT_ASSERT_EQUAL(decoded, value);
- }
-
- template <NUdf::EDataSlot Slot, typename T>
- void ValidateEncoding(bool isDesc, const std::pair<T, ui16>& value, const TString& hex) {
- TPresortEncoder encoder;
- encoder.AddType(Slot, false, isDesc);
-
- TPresortDecoder decoder;
- decoder.AddType(Slot, false, isDesc);
-
- NUdf::TUnboxedValuePod uv(value.first);
- uv.SetTimezoneId(value.second);
-
- encoder.Start();
- encoder.Encode(uv);
- auto bytes = encoder.Finish();
-
- UNIT_ASSERT_EQUAL(HexEncode(bytes.data(), bytes.size()), hex);
-
- decoder.Start(bytes);
- auto decoded = decoder.Decode();
- decoder.Finish();
-
- UNIT_ASSERT_EQUAL(decoded.Get<T>(), value.first);
- UNIT_ASSERT_EQUAL(decoded.GetTimezoneId(), value.second);
- };
-
+ auto bytes = encoder.Finish();
+
+ UNIT_ASSERT_EQUAL(HexEncode(bytes.data(), bytes.size()), hex);
+
+ decoder.Start(bytes);
+ auto uv = decoder.Decode();
+ decoder.Finish();
+
+ auto stringRef = uv.AsStringRef();
+ auto decoded = TStringBuf(stringRef.Data(), stringRef.Size());
+
+ UNIT_ASSERT_EQUAL(decoded, value);
+ }
+
+ template <NUdf::EDataSlot Slot, typename T>
+ void ValidateEncoding(bool isDesc, const std::pair<T, ui16>& value, const TString& hex) {
+ TPresortEncoder encoder;
+ encoder.AddType(Slot, false, isDesc);
+
+ TPresortDecoder decoder;
+ decoder.AddType(Slot, false, isDesc);
+
+ NUdf::TUnboxedValuePod uv(value.first);
+ uv.SetTimezoneId(value.second);
+
+ encoder.Start();
+ encoder.Encode(uv);
+ auto bytes = encoder.Finish();
+
+ UNIT_ASSERT_EQUAL(HexEncode(bytes.data(), bytes.size()), hex);
+
+ decoder.Start(bytes);
+ auto decoded = decoder.Decode();
+ decoder.Finish();
+
+ UNIT_ASSERT_EQUAL(decoded.Get<T>(), value.first);
+ UNIT_ASSERT_EQUAL(decoded.GetTimezoneId(), value.second);
+ };
+
void ValidateEncoding(bool isDesc, NYql::NDecimal::TInt128 value, const TString& hex) {
- TPresortEncoder encoder;
- encoder.AddType(NUdf::EDataSlot::Decimal, false, isDesc);
-
- TPresortDecoder decoder;
- decoder.AddType(NUdf::EDataSlot::Decimal, false, isDesc);
-
- encoder.Start();
- encoder.Encode(NUdf::TUnboxedValuePod(value));
- auto bytes = encoder.Finish();
-
- UNIT_ASSERT_EQUAL(HexEncode(bytes.data(), bytes.size()), hex);
-
- decoder.Start(bytes);
- auto decoded = decoder.Decode().GetInt128();
- decoder.Finish();
-
- UNIT_ASSERT_EQUAL(decoded, value);
- };
-
- template <typename T>
- void ValidateEncoding(const TVector<T>& values) {
- for (auto& value : values) {
- ValidateEncoding(false, std::get<0>(value), std::get<1>(value));
- ValidateEncoding(true, std::get<0>(value), std::get<2>(value));
- }
- }
-
- template <NUdf::EDataSlot Slot, typename T>
- void ValidateEncoding(const TVector<T>& values) {
- for (auto& value : values) {
- ValidateEncoding<Slot>(false, std::get<0>(value), std::get<1>(value));
- ValidateEncoding<Slot>(true, std::get<0>(value), std::get<2>(value));
- }
- }
-};
-
-}
-
+ TPresortEncoder encoder;
+ encoder.AddType(NUdf::EDataSlot::Decimal, false, isDesc);
+
+ TPresortDecoder decoder;
+ decoder.AddType(NUdf::EDataSlot::Decimal, false, isDesc);
+
+ encoder.Start();
+ encoder.Encode(NUdf::TUnboxedValuePod(value));
+ auto bytes = encoder.Finish();
+
+ UNIT_ASSERT_EQUAL(HexEncode(bytes.data(), bytes.size()), hex);
+
+ decoder.Start(bytes);
+ auto decoded = decoder.Decode().GetInt128();
+ decoder.Finish();
+
+ UNIT_ASSERT_EQUAL(decoded, value);
+ };
+
+ template <typename T>
+ void ValidateEncoding(const TVector<T>& values) {
+ for (auto& value : values) {
+ ValidateEncoding(false, std::get<0>(value), std::get<1>(value));
+ ValidateEncoding(true, std::get<0>(value), std::get<2>(value));
+ }
+ }
+
+ template <NUdf::EDataSlot Slot, typename T>
+ void ValidateEncoding(const TVector<T>& values) {
+ for (auto& value : values) {
+ ValidateEncoding<Slot>(false, std::get<0>(value), std::get<1>(value));
+ ValidateEncoding<Slot>(true, std::get<0>(value), std::get<2>(value));
+ }
+ }
+};
+
+}
+
Y_UNIT_TEST_SUITE(TPresortCodecTest) {
-
+
Y_UNIT_TEST(SimpleTypes) {
- TPresortTest test;
-
- TSimpleTypes values = {false, 1u, 2u, 3u, 4u, 5, 6, 7, 8, 9.f, 10.0};
-
- auto validateSimpleTypes = [&] (bool isOptional, bool isDesc) {
- TPresortEncoder encoder;
- AddTypes(encoder, isOptional, isDesc);
-
+ TPresortTest test;
+
+ TSimpleTypes values = {false, 1u, 2u, 3u, 4u, 5, 6, 7, 8, 9.f, 10.0};
+
+ auto validateSimpleTypes = [&] (bool isOptional, bool isDesc) {
+ TPresortEncoder encoder;
+ AddTypes(encoder, isOptional, isDesc);
+
TPresortDecoder decoder;
- AddTypes(decoder, isOptional, isDesc);
-
- auto bytes = Encode(encoder, values);
- Decode(decoder, bytes, values);
- };
-
- validateSimpleTypes(false, false);
- validateSimpleTypes(false, true);
- validateSimpleTypes(true, false);
- validateSimpleTypes(true, true);
-}
-
-
+ AddTypes(decoder, isOptional, isDesc);
+
+ auto bytes = Encode(encoder, values);
+ Decode(decoder, bytes, values);
+ };
+
+ validateSimpleTypes(false, false);
+ validateSimpleTypes(false, true);
+ validateSimpleTypes(true, false);
+ validateSimpleTypes(true, true);
+}
+
+
Y_UNIT_TEST(Bool) {
- const TVector<std::tuple<bool, TString, TString>> values = {
- {false, "00", "FF"},
- {true, "01", "FE"}
- };
- TPresortTest().ValidateEncoding(values);
-}
-
+ const TVector<std::tuple<bool, TString, TString>> values = {
+ {false, "00", "FF"},
+ {true, "01", "FE"}
+ };
+ TPresortTest().ValidateEncoding(values);
+}
+
Y_UNIT_TEST(Int8) {
- const TVector<std::tuple<i8, TString, TString>> values = {
- {-0x80, "00", "FF"},
- {-1, "7F", "80"},
- {0, "80", "7F"},
- {1, "81", "7E"},
- {0x7F, "FF", "00"}
- };
- TPresortTest().ValidateEncoding(values);
-}
-
+ const TVector<std::tuple<i8, TString, TString>> values = {
+ {-0x80, "00", "FF"},
+ {-1, "7F", "80"},
+ {0, "80", "7F"},
+ {1, "81", "7E"},
+ {0x7F, "FF", "00"}
+ };
+ TPresortTest().ValidateEncoding(values);
+}
+
Y_UNIT_TEST(Uint8) {
- const TVector<std::tuple<ui8, TString, TString>> values = {
- {0u, "00", "FF"},
- {0x80u, "80", "7F"},
- {0xFFu, "FF", "00"},
- };
- TPresortTest().ValidateEncoding(values);
-}
-
+ const TVector<std::tuple<ui8, TString, TString>> values = {
+ {0u, "00", "FF"},
+ {0x80u, "80", "7F"},
+ {0xFFu, "FF", "00"},
+ };
+ TPresortTest().ValidateEncoding(values);
+}
+
Y_UNIT_TEST(Int16) {
- const TVector<std::tuple<i16, TString, TString>> values = {
- {-0x8000, "0000", "FFFF"},
- {-1, "7FFF", "8000"},
- {0, "8000", "7FFF"},
- {1, "8001", "7FFE"},
- {0x7FFF, "FFFF", "0000"}
- };
- TPresortTest().ValidateEncoding(values);
-}
-
+ const TVector<std::tuple<i16, TString, TString>> values = {
+ {-0x8000, "0000", "FFFF"},
+ {-1, "7FFF", "8000"},
+ {0, "8000", "7FFF"},
+ {1, "8001", "7FFE"},
+ {0x7FFF, "FFFF", "0000"}
+ };
+ TPresortTest().ValidateEncoding(values);
+}
+
Y_UNIT_TEST(Uint16) {
- const TVector<std::tuple<ui16, TString, TString>> values = {
- {0, "0000", "FFFF"},
- {0x8000u, "8000", "7FFF"},
- {0xFFFFu, "FFFF", "0000"},
- };
- TPresortTest().ValidateEncoding(values);
-}
-
+ const TVector<std::tuple<ui16, TString, TString>> values = {
+ {0, "0000", "FFFF"},
+ {0x8000u, "8000", "7FFF"},
+ {0xFFFFu, "FFFF", "0000"},
+ };
+ TPresortTest().ValidateEncoding(values);
+}
+
Y_UNIT_TEST(Int32) {
- const TVector<std::tuple<i32, TString, TString>> values = {
- {-0x80000000, "00000000", "FFFFFFFF"},
- {-1, "7FFFFFFF", "80000000"},
- {0, "80000000", "7FFFFFFF"},
- {1, "80000001", "7FFFFFFE"},
- {0x7FFFFFFF, "FFFFFFFF", "00000000"}
- };
- TPresortTest().ValidateEncoding(values);
-}
-
+ const TVector<std::tuple<i32, TString, TString>> values = {
+ {-0x80000000, "00000000", "FFFFFFFF"},
+ {-1, "7FFFFFFF", "80000000"},
+ {0, "80000000", "7FFFFFFF"},
+ {1, "80000001", "7FFFFFFE"},
+ {0x7FFFFFFF, "FFFFFFFF", "00000000"}
+ };
+ TPresortTest().ValidateEncoding(values);
+}
+
Y_UNIT_TEST(Uint32) {
- const TVector<std::tuple<ui32, TString, TString>> values = {
- {0u, "00000000", "FFFFFFFF"},
- {0x80000000u, "80000000", "7FFFFFFF"},
- {0xFFFFFFFFu, "FFFFFFFF", "00000000"},
- };
- TPresortTest().ValidateEncoding(values);
-}
-
+ const TVector<std::tuple<ui32, TString, TString>> values = {
+ {0u, "00000000", "FFFFFFFF"},
+ {0x80000000u, "80000000", "7FFFFFFF"},
+ {0xFFFFFFFFu, "FFFFFFFF", "00000000"},
+ };
+ TPresortTest().ValidateEncoding(values);
+}
+
Y_UNIT_TEST(Int64) {
- const TVector<std::tuple<i64, TString, TString>> values = {
- {-0x8000000000000000, "0000000000000000", "FFFFFFFFFFFFFFFF"},
- {-1, "7FFFFFFFFFFFFFFF", "8000000000000000"},
- {0, "8000000000000000", "7FFFFFFFFFFFFFFF"},
- {1, "8000000000000001", "7FFFFFFFFFFFFFFE"},
- {0x7FFFFFFFFFFFFFFF, "FFFFFFFFFFFFFFFF", "0000000000000000"}
- };
- TPresortTest().ValidateEncoding(values);
-}
-
+ const TVector<std::tuple<i64, TString, TString>> values = {
+ {-0x8000000000000000, "0000000000000000", "FFFFFFFFFFFFFFFF"},
+ {-1, "7FFFFFFFFFFFFFFF", "8000000000000000"},
+ {0, "8000000000000000", "7FFFFFFFFFFFFFFF"},
+ {1, "8000000000000001", "7FFFFFFFFFFFFFFE"},
+ {0x7FFFFFFFFFFFFFFF, "FFFFFFFFFFFFFFFF", "0000000000000000"}
+ };
+ TPresortTest().ValidateEncoding(values);
+}
+
Y_UNIT_TEST(Uint64) {
- const TVector<std::tuple<ui64, TString, TString>> values = {
- {0u, "0000000000000000", "FFFFFFFFFFFFFFFF"},
- {0x8000000000000000u, "8000000000000000", "7FFFFFFFFFFFFFFF"},
- {0xFFFFFFFFFFFFFFFFu, "FFFFFFFFFFFFFFFF", "0000000000000000"}
- };
- TPresortTest().ValidateEncoding(values);
-}
-
+ const TVector<std::tuple<ui64, TString, TString>> values = {
+ {0u, "0000000000000000", "FFFFFFFFFFFFFFFF"},
+ {0x8000000000000000u, "8000000000000000", "7FFFFFFFFFFFFFFF"},
+ {0xFFFFFFFFFFFFFFFFu, "FFFFFFFFFFFFFFFF", "0000000000000000"}
+ };
+ TPresortTest().ValidateEncoding(values);
+}
+
Y_UNIT_TEST(Float) {
- using TLimits = std::numeric_limits<float>;
-
- const TVector<std::tuple<float, TString, TString>> values = {
- {-TLimits::infinity(), "00", "FF"},
- {-TLimits::max(), "0100800000", "FEFF7FFFFF"},
- {-1.f, "01407FFFFF", "FEBF800000"},
- {-TLimits::min(), "017F7FFFFF", "FE80800000"},
- {-TLimits::min()/8.f, "017FEFFFFF", "FE80100000"},
- {0.f, "02", "FD"},
- {TLimits::min()/8.f, "0300100000", "FCFFEFFFFF"},
- {TLimits::min(), "0300800000", "FCFF7FFFFF"},
- {1.f, "033F800000", "FCC07FFFFF"},
- {TLimits::max(), "037F7FFFFF", "FC80800000"},
- {TLimits::infinity(), "04", "FB"},
- };
- TPresortTest().ValidateEncoding(values);
-}
-
+ using TLimits = std::numeric_limits<float>;
+
+ const TVector<std::tuple<float, TString, TString>> values = {
+ {-TLimits::infinity(), "00", "FF"},
+ {-TLimits::max(), "0100800000", "FEFF7FFFFF"},
+ {-1.f, "01407FFFFF", "FEBF800000"},
+ {-TLimits::min(), "017F7FFFFF", "FE80800000"},
+ {-TLimits::min()/8.f, "017FEFFFFF", "FE80100000"},
+ {0.f, "02", "FD"},
+ {TLimits::min()/8.f, "0300100000", "FCFFEFFFFF"},
+ {TLimits::min(), "0300800000", "FCFF7FFFFF"},
+ {1.f, "033F800000", "FCC07FFFFF"},
+ {TLimits::max(), "037F7FFFFF", "FC80800000"},
+ {TLimits::infinity(), "04", "FB"},
+ };
+ TPresortTest().ValidateEncoding(values);
+}
+
Y_UNIT_TEST(Double) {
- using TLimits = std::numeric_limits<double>;
-
- const TVector<std::tuple<double, TString, TString>> values = {
- {-TLimits::infinity(), "00", "FF"},
- {-TLimits::max(), "010010000000000000", "FEFFEFFFFFFFFFFFFF"},
- {-1., "01400FFFFFFFFFFFFF", "FEBFF0000000000000"},
- {-TLimits::min(), "017FEFFFFFFFFFFFFF", "FE8010000000000000"},
- {-TLimits::min()/8., "017FFDFFFFFFFFFFFF", "FE8002000000000000"},
- {0., "02", "FD"},
- {TLimits::min()/8., "030002000000000000", "FCFFFDFFFFFFFFFFFF"},
- {TLimits::min(), "030010000000000000", "FCFFEFFFFFFFFFFFFF"},
- {1., "033FF0000000000000", "FCC00FFFFFFFFFFFFF"},
- {TLimits::max(), "037FEFFFFFFFFFFFFF", "FC8010000000000000"},
- {TLimits::infinity(), "04", "FB"},
- };
- TPresortTest().ValidateEncoding(values);
-}
-
+ using TLimits = std::numeric_limits<double>;
+
+ const TVector<std::tuple<double, TString, TString>> values = {
+ {-TLimits::infinity(), "00", "FF"},
+ {-TLimits::max(), "010010000000000000", "FEFFEFFFFFFFFFFFFF"},
+ {-1., "01400FFFFFFFFFFFFF", "FEBFF0000000000000"},
+ {-TLimits::min(), "017FEFFFFFFFFFFFFF", "FE8010000000000000"},
+ {-TLimits::min()/8., "017FFDFFFFFFFFFFFF", "FE8002000000000000"},
+ {0., "02", "FD"},
+ {TLimits::min()/8., "030002000000000000", "FCFFFDFFFFFFFFFFFF"},
+ {TLimits::min(), "030010000000000000", "FCFFEFFFFFFFFFFFFF"},
+ {1., "033FF0000000000000", "FCC00FFFFFFFFFFFFF"},
+ {TLimits::max(), "037FEFFFFFFFFFFFFF", "FC8010000000000000"},
+ {TLimits::infinity(), "04", "FB"},
+ };
+ TPresortTest().ValidateEncoding(values);
+}
+
Y_UNIT_TEST(String) {
- const TVector<std::tuple<TStringBuf, TString, TString>> values = {
+ const TVector<std::tuple<TStringBuf, TString, TString>> values = {
{TStringBuf(""), "00", "FF"},
{"\x00"sv, "1F00000000000000000000000000000001",
- "E0FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"},
+ "E0FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"},
{"\x01", "1F01000000000000000000000000000001",
- "E0FEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"},
+ "E0FEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"},
{"0", "1F30000000000000000000000000000001",
- "E0CFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"},
+ "E0CFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"},
{"0123", "1F30313233000000000000000000000004",
- "E0CFCECDCCFFFFFFFFFFFFFFFFFFFFFFFB"},
+ "E0CFCECDCCFFFFFFFFFFFFFFFFFFFFFFFB"},
{"0123456789abcde", "1F3031323334353637383961626364650F",
- "E0CFCECDCCCBCAC9C8C7C69E9D9C9B9AF0"},
+ "E0CFCECDCCCBCAC9C8C7C69E9D9C9B9AF0"},
{"a", "1F61000000000000000000000000000001",
- "E09EFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"},
+ "E09EFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"},
{"a\x00"sv, "1F61000000000000000000000000000002",
- "E09EFFFFFFFFFFFFFFFFFFFFFFFFFFFFFD"},
+ "E09EFFFFFFFFFFFFFFFFFFFFFFFFFFFFFD"},
{"abc", "1F61626300000000000000000000000003",
- "E09E9D9CFFFFFFFFFFFFFFFFFFFFFFFFFC"},
+ "E09E9D9CFFFFFFFFFFFFFFFFFFFFFFFFFC"},
{"b", "1F62000000000000000000000000000001",
- "E09DFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"},
- };
- TPresortTest().ValidateEncoding<NUdf::EDataSlot::String>(values);
-}
-
-Y_UNIT_TEST(Uuid) {
- const TVector<std::tuple<TStringBuf, TString, TString>> values = {
+ "E09DFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"},
+ };
+ TPresortTest().ValidateEncoding<NUdf::EDataSlot::String>(values);
+}
+
+Y_UNIT_TEST(Uuid) {
+ const TVector<std::tuple<TStringBuf, TString, TString>> values = {
{"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"sv,
- "00000000000000000000000000000000",
- "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"},
+ "00000000000000000000000000000000",
+ "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"},
{"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\1"sv,
- "00000000000000000000000000000001",
- "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"},
+ "00000000000000000000000000000001",
+ "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"},
{"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff",
- "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF",
- "00000000000000000000000000000000"},
- };
- TPresortTest().ValidateEncoding<NUdf::EDataSlot::Uuid>(values);
-}
-
-Y_UNIT_TEST(TzDate) {
- const TVector<std::tuple<std::pair<ui16, ui16>, TString, TString>> values = {
- {{0u, 0u}, "00000000", "FFFFFFFF"},
- {{0u, 1u}, "00000001", "FFFFFFFE"},
- {{1u, 0u}, "00010000", "FFFEFFFF"},
- {{NUdf::MAX_DATE, 0u}, "C2090000", "3DF6FFFF"},
- };
- TPresortTest().ValidateEncoding<NUdf::EDataSlot::TzDate>(values);
-}
-
-Y_UNIT_TEST(TzDatetime) {
- const TVector<std::tuple<std::pair<ui32, ui16>, TString, TString>> values = {
- {{0u, 0u}, "000000000000", "FFFFFFFFFFFF"},
- {{0u, 1u}, "000000000001", "FFFFFFFFFFFE"},
- {{1u, 0u}, "000000010000", "FFFFFFFEFFFF"},
- {{NUdf::MAX_DATETIME, 0u}, "FFCEDD800000", "0031227FFFFF"},
- };
- TPresortTest().ValidateEncoding<NUdf::EDataSlot::TzDatetime>(values);
-}
-
-Y_UNIT_TEST(TzTimestamp) {
- const TVector<std::tuple<std::pair<ui64, ui16>, TString, TString>> values = {
- {{0u, 0u}, "00000000000000000000", "FFFFFFFFFFFFFFFFFFFF"},
- {{0u, 1u}, "00000000000000000001", "FFFFFFFFFFFFFFFFFFFE"},
- {{1u, 0u}, "00000000000000010000", "FFFFFFFFFFFFFFFEFFFF"},
- {{NUdf::MAX_TIMESTAMP, 0u}, "000F3F52435260000000", "FFF0C0ADBCAD9FFFFFFF"},
- };
- TPresortTest().ValidateEncoding<NUdf::EDataSlot::TzTimestamp>(values);
-}
-
-Y_UNIT_TEST(Decimal) {
+ "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF",
+ "00000000000000000000000000000000"},
+ };
+ TPresortTest().ValidateEncoding<NUdf::EDataSlot::Uuid>(values);
+}
+
+Y_UNIT_TEST(TzDate) {
+ const TVector<std::tuple<std::pair<ui16, ui16>, TString, TString>> values = {
+ {{0u, 0u}, "00000000", "FFFFFFFF"},
+ {{0u, 1u}, "00000001", "FFFFFFFE"},
+ {{1u, 0u}, "00010000", "FFFEFFFF"},
+ {{NUdf::MAX_DATE, 0u}, "C2090000", "3DF6FFFF"},
+ };
+ TPresortTest().ValidateEncoding<NUdf::EDataSlot::TzDate>(values);
+}
+
+Y_UNIT_TEST(TzDatetime) {
+ const TVector<std::tuple<std::pair<ui32, ui16>, TString, TString>> values = {
+ {{0u, 0u}, "000000000000", "FFFFFFFFFFFF"},
+ {{0u, 1u}, "000000000001", "FFFFFFFFFFFE"},
+ {{1u, 0u}, "000000010000", "FFFFFFFEFFFF"},
+ {{NUdf::MAX_DATETIME, 0u}, "FFCEDD800000", "0031227FFFFF"},
+ };
+ TPresortTest().ValidateEncoding<NUdf::EDataSlot::TzDatetime>(values);
+}
+
+Y_UNIT_TEST(TzTimestamp) {
+ const TVector<std::tuple<std::pair<ui64, ui16>, TString, TString>> values = {
+ {{0u, 0u}, "00000000000000000000", "FFFFFFFFFFFFFFFFFFFF"},
+ {{0u, 1u}, "00000000000000000001", "FFFFFFFFFFFFFFFFFFFE"},
+ {{1u, 0u}, "00000000000000010000", "FFFFFFFFFFFFFFFEFFFF"},
+ {{NUdf::MAX_TIMESTAMP, 0u}, "000F3F52435260000000", "FFF0C0ADBCAD9FFFFFFF"},
+ };
+ TPresortTest().ValidateEncoding<NUdf::EDataSlot::TzTimestamp>(values);
+}
+
+Y_UNIT_TEST(Decimal) {
const TVector<std::tuple<NYql::NDecimal::TInt128, TString, TString>> values = {
{-NYql::NDecimal::Nan(),
"00",
@@ -428,10 +428,10 @@ Y_UNIT_TEST(Decimal) {
{NYql::NDecimal::Nan(),
"FF",
"00"},
- };
- TPresortTest().ValidateEncoding(values);
-}
-
+ };
+ TPresortTest().ValidateEncoding(values);
+}
+
Y_UNIT_TEST(GenericVoid) {
TScopedAlloc alloc;
TTypeEnvironment env(alloc);
@@ -440,8 +440,8 @@ Y_UNIT_TEST(GenericVoid) {
TGenericPresortEncoder encoder(type);
auto buf = encoder.Encode(value, false);
UNIT_ASSERT_NO_DIFF(buf, TStringBuf(""));
-}
-
+}
+
Y_UNIT_TEST(GenericBool) {
TScopedAlloc alloc;
TTypeEnvironment env(alloc);
@@ -642,5 +642,5 @@ Y_UNIT_TEST(GenericDict) {
}
-} // NMiniKQL
-} // NKikimr
+} // NMiniKQL
+} // NKikimr
diff --git a/ydb/library/yql/minikql/computation/ut/ya.make b/ydb/library/yql/minikql/computation/ut/ya.make
index ab27689c7b0..d59d23d2ec4 100644
--- a/ydb/library/yql/minikql/computation/ut/ya.make
+++ b/ydb/library/yql/minikql/computation/ut/ya.make
@@ -20,7 +20,7 @@ SRCS(
mkql_computation_node_pack_ut.cpp
mkql_computation_node_list_ut.cpp
mkql_computation_node_dict_ut.cpp
- mkql_computation_node_graph_saveload_ut.cpp
+ mkql_computation_node_graph_saveload_ut.cpp
mkql_validate_ut.cpp
mkql_value_builder_ut.cpp
presort_ut.cpp
diff --git a/ydb/library/yql/minikql/computation/ya.make b/ydb/library/yql/minikql/computation/ya.make
index ae419b3b232..6bd282409db 100644
--- a/ydb/library/yql/minikql/computation/ya.make
+++ b/ydb/library/yql/minikql/computation/ya.make
@@ -13,8 +13,8 @@ SRCS(
mkql_computation_node_codegen.h
mkql_computation_node_codegen.cpp
mkql_computation_node_graph.cpp
- mkql_computation_node_graph_saveload.cpp
- mkql_computation_node_graph_saveload.h
+ mkql_computation_node_graph_saveload.cpp
+ mkql_computation_node_graph_saveload.h
mkql_computation_node_holders.cpp
mkql_computation_node_impl.h
mkql_computation_node_impl.cpp
diff --git a/ydb/library/yql/minikql/mkql_program_builder.cpp b/ydb/library/yql/minikql/mkql_program_builder.cpp
index 47ae6e4c8fc..7717ac582a0 100644
--- a/ydb/library/yql/minikql/mkql_program_builder.cpp
+++ b/ydb/library/yql/minikql/mkql_program_builder.cpp
@@ -654,26 +654,26 @@ TRuntimeNode TProgramBuilder::Squeeze(TRuntimeNode stream, TRuntimeNode state,
const auto newState = handler(itemArg, stateNodeArg);
MKQL_ENSURE(newState.GetStaticType()->IsSameType(*state.GetStaticType()), "State type is changed by the handler");
- TRuntimeNode saveArg, outSave, loadArg, outLoad;
-
+ TRuntimeNode saveArg, outSave, loadArg, outLoad;
+
if (save && load) {
outSave = save(saveArg = Arg(state.GetStaticType()));
outLoad = load(loadArg = Arg(outSave.GetStaticType()));
- MKQL_ENSURE(outLoad.GetStaticType()->IsSameType(*state.GetStaticType()), "Loaded type is changed by the load handler");
- } else {
- saveArg = outSave = loadArg = outLoad = NewVoid();
- }
-
+ MKQL_ENSURE(outLoad.GetStaticType()->IsSameType(*state.GetStaticType()), "Loaded type is changed by the load handler");
+ } else {
+ saveArg = outSave = loadArg = outLoad = NewVoid();
+ }
+
TCallableBuilder callableBuilder(Env, __func__, TStreamType::Create(state.GetStaticType(), Env));
callableBuilder.Add(stream);
callableBuilder.Add(state);
callableBuilder.Add(itemArg);
callableBuilder.Add(stateNodeArg);
callableBuilder.Add(newState);
- callableBuilder.Add(saveArg);
- callableBuilder.Add(outSave);
- callableBuilder.Add(loadArg);
- callableBuilder.Add(outLoad);
+ callableBuilder.Add(saveArg);
+ callableBuilder.Add(outSave);
+ callableBuilder.Add(loadArg);
+ callableBuilder.Add(outLoad);
return TRuntimeNode(callableBuilder.Build(), false);
}
@@ -694,26 +694,26 @@ TRuntimeNode TProgramBuilder::Squeeze1(TRuntimeNode stream, const TUnaryLambda&
const auto newState = handler(itemArg, stateNodeArg);
MKQL_ENSURE(newState.GetStaticType()->IsSameType(*initState.GetStaticType()), "State type is changed by the handler");
- TRuntimeNode saveArg, outSave, loadArg, outLoad;
-
+ TRuntimeNode saveArg, outSave, loadArg, outLoad;
+
if (save && load) {
outSave = save(saveArg = Arg(initState.GetStaticType()));
outLoad = load(loadArg = Arg(outSave.GetStaticType()));
- MKQL_ENSURE(outLoad.GetStaticType()->IsSameType(*initState.GetStaticType()), "Loaded type is changed by the load handler");
- } else {
- saveArg = outSave = loadArg = outLoad = NewVoid();
- }
-
+ MKQL_ENSURE(outLoad.GetStaticType()->IsSameType(*initState.GetStaticType()), "Loaded type is changed by the load handler");
+ } else {
+ saveArg = outSave = loadArg = outLoad = NewVoid();
+ }
+
TCallableBuilder callableBuilder(Env, __func__, NewStreamType(newState.GetStaticType()));
callableBuilder.Add(stream);
callableBuilder.Add(itemArg);
callableBuilder.Add(initState);
callableBuilder.Add(stateNodeArg);
callableBuilder.Add(newState);
- callableBuilder.Add(saveArg);
- callableBuilder.Add(outSave);
- callableBuilder.Add(loadArg);
- callableBuilder.Add(outLoad);
+ callableBuilder.Add(saveArg);
+ callableBuilder.Add(outSave);
+ callableBuilder.Add(loadArg);
+ callableBuilder.Add(outLoad);
return TRuntimeNode(callableBuilder.Build(), false);
}
@@ -4589,7 +4589,7 @@ TRuntimeNode TProgramBuilder::WideChopper(TRuntimeNode flow, const TWideLambda&
return TRuntimeNode(callableBuilder.Build(), false);
}
-TRuntimeNode TProgramBuilder::HoppingCore(TRuntimeNode list,
+TRuntimeNode TProgramBuilder::HoppingCore(TRuntimeNode list,
const TUnaryLambda& timeExtractor,
const TUnaryLambda& init,
const TBinaryLambda& update,
@@ -4597,69 +4597,69 @@ TRuntimeNode TProgramBuilder::HoppingCore(TRuntimeNode list,
const TUnaryLambda& load,
const TBinaryLambda& merge,
const TBinaryLambda& finish,
- TRuntimeNode hop, TRuntimeNode interval, TRuntimeNode delay)
-{
- auto streamType = AS_TYPE(TStreamType, list);
- auto itemType = AS_TYPE(TStructType, streamType->GetItemType());
- auto timestampType = TOptionalType::Create(TDataType::Create(NUdf::TDataType<NUdf::TTimestamp>::Id, Env), Env);
-
+ TRuntimeNode hop, TRuntimeNode interval, TRuntimeNode delay)
+{
+ auto streamType = AS_TYPE(TStreamType, list);
+ auto itemType = AS_TYPE(TStructType, streamType->GetItemType());
+ auto timestampType = TOptionalType::Create(TDataType::Create(NUdf::TDataType<NUdf::TTimestamp>::Id, Env), Env);
+
TRuntimeNode itemArg = Arg(itemType);
-
- auto outTime = timeExtractor(itemArg);
- auto outStateInit = init(itemArg);
-
- auto stateType = outStateInit.GetStaticType();
+
+ auto outTime = timeExtractor(itemArg);
+ auto outStateInit = init(itemArg);
+
+ auto stateType = outStateInit.GetStaticType();
TRuntimeNode stateArg = Arg(stateType);
-
- auto outStateUpdate = update(itemArg, stateArg);
-
- auto hasSaveLoad = (bool)save;
- TRuntimeNode saveArg, outSave, loadArg, outLoad;
- if (hasSaveLoad) {
+
+ auto outStateUpdate = update(itemArg, stateArg);
+
+ auto hasSaveLoad = (bool)save;
+ TRuntimeNode saveArg, outSave, loadArg, outLoad;
+ if (hasSaveLoad) {
saveArg = Arg(stateType);
- outSave = save(saveArg);
-
+ outSave = save(saveArg);
+
loadArg = Arg(outSave.GetStaticType());
- outLoad = load(loadArg);
-
- MKQL_ENSURE(outLoad.GetStaticType()->IsSameType(*stateType), "Loaded type is changed by the load handler");
- } else {
- saveArg = outSave = loadArg = outLoad = NewVoid();
- }
-
+ outLoad = load(loadArg);
+
+ MKQL_ENSURE(outLoad.GetStaticType()->IsSameType(*stateType), "Loaded type is changed by the load handler");
+ } else {
+ saveArg = outSave = loadArg = outLoad = NewVoid();
+ }
+
TRuntimeNode state2Arg = Arg(stateType);
- TRuntimeNode timeArg = Arg(timestampType);
-
- auto outStateMerge = merge(stateArg, state2Arg);
- auto outItemFinish = finish(stateArg, timeArg);
-
- auto finishType = outItemFinish.GetStaticType();
- MKQL_ENSURE(finishType->IsStruct(), "Expected struct type as finish lambda output");
-
- auto resultType = TStreamType::Create(outItemFinish.GetStaticType(), Env);
-
+ TRuntimeNode timeArg = Arg(timestampType);
+
+ auto outStateMerge = merge(stateArg, state2Arg);
+ auto outItemFinish = finish(stateArg, timeArg);
+
+ auto finishType = outItemFinish.GetStaticType();
+ MKQL_ENSURE(finishType->IsStruct(), "Expected struct type as finish lambda output");
+
+ auto resultType = TStreamType::Create(outItemFinish.GetStaticType(), Env);
+
TCallableBuilder callableBuilder(Env, __func__, resultType);
- callableBuilder.Add(list);
- callableBuilder.Add(itemArg);
- callableBuilder.Add(stateArg);
- callableBuilder.Add(state2Arg);
- callableBuilder.Add(timeArg);
- callableBuilder.Add(saveArg);
- callableBuilder.Add(loadArg);
- callableBuilder.Add(outTime);
- callableBuilder.Add(outStateInit);
- callableBuilder.Add(outStateUpdate);
- callableBuilder.Add(outSave);
- callableBuilder.Add(outLoad);
- callableBuilder.Add(outStateMerge);
- callableBuilder.Add(outItemFinish);
- callableBuilder.Add(hop);
- callableBuilder.Add(interval);
- callableBuilder.Add(delay);
-
- return TRuntimeNode(callableBuilder.Build(), false);
-}
-
+ callableBuilder.Add(list);
+ callableBuilder.Add(itemArg);
+ callableBuilder.Add(stateArg);
+ callableBuilder.Add(state2Arg);
+ callableBuilder.Add(timeArg);
+ callableBuilder.Add(saveArg);
+ callableBuilder.Add(loadArg);
+ callableBuilder.Add(outTime);
+ callableBuilder.Add(outStateInit);
+ callableBuilder.Add(outStateUpdate);
+ callableBuilder.Add(outSave);
+ callableBuilder.Add(outLoad);
+ callableBuilder.Add(outStateMerge);
+ callableBuilder.Add(outItemFinish);
+ callableBuilder.Add(hop);
+ callableBuilder.Add(interval);
+ callableBuilder.Add(delay);
+
+ return TRuntimeNode(callableBuilder.Build(), false);
+}
+
TRuntimeNode TProgramBuilder::MultiHoppingCore(TRuntimeNode list,
const TUnaryLambda& keyExtractor,
const TUnaryLambda& timeExtractor,
diff --git a/ydb/library/yql/minikql/mkql_program_builder.h b/ydb/library/yql/minikql/mkql_program_builder.h
index 10f1ad7ccf1..9939e2d2914 100644
--- a/ydb/library/yql/minikql/mkql_program_builder.h
+++ b/ydb/library/yql/minikql/mkql_program_builder.h
@@ -416,7 +416,7 @@ public:
const TBinaryLambda& groupSwitch,
const TUnaryLambda& keyExtractor,
const TUnaryLambda& handler = {});
- TRuntimeNode HoppingCore(TRuntimeNode list,
+ TRuntimeNode HoppingCore(TRuntimeNode list,
const TUnaryLambda& timeExtractor,
const TUnaryLambda& init,
const TBinaryLambda& update,
@@ -424,7 +424,7 @@ public:
const TUnaryLambda& load,
const TBinaryLambda& merge,
const TBinaryLambda& finish,
- TRuntimeNode hop, TRuntimeNode interval, TRuntimeNode delay);
+ TRuntimeNode hop, TRuntimeNode interval, TRuntimeNode delay);
TRuntimeNode MultiHoppingCore(TRuntimeNode list,
const TUnaryLambda& keyExtractor,
const TUnaryLambda& timeExtractor,
diff --git a/ydb/library/yql/minikql/mkql_type_builder.cpp b/ydb/library/yql/minikql/mkql_type_builder.cpp
index 4aecb564bf7..e8b3f8bcb92 100644
--- a/ydb/library/yql/minikql/mkql_type_builder.cpp
+++ b/ydb/library/yql/minikql/mkql_type_builder.cpp
@@ -3,7 +3,7 @@
#include "mkql_node_builder.h"
#include <ydb/library/yql/public/udf/udf_type_ops.h>
-
+
#include <library/cpp/containers/stack_vector/stack_vec.h>
#include <ydb/library/yql/minikql/computation/mkql_computation_node_impl.h>
@@ -598,13 +598,13 @@ private:
TVector<NMiniKQL::TArgInfo>& Args_;
};
-//////////////////////////////////////////////////////////////////////////////
-// THash
-//////////////////////////////////////////////////////////////////////////////
-
-struct TTypeNotSupported : public yexception
-{};
-
+//////////////////////////////////////////////////////////////////////////////
+// THash
+//////////////////////////////////////////////////////////////////////////////
+
+struct TTypeNotSupported : public yexception
+{};
+
class TEmptyHash final : public NUdf::IHash {
public:
ui64 Hash(NUdf::TUnboxedValuePod value) const override {
@@ -613,35 +613,35 @@ public:
}
};
-template <NMiniKQL::TType::EKind Kind, NUdf::EDataSlot Slot = NUdf::EDataSlot::Bool>
-class THash;
-
-template <NUdf::EDataSlot Slot>
-class THash<NMiniKQL::TType::EKind::Data, Slot> final : public NUdf::IHash {
-public:
- ui64 Hash(NUdf::TUnboxedValuePod value) const override {
+template <NMiniKQL::TType::EKind Kind, NUdf::EDataSlot Slot = NUdf::EDataSlot::Bool>
+class THash;
+
+template <NUdf::EDataSlot Slot>
+class THash<NMiniKQL::TType::EKind::Data, Slot> final : public NUdf::IHash {
+public:
+ ui64 Hash(NUdf::TUnboxedValuePod value) const override {
return NUdf::GetValueHash<Slot>(std::move(value));
- }
-};
-
-template <>
-class THash<NMiniKQL::TType::EKind::Optional> final : public NUdf::IHash {
-public:
+ }
+};
+
+template <>
+class THash<NMiniKQL::TType::EKind::Optional> final : public NUdf::IHash {
+public:
explicit THash(const NMiniKQL::TType* type)
: Hash_(MakeHashImpl(static_cast<const NMiniKQL::TOptionalType*>(type)->GetItemType()))
{}
-
- ui64 Hash(NUdf::TUnboxedValuePod value) const override {
- if (!value) {
- return 0;
- }
+
+ ui64 Hash(NUdf::TUnboxedValuePod value) const override {
+ if (!value) {
+ return 0;
+ }
return CombineHashes(ui64(1), Hash_->Hash(value.GetOptionalValue()));
- }
-
-private:
+ }
+
+private:
const NUdf::IHash::TPtr Hash_;
-};
-
+};
+
template <>
class THash<NMiniKQL::TType::EKind::List> final : public NUdf::IHash {
public:
@@ -711,9 +711,9 @@ private:
NUdf::IHash::TPtr PayloadHash_;
};
-class TVectorHash : public NUdf::IHash {
-public:
- ui64 Hash(NUdf::TUnboxedValuePod value) const override {
+class TVectorHash : public NUdf::IHash {
+public:
+ ui64 Hash(NUdf::TUnboxedValuePod value) const override {
ui64 result = 0ULL;
auto elements = value.GetElements();
if (elements) {
@@ -727,39 +727,39 @@ public:
}
}
- return result;
- }
-
-protected:
+ return result;
+ }
+
+protected:
std::vector<NUdf::IHash::TPtr, NKikimr::NMiniKQL::TMKQLAllocator<NUdf::IHash::TPtr>> Hash_;
-};
-
-template <>
-class THash<NMiniKQL::TType::EKind::Tuple> final : public TVectorHash {
-public:
- explicit THash(const NMiniKQL::TType* type) {
- auto tupleType = static_cast<const NMiniKQL::TTupleType*>(type);
- auto count = tupleType->GetElementsCount();
- Hash_.reserve(count);
- for (ui32 i = 0; i < count; ++i) {
- Hash_.push_back(MakeHashImpl(tupleType->GetElementType(i)));
- }
- }
-};
-
-template <>
-class THash<NMiniKQL::TType::EKind::Struct> final : public TVectorHash {
-public:
- explicit THash(const NMiniKQL::TType* type) {
- auto structType = static_cast<const NMiniKQL::TStructType*>(type);
- auto count = structType->GetMembersCount();
- Hash_.reserve(count);
- for (ui32 i = 0; i < count; ++i) {
- Hash_.push_back(MakeHashImpl(structType->GetMemberType(i)));
- }
- }
-};
-
+};
+
+template <>
+class THash<NMiniKQL::TType::EKind::Tuple> final : public TVectorHash {
+public:
+ explicit THash(const NMiniKQL::TType* type) {
+ auto tupleType = static_cast<const NMiniKQL::TTupleType*>(type);
+ auto count = tupleType->GetElementsCount();
+ Hash_.reserve(count);
+ for (ui32 i = 0; i < count; ++i) {
+ Hash_.push_back(MakeHashImpl(tupleType->GetElementType(i)));
+ }
+ }
+};
+
+template <>
+class THash<NMiniKQL::TType::EKind::Struct> final : public TVectorHash {
+public:
+ explicit THash(const NMiniKQL::TType* type) {
+ auto structType = static_cast<const NMiniKQL::TStructType*>(type);
+ auto count = structType->GetMembersCount();
+ Hash_.reserve(count);
+ for (ui32 i = 0; i < count; ++i) {
+ Hash_.push_back(MakeHashImpl(structType->GetMemberType(i)));
+ }
+ }
+};
+
template <>
class THash<NMiniKQL::TType::EKind::Variant> final : public NUdf::IHash {
public:
@@ -793,9 +793,9 @@ private:
std::vector<NUdf::IHash::TPtr, NKikimr::NMiniKQL::TMKQLAllocator<NUdf::IHash::TPtr>> Hash_;
};
-//////////////////////////////////////////////////////////////////////////////
-// TEquate
-//////////////////////////////////////////////////////////////////////////////
+//////////////////////////////////////////////////////////////////////////////
+// TEquate
+//////////////////////////////////////////////////////////////////////////////
class TEmptyEquate final : public NUdf::IEquate {
public:
bool Equals(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override {
@@ -806,42 +806,42 @@ public:
};
-template <NMiniKQL::TType::EKind Kind, NUdf::EDataSlot Slot = NUdf::EDataSlot::Bool>
-class TEquate;
-
-template <NUdf::EDataSlot Slot>
-class TEquate<NMiniKQL::TType::EKind::Data, Slot> final : public NUdf::IEquate {
-public:
- bool Equals(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override {
+template <NMiniKQL::TType::EKind Kind, NUdf::EDataSlot Slot = NUdf::EDataSlot::Bool>
+class TEquate;
+
+template <NUdf::EDataSlot Slot>
+class TEquate<NMiniKQL::TType::EKind::Data, Slot> final : public NUdf::IEquate {
+public:
+ bool Equals(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override {
return NUdf::EquateValues<Slot>(std::move(lhs), std::move(rhs));
- }
-};
-
-template <>
-class TEquate<NMiniKQL::TType::EKind::Optional> final : public NUdf::IEquate {
-public:
+ }
+};
+
+template <>
+class TEquate<NMiniKQL::TType::EKind::Optional> final : public NUdf::IEquate {
+public:
explicit TEquate(const NMiniKQL::TType* type)
: Equate_(MakeEquateImpl(static_cast<const NMiniKQL::TOptionalType*>(type)->GetItemType()))
{}
-
- bool Equals(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override {
- if (!lhs) {
- if (!rhs) {
- return true;
- }
- return false;
- } else {
- if (!rhs) {
- return false;
- }
+
+ bool Equals(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override {
+ if (!lhs) {
+ if (!rhs) {
+ return true;
+ }
+ return false;
+ } else {
+ if (!rhs) {
+ return false;
+ }
return Equate_->Equals(lhs.GetOptionalValue(), rhs.GetOptionalValue());
- }
- }
-
-private:
+ }
+ }
+
+private:
const NUdf::IEquate::TPtr Equate_;
-};
-
+};
+
template <>
class TEquate<NMiniKQL::TType::EKind::List> final : public NUdf::IEquate {
public:
@@ -944,49 +944,49 @@ private:
};
-class TVectorEquate : public NUdf::IEquate {
-public:
- bool Equals(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override {
- for (size_t i = 0; i < Equate_.size(); ++i) {
+class TVectorEquate : public NUdf::IEquate {
+public:
+ bool Equals(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override {
+ for (size_t i = 0; i < Equate_.size(); ++i) {
if (!Equate_[i]->Equals(
static_cast<const NUdf::TUnboxedValuePod&>(lhs.GetElement(i)),
static_cast<const NUdf::TUnboxedValuePod&>(rhs.GetElement(i)))) {
- return false;
- }
- }
- return true;
- }
-
-protected:
+ return false;
+ }
+ }
+ return true;
+ }
+
+protected:
std::vector<NUdf::IEquate::TPtr, NKikimr::NMiniKQL::TMKQLAllocator<NUdf::IEquate::TPtr>> Equate_;
-};
-
-template <>
-class TEquate<NMiniKQL::TType::EKind::Tuple> final : public TVectorEquate {
-public:
- explicit TEquate(const NMiniKQL::TType* type) {
- auto tupleType = static_cast<const NMiniKQL::TTupleType*>(type);
- auto count = tupleType->GetElementsCount();
- Equate_.reserve(count);
- for (ui32 i = 0; i < count; ++i) {
- Equate_.push_back(MakeEquateImpl(tupleType->GetElementType(i)));
- }
- }
-};
-
-template <>
-class TEquate<NMiniKQL::TType::EKind::Struct> final : public TVectorEquate {
-public:
- explicit TEquate(const NMiniKQL::TType* type) {
- auto structType = static_cast<const NMiniKQL::TStructType*>(type);
- auto count = structType->GetMembersCount();
- Equate_.reserve(count);
- for (ui32 i = 0; i < count; ++i) {
- Equate_.push_back(MakeEquateImpl(structType->GetMemberType(i)));
- }
- }
-};
-
+};
+
+template <>
+class TEquate<NMiniKQL::TType::EKind::Tuple> final : public TVectorEquate {
+public:
+ explicit TEquate(const NMiniKQL::TType* type) {
+ auto tupleType = static_cast<const NMiniKQL::TTupleType*>(type);
+ auto count = tupleType->GetElementsCount();
+ Equate_.reserve(count);
+ for (ui32 i = 0; i < count; ++i) {
+ Equate_.push_back(MakeEquateImpl(tupleType->GetElementType(i)));
+ }
+ }
+};
+
+template <>
+class TEquate<NMiniKQL::TType::EKind::Struct> final : public TVectorEquate {
+public:
+ explicit TEquate(const NMiniKQL::TType* type) {
+ auto structType = static_cast<const NMiniKQL::TStructType*>(type);
+ auto count = structType->GetMembersCount();
+ Equate_.reserve(count);
+ for (ui32 i = 0; i < count; ++i) {
+ Equate_.push_back(MakeEquateImpl(structType->GetMemberType(i)));
+ }
+ }
+};
+
template <>
class TEquate<NMiniKQL::TType::EKind::Variant> final : public NUdf::IEquate {
public:
@@ -1027,9 +1027,9 @@ private:
std::vector<NUdf::IEquate::TPtr, NKikimr::NMiniKQL::TMKQLAllocator<NUdf::IEquate::TPtr>> Equate_;
};
-//////////////////////////////////////////////////////////////////////////////
-// TCompare
-//////////////////////////////////////////////////////////////////////////////
+//////////////////////////////////////////////////////////////////////////////
+// TCompare
+//////////////////////////////////////////////////////////////////////////////
class TEmptyCompare final : public NUdf::ICompare {
public:
bool Less(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override {
@@ -1045,92 +1045,92 @@ public:
}
};
-template <NMiniKQL::TType::EKind Kind, NUdf::EDataSlot Slot = NUdf::EDataSlot::Bool>
-class TCompare;
-
-template <NUdf::EDataSlot Slot>
-class TCompare<NMiniKQL::TType::EKind::Data, Slot> final : public NUdf::ICompare {
-public:
- bool Less(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override {
+template <NMiniKQL::TType::EKind Kind, NUdf::EDataSlot Slot = NUdf::EDataSlot::Bool>
+class TCompare;
+
+template <NUdf::EDataSlot Slot>
+class TCompare<NMiniKQL::TType::EKind::Data, Slot> final : public NUdf::ICompare {
+public:
+ bool Less(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override {
return NUdf::CompareValues<Slot>(std::move(lhs), std::move(rhs)) < 0;
- }
-
- int Compare(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override {
+ }
+
+ int Compare(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override {
return NUdf::CompareValues<Slot>(std::move(lhs), std::move(rhs));
- }
-};
-
-template <>
-class TCompare<NMiniKQL::TType::EKind::Optional> final : public NUdf::ICompare {
-public:
+ }
+};
+
+template <>
+class TCompare<NMiniKQL::TType::EKind::Optional> final : public NUdf::ICompare {
+public:
explicit TCompare(const NMiniKQL::TType* type)
: Compare_(MakeCompareImpl(static_cast<const NMiniKQL::TOptionalType*>(type)->GetItemType()))
{}
-
- bool Less(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override {
- if (!lhs) {
- if (!rhs) {
- return false;
- }
- return true;
- } else {
- if (!rhs) {
- return false;
- }
+
+ bool Less(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override {
+ if (!lhs) {
+ if (!rhs) {
+ return false;
+ }
+ return true;
+ } else {
+ if (!rhs) {
+ return false;
+ }
return Compare_->Less(lhs.GetOptionalValue(), rhs.GetOptionalValue());
- }
- }
-
- int Compare(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override {
- if (!lhs) {
- if (!rhs) {
- return 0;
- }
- return -1;
- } else {
- if (!rhs) {
- return 1;
- }
+ }
+ }
+
+ int Compare(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override {
+ if (!lhs) {
+ if (!rhs) {
+ return 0;
+ }
+ return -1;
+ } else {
+ if (!rhs) {
+ return 1;
+ }
return Compare_->Compare(lhs.GetOptionalValue(), rhs.GetOptionalValue());
- }
- }
-
-private:
+ }
+ }
+
+private:
const NUdf::ICompare::TPtr Compare_;
-};
-
-template <>
-class TCompare<NMiniKQL::TType::EKind::Tuple> final : public NUdf::ICompare {
-public:
+};
+
+template <>
+class TCompare<NMiniKQL::TType::EKind::Tuple> final : public NUdf::ICompare {
+public:
explicit TCompare(const NMiniKQL::TType* type) {
- auto tupleType = static_cast<const NMiniKQL::TTupleType*>(type);
- auto count = tupleType->GetElementsCount();
- Compare_.reserve(count);
- for (ui32 i = 0; i < count; ++i) {
+ auto tupleType = static_cast<const NMiniKQL::TTupleType*>(type);
+ auto count = tupleType->GetElementsCount();
+ Compare_.reserve(count);
+ for (ui32 i = 0; i < count; ++i) {
Compare_.push_back(MakeCompareImpl(tupleType->GetElementType(i)));
- }
- }
-
- bool Less(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override {
- return Compare(lhs, rhs) < 0;
- }
-
- int Compare(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override {
- for (size_t i = 0; i < Compare_.size(); ++i) {
+ }
+ }
+
+ bool Less(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override {
+ return Compare(lhs, rhs) < 0;
+ }
+
+ int Compare(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override {
+ for (size_t i = 0; i < Compare_.size(); ++i) {
auto cmp = Compare_[i]->Compare(
static_cast<const NUdf::TUnboxedValuePod&>(lhs.GetElement(i)),
static_cast<const NUdf::TUnboxedValuePod&>(rhs.GetElement(i)));
- if (cmp != 0) {
- return cmp;
- }
- }
- return 0;
- }
-
-private:
+ if (cmp != 0) {
+ return cmp;
+ }
+ }
+ return 0;
+ }
+
+private:
std::vector<NUdf::ICompare::TPtr, NKikimr::NMiniKQL::TMKQLAllocator<NUdf::ICompare::TPtr>> Compare_;
-};
-
+};
+
template <>
class TCompare<NMiniKQL::TType::EKind::Variant> final : public NUdf::ICompare {
public:
@@ -1517,36 +1517,36 @@ NUdf::TSourcePosition TFunctionTypeInfoBuilder::GetSourcePosition() {
return Pos_;
}
-NUdf::IHash::TPtr TFunctionTypeInfoBuilder::MakeHash(const NUdf::TType* type) {
- try {
- auto mkqlType = static_cast<const NMiniKQL::TType*>(type);
- return MakeHashImpl(mkqlType);
- } catch (const TTypeNotSupported& ex) {
- SetError(TStringBuf(ex.what()));
- return nullptr;
- }
-}
-
-NUdf::IEquate::TPtr TFunctionTypeInfoBuilder::MakeEquate(const NUdf::TType* type) {
- try {
- auto mkqlType = static_cast<const NMiniKQL::TType*>(type);
- return MakeEquateImpl(mkqlType);
- } catch (const TTypeNotSupported& ex) {
- SetError(TStringBuf(ex.what()));
- return nullptr;
- }
-}
-
-NUdf::ICompare::TPtr TFunctionTypeInfoBuilder::MakeCompare(const NUdf::TType* type) {
- try {
- auto mkqlType = static_cast<const NMiniKQL::TType*>(type);
+NUdf::IHash::TPtr TFunctionTypeInfoBuilder::MakeHash(const NUdf::TType* type) {
+ try {
+ auto mkqlType = static_cast<const NMiniKQL::TType*>(type);
+ return MakeHashImpl(mkqlType);
+ } catch (const TTypeNotSupported& ex) {
+ SetError(TStringBuf(ex.what()));
+ return nullptr;
+ }
+}
+
+NUdf::IEquate::TPtr TFunctionTypeInfoBuilder::MakeEquate(const NUdf::TType* type) {
+ try {
+ auto mkqlType = static_cast<const NMiniKQL::TType*>(type);
+ return MakeEquateImpl(mkqlType);
+ } catch (const TTypeNotSupported& ex) {
+ SetError(TStringBuf(ex.what()));
+ return nullptr;
+ }
+}
+
+NUdf::ICompare::TPtr TFunctionTypeInfoBuilder::MakeCompare(const NUdf::TType* type) {
+ try {
+ auto mkqlType = static_cast<const NMiniKQL::TType*>(type);
return MakeCompareImpl(mkqlType);
- } catch (const TTypeNotSupported& ex) {
- SetError(TStringBuf(ex.what()));
- return nullptr;
- }
-}
-
+ } catch (const TTypeNotSupported& ex) {
+ SetError(TStringBuf(ex.what()));
+ return nullptr;
+ }
+}
+
NUdf::ETypeKind TTypeInfoHelper::GetTypeKind(const NUdf::TType* type) const {
if (!type) {
return NUdf::ETypeKind::Unknown;
diff --git a/ydb/library/yql/minikql/mkql_type_builder.h b/ydb/library/yql/minikql/mkql_type_builder.h
index a93fb41cee3..4a9a280218c 100644
--- a/ydb/library/yql/minikql/mkql_type_builder.h
+++ b/ydb/library/yql/minikql/mkql_type_builder.h
@@ -95,10 +95,10 @@ public:
NUdf::TScopedProbe GetScopedProbe(const NUdf::TStringRef& name) override;
NUdf::TSourcePosition GetSourcePosition() override;
- NUdf::IHash::TPtr MakeHash(const NUdf::TType* type) override;
- NUdf::IEquate::TPtr MakeEquate(const NUdf::TType* type) override;
- NUdf::ICompare::TPtr MakeCompare(const NUdf::TType* type) override;
-
+ NUdf::IHash::TPtr MakeHash(const NUdf::TType* type) override;
+ NUdf::IEquate::TPtr MakeEquate(const NUdf::TType* type) override;
+ NUdf::ICompare::TPtr MakeCompare(const NUdf::TType* type) override;
+
NUdf::TType* Decimal(ui8 precision, ui8 scale) const override;
NUdf::IFunctionTypeInfoBuilder7& IRImplementationImpl(
diff --git a/ydb/library/yql/minikql/mkql_type_ops.cpp b/ydb/library/yql/minikql/mkql_type_ops.cpp
index d559bb108ce..1962eab8e99 100644
--- a/ydb/library/yql/minikql/mkql_type_ops.cpp
+++ b/ydb/library/yql/minikql/mkql_type_ops.cpp
@@ -667,32 +667,32 @@ public:
TDateTable() {
ui32 prevYear = NUdf::MIN_YEAR - 1;
YearsOffsets_[0] = 0;
-
+
ui32 dayOfYear = 365;
ui32 dayOfWeek = 2;
ui32 weekOfYear = 52;
ui32 weekOfYearIso8601 = 1;
-
+
for (ui16 date = 0; date < Days_.size(); ++date) {
ui32 year, month, day;
Y_VERIFY(SplitDateUncached(date, year, month, day));
-
- ++dayOfYear;
- if (++dayOfWeek > 7) {
- dayOfWeek = 1;
- ++weekOfYear;
+
+ ++dayOfYear;
+ if (++dayOfWeek > 7) {
+ dayOfWeek = 1;
+ ++weekOfYear;
if ((month == 12 && day >= 29) || (month == 1 && day <= 4)) {
weekOfYearIso8601 = 1;
} else {
++weekOfYearIso8601;
}
- }
-
+ }
+
if (year > prevYear) {
YearsOffsets_[year - NUdf::MIN_YEAR] = date;
prevYear = year;
- dayOfYear = 1;
- weekOfYear = 1;
+ dayOfYear = 1;
+ weekOfYear = 1;
}
Days_[date] = TDayInfo{month, day, dayOfYear, weekOfYear, weekOfYearIso8601, dayOfWeek};
@@ -705,9 +705,9 @@ public:
}
year = NUdf::MIN_YEAR + std::distance(YearsOffsets_.cbegin(), std::upper_bound(YearsOffsets_.cbegin(), YearsOffsets_.cend(), value)) - 1;
- auto& info = Days_[value];
- month = info.Month;
- day = info.Day;
+ auto& info = Days_[value];
+ month = info.Month;
+ day = info.Day;
return true;
}
@@ -726,14 +726,14 @@ public:
const auto ptr = YearsOffsets_.data() + year - NUdf::MIN_YEAR;
const auto begin = Days_.cbegin() + ptr[0];
const auto end = year == NUdf::MAX_YEAR ? Days_.cend() : Days_.cbegin() + ptr[1];
-
+
// search valid month/day in this year
const auto target = PackMonthDay(month, day);
const auto it = std::lower_bound(begin, end, target, [](const TDayInfo& info, ui16 y) {
- return PackMonthDay(info.Month, info.Day) < y;
+ return PackMonthDay(info.Month, info.Day) < y;
});
- if (Y_UNLIKELY(it == end || PackMonthDay(it->Month, it->Day) != target)) {
+ if (Y_UNLIKELY(it == end || PackMonthDay(it->Month, it->Day) != target)) {
return false;
}
@@ -757,17 +757,17 @@ public:
bool EnrichByOffset(ui16 value, ui32& dayOfYear, ui32& weekOfYear, ui32& weekOfYearIso8601, ui32& dayOfWeek) const {
if (Y_UNLIKELY(value >= Days_.size())) {
- return false;
- }
-
- auto& info = Days_[value];
- dayOfYear = info.DayOfYear;
- weekOfYear = info.WeekOfYear;
+ return false;
+ }
+
+ auto& info = Days_[value];
+ dayOfYear = info.DayOfYear;
+ weekOfYear = info.WeekOfYear;
weekOfYearIso8601 = info.WeekOfYearIso8601;
- dayOfWeek = info.DayOfWeek;
- return true;
- }
-
+ dayOfWeek = info.DayOfWeek;
+ return true;
+ }
+
bool EnrichDate(ui16 value, ui32& dayOfYear, ui32& weekOfYear, ui32& weekOfYearIso8601, ui32& dayOfWeek) const {
return EnrichByOffset(++value, dayOfYear, weekOfYear, weekOfYearIso8601, dayOfWeek);
}
@@ -781,15 +781,15 @@ public:
}
private:
- struct TDayInfo {
- ui32 Month : 4;
- ui32 Day : 5;
- ui32 DayOfYear : 9;
- ui32 WeekOfYear : 6;
+ struct TDayInfo {
+ ui32 Month : 4;
+ ui32 Day : 5;
+ ui32 DayOfYear : 9;
+ ui32 WeekOfYear : 6;
ui32 WeekOfYearIso8601: 6;
- ui32 DayOfWeek : 3;
- };
-
+ ui32 DayOfWeek : 3;
+ };
+
std::array<ui16, NUdf::MAX_YEAR - NUdf::MIN_YEAR + 1> YearsOffsets_; // start of linear date for each year
std::array<TDayInfo, NUdf::MAX_DATE + 2> Days_; // packed info for each date
};
@@ -853,44 +853,44 @@ bool SplitInterval(i64 value, bool& sign, ui32& day, ui32& hour, ui32& min, ui32
return true;
}
-bool MakeTzDatetime(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 min, ui32 sec, ui32& value, ui16 tzId) {
- if (tzId) {
- const auto& tz = Singleton<TTimezones>()->GetZone(tzId);
- cctz::civil_second cs(year, month, day, hour, min, sec);
+bool MakeTzDatetime(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 min, ui32 sec, ui32& value, ui16 tzId) {
+ if (tzId) {
+ const auto& tz = Singleton<TTimezones>()->GetZone(tzId);
+ cctz::civil_second cs(year, month, day, hour, min, sec);
auto utcSeconds = cctz::TimePointToUnixSeconds(tz.lookup(cs).pre);
if (utcSeconds < 0 || utcSeconds >= (std::int_fast64_t) NUdf::MAX_DATETIME) {
- return false;
- }
-
- value = (ui32)utcSeconds;
- return true;
-
- } else {
- ui16 date;
- ui32 time;
- if (!MakeDate(year, month, day, date)) {
- return false;
- }
- if (!MakeTime(hour, min, sec, time)) {
- return false;
- }
- value = date * 86400u + time;
- return true;
- }
-}
-
-bool SplitTzDatetime(ui32 value, ui32& year, ui32& month, ui32& day, ui32& hour, ui32& min, ui32& sec, ui16 tzId) {
- if (tzId) {
- if (value >= NUdf::MAX_DATETIME) {
- return false;
- }
- ToLocalTime(value, tzId, year, month, day, hour, min, sec);
- return true;
- } else {
- return SplitDatetime(value, year, month, day, hour, min, sec);
- }
-}
-
+ return false;
+ }
+
+ value = (ui32)utcSeconds;
+ return true;
+
+ } else {
+ ui16 date;
+ ui32 time;
+ if (!MakeDate(year, month, day, date)) {
+ return false;
+ }
+ if (!MakeTime(hour, min, sec, time)) {
+ return false;
+ }
+ value = date * 86400u + time;
+ return true;
+ }
+}
+
+bool SplitTzDatetime(ui32 value, ui32& year, ui32& month, ui32& day, ui32& hour, ui32& min, ui32& sec, ui16 tzId) {
+ if (tzId) {
+ if (value >= NUdf::MAX_DATETIME) {
+ return false;
+ }
+ ToLocalTime(value, tzId, year, month, day, hour, min, sec);
+ return true;
+ } else {
+ return SplitDatetime(value, year, month, day, hour, min, sec);
+ }
+}
+
bool SplitTzDate(ui16 value, ui32& year, ui32& month, ui32& day, ui32& dayOfYear, ui32& weekOfYear, ui32& weekOfYearIso8601, ui32& dayOfWeek, ui16 tzId) {
if (tzId) {
if (value >= NUdf::MAX_DATE) {
@@ -931,55 +931,55 @@ bool SplitTzDatetime(ui32 value, ui32& year, ui32& month, ui32& day, ui32& hour,
bool EnrichDate(ui16 date, ui32& dayOfYear, ui32& weekOfYear, ui32& weekOfYearIso8601, ui32& dayOfWeek) {
return TDateTable::Instance().EnrichDate(date, dayOfYear, weekOfYear, weekOfYearIso8601, dayOfWeek);
-}
-
-bool GetTimezoneShift(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 min, ui32 sec, ui16 tzId, i32& value) {
- if (!tzId) {
- value = 0;
- return true;
- }
-
- const auto& tz = Singleton<TTimezones>()->GetZone(tzId);
- cctz::civil_second cs(year, month, day, hour, min, sec);
- value = tz.lookup(tz.lookup(cs).pre).offset / 60;
- return true;
-}
-
-namespace {
-
-bool FromLocalTimeValidated(ui16 tzId, ui32 year, ui32 month, ui32 day, ui32 hour, ui32 minute, ui32 second, ui32& value) {
- if (year < NUdf::MIN_YEAR - 1 || year > NUdf::MAX_YEAR) {
- return false;
- } else if (year == NUdf::MIN_YEAR - 1) {
- if (month != 12 || day != 31) {
- return false;
- }
- } else if (year == NUdf::MAX_YEAR) {
- if (month != 1 || day != 1) {
- return false;
- }
- }
-
- const auto& tz = Singleton<TTimezones>()->GetZone(tzId);
-
- cctz::civil_second sec(year, month, day, hour, minute, second);
+}
+
+bool GetTimezoneShift(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 min, ui32 sec, ui16 tzId, i32& value) {
+ if (!tzId) {
+ value = 0;
+ return true;
+ }
+
+ const auto& tz = Singleton<TTimezones>()->GetZone(tzId);
+ cctz::civil_second cs(year, month, day, hour, min, sec);
+ value = tz.lookup(tz.lookup(cs).pre).offset / 60;
+ return true;
+}
+
+namespace {
+
+bool FromLocalTimeValidated(ui16 tzId, ui32 year, ui32 month, ui32 day, ui32 hour, ui32 minute, ui32 second, ui32& value) {
+ if (year < NUdf::MIN_YEAR - 1 || year > NUdf::MAX_YEAR) {
+ return false;
+ } else if (year == NUdf::MIN_YEAR - 1) {
+ if (month != 12 || day != 31) {
+ return false;
+ }
+ } else if (year == NUdf::MAX_YEAR) {
+ if (month != 1 || day != 1) {
+ return false;
+ }
+ }
+
+ const auto& tz = Singleton<TTimezones>()->GetZone(tzId);
+
+ cctz::civil_second sec(year, month, day, hour, minute, second);
if ((ui32)sec.year() != year || (ui32)sec.month() != month || (ui32)sec.day() != day ||
(ui32)sec.hour() != hour || (ui32)sec.minute() != minute || (ui32)sec.second() != second) {
// normalized
return false;
}
- const auto absoluteSeconds = std::chrono::system_clock::to_time_t(tz.lookup(sec).pre);
- if (absoluteSeconds < 0 || (ui32)absoluteSeconds >= NUdf::MAX_DATETIME) {
- return false;
- }
-
- value = (ui32)absoluteSeconds;
- return true;
-}
-
-} // namespace
-
+ const auto absoluteSeconds = std::chrono::system_clock::to_time_t(tz.lookup(sec).pre);
+ if (absoluteSeconds < 0 || (ui32)absoluteSeconds >= NUdf::MAX_DATETIME) {
+ return false;
+ }
+
+ value = (ui32)absoluteSeconds;
+ return true;
+}
+
+} // namespace
+
ui32 ParseNumber(ui32& pos, NUdf::TStringRef buf, ui32& value) {
value = 0;
ui32 count = 0;
@@ -1157,12 +1157,12 @@ NUdf::TUnboxedValuePod ParseTzDate(NUdf::TStringRef str) {
return NUdf::TUnboxedValuePod();
}
- ui32 absoluteSeconds;
+ ui32 absoluteSeconds;
if (!FromLocalTimeValidated(*tzId, year, month, day, 0, 0, 0, absoluteSeconds)) {
return NUdf::TUnboxedValuePod();
}
- ui16 value = absoluteSeconds / 86400u;
+ ui16 value = absoluteSeconds / 86400u;
NUdf::TUnboxedValuePod out(value);
out.SetTimezoneId(*tzId);
return out;
@@ -1276,12 +1276,12 @@ NUdf::TUnboxedValuePod ParseTzDatetime(NUdf::TStringRef str) {
return NUdf::TUnboxedValuePod();
}
- ui32 absoluteSeconds;
- if (!FromLocalTimeValidated(*tzId, year, month, day, hour, minute, second, absoluteSeconds)) {
+ ui32 absoluteSeconds;
+ if (!FromLocalTimeValidated(*tzId, year, month, day, hour, minute, second, absoluteSeconds)) {
return NUdf::TUnboxedValuePod();
}
- ui32 value = absoluteSeconds;
+ ui32 value = absoluteSeconds;
NUdf::TUnboxedValuePod out(value);
out.SetTimezoneId(*tzId);
return out;
@@ -1449,8 +1449,8 @@ NUdf::TUnboxedValuePod ParseTzTimestamp(NUdf::TStringRef str) {
}
}
- ui32 absoluteSeconds;
- if (!FromLocalTimeValidated(*tzId, year, month, day, hour, minute, second, absoluteSeconds)) {
+ ui32 absoluteSeconds;
+ if (!FromLocalTimeValidated(*tzId, year, month, day, hour, minute, second, absoluteSeconds)) {
return NUdf::TUnboxedValuePod();
}
@@ -2013,7 +2013,7 @@ ui32 FromLocalTime(ui16 tzId, ui32 year, ui32 month, ui32 day, ui32 hour, ui32 m
return absoluteSeconds;
}
-
+
void SerializeTzDate(ui16 date, ui16 tzId, IOutputStream& out) {
date = SwapBytes(date);
tzId = SwapBytes(tzId);
diff --git a/ydb/library/yql/minikql/mkql_type_ops.h b/ydb/library/yql/minikql/mkql_type_ops.h
index 8ff9b258ddf..93b5e88a56a 100644
--- a/ydb/library/yql/minikql/mkql_type_ops.h
+++ b/ydb/library/yql/minikql/mkql_type_ops.h
@@ -43,11 +43,11 @@ bool SplitInterval(i64 value, bool& sign, ui32& day, ui32& hour, ui32& min, ui32
bool SplitTzDate(ui16 value, ui32& year, ui32& month, ui32& day, ui32& dayOfYear, ui32& weekOfYear, ui32& weekOfYearIso8601, ui32& dayOfWeek, ui16 tzId);
bool SplitTzDatetime(ui32 value, ui32& year, ui32& month, ui32& day, ui32& hour, ui32& min, ui32& sec, ui32& dayOfYear, ui32& weekOfYear, ui32& weekOfYearIso8601, ui32& dayOfWeek, ui16 tzId);
-bool MakeTzDatetime(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 min, ui32 sec, ui32& value, ui16 tzId);
-bool SplitTzDatetime(ui32 value, ui32& year, ui32& month, ui32& day, ui32& hour, ui32& min, ui32& sec, ui16 tzId);
+bool MakeTzDatetime(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 min, ui32 sec, ui32& value, ui16 tzId);
+bool SplitTzDatetime(ui32 value, ui32& year, ui32& month, ui32& day, ui32& hour, ui32& min, ui32& sec, ui16 tzId);
bool EnrichDate(ui16 date, ui32& dayOfYear, ui32& weekOfYear, ui32& weekOfYearIso8601, ui32& dayOfWeek);
-bool GetTimezoneShift(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 min, ui32 sec, ui16 tzId, i32& value);
-
+bool GetTimezoneShift(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 min, ui32 sec, ui16 tzId, i32& value);
+
ui16 InitTimezones();
bool IsValidTimezoneId(ui16 id);
TMaybe<ui16> FindTimezoneId(TStringBuf ianaName);
@@ -55,10 +55,10 @@ ui16 GetTimezoneId(TStringBuf ianaName);
TMaybe<TStringBuf> FindTimezoneIANAName(ui16 id);
TStringBuf GetTimezoneIANAName(ui16 id);
std::vector<ui16> GetTzBlackList();
-
+
void ToLocalTime(ui32 utcSeconds, ui16 tzId, ui32& year, ui32& month, ui32& day, ui32& hour, ui32& min, ui32& sec);
ui32 FromLocalTime(ui16 tzId, ui32 year, ui32 month, ui32 day, ui32 hour, ui32 min, ui32 sec);
-
+
void SerializeTzDate(ui16 date, ui16 tzId, IOutputStream& out);
void SerializeTzDatetime(ui32 datetime, ui16 tzId, IOutputStream& out);
void SerializeTzTimestamp(ui64 timestamp, ui16 tzId, IOutputStream& out);
diff --git a/ydb/library/yql/minikql/perf/presort/presort.cpp b/ydb/library/yql/minikql/perf/presort/presort.cpp
index cfe4ddfb8f3..278250ba191 100644
--- a/ydb/library/yql/minikql/perf/presort/presort.cpp
+++ b/ydb/library/yql/minikql/perf/presort/presort.cpp
@@ -4,298 +4,298 @@
#include <ydb/library/yql/minikql/mkql_alloc.h>
#include <ydb/library/yql/minikql/mkql_string_util.h>
#include <ydb/library/yql/public/udf/udf_types.h>
-
+
#include <library/cpp/presort/presort.h>
-
-#include <util/random/random.h>
-#include <util/datetime/cputimer.h>
+
+#include <util/random/random.h>
+#include <util/datetime/cputimer.h>
#include <util/string/builder.h>
-
-using namespace NKikimr;
-using namespace NKikimr::NMiniKQL;
-
-namespace {
-
-struct TSettings {
- ui32 Index;
- bool IsOptional;
- NKikimr::NUdf::EDataSlot Slot;
-};
-
-template <bool Desc>
-struct TPresortOps : public NPresort::TResultOps {
- const TVector<TSettings>& Settings;
- NUdf::TUnboxedValue* Items;
-
- size_t Current = 0;
-
- TPresortOps(
- const TVector<TSettings>& settings,
- NUdf::TUnboxedValue* items)
+
+using namespace NKikimr;
+using namespace NKikimr::NMiniKQL;
+
+namespace {
+
+struct TSettings {
+ ui32 Index;
+ bool IsOptional;
+ NKikimr::NUdf::EDataSlot Slot;
+};
+
+template <bool Desc>
+struct TPresortOps : public NPresort::TResultOps {
+ const TVector<TSettings>& Settings;
+ NUdf::TUnboxedValue* Items;
+
+ size_t Current = 0;
+
+ TPresortOps(
+ const TVector<TSettings>& settings,
+ NUdf::TUnboxedValue* items)
: Settings(settings)
- , Items(items)
- {}
-
- void Encode(IOutputStream& out) {
- for (const auto& setting : Settings) {
- auto& value = Items[setting.Index];
-
- switch (setting.Slot) {
- case NUdf::EDataSlot::Bool:
- NPresort::EncodeUnsignedInt(out, value.template Get<bool>(), Desc);
- break;
- case NUdf::EDataSlot::Uint8:
- NPresort::EncodeUnsignedInt(out, value.template Get<ui8>(), Desc);
- break;
- case NUdf::EDataSlot::Uint16:
- case NUdf::EDataSlot::Date:
- NPresort::EncodeUnsignedInt(out, value.template Get<ui16>(), Desc);
- break;
- case NUdf::EDataSlot::Uint32:
- case NUdf::EDataSlot::Datetime:
- NPresort::EncodeUnsignedInt(out, value.template Get<ui32>(), Desc);
- break;
- case NUdf::EDataSlot::Uint64:
- case NUdf::EDataSlot::Timestamp:
- NPresort::EncodeUnsignedInt(out, value.template Get<ui64>(), Desc);
- break;
- case NUdf::EDataSlot::Int8:
- NPresort::EncodeSignedInt(out, value.template Get<i8>(), Desc);
- break;
- case NUdf::EDataSlot::Int16:
- NPresort::EncodeSignedInt(out, value.template Get<i16>(), Desc);
- break;
- case NUdf::EDataSlot::Int32:
- NPresort::EncodeSignedInt(out, value.template Get<i32>(), Desc);
- break;
- case NUdf::EDataSlot::Int64:
- case NUdf::EDataSlot::Interval:
- NPresort::EncodeSignedInt(out, value.template Get<i64>(), Desc);
- break;
- case NUdf::EDataSlot::Float:
- NPresort::EncodeFloating(out, value.template Get<float>(), Desc);
- break;
- case NUdf::EDataSlot::Double:
- NPresort::EncodeFloating(out, value.template Get<double>(), Desc);
- break;
- case NUdf::EDataSlot::String:
- case NUdf::EDataSlot::Utf8: {
- auto strRef = value.AsStringRef();
- NPresort::EncodeString(out, TStringBuf(strRef.Data(), strRef.Size()), Desc);
- break;
- }
- default:
- MKQL_ENSURE(false, TStringBuilder() << "Unknown slot: " << setting.Slot);
- }
- }
- }
-
- void SetError(const TString& err) {
- MKQL_ENSURE(false, TStringBuilder() << "Presort decoding error: " << err);
- }
-
- void SetUnsignedInt(ui64 value) {
- const auto& setting = Settings[Current++];
- switch (setting.Slot) {
- case NUdf::EDataSlot::Bool:
- Items[setting.Index] = NUdf::TUnboxedValuePod(value != 0);
- break;
- case NUdf::EDataSlot::Uint8:
- Items[setting.Index] = NUdf::TUnboxedValuePod(static_cast<ui8>(value));
- break;
- case NUdf::EDataSlot::Uint16:
- case NUdf::EDataSlot::Date:
- Items[setting.Index] = NUdf::TUnboxedValuePod(static_cast<ui16>(value));
- break;
- case NUdf::EDataSlot::Uint32:
- case NUdf::EDataSlot::Datetime:
- Items[setting.Index] = NUdf::TUnboxedValuePod(static_cast<ui32>(value));
- break;
- case NUdf::EDataSlot::Uint64:
- case NUdf::EDataSlot::Timestamp:
- Items[setting.Index] = NUdf::TUnboxedValuePod(value);
- break;
- default:
- MKQL_ENSURE(false, TStringBuilder() << "Unknown slot: " << setting.Slot);
- }
- }
-
- void SetSignedInt(i64 value) {
- const auto& setting = Settings[Current++];
- switch (setting.Slot) {
- case NUdf::EDataSlot::Int8:
- Items[setting.Index] = NUdf::TUnboxedValuePod(static_cast<i8>(value));
- break;
- case NUdf::EDataSlot::Int16:
- Items[setting.Index] = NUdf::TUnboxedValuePod(static_cast<i16>(value));
- break;
- case NUdf::EDataSlot::Int32:
- Items[setting.Index] = NUdf::TUnboxedValuePod(static_cast<i32>(value));
- break;
- case NUdf::EDataSlot::Int64:
- case NUdf::EDataSlot::Interval:
- Items[setting.Index] = NUdf::TUnboxedValuePod(value);
- break;
- default:
- MKQL_ENSURE(false, "Unknown slot: " << setting.Slot);
- }
- }
-
- void SetFloat(float value) {
- Items[Settings[Current++].Index] = NUdf::TUnboxedValuePod(value);
- }
-
- void SetDouble(double value) {
- Items[Settings[Current++].Index] = NUdf::TUnboxedValuePod(value);
- }
-
- void SetString(const TString& value) {
+ , Items(items)
+ {}
+
+ void Encode(IOutputStream& out) {
+ for (const auto& setting : Settings) {
+ auto& value = Items[setting.Index];
+
+ switch (setting.Slot) {
+ case NUdf::EDataSlot::Bool:
+ NPresort::EncodeUnsignedInt(out, value.template Get<bool>(), Desc);
+ break;
+ case NUdf::EDataSlot::Uint8:
+ NPresort::EncodeUnsignedInt(out, value.template Get<ui8>(), Desc);
+ break;
+ case NUdf::EDataSlot::Uint16:
+ case NUdf::EDataSlot::Date:
+ NPresort::EncodeUnsignedInt(out, value.template Get<ui16>(), Desc);
+ break;
+ case NUdf::EDataSlot::Uint32:
+ case NUdf::EDataSlot::Datetime:
+ NPresort::EncodeUnsignedInt(out, value.template Get<ui32>(), Desc);
+ break;
+ case NUdf::EDataSlot::Uint64:
+ case NUdf::EDataSlot::Timestamp:
+ NPresort::EncodeUnsignedInt(out, value.template Get<ui64>(), Desc);
+ break;
+ case NUdf::EDataSlot::Int8:
+ NPresort::EncodeSignedInt(out, value.template Get<i8>(), Desc);
+ break;
+ case NUdf::EDataSlot::Int16:
+ NPresort::EncodeSignedInt(out, value.template Get<i16>(), Desc);
+ break;
+ case NUdf::EDataSlot::Int32:
+ NPresort::EncodeSignedInt(out, value.template Get<i32>(), Desc);
+ break;
+ case NUdf::EDataSlot::Int64:
+ case NUdf::EDataSlot::Interval:
+ NPresort::EncodeSignedInt(out, value.template Get<i64>(), Desc);
+ break;
+ case NUdf::EDataSlot::Float:
+ NPresort::EncodeFloating(out, value.template Get<float>(), Desc);
+ break;
+ case NUdf::EDataSlot::Double:
+ NPresort::EncodeFloating(out, value.template Get<double>(), Desc);
+ break;
+ case NUdf::EDataSlot::String:
+ case NUdf::EDataSlot::Utf8: {
+ auto strRef = value.AsStringRef();
+ NPresort::EncodeString(out, TStringBuf(strRef.Data(), strRef.Size()), Desc);
+ break;
+ }
+ default:
+ MKQL_ENSURE(false, TStringBuilder() << "Unknown slot: " << setting.Slot);
+ }
+ }
+ }
+
+ void SetError(const TString& err) {
+ MKQL_ENSURE(false, TStringBuilder() << "Presort decoding error: " << err);
+ }
+
+ void SetUnsignedInt(ui64 value) {
+ const auto& setting = Settings[Current++];
+ switch (setting.Slot) {
+ case NUdf::EDataSlot::Bool:
+ Items[setting.Index] = NUdf::TUnboxedValuePod(value != 0);
+ break;
+ case NUdf::EDataSlot::Uint8:
+ Items[setting.Index] = NUdf::TUnboxedValuePod(static_cast<ui8>(value));
+ break;
+ case NUdf::EDataSlot::Uint16:
+ case NUdf::EDataSlot::Date:
+ Items[setting.Index] = NUdf::TUnboxedValuePod(static_cast<ui16>(value));
+ break;
+ case NUdf::EDataSlot::Uint32:
+ case NUdf::EDataSlot::Datetime:
+ Items[setting.Index] = NUdf::TUnboxedValuePod(static_cast<ui32>(value));
+ break;
+ case NUdf::EDataSlot::Uint64:
+ case NUdf::EDataSlot::Timestamp:
+ Items[setting.Index] = NUdf::TUnboxedValuePod(value);
+ break;
+ default:
+ MKQL_ENSURE(false, TStringBuilder() << "Unknown slot: " << setting.Slot);
+ }
+ }
+
+ void SetSignedInt(i64 value) {
+ const auto& setting = Settings[Current++];
+ switch (setting.Slot) {
+ case NUdf::EDataSlot::Int8:
+ Items[setting.Index] = NUdf::TUnboxedValuePod(static_cast<i8>(value));
+ break;
+ case NUdf::EDataSlot::Int16:
+ Items[setting.Index] = NUdf::TUnboxedValuePod(static_cast<i16>(value));
+ break;
+ case NUdf::EDataSlot::Int32:
+ Items[setting.Index] = NUdf::TUnboxedValuePod(static_cast<i32>(value));
+ break;
+ case NUdf::EDataSlot::Int64:
+ case NUdf::EDataSlot::Interval:
+ Items[setting.Index] = NUdf::TUnboxedValuePod(value);
+ break;
+ default:
+ MKQL_ENSURE(false, "Unknown slot: " << setting.Slot);
+ }
+ }
+
+ void SetFloat(float value) {
+ Items[Settings[Current++].Index] = NUdf::TUnboxedValuePod(value);
+ }
+
+ void SetDouble(double value) {
+ Items[Settings[Current++].Index] = NUdf::TUnboxedValuePod(value);
+ }
+
+ void SetString(const TString& value) {
Items[Settings[Current++].Index] = MakeString(NUdf::TStringRef(value.data(), value.size()));
- }
-
- void SetOptional(bool) {}
-};
-
-template <typename T>
+ }
+
+ void SetOptional(bool) {}
+};
+
+template <typename T>
NUdf::TUnboxedValue RandomValue() {
- return NUdf::TUnboxedValuePod(RandomNumber<T>());
-}
-
-template <>
+ return NUdf::TUnboxedValuePod(RandomNumber<T>());
+}
+
+template <>
NUdf::TUnboxedValue RandomValue<char*>() {
- auto length = RandomNumber<ui64>(64);
+ auto length = RandomNumber<ui64>(64);
return MakeStringNotFilled(length);
-}
-
-template <typename T, NUdf::EDataSlot Slot, bool Desc>
-std::pair<ui64, ui64> MeasureOld() {
- constexpr size_t count = 1000;
- constexpr size_t rowCount = 100000;
-
- TScopedAlloc alloc;
- TMemoryUsageInfo memInfo("Memory");
-
- TVector<NUdf::TUnboxedValue> values;
- TVector<TSettings> settings;
-
- for (ui32 i = 0; i < count; ++i) {
+}
+
+template <typename T, NUdf::EDataSlot Slot, bool Desc>
+std::pair<ui64, ui64> MeasureOld() {
+ constexpr size_t count = 1000;
+ constexpr size_t rowCount = 100000;
+
+ TScopedAlloc alloc;
+ TMemoryUsageInfo memInfo("Memory");
+
+ TVector<NUdf::TUnboxedValue> values;
+ TVector<TSettings> settings;
+
+ for (ui32 i = 0; i < count; ++i) {
values.push_back(RandomValue<T>());
- settings.push_back({i, false, NUdf::TDataType<T>::Slot});
- }
-
- TSimpleTimer timer;
-
- TStringStream stream;
- for (size_t n = 0; n < rowCount; ++n) {
- stream.clear();
+ settings.push_back({i, false, NUdf::TDataType<T>::Slot});
+ }
+
+ TSimpleTimer timer;
+
+ TStringStream stream;
+ for (size_t n = 0; n < rowCount; ++n) {
+ stream.clear();
TPresortOps<Desc> ops{settings, values.begin()};
- ops.Encode(stream);
- }
- auto encodeTime = timer.Get().MicroSeconds();
-
- auto rowSize = stream.Str().size();
- Cerr << "row size " << rowSize << ", row count " << rowCount << Endl;
- Cerr << "encoding " << rowSize * rowCount * 1000000 / encodeTime << " bytes per sec ("
- << encodeTime << " us)" << Endl;
-
- timer.Reset();
- for (size_t n = 0; n < rowCount; ++n) {
+ ops.Encode(stream);
+ }
+ auto encodeTime = timer.Get().MicroSeconds();
+
+ auto rowSize = stream.Str().size();
+ Cerr << "row size " << rowSize << ", row count " << rowCount << Endl;
+ Cerr << "encoding " << rowSize * rowCount * 1000000 / encodeTime << " bytes per sec ("
+ << encodeTime << " us)" << Endl;
+
+ timer.Reset();
+ for (size_t n = 0; n < rowCount; ++n) {
TPresortOps<Desc> ops{settings, values.begin()};
- auto str = stream.Str();
- NPresort::Decode(ops, TStringBuf(str.data(), str.size()));
- }
- auto decodeTime = timer.Get().MicroSeconds();
-
- Cerr << "decoding " << rowSize * rowCount * 1000000 / decodeTime << " bytes per sec ("
- << decodeTime << " us)" << Endl;
- Cerr << Endl;
-
- return std::make_pair(encodeTime, decodeTime);
-}
-
-template <typename T, NUdf::EDataSlot Slot, bool Desc>
-std::pair<ui64, ui64> MeasureNew() {
- constexpr size_t count = 1000;
- constexpr size_t rowCount = 100000;
-
- TScopedAlloc alloc;
- TMemoryUsageInfo memInfo("Memory");
-
- TVector<NUdf::TUnboxedValuePod> values;
- TPresortEncoder encoder;
+ auto str = stream.Str();
+ NPresort::Decode(ops, TStringBuf(str.data(), str.size()));
+ }
+ auto decodeTime = timer.Get().MicroSeconds();
+
+ Cerr << "decoding " << rowSize * rowCount * 1000000 / decodeTime << " bytes per sec ("
+ << decodeTime << " us)" << Endl;
+ Cerr << Endl;
+
+ return std::make_pair(encodeTime, decodeTime);
+}
+
+template <typename T, NUdf::EDataSlot Slot, bool Desc>
+std::pair<ui64, ui64> MeasureNew() {
+ constexpr size_t count = 1000;
+ constexpr size_t rowCount = 100000;
+
+ TScopedAlloc alloc;
+ TMemoryUsageInfo memInfo("Memory");
+
+ TVector<NUdf::TUnboxedValuePod> values;
+ TPresortEncoder encoder;
TPresortDecoder decoder;
-
- for (size_t i = 0; i < count; ++i) {
+
+ for (size_t i = 0; i < count; ++i) {
values.push_back(RandomValue<T>());
- encoder.AddType(Slot, false, Desc);
- decoder.AddType(Slot, false, Desc);
- }
-
- TSimpleTimer timer;
- TStringBuf buffer;
-
- for (size_t n = 0; n < rowCount; ++n) {
- encoder.Start();
- for (size_t i = 0; i < count; ++i) {
- encoder.Encode(values[i]);
- }
- buffer = encoder.Finish();
- }
- auto encodeTime = timer.Get().MicroSeconds();
-
- auto rowSize = buffer.size();
-
- Cerr << "row size " << rowSize << ", row count " << rowCount << Endl;
- Cerr << "encoding " << rowSize * rowCount * 1000000 / encodeTime << " bytes per sec ("
- << encodeTime << " us)" << Endl;
-
- timer.Reset();
- for (size_t n = 0; n < rowCount; ++n) {
- decoder.Start(buffer);
- for (size_t i = 0; i < count; ++i) {
- decoder.Decode();
- }
- encoder.Finish();
- }
- auto decodeTime = timer.Get().MicroSeconds();
-
- Cerr << "decoding " << rowSize * rowCount * 1000000 / decodeTime << " bytes per sec ("
- << decodeTime << " us)" << Endl;
- Cerr << Endl;
-
- return std::make_pair(encodeTime, decodeTime);
-}
-
-template <typename T, NUdf::EDataSlot Slot, bool Desc>
-void Compare() {
- auto newTimes = MeasureNew<T, Slot, Desc>();
- auto oldTimes = MeasureOld<T, Slot, Desc>();
-
- Cerr << "encoding speedup " << (double)oldTimes.first / (double)newTimes.first << Endl;
- Cerr << "decoding speedup " << (double)oldTimes.second / (double)newTimes.second << Endl;
- Cerr << "--------" << Endl << Endl;
-}
-
-template <typename T, NUdf::EDataSlot Slot>
-void CompareType(const char* type) {
- Cerr << type << Endl;
- Compare<T, Slot, false>();
-
- Cerr << type << " desc" << Endl;
- Compare<T, Slot, true>();
-}
-
-}
-
-int main(int, char**) {
- CompareType<bool, NUdf::EDataSlot::Bool>("bool");
- CompareType<ui8, NUdf::EDataSlot::Uint8>("ui8");
- CompareType<ui16, NUdf::EDataSlot::Uint16>("ui16");
- CompareType<ui32, NUdf::EDataSlot::Uint32>("ui32");
- CompareType<ui64, NUdf::EDataSlot::Uint64>("ui64");
- CompareType<float, NUdf::EDataSlot::Float>("float");
- CompareType<double, NUdf::EDataSlot::Double>("double");
- CompareType<char*, NUdf::EDataSlot::String>("string");
-
- return 0;
-}
+ encoder.AddType(Slot, false, Desc);
+ decoder.AddType(Slot, false, Desc);
+ }
+
+ TSimpleTimer timer;
+ TStringBuf buffer;
+
+ for (size_t n = 0; n < rowCount; ++n) {
+ encoder.Start();
+ for (size_t i = 0; i < count; ++i) {
+ encoder.Encode(values[i]);
+ }
+ buffer = encoder.Finish();
+ }
+ auto encodeTime = timer.Get().MicroSeconds();
+
+ auto rowSize = buffer.size();
+
+ Cerr << "row size " << rowSize << ", row count " << rowCount << Endl;
+ Cerr << "encoding " << rowSize * rowCount * 1000000 / encodeTime << " bytes per sec ("
+ << encodeTime << " us)" << Endl;
+
+ timer.Reset();
+ for (size_t n = 0; n < rowCount; ++n) {
+ decoder.Start(buffer);
+ for (size_t i = 0; i < count; ++i) {
+ decoder.Decode();
+ }
+ encoder.Finish();
+ }
+ auto decodeTime = timer.Get().MicroSeconds();
+
+ Cerr << "decoding " << rowSize * rowCount * 1000000 / decodeTime << " bytes per sec ("
+ << decodeTime << " us)" << Endl;
+ Cerr << Endl;
+
+ return std::make_pair(encodeTime, decodeTime);
+}
+
+template <typename T, NUdf::EDataSlot Slot, bool Desc>
+void Compare() {
+ auto newTimes = MeasureNew<T, Slot, Desc>();
+ auto oldTimes = MeasureOld<T, Slot, Desc>();
+
+ Cerr << "encoding speedup " << (double)oldTimes.first / (double)newTimes.first << Endl;
+ Cerr << "decoding speedup " << (double)oldTimes.second / (double)newTimes.second << Endl;
+ Cerr << "--------" << Endl << Endl;
+}
+
+template <typename T, NUdf::EDataSlot Slot>
+void CompareType(const char* type) {
+ Cerr << type << Endl;
+ Compare<T, Slot, false>();
+
+ Cerr << type << " desc" << Endl;
+ Compare<T, Slot, true>();
+}
+
+}
+
+int main(int, char**) {
+ CompareType<bool, NUdf::EDataSlot::Bool>("bool");
+ CompareType<ui8, NUdf::EDataSlot::Uint8>("ui8");
+ CompareType<ui16, NUdf::EDataSlot::Uint16>("ui16");
+ CompareType<ui32, NUdf::EDataSlot::Uint32>("ui32");
+ CompareType<ui64, NUdf::EDataSlot::Uint64>("ui64");
+ CompareType<float, NUdf::EDataSlot::Float>("float");
+ CompareType<double, NUdf::EDataSlot::Double>("double");
+ CompareType<char*, NUdf::EDataSlot::String>("string");
+
+ return 0;
+}
diff --git a/ydb/library/yql/minikql/perf/presort/ya.make b/ydb/library/yql/minikql/perf/presort/ya.make
index 148198f13df..f30b3c4ba2b 100644
--- a/ydb/library/yql/minikql/perf/presort/ya.make
+++ b/ydb/library/yql/minikql/perf/presort/ya.make
@@ -1,20 +1,20 @@
-PROGRAM()
-
+PROGRAM()
+
OWNER(monster)
-
-PEERDIR(
+
+PEERDIR(
library/cpp/presort
ydb/library/yql/minikql
ydb/library/yql/minikql/computation
ydb/library/yql/minikql/invoke_builtins
ydb/library/yql/public/udf
ydb/library/yql/public/udf/service/exception_policy
-)
-
+)
+
YQL_LAST_ABI_VERSION()
-SRCS(
- presort.cpp
-)
-
-END()
+SRCS(
+ presort.cpp
+)
+
+END()
diff --git a/ydb/library/yql/mount/lib/yql/aggregate.yql b/ydb/library/yql/mount/lib/yql/aggregate.yql
index a1c231f3392..049132c1b30 100644
--- a/ydb/library/yql/mount/lib/yql/aggregate.yql
+++ b/ydb/library/yql/mount/lib/yql/aggregate.yql
@@ -217,117 +217,117 @@
(return (AggregationTraits (ListItemType list_type) init update save load merge finish (Null)))
))))
-# list_type:type count:ui32
-# doesn't support optional values
-(let set_traits_factory_raw (lambda '(list_type count) (block '(
- (let value_type (ListItemType list_type))
- (let serialized_type (TupleType (DataType 'Uint32) (ListType value_type)))
-
- (let UdfSetCreate (Udf 'Set.Create (Void) (TupleType (TupleType value_type (DataType 'Uint32)) (StructType) value_type)))
-
- (let resource_type (TypeOf (Apply UdfSetCreate (InstanceOf value_type) (Uint32 '0))))
-
- (let UdfSetAddValue (Udf 'Set.AddValue (Void) (TupleType (TupleType resource_type value_type) (StructType) value_type)))
- (let UdfSetMerge (Udf 'Set.Merge (Void) (TupleType (TupleType resource_type resource_type) (StructType) value_type)))
- (let UdfSetSerialize (Udf 'Set.Serialize (Void) (TupleType (TupleType resource_type) (StructType) value_type)))
- (let UdfSetDeserialize (Udf 'Set.Deserialize (Void) (TupleType (TupleType serialized_type) (StructType) value_type)))
- (let UdfSetGetResult (Udf 'Set.GetResult (Void) (TupleType (TupleType resource_type) (StructType) value_type)))
-
- (let init (lambda '(value parent) (NamedApply UdfSetCreate '(value count) (AsStruct) (DependsOn parent))))
- (let update (lambda '(value state parent) (NamedApply UdfSetAddValue '(state value) (AsStruct) (DependsOn parent))))
- (let save (lambda '(state) (Apply UdfSetSerialize state)))
- (let load (lambda '(state) (Apply UdfSetDeserialize state)))
- (let merge (lambda '(one two) (Apply UdfSetMerge one two)))
- (let finish (lambda '(state) (Apply UdfSetGetResult state)))
-
+# list_type:type count:ui32
+# doesn't support optional values
+(let set_traits_factory_raw (lambda '(list_type count) (block '(
+ (let value_type (ListItemType list_type))
+ (let serialized_type (TupleType (DataType 'Uint32) (ListType value_type)))
+
+ (let UdfSetCreate (Udf 'Set.Create (Void) (TupleType (TupleType value_type (DataType 'Uint32)) (StructType) value_type)))
+
+ (let resource_type (TypeOf (Apply UdfSetCreate (InstanceOf value_type) (Uint32 '0))))
+
+ (let UdfSetAddValue (Udf 'Set.AddValue (Void) (TupleType (TupleType resource_type value_type) (StructType) value_type)))
+ (let UdfSetMerge (Udf 'Set.Merge (Void) (TupleType (TupleType resource_type resource_type) (StructType) value_type)))
+ (let UdfSetSerialize (Udf 'Set.Serialize (Void) (TupleType (TupleType resource_type) (StructType) value_type)))
+ (let UdfSetDeserialize (Udf 'Set.Deserialize (Void) (TupleType (TupleType serialized_type) (StructType) value_type)))
+ (let UdfSetGetResult (Udf 'Set.GetResult (Void) (TupleType (TupleType resource_type) (StructType) value_type)))
+
+ (let init (lambda '(value parent) (NamedApply UdfSetCreate '(value count) (AsStruct) (DependsOn parent))))
+ (let update (lambda '(value state parent) (NamedApply UdfSetAddValue '(state value) (AsStruct) (DependsOn parent))))
+ (let save (lambda '(state) (Apply UdfSetSerialize state)))
+ (let load (lambda '(state) (Apply UdfSetDeserialize state)))
+ (let merge (lambda '(one two) (Apply UdfSetMerge one two)))
+ (let finish (lambda '(state) (Apply UdfSetGetResult state)))
+
(return (AggregationTraits value_type init update save load merge finish (EmptyList)))
-))))
-
+))))
+
# list_type:type n:ui32 buffer:ui32
# doesn't support optional values
(let topfreq_traits_factory_raw (lambda '(list_type n buffer) (block '(
(let value_type (ListItemType list_type))
- (let vector_element_type (TupleType (DataType 'Uint64) value_type))
- (let serialized_type (TupleType (DataType 'Uint32) (DataType 'Uint32) (ListType vector_element_type)))
-
- (let UdfTopFreqCreate (Udf 'TopFreq.TopFreq_Create (Void) (TupleType (TupleType value_type (DataType 'Uint32)) (StructType) value_type)))
-
- (let resource_type (TypeOf (Apply UdfTopFreqCreate (InstanceOf value_type) (Uint32 '1))))
-
- (let UdfTopFreqAddValue (Udf 'TopFreq.TopFreq_AddValue (Void) (TupleType (TupleType resource_type value_type) (StructType) value_type)))
- (let UdfTopFreqMerge (Udf 'TopFreq.TopFreq_Merge (Void) (TupleType (TupleType resource_type resource_type) (StructType) value_type)))
- (let UdfTopFreqSerialize (Udf 'TopFreq.TopFreq_Serialize (Void) (TupleType (TupleType resource_type) (StructType) value_type)))
- (let UdfTopFreqDeserialize (Udf 'TopFreq.TopFreq_Deserialize (Void) (TupleType (TupleType serialized_type) (StructType) value_type)))
- (let UdfTopFreqGet (Udf 'TopFreq.TopFreq_Get (Void) (TupleType (TupleType resource_type (DataType 'Uint32)) (StructType) value_type)))
-
+ (let vector_element_type (TupleType (DataType 'Uint64) value_type))
+ (let serialized_type (TupleType (DataType 'Uint32) (DataType 'Uint32) (ListType vector_element_type)))
+
+ (let UdfTopFreqCreate (Udf 'TopFreq.TopFreq_Create (Void) (TupleType (TupleType value_type (DataType 'Uint32)) (StructType) value_type)))
+
+ (let resource_type (TypeOf (Apply UdfTopFreqCreate (InstanceOf value_type) (Uint32 '1))))
+
+ (let UdfTopFreqAddValue (Udf 'TopFreq.TopFreq_AddValue (Void) (TupleType (TupleType resource_type value_type) (StructType) value_type)))
+ (let UdfTopFreqMerge (Udf 'TopFreq.TopFreq_Merge (Void) (TupleType (TupleType resource_type resource_type) (StructType) value_type)))
+ (let UdfTopFreqSerialize (Udf 'TopFreq.TopFreq_Serialize (Void) (TupleType (TupleType resource_type) (StructType) value_type)))
+ (let UdfTopFreqDeserialize (Udf 'TopFreq.TopFreq_Deserialize (Void) (TupleType (TupleType serialized_type) (StructType) value_type)))
+ (let UdfTopFreqGet (Udf 'TopFreq.TopFreq_Get (Void) (TupleType (TupleType resource_type (DataType 'Uint32)) (StructType) value_type)))
+
(let init (lambda '(value parent) (NamedApply UdfTopFreqCreate '(value buffer) (AsStruct) (DependsOn parent))))
(let update (lambda '(value state parent) (NamedApply UdfTopFreqAddValue '(state value) (AsStruct) (DependsOn parent))))
(let save (lambda '(state) (Apply UdfTopFreqSerialize state)))
(let load (lambda '(state) (Apply UdfTopFreqDeserialize state)))
(let merge (lambda '(one two) (Apply UdfTopFreqMerge one two)))
(let finish (lambda '(state) (Apply UdfTopFreqGet state n)))
-
+
(return (AggregationTraits value_type init update save load merge finish (EmptyList)))
))))
-# list_type:type extractor:lambda count:ui32 is_top:atom
-# doesn't support optional values
-(let top_traits_factory_raw (lambda '(list_type extractor count is_top) (block '(
- (let value_type (ListItemType list_type))
- (let serialized_type (TupleType (DataType 'Uint32) (ListType value_type)))
- (let type_config (Combine '0 is_top))
-
- (let UdfTopCreate (Udf 'Top.Create (Void) (TupleType (TupleType value_type (DataType 'Uint32)) (StructType) value_type) type_config))
-
- (let resource_type (TypeOf (Apply UdfTopCreate (InstanceOf value_type) (Uint32 '0))))
-
- (let UdfTopAddValue (Udf 'Top.AddValue (Void) (TupleType (TupleType resource_type value_type) (StructType) value_type) type_config))
- (let UdfTopMerge (Udf 'Top.Merge (Void) (TupleType (TupleType resource_type resource_type) (StructType) value_type) type_config))
- (let UdfTopSerialize (Udf 'Top.Serialize (Void) (TupleType (TupleType resource_type) (StructType) value_type) type_config))
- (let UdfTopDeserialize (Udf 'Top.Deserialize (Void) (TupleType (TupleType serialized_type) (StructType) value_type) type_config))
- (let UdfTopGetResult (Udf 'Top.GetResult (Void) (TupleType (TupleType resource_type) (StructType) value_type) type_config))
-
- (let init (lambda '(value parent) (NamedApply UdfTopCreate '(value count) (AsStruct) (DependsOn parent))))
- (let update (lambda '(value state parent) (NamedApply UdfTopAddValue '(state value) (AsStruct) (DependsOn parent))))
- (let save (lambda '(state) (Apply UdfTopSerialize state)))
- (let load (lambda '(state) (Apply UdfTopDeserialize state)))
- (let merge (lambda '(one two) (Apply UdfTopMerge one two)))
- (let finish (lambda '(state) (Apply UdfTopGetResult state)))
-
+# list_type:type extractor:lambda count:ui32 is_top:atom
+# doesn't support optional values
+(let top_traits_factory_raw (lambda '(list_type extractor count is_top) (block '(
+ (let value_type (ListItemType list_type))
+ (let serialized_type (TupleType (DataType 'Uint32) (ListType value_type)))
+ (let type_config (Combine '0 is_top))
+
+ (let UdfTopCreate (Udf 'Top.Create (Void) (TupleType (TupleType value_type (DataType 'Uint32)) (StructType) value_type) type_config))
+
+ (let resource_type (TypeOf (Apply UdfTopCreate (InstanceOf value_type) (Uint32 '0))))
+
+ (let UdfTopAddValue (Udf 'Top.AddValue (Void) (TupleType (TupleType resource_type value_type) (StructType) value_type) type_config))
+ (let UdfTopMerge (Udf 'Top.Merge (Void) (TupleType (TupleType resource_type resource_type) (StructType) value_type) type_config))
+ (let UdfTopSerialize (Udf 'Top.Serialize (Void) (TupleType (TupleType resource_type) (StructType) value_type) type_config))
+ (let UdfTopDeserialize (Udf 'Top.Deserialize (Void) (TupleType (TupleType serialized_type) (StructType) value_type) type_config))
+ (let UdfTopGetResult (Udf 'Top.GetResult (Void) (TupleType (TupleType resource_type) (StructType) value_type) type_config))
+
+ (let init (lambda '(value parent) (NamedApply UdfTopCreate '(value count) (AsStruct) (DependsOn parent))))
+ (let update (lambda '(value state parent) (NamedApply UdfTopAddValue '(state value) (AsStruct) (DependsOn parent))))
+ (let save (lambda '(state) (Apply UdfTopSerialize state)))
+ (let load (lambda '(state) (Apply UdfTopDeserialize state)))
+ (let merge (lambda '(one two) (Apply UdfTopMerge one two)))
+ (let finish (lambda '(state) (Apply UdfTopGetResult state)))
+
(return (AggregationTraits value_type init update save load merge finish (EmptyList)))
-))))
-
-# list_type:type key_extractor:lambda payload_extractor:lambda count:ui32 is_top:atom
-# doesn't support optional values
-(let top_by_traits_factory_raw (lambda '(list_type key_extractor payload_extractor count is_top) (block '(
- (let value_type (ListItemType list_type))
- (let key_type (TypeOf (Apply key_extractor (InstanceOf value_type))))
- (let payload_type (TypeOf (Apply payload_extractor (InstanceOf value_type))))
- (let key_payload_type (TupleType key_type payload_type))
- (let serialized_type (TupleType (DataType 'Uint32) (ListType key_payload_type)))
- (let type_config (Combine '1 is_top))
-
- (let UdfTopCreate (Udf 'Top.Create (Void) (TupleType (TupleType key_type payload_type (DataType 'Uint32)) (StructType) key_payload_type) type_config))
-
- (let resource_type (TypeOf (Apply UdfTopCreate (InstanceOf key_type) (InstanceOf payload_type) (Uint32 '0))))
-
- (let UdfTopAddValue (Udf 'Top.AddValue (Void) (TupleType (TupleType resource_type key_type payload_type) (StructType) key_payload_type) type_config))
- (let UdfTopMerge (Udf 'Top.Merge (Void) (TupleType (TupleType resource_type resource_type) (StructType) key_payload_type) type_config))
- (let UdfTopSerialize (Udf 'Top.Serialize (Void) (TupleType (TupleType resource_type) (StructType) key_payload_type) type_config))
- (let UdfTopDeserialize (Udf 'Top.Deserialize (Void) (TupleType (TupleType serialized_type) (StructType) key_payload_type) type_config))
- (let UdfTopGetResult (Udf 'Top.GetResult (Void) (TupleType (TupleType resource_type) (StructType) key_payload_type) type_config))
-
- (let init (lambda '(value parent) (NamedApply UdfTopCreate '((Apply key_extractor value) (Apply payload_extractor value) count) (AsStruct) (DependsOn parent))))
- (let update (lambda '(value state parent) (NamedApply UdfTopAddValue '(state (Apply key_extractor value) (Apply payload_extractor value)) (AsStruct) (DependsOn parent))))
- (let save (lambda '(state) (Apply UdfTopSerialize state)))
- (let load (lambda '(state) (Apply UdfTopDeserialize state)))
- (let merge (lambda '(one two) (Apply UdfTopMerge one two)))
- (let finish (lambda '(state) (Apply UdfTopGetResult state)))
-
+))))
+
+# list_type:type key_extractor:lambda payload_extractor:lambda count:ui32 is_top:atom
+# doesn't support optional values
+(let top_by_traits_factory_raw (lambda '(list_type key_extractor payload_extractor count is_top) (block '(
+ (let value_type (ListItemType list_type))
+ (let key_type (TypeOf (Apply key_extractor (InstanceOf value_type))))
+ (let payload_type (TypeOf (Apply payload_extractor (InstanceOf value_type))))
+ (let key_payload_type (TupleType key_type payload_type))
+ (let serialized_type (TupleType (DataType 'Uint32) (ListType key_payload_type)))
+ (let type_config (Combine '1 is_top))
+
+ (let UdfTopCreate (Udf 'Top.Create (Void) (TupleType (TupleType key_type payload_type (DataType 'Uint32)) (StructType) key_payload_type) type_config))
+
+ (let resource_type (TypeOf (Apply UdfTopCreate (InstanceOf key_type) (InstanceOf payload_type) (Uint32 '0))))
+
+ (let UdfTopAddValue (Udf 'Top.AddValue (Void) (TupleType (TupleType resource_type key_type payload_type) (StructType) key_payload_type) type_config))
+ (let UdfTopMerge (Udf 'Top.Merge (Void) (TupleType (TupleType resource_type resource_type) (StructType) key_payload_type) type_config))
+ (let UdfTopSerialize (Udf 'Top.Serialize (Void) (TupleType (TupleType resource_type) (StructType) key_payload_type) type_config))
+ (let UdfTopDeserialize (Udf 'Top.Deserialize (Void) (TupleType (TupleType serialized_type) (StructType) key_payload_type) type_config))
+ (let UdfTopGetResult (Udf 'Top.GetResult (Void) (TupleType (TupleType resource_type) (StructType) key_payload_type) type_config))
+
+ (let init (lambda '(value parent) (NamedApply UdfTopCreate '((Apply key_extractor value) (Apply payload_extractor value) count) (AsStruct) (DependsOn parent))))
+ (let update (lambda '(value state parent) (NamedApply UdfTopAddValue '(state (Apply key_extractor value) (Apply payload_extractor value)) (AsStruct) (DependsOn parent))))
+ (let save (lambda '(state) (Apply UdfTopSerialize state)))
+ (let load (lambda '(state) (Apply UdfTopDeserialize state)))
+ (let merge (lambda '(one two) (Apply UdfTopMerge one two)))
+ (let finish (lambda '(state) (Apply UdfTopGetResult state)))
+
(return (AggregationTraits value_type init update save load merge finish (List (ListType payload_type))))
-))))
-
-
+))))
+
+
# list_type:type n:double
# doesn't support optional values
(let percentile_traits_factory_raw (lambda '(list_type n) (block '(
@@ -578,10 +578,10 @@
# support optional values
(let percentile_traits_factory_opt (lambda '(list_type n) (Apply optional_traits_factory_parent list_type (lambda '(list_type) (Apply percentile_traits_factory_raw list_type n)))))
-# list_type:type count:ui32
-# support optional values
+# list_type:type count:ui32
+# support optional values
(let set_traits_factory_opt (lambda '(list_type count) (Apply flatten_traits_factory_parent list_type (lambda '(list_type) (Apply set_traits_factory_raw list_type count)))))
-
+
# list_type:type n:ui32 buffer:ui32
# support optional values
(let topfreq_traits_factory_opt (lambda '(list_type n buffer) (Apply flatten_traits_factory_parent list_type (lambda '(list_type) (Apply topfreq_traits_factory_raw list_type n buffer)))))
@@ -655,10 +655,10 @@
# support optional values
(let percentile_traits_factory (lambda '(list_type extractor n) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply percentile_traits_factory_opt list_type n)))))
-# list_type:type extractor:lambda count:ui32
-# support optional values
-(let set_traits_factory (lambda '(list_type extractor count) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply set_traits_factory_opt list_type count)))))
-
+# list_type:type extractor:lambda count:ui32
+# support optional values
+(let set_traits_factory (lambda '(list_type extractor count) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply set_traits_factory_opt list_type count)))))
+
# list_type:type extractor:lambda n:ui32 buffer:ui32
# support optional values
(let topfreq_traits_factory (lambda '(list_type extractor n buffer) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply topfreq_traits_factory_opt list_type n buffer)))))
@@ -689,10 +689,10 @@
(let compare_traits_factory_list_raw (lambda '(list_type compare first second limit) (block '(
(let key_type (TypeOf (Apply first (InstanceOf (ListItemType list_type)))))
(let init (lambda '(row) '((AsList (Apply second row)) (Apply first row))))
- (let update (lambda '(row state) (If (AggrEquals (Apply first row) (Nth state '1)) '((Take (Insert (Nth state '0) (Apply second row)) limit) (Nth state '1)) (If (Apply compare (Apply first row) (Nth state '1)) (Apply init row) state))))
+ (let update (lambda '(row state) (If (AggrEquals (Apply first row) (Nth state '1)) '((Take (Insert (Nth state '0) (Apply second row)) limit) (Nth state '1)) (If (Apply compare (Apply first row) (Nth state '1)) (Apply init row) state))))
(let save (lambda '(state) state))
(let load (lambda '(state) state))
- (let merge (lambda '(one two) (If (AggrEquals (Nth one '1) (Nth two '1)) '((Take (Extend (Nth one '0) (Nth two '0)) limit) (Nth one '1)) (If (Apply compare (Nth one '1) (Nth two '1)) one two))))
+ (let merge (lambda '(one two) (If (AggrEquals (Nth one '1) (Nth two '1)) '((Take (Extend (Nth one '0) (Nth two '0)) limit) (Nth one '1)) (If (Apply compare (Nth one '1) (Nth two '1)) one two))))
(let finish (lambda '(state) (Nth state '0)))
(return (AggregationTraits (ListItemType list_type) init update save load merge finish (Null)))
))))
@@ -849,8 +849,8 @@
)))
# deprecated
-(let min_by_traits_factory (lambda '(list_type first second limit) (Apply compare_by_traits_factory list_type (lambda '(one two) (AggrLess one two)) first second limit)))
-(let max_by_traits_factory (lambda '(list_type first second limit) (Apply compare_by_traits_factory list_type (lambda '(one two) (AggrLess two one)) first second limit)))
+(let min_by_traits_factory (lambda '(list_type first second limit) (Apply compare_by_traits_factory list_type (lambda '(one two) (AggrLess one two)) first second limit)))
+(let max_by_traits_factory (lambda '(list_type first second limit) (Apply compare_by_traits_factory list_type (lambda '(one two) (AggrLess two one)) first second limit)))
(let min_by_traits_factory1 (lambda '(list_type first second) (Apply compare_by_traits_factory_opt list_type (lambda '(one two) (AggrLess one two)) first second (Void))))
(let max_by_traits_factory1 (lambda '(list_type first second) (Apply compare_by_traits_factory_opt list_type (lambda '(one two) (AggrLess two one)) first second (Void))))
@@ -867,27 +867,27 @@
(let pl (Apply payload value))
(return (MatchType pl 'Optional (lambda '() pl) (lambda '() (Just pl)))))))))))
-(let top_bottom_traits_factory (lambda '(list_type extractor count is_top)
- (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type)
+(let top_bottom_traits_factory (lambda '(list_type extractor count is_top)
+ (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type)
(Apply flatten_traits_factory_parent list_type (lambda '(list_type)
- (Apply top_traits_factory_raw list_type extractor count is_top)))))))
-
-(let top_traits_factory (lambda '(list_type extractor count)
- (Apply top_bottom_traits_factory list_type extractor count '1)))
-
-(let bottom_traits_factory (lambda '(list_type extractor count)
- (Apply top_bottom_traits_factory list_type extractor count '0)))
-
-(let top_bottom_by_traits_factory (lambda '(list_type key_extractor payload_extractor count is_top)
+ (Apply top_traits_factory_raw list_type extractor count is_top)))))))
+
+(let top_traits_factory (lambda '(list_type extractor count)
+ (Apply top_bottom_traits_factory list_type extractor count '1)))
+
+(let bottom_traits_factory (lambda '(list_type extractor count)
+ (Apply top_bottom_traits_factory list_type extractor count '0)))
+
+(let top_bottom_by_traits_factory (lambda '(list_type key_extractor payload_extractor count is_top)
(Apply list_by_traits_factory_opt_parent (lambda '(list_type key_extractor payload_extractor)
- (Apply top_by_traits_factory_raw list_type key_extractor payload_extractor count is_top)) list_type key_extractor payload_extractor)))
-
-(let top_by_traits_factory (lambda '(list_type key_extractor payload_extractor count)
- (Apply top_bottom_by_traits_factory list_type key_extractor payload_extractor count '1)))
-
-(let bottom_by_traits_factory (lambda '(list_type key_extractor payload_extractor count)
- (Apply top_bottom_by_traits_factory list_type key_extractor payload_extractor count '0)))
-
+ (Apply top_by_traits_factory_raw list_type key_extractor payload_extractor count is_top)) list_type key_extractor payload_extractor)))
+
+(let top_by_traits_factory (lambda '(list_type key_extractor payload_extractor count)
+ (Apply top_bottom_by_traits_factory list_type key_extractor payload_extractor count '1)))
+
+(let bottom_by_traits_factory (lambda '(list_type key_extractor payload_extractor count)
+ (Apply top_bottom_by_traits_factory list_type key_extractor payload_extractor count '0)))
+
(export min_traits_factory)
(export max_traits_factory)
(export sum_traits_factory)
@@ -928,12 +928,12 @@
(export histogram_logarithmic_traits_factory)
(export udaf_traits_factory)
(export percentile_traits_factory)
-(export set_traits_factory)
+(export set_traits_factory)
(export topfreq_traits_factory)
(export hyperloglog_traits_factory)
-(export top_traits_factory)
-(export bottom_traits_factory)
-(export top_by_traits_factory)
-(export bottom_by_traits_factory)
+(export top_traits_factory)
+(export bottom_traits_factory)
+(export top_by_traits_factory)
+(export bottom_by_traits_factory)
)
diff --git a/ydb/library/yql/mount/lib/yql/window.yql b/ydb/library/yql/mount/lib/yql/window.yql
index 371f4c687c8..627d31a153c 100644
--- a/ydb/library/yql/mount/lib/yql/window.yql
+++ b/ydb/library/yql/mount/lib/yql/window.yql
@@ -224,89 +224,89 @@
# list_type:type n:ui32 buffer:ui32
# doesn't support optional values
-(let set_traits_factory_raw (lambda '(list_type count) (block '(
- (let value_type (ListItemType list_type))
-
- (let UdfSetCreate (Udf 'Set.Create (Void) (TupleType (TupleType value_type (DataType 'Uint32)) (StructType) value_type)))
-
- (let resource_type (TypeOf (Apply UdfSetCreate (InstanceOf value_type) (Uint32 '0))))
-
- (let UdfSetAddValue (Udf 'Set.AddValue (Void) (TupleType (TupleType resource_type value_type) (StructType) value_type)))
- (let UdfSetGetResult (Udf 'Set.GetResult (Void) (TupleType (TupleType resource_type) (StructType) value_type)))
-
- (let init (lambda '(value parent) (NamedApply UdfSetCreate '(value count) (AsStruct) (DependsOn parent))))
- (let update (lambda '(value state parent) (NamedApply UdfSetAddValue '(state value) (AsStruct) (DependsOn parent))))
- (let shift (lambda '(value state) (Void)))
- (let current (lambda '(state) (Apply UdfSetGetResult state)))
-
+(let set_traits_factory_raw (lambda '(list_type count) (block '(
+ (let value_type (ListItemType list_type))
+
+ (let UdfSetCreate (Udf 'Set.Create (Void) (TupleType (TupleType value_type (DataType 'Uint32)) (StructType) value_type)))
+
+ (let resource_type (TypeOf (Apply UdfSetCreate (InstanceOf value_type) (Uint32 '0))))
+
+ (let UdfSetAddValue (Udf 'Set.AddValue (Void) (TupleType (TupleType resource_type value_type) (StructType) value_type)))
+ (let UdfSetGetResult (Udf 'Set.GetResult (Void) (TupleType (TupleType resource_type) (StructType) value_type)))
+
+ (let init (lambda '(value parent) (NamedApply UdfSetCreate '(value count) (AsStruct) (DependsOn parent))))
+ (let update (lambda '(value state parent) (NamedApply UdfSetAddValue '(state value) (AsStruct) (DependsOn parent))))
+ (let shift (lambda '(value state) (Void)))
+ (let current (lambda '(state) (Apply UdfSetGetResult state)))
+
(return (WindowTraits value_type init update shift current (EmptyList)))
-))))
-
-# list_type:type n:ui32 buffer:ui32
-# doesn't support optional values
+))))
+
+# list_type:type n:ui32 buffer:ui32
+# doesn't support optional values
(let topfreq_traits_factory_raw (lambda '(list_type n buffer) (block '(
(let value_type (ListItemType list_type))
-
- (let UdfTopFreqCreate (Udf 'TopFreq.TopFreq_Create (Void) (TupleType (TupleType value_type (DataType 'Uint32)) (StructType) value_type)))
-
- (let resource_type (TypeOf (Apply UdfTopFreqCreate (InstanceOf value_type) (Uint32 '0))))
-
- (let UdfTopFreqAddValue (Udf 'TopFreq.TopFreq_AddValue (Void) (TupleType (TupleType resource_type value_type) (StructType) value_type)))
- (let UdfTopFreqGet (Udf 'TopFreq.TopFreq_Get (Void) (TupleType (TupleType resource_type (DataType 'Uint32)) (StructType) value_type)))
-
+
+ (let UdfTopFreqCreate (Udf 'TopFreq.TopFreq_Create (Void) (TupleType (TupleType value_type (DataType 'Uint32)) (StructType) value_type)))
+
+ (let resource_type (TypeOf (Apply UdfTopFreqCreate (InstanceOf value_type) (Uint32 '0))))
+
+ (let UdfTopFreqAddValue (Udf 'TopFreq.TopFreq_AddValue (Void) (TupleType (TupleType resource_type value_type) (StructType) value_type)))
+ (let UdfTopFreqGet (Udf 'TopFreq.TopFreq_Get (Void) (TupleType (TupleType resource_type (DataType 'Uint32)) (StructType) value_type)))
+
(let init (lambda '(value parent) (NamedApply UdfTopFreqCreate '(value buffer) (AsStruct) (DependsOn parent))))
(let update (lambda '(value state parent) (NamedApply UdfTopFreqAddValue '(state value) (AsStruct) (DependsOn parent))))
(let shift (lambda '(value state) (Void)))
(let current (lambda '(state) (Apply UdfTopFreqGet state n)))
-
+
(return (WindowTraits value_type init update shift current (EmptyList)))
))))
-# list_type:type extractor:lambda count:ui32 is_top:atom
-# doesn't support optional values
-(let top_traits_factory_raw (lambda '(list_type extractor count is_top) (block '(
- (let value_type (ListItemType list_type))
- (let type_config (Combine '0 is_top))
-
- (let UdfTopCreate (Udf 'Top.Create (Void) (TupleType (TupleType value_type (DataType 'Uint32)) (StructType) value_type) type_config))
-
- (let resource_type (TypeOf (Apply UdfTopCreate (InstanceOf value_type) (Uint32 '0))))
-
- (let UdfTopAddValue (Udf 'Top.AddValue (Void) (TupleType (TupleType resource_type value_type) (StructType) value_type) type_config))
- (let UdfTopGetResult (Udf 'Top.GetResult (Void) (TupleType (TupleType resource_type) (StructType) value_type) type_config))
-
- (let init (lambda '(value parent) (NamedApply UdfTopCreate '(value count) (AsStruct) (DependsOn parent))))
- (let update (lambda '(value state parent) (NamedApply UdfTopAddValue '(state value) (AsStruct) (DependsOn parent))))
- (let shift (lambda '(value state) (Void)))
- (let current (lambda '(state) (Apply UdfTopGetResult state)))
-
+# list_type:type extractor:lambda count:ui32 is_top:atom
+# doesn't support optional values
+(let top_traits_factory_raw (lambda '(list_type extractor count is_top) (block '(
+ (let value_type (ListItemType list_type))
+ (let type_config (Combine '0 is_top))
+
+ (let UdfTopCreate (Udf 'Top.Create (Void) (TupleType (TupleType value_type (DataType 'Uint32)) (StructType) value_type) type_config))
+
+ (let resource_type (TypeOf (Apply UdfTopCreate (InstanceOf value_type) (Uint32 '0))))
+
+ (let UdfTopAddValue (Udf 'Top.AddValue (Void) (TupleType (TupleType resource_type value_type) (StructType) value_type) type_config))
+ (let UdfTopGetResult (Udf 'Top.GetResult (Void) (TupleType (TupleType resource_type) (StructType) value_type) type_config))
+
+ (let init (lambda '(value parent) (NamedApply UdfTopCreate '(value count) (AsStruct) (DependsOn parent))))
+ (let update (lambda '(value state parent) (NamedApply UdfTopAddValue '(state value) (AsStruct) (DependsOn parent))))
+ (let shift (lambda '(value state) (Void)))
+ (let current (lambda '(state) (Apply UdfTopGetResult state)))
+
(return (WindowTraits value_type init update shift current (EmptyList)))
-))))
-
-# list_type:type key_extractor:lambda payload_extractor:lambda count:ui32 is_top:atom
-# doesn't support optional values
-(let top_by_traits_factory_raw (lambda '(list_type key_extractor payload_extractor count is_top) (block '(
- (let value_type (ListItemType list_type))
- (let key_type (TypeOf (Apply key_extractor (InstanceOf value_type))))
- (let payload_type (TypeOf (Apply payload_extractor (InstanceOf value_type))))
- (let key_payload_type (TupleType key_type payload_type))
- (let type_config (Combine '1 is_top))
-
- (let UdfTopCreate (Udf 'Top.Create (Void) (TupleType (TupleType key_type payload_type (DataType 'Uint32)) (StructType) key_payload_type) type_config))
-
- (let resource_type (TypeOf (Apply UdfTopCreate (InstanceOf key_type) (InstanceOf payload_type) (Uint32 '0))))
-
- (let UdfTopAddValue (Udf 'Top.AddValue (Void) (TupleType (TupleType resource_type key_type payload_type) (StructType) key_payload_type) type_config))
- (let UdfTopGetResult (Udf 'Top.GetResult (Void) (TupleType (TupleType resource_type) (StructType) key_payload_type) type_config))
-
- (let init (lambda '(value parent) (NamedApply UdfTopCreate '((Apply key_extractor value) (Apply payload_extractor value) count) (AsStruct) (DependsOn parent))))
- (let update (lambda '(value state parent) (NamedApply UdfTopAddValue '(state (Apply key_extractor value) (Apply payload_extractor value)) (AsStruct) (DependsOn parent))))
- (let shift (lambda '(value state) (Void)))
- (let current (lambda '(state) (Apply UdfTopGetResult state)))
-
+))))
+
+# list_type:type key_extractor:lambda payload_extractor:lambda count:ui32 is_top:atom
+# doesn't support optional values
+(let top_by_traits_factory_raw (lambda '(list_type key_extractor payload_extractor count is_top) (block '(
+ (let value_type (ListItemType list_type))
+ (let key_type (TypeOf (Apply key_extractor (InstanceOf value_type))))
+ (let payload_type (TypeOf (Apply payload_extractor (InstanceOf value_type))))
+ (let key_payload_type (TupleType key_type payload_type))
+ (let type_config (Combine '1 is_top))
+
+ (let UdfTopCreate (Udf 'Top.Create (Void) (TupleType (TupleType key_type payload_type (DataType 'Uint32)) (StructType) key_payload_type) type_config))
+
+ (let resource_type (TypeOf (Apply UdfTopCreate (InstanceOf key_type) (InstanceOf payload_type) (Uint32 '0))))
+
+ (let UdfTopAddValue (Udf 'Top.AddValue (Void) (TupleType (TupleType resource_type key_type payload_type) (StructType) key_payload_type) type_config))
+ (let UdfTopGetResult (Udf 'Top.GetResult (Void) (TupleType (TupleType resource_type) (StructType) key_payload_type) type_config))
+
+ (let init (lambda '(value parent) (NamedApply UdfTopCreate '((Apply key_extractor value) (Apply payload_extractor value) count) (AsStruct) (DependsOn parent))))
+ (let update (lambda '(value state parent) (NamedApply UdfTopAddValue '(state (Apply key_extractor value) (Apply payload_extractor value)) (AsStruct) (DependsOn parent))))
+ (let shift (lambda '(value state) (Void)))
+ (let current (lambda '(state) (Apply UdfTopGetResult state)))
+
(return (WindowTraits value_type init update shift current (List (ListType payload_type))))
-))))
-
+))))
+
# list_type:type n:double
# doesn't support optional values
(let percentile_traits_factory_raw (lambda '(list_type n) (block '(
@@ -550,10 +550,10 @@
# support optional values
(let percentile_traits_factory_opt (lambda '(list_type n) (Apply optional_traits_factory_parent list_type (lambda '(list_type) (Apply percentile_traits_factory_raw list_type n)))))
-# list_type:type count:ui32
-# support optional values
+# list_type:type count:ui32
+# support optional values
(let set_traits_factory_opt (lambda '(list_type count) (Apply flatten_traits_factory_parent list_type (lambda '(list_type) (Apply set_traits_factory_raw list_type count)))))
-
+
# list_type:type n:ui32 buffer:ui32
# support optional values
(let topfreq_traits_factory_opt (lambda '(list_type n buffer) (Apply flatten_traits_factory_parent list_type (lambda '(list_type) (Apply topfreq_traits_factory_raw list_type n buffer)))))
@@ -630,10 +630,10 @@
# support optional values
(let percentile_traits_factory (lambda '(list_type extractor n) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply percentile_traits_factory_opt list_type n)))))
-# list_type:type extractor:lambda count:ui32
-# support optional values
-(let set_traits_factory (lambda '(list_type extractor count) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply set_traits_factory_opt list_type count)))))
-
+# list_type:type extractor:lambda count:ui32
+# support optional values
+(let set_traits_factory (lambda '(list_type extractor count) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply set_traits_factory_opt list_type count)))))
+
# list_type:type extractor:lambda n:ui32 buffer:ui32
# support optional values
(let topfreq_traits_factory (lambda '(list_type extractor n buffer) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply topfreq_traits_factory_opt list_type n buffer)))))
@@ -662,7 +662,7 @@
(let compare_traits_factory_list_raw (lambda '(list_type compare first second limit) (block '(
(let key_type (TypeOf (Apply first (InstanceOf (ListItemType list_type)))))
(let init (lambda '(row) '((AsList (Apply second row)) (Apply first row))))
- (let update (lambda '(row state) (If (AggrEquals (Apply first row) (Nth state '1)) '((Take (Insert (Nth state '0) (Apply second row)) limit) (Nth state '1)) (If (Apply compare (Apply first row) (Nth state '1)) (Apply init row) state))))
+ (let update (lambda '(row state) (If (AggrEquals (Apply first row) (Nth state '1)) '((Take (Insert (Nth state '0) (Apply second row)) limit) (Nth state '1)) (If (Apply compare (Apply first row) (Nth state '1)) (Apply init row) state))))
(let shift (lambda '(value state) (Void)))
(let current (lambda '(state) (Nth state '0)))
(return (WindowTraits (ListItemType list_type) init update shift current (Null)))
@@ -798,18 +798,18 @@
(let histogram_linear_traits_factory (lambda '(list_type value binsize minimum maximum) (Apply double_traits_factory_opt_parent (lambda '(list_type value binsize) (Apply histogram_linear_traits_factory_raw list_type value binsize minimum maximum)) list_type value binsize)))
(let histogram_logarithmic_traits_factory (lambda '(list_type value binsize minimum maximum) (Apply double_traits_factory_opt_parent (lambda '(list_type value binsize) (Apply histogram_logarithmic_traits_factory_raw list_type value binsize minimum maximum)) list_type value binsize)))
-(let compare_by_traits_factory_opt (lambda '(list_type compare first second limit) (Apply double_traits_factory_opt (lambda '(list_type first second) (Apply compare_traits_factory_raw list_type compare first second limit)) list_type first second)))
-(let list_compare_by_traits_factory_opt (lambda '(list_type compare first second limit) (Apply double_traits_factory_opt (lambda '(list_type first second) (Apply compare_traits_factory_list_raw list_type compare first second limit)) list_type first second)))
+(let compare_by_traits_factory_opt (lambda '(list_type compare first second limit) (Apply double_traits_factory_opt (lambda '(list_type first second) (Apply compare_traits_factory_raw list_type compare first second limit)) list_type first second)))
+(let list_compare_by_traits_factory_opt (lambda '(list_type compare first second limit) (Apply double_traits_factory_opt (lambda '(list_type first second) (Apply compare_traits_factory_list_raw list_type compare first second limit)) list_type first second)))
#deprecated
(let compare_by_traits_factory (lambda '(list_type compare first second limit) (IfType limit (VoidType)
- (lambda '() (Apply compare_by_traits_factory_opt list_type compare first second limit))
- (lambda '() (Apply list_compare_by_traits_factory_opt list_type compare first second limit))
+ (lambda '() (Apply compare_by_traits_factory_opt list_type compare first second limit))
+ (lambda '() (Apply list_compare_by_traits_factory_opt list_type compare first second limit))
)))
#deprecated
-(let min_by_traits_factory (lambda '(list_type first second limit) (Apply compare_by_traits_factory list_type (lambda '(one two) (AggrLess one two)) first second limit)))
-(let max_by_traits_factory (lambda '(list_type first second limit) (Apply compare_by_traits_factory list_type (lambda '(one two) (AggrLess two one)) first second limit)))
+(let min_by_traits_factory (lambda '(list_type first second limit) (Apply compare_by_traits_factory list_type (lambda '(one two) (AggrLess one two)) first second limit)))
+(let max_by_traits_factory (lambda '(list_type first second limit) (Apply compare_by_traits_factory list_type (lambda '(one two) (AggrLess two one)) first second limit)))
(let min_by_traits_factory1 (lambda '(list_type first second) (Apply compare_by_traits_factory_opt list_type (lambda '(one two) (AggrLess one two)) first second (Void))))
(let max_by_traits_factory1 (lambda '(list_type first second) (Apply compare_by_traits_factory_opt list_type (lambda '(one two) (AggrLess two one)) first second (Void))))
@@ -832,27 +832,27 @@
(let udaf_traits_factory (lambda '(list_type extractor init update merge finish save load defval) (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type) (Apply udaf_traits_factory_opt list_type init update finish defval)))))
-(let top_bottom_traits_factory (lambda '(list_type extractor count is_top)
- (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type)
+(let top_bottom_traits_factory (lambda '(list_type extractor count is_top)
+ (Apply extractor_traits_factory_parent list_type extractor (lambda '(list_type)
(Apply flatten_traits_factory_parent list_type (lambda '(list_type)
- (Apply top_traits_factory_raw list_type extractor count is_top)))))))
-
-(let top_traits_factory (lambda '(list_type extractor count)
- (Apply top_bottom_traits_factory list_type extractor count '1)))
-
-(let bottom_traits_factory (lambda '(list_type extractor count)
- (Apply top_bottom_traits_factory list_type extractor count '0)))
-
-(let top_bottom_by_traits_factory (lambda '(list_type key_extractor payload_extractor count is_top)
+ (Apply top_traits_factory_raw list_type extractor count is_top)))))))
+
+(let top_traits_factory (lambda '(list_type extractor count)
+ (Apply top_bottom_traits_factory list_type extractor count '1)))
+
+(let bottom_traits_factory (lambda '(list_type extractor count)
+ (Apply top_bottom_traits_factory list_type extractor count '0)))
+
+(let top_bottom_by_traits_factory (lambda '(list_type key_extractor payload_extractor count is_top)
(Apply list_by_traits_factory_opt_parent (lambda '(list_type key_extractor payload_extractor)
- (Apply top_by_traits_factory_raw list_type key_extractor payload_extractor count is_top)) list_type key_extractor payload_extractor)))
-
-(let top_by_traits_factory (lambda '(list_type key_extractor payload_extractor count)
- (Apply top_bottom_by_traits_factory list_type key_extractor payload_extractor count '1)))
-
-(let bottom_by_traits_factory (lambda '(list_type key_extractor payload_extractor count)
- (Apply top_bottom_by_traits_factory list_type key_extractor payload_extractor count '0)))
-
+ (Apply top_by_traits_factory_raw list_type key_extractor payload_extractor count is_top)) list_type key_extractor payload_extractor)))
+
+(let top_by_traits_factory (lambda '(list_type key_extractor payload_extractor count)
+ (Apply top_bottom_by_traits_factory list_type key_extractor payload_extractor count '1)))
+
+(let bottom_by_traits_factory (lambda '(list_type key_extractor payload_extractor count)
+ (Apply top_bottom_by_traits_factory list_type key_extractor payload_extractor count '0)))
+
(export min_traits_factory)
(export max_traits_factory)
(export sum_traits_factory)
@@ -893,7 +893,7 @@
(export histogram_logarithmic_traits_factory)
(export udwf_traits_factory)
(export percentile_traits_factory)
-(export set_traits_factory)
+(export set_traits_factory)
(export topfreq_traits_factory)
(export hyperloglog_traits_factory)
(export rank_traits_factory)
@@ -903,9 +903,9 @@
(export first_value_ignore_nulls_traits_factory)
(export last_value_ignore_nulls_traits_factory)
(export udaf_traits_factory)
-(export top_traits_factory)
-(export bottom_traits_factory)
-(export top_by_traits_factory)
-(export bottom_by_traits_factory)
+(export top_traits_factory)
+(export bottom_traits_factory)
+(export top_by_traits_factory)
+(export bottom_by_traits_factory)
)
diff --git a/ydb/library/yql/providers/common/codec/yql_codec.cpp b/ydb/library/yql/providers/common/codec/yql_codec.cpp
index 7587e188fab..dea9c6350a2 100644
--- a/ydb/library/yql/providers/common/codec/yql_codec.cpp
+++ b/ydb/library/yql/providers/common/codec/yql_codec.cpp
@@ -51,22 +51,22 @@ void WriteYsonValueImpl(TYsonResultWriter& writer, const NUdf::TUnboxedValuePod&
auto dataType = AS_TYPE(TDataType, type);
switch (dataType->GetSchemeType()) {
case NUdf::TDataType<bool>::Id:
- writer.OnBooleanScalar(value.Get<bool>());
+ writer.OnBooleanScalar(value.Get<bool>());
return;
case NUdf::TDataType<i32>::Id:
- writer.OnInt64Scalar(value.Get<i32>());
+ writer.OnInt64Scalar(value.Get<i32>());
return;
case NUdf::TDataType<ui32>::Id:
- writer.OnUint64Scalar(value.Get<ui32>());
+ writer.OnUint64Scalar(value.Get<ui32>());
return;
case NUdf::TDataType<i64>::Id:
- writer.OnInt64Scalar(value.Get<i64>());
+ writer.OnInt64Scalar(value.Get<i64>());
return;
case NUdf::TDataType<ui64>::Id:
- writer.OnUint64Scalar(value.Get<ui64>());
+ writer.OnUint64Scalar(value.Get<ui64>());
return;
case NUdf::TDataType<ui8>::Id:
- writer.OnUint64Scalar(value.Get<ui8>());
+ writer.OnUint64Scalar(value.Get<ui8>());
return;
case NUdf::TDataType<i8>::Id:
writer.OnInt64Scalar(value.Get<i8>());
diff --git a/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp b/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp
index dd1f162d784..a2817179ae1 100644
--- a/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp
+++ b/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp
@@ -1568,10 +1568,10 @@ TMkqlCommonCallableCompiler::TShared::TShared() {
AddCallable("HoppingCore", [](const TExprNode& node, TMkqlBuildContext& ctx) {
const auto stream = MkqlBuildExpr(node.Head(), ctx);
-
+
const auto timeExtractor = [&](TRuntimeNode item) {
return MkqlBuildLambda(*node.Child(1), ctx, {item});
- };
+ };
const auto hop = MkqlBuildExpr(*node.Child(2), ctx);
const auto interval = MkqlBuildExpr(*node.Child(3), ctx);
const auto delay = MkqlBuildExpr(*node.Child(4), ctx);
@@ -1581,24 +1581,24 @@ TMkqlCommonCallableCompiler::TShared::TShared() {
};
const auto update = [&](TRuntimeNode item, TRuntimeNode state) {
return MkqlBuildLambda(*node.Child(6), ctx, {item, state});
- };
+ };
const auto save = node.Child(3)->IsCallable("Void") ? std::function<TRuntimeNode(TRuntimeNode)>() : [&](TRuntimeNode state) {
return MkqlBuildLambda(*node.Child(7), ctx, {state});
- };
+ };
const auto load = node.Child(4)->IsCallable("Void") ? std::function<TRuntimeNode(TRuntimeNode)>() : [&](TRuntimeNode state) {
return MkqlBuildLambda(*node.Child(8), ctx, {state});
- };
+ };
const auto merge = [&](TRuntimeNode state1, TRuntimeNode state2) {
return MkqlBuildLambda(*node.Child(9), ctx, {state1, state2});
- };
+ };
const auto finish = [&](TRuntimeNode state, TRuntimeNode time) {
return MkqlBuildLambda(*node.Child(10), ctx, {state, time});
- };
-
+ };
+
return ctx.ProgramBuilder.HoppingCore(
stream, timeExtractor, init, update, save, load, merge, finish, hop, interval, delay);
- });
-
+ });
+
AddCallable("MultiHoppingCore", [](const TExprNode& node, TMkqlBuildContext& ctx) {
const auto stream = MkqlBuildExpr(node.Head(), ctx);
diff --git a/ydb/library/yql/providers/common/proto/gateways_config.proto b/ydb/library/yql/providers/common/proto/gateways_config.proto
index 8561fe25368..1f6b9de3393 100644
--- a/ydb/library/yql/providers/common/proto/gateways_config.proto
+++ b/ydb/library/yql/providers/common/proto/gateways_config.proto
@@ -112,7 +112,7 @@ message TKikimrClusterConfig {
repeated THostPort MessageBus = 3;
optional TKikimrGrpcData Grpc = 4;
optional string Database = 5;
- optional uint32 TvmId = 6 [default = 0];
+ optional uint32 TvmId = 6 [default = 0];
optional string Token = 7;
optional string ProxyUrl = 8;
optional string Location = 9;
@@ -132,7 +132,7 @@ message TKikimrMvpProxyConfig {
optional string Host = 1;
optional uint32 Port = 2;
optional string BasePath = 3;
- optional uint32 TvmId = 4 [default = 0];
+ optional uint32 TvmId = 4 [default = 0];
}
message TKikimrMvpGatewayConfig {
@@ -187,13 +187,13 @@ message TClickHouseGatewayConfig {
repeated TAttr DefaultSettings = 2;
}
-///////////////////////////// RTMR /////////////////////////////
+///////////////////////////// RTMR /////////////////////////////
enum EYfArtifactType {
AT_NONE = 0;
AT_FILE = 1;
AT_LAYER = 2;
};
-
+
enum ERtmrOperationType {
OT_NONE = 0;
OT_LF_PARSE = 1;
@@ -218,16 +218,16 @@ message TRtmrPqConsumerInfo {
optional string Consumer = 2;
}
-message TRtmrClusterConfig {
- optional string Name = 1;
- optional bool Default = 2 [default = false];
- optional string Cluster = 3;
- optional string RemoteName = 4;
- optional string YfCluster = 5;
- optional string S3TokenPath = 6;
- optional string S3FileCachePath = 7;
- optional uint32 TvmId = 8 [default = 0];
- optional string TvmSecretPath = 9;
+message TRtmrClusterConfig {
+ optional string Name = 1;
+ optional bool Default = 2 [default = false];
+ optional string Cluster = 3;
+ optional string RemoteName = 4;
+ optional string YfCluster = 5;
+ optional string S3TokenPath = 6;
+ optional string S3FileCachePath = 7;
+ optional uint32 TvmId = 8 [default = 0];
+ optional string TvmSecretPath = 9;
repeated string Users = 10;
optional bool UploadViaYfClient = 11 [default = false];
optional string MdsTorrentUrl = 12;
@@ -236,20 +236,20 @@ message TRtmrClusterConfig {
repeated TRtmrPqConsumerInfo PqConsumerMapping = 15;
optional uint32 MaxPqPartitions = 16;
optional uint32 PreviewCollectTimeoutMs = 17;
- repeated TAttr Settings = 101;
-}
-
-message TRtmrGatewayConfig {
- repeated TRtmrClusterConfig ClusterMapping = 1;
- repeated TAttr DefaultSettings = 2;
- optional string YqlRtmrDynLib = 3; // path to libyql-dynlib.so
+ repeated TAttr Settings = 101;
+}
+
+message TRtmrGatewayConfig {
+ repeated TRtmrClusterConfig ClusterMapping = 1;
+ repeated TAttr DefaultSettings = 2;
+ optional string YqlRtmrDynLib = 3; // path to libyql-dynlib.so
optional bool UseFakeYfUpload = 4 [default = false]; // use in tests only to speedup upload
repeated string Artifacts = 5; /// default artifacts to put into every function
repeated TRtmrOperationArtifactsInfo CommonArtifacts = 6; // common artifacts for all clusters
optional uint32 MaxPqPartitions = 7 [default = 10];
optional uint32 PreviewCollectTimeoutMs = 8 [default = 2000];
-}
-
+}
+
///////////////////////////// Pq //////////////////////////////
message TPqClusterConfig {
@@ -331,9 +331,9 @@ message TS3GatewayConfig {
repeated TAttr DefaultSettings = 100;
}
-///////////////////////////// Solomon /////////////////////////////
-
-message TSolomonClusterConfig {
+///////////////////////////// Solomon /////////////////////////////
+
+message TSolomonClusterConfig {
enum ESolomonClusterType {
SCT_UNSPECIFIED = 0;
SCT_SOLOMON = 1;
@@ -345,23 +345,23 @@ message TSolomonClusterConfig {
required string Cluster = 2; // FolderId for YC
}
- optional string Name = 1;
- optional string Cluster = 2;
+ optional string Name = 1;
+ optional string Cluster = 2;
optional bool UseSsl = 3;
optional ESolomonClusterType ClusterType = 4 [default = SCT_SOLOMON];
optional string Token = 5;
optional string ServiceAccountId = 6;
optional string ServiceAccountIdSignature = 7;
optional TShardPath Path = 8;
-
- repeated TAttr Settings = 100;
-}
-
-message TSolomonGatewayConfig {
- repeated TSolomonClusterConfig ClusterMapping = 1;
- repeated TAttr DefaultSettings = 2;
-}
-
+
+ repeated TAttr Settings = 100;
+}
+
+message TSolomonGatewayConfig {
+ repeated TSolomonClusterConfig ClusterMapping = 1;
+ repeated TAttr DefaultSettings = 2;
+}
+
message TFileStorageAdditionalConfig {
repeated string AllowedUrlPatterns = 1;
}
@@ -439,11 +439,11 @@ message TGatewaysConfig {
optional TYtGatewayConfig Yt = 1;
optional TKikimrGatewayConfig Kikimr = 2;
optional TClickHouseGatewayConfig ClickHouse = 3;
- optional TRtmrGatewayConfig Rtmr = 4;
+ optional TRtmrGatewayConfig Rtmr = 4;
optional TKikimrMvpGatewayConfig KikimrMvp = 5;
optional TStatGatewayConfig Stat = 6;
optional TChytGatewayConfig Chyt = 7;
- optional TSolomonGatewayConfig Solomon = 8;
+ optional TSolomonGatewayConfig Solomon = 8;
optional TFileStorageAdditionalConfig Fs = 9;
optional TYqlCoreConfig YqlCore = 10;
optional TPostgresqlGatewayConfig Postgresql = 11;
diff --git a/ydb/library/yql/providers/common/provider/yql_data_provider_impl.cpp b/ydb/library/yql/providers/common/provider/yql_data_provider_impl.cpp
index 8de3d57fd3a..26cfc79ccdc 100644
--- a/ydb/library/yql/providers/common/provider/yql_data_provider_impl.cpp
+++ b/ydb/library/yql/providers/common/provider/yql_data_provider_impl.cpp
@@ -82,12 +82,12 @@ IGraphTransformer& TDataProviderBase::GetConfigurationTransformer() {
return NullTransformer_;
}
-TExprNode::TPtr TDataProviderBase::GetClusterInfo(const TString& cluster, TExprContext& ctx) {
- Y_UNUSED(cluster);
- Y_UNUSED(ctx);
- return {};
-}
-
+TExprNode::TPtr TDataProviderBase::GetClusterInfo(const TString& cluster, TExprContext& ctx) {
+ Y_UNUSED(cluster);
+ Y_UNUSED(ctx);
+ return {};
+}
+
const THashMap<TString, TString>* TDataProviderBase::GetClusterTokens() {
return nullptr;
}
diff --git a/ydb/library/yql/providers/common/provider/yql_data_provider_impl.h b/ydb/library/yql/providers/common/provider/yql_data_provider_impl.h
index 242a39877e1..bdef7492b9e 100644
--- a/ydb/library/yql/providers/common/provider/yql_data_provider_impl.h
+++ b/ydb/library/yql/providers/common/provider/yql_data_provider_impl.h
@@ -43,7 +43,7 @@ public:
bool Initialize(TExprContext& ctx) override;
IGraphTransformer& GetConfigurationTransformer() override;
- TExprNode::TPtr GetClusterInfo(const TString& cluster, TExprContext& ctx) override;
+ TExprNode::TPtr GetClusterInfo(const TString& cluster, TExprContext& ctx) override;
const THashMap<TString, TString>* GetClusterTokens() override;
IGraphTransformer& GetIODiscoveryTransformer() override;
IGraphTransformer& GetEpochsTransformer() override;
diff --git a/ydb/library/yql/providers/common/provider/yql_provider.cpp b/ydb/library/yql/providers/common/provider/yql_provider.cpp
index aa782ee93ff..ae524b0d99c 100644
--- a/ydb/library/yql/providers/common/provider/yql_provider.cpp
+++ b/ydb/library/yql/providers/common/provider/yql_provider.cpp
@@ -152,8 +152,8 @@ TWriteTableSettings ParseWriteTableSettings(TExprList node, TExprContext& ctx) {
TMaybeNode<TCoAtom> mode;
TMaybeNode<TExprList> columns;
TMaybeNode<TCoAtomList> primaryKey;
- TMaybeNode<TCoAtomList> partitionBy;
- TMaybeNode<TCoNameValueTupleList> orderBy;
+ TMaybeNode<TCoAtomList> partitionBy;
+ TMaybeNode<TCoNameValueTupleList> orderBy;
TMaybeNode<TCoLambda> filter;
TMaybeNode<TCoLambda> update;
TVector<TCoNameValueTuple> other;
@@ -176,12 +176,12 @@ TWriteTableSettings ParseWriteTableSettings(TExprList node, TExprContext& ctx) {
} else if (name == "primarykey") {
YQL_ENSURE(tuple.Value().Maybe<TCoAtomList>());
primaryKey = tuple.Value().Cast<TCoAtomList>();
- } else if (name == "partitionby") {
- YQL_ENSURE(tuple.Value().Maybe<TCoAtomList>());
- partitionBy = tuple.Value().Cast<TCoAtomList>();
- } else if (name == "orderby") {
- YQL_ENSURE(tuple.Value().Maybe<TCoNameValueTupleList>());
- orderBy = tuple.Value().Cast<TCoNameValueTupleList>();
+ } else if (name == "partitionby") {
+ YQL_ENSURE(tuple.Value().Maybe<TCoAtomList>());
+ partitionBy = tuple.Value().Cast<TCoAtomList>();
+ } else if (name == "orderby") {
+ YQL_ENSURE(tuple.Value().Maybe<TCoNameValueTupleList>());
+ orderBy = tuple.Value().Cast<TCoNameValueTupleList>();
} else if (name == "filter") {
YQL_ENSURE(tuple.Value().Maybe<TCoLambda>());
filter = tuple.Value().Cast<TCoLambda>();
@@ -270,8 +270,8 @@ TWriteTableSettings ParseWriteTableSettings(TExprList node, TExprContext& ctx) {
ret.Mode = mode;
ret.Columns = columns;
ret.PrimaryKey = primaryKey;
- ret.PartitionBy = partitionBy;
- ret.OrderBy = orderBy;
+ ret.PartitionBy = partitionBy;
+ ret.OrderBy = orderBy;
ret.Filter = filter;
ret.Update = update;
ret.Indexes = idx;
diff --git a/ydb/library/yql/providers/common/provider/yql_provider.h b/ydb/library/yql/providers/common/provider/yql_provider.h
index 9e80cfd1355..e562c40ce7b 100644
--- a/ydb/library/yql/providers/common/provider/yql_provider.h
+++ b/ydb/library/yql/providers/common/provider/yql_provider.h
@@ -36,8 +36,8 @@ struct TWriteTableSettings {
NNodes::TMaybeNode<NNodes::TCoAtom> Mode;
NNodes::TMaybeNode<NNodes::TExprList> Columns;
NNodes::TMaybeNode<NNodes::TCoAtomList> PrimaryKey;
- NNodes::TMaybeNode<NNodes::TCoAtomList> PartitionBy;
- NNodes::TMaybeNode<NNodes::TCoNameValueTupleList> OrderBy;
+ NNodes::TMaybeNode<NNodes::TCoAtomList> PartitionBy;
+ NNodes::TMaybeNode<NNodes::TCoNameValueTupleList> OrderBy;
NNodes::TMaybeNode<NNodes::TCoLambda> Filter;
NNodes::TMaybeNode<NNodes::TCoLambda> Update;
NNodes::TMaybeNode<NNodes::TCoIndexList> Indexes;
diff --git a/ydb/library/yql/providers/config/yql_config_provider.cpp b/ydb/library/yql/providers/config/yql_config_provider.cpp
index 6952e96cf07..086fcce08cb 100644
--- a/ydb/library/yql/providers/config/yql_config_provider.cpp
+++ b/ydb/library/yql/providers/config/yql_config_provider.cpp
@@ -167,7 +167,7 @@ namespace {
return (node.Child(1)->Child(0)->Content() == ConfigProviderName);
}
- bool CanParse(const TExprNode& node) override {
+ bool CanParse(const TExprNode& node) override {
if (ConfigProviderFunctions().contains(node.Content()) ||
node.Content() == ConfigureName)
{
@@ -246,8 +246,8 @@ namespace {
});
return *ConfigurationTransformer;
- }
-
+ }
+
IGraphTransformer& GetTypeAnnotationTransformer(bool instantOnly) override {
Y_UNUSED(instantOnly);
if (!TypeAnnotationTransformer) {
diff --git a/ydb/library/yql/providers/result/provider/yql_result_provider.cpp b/ydb/library/yql/providers/result/provider/yql_result_provider.cpp
index 13bfd5e0f74..5834937736d 100644
--- a/ydb/library/yql/providers/result/provider/yql_result_provider.cpp
+++ b/ydb/library/yql/providers/result/provider/yql_result_provider.cpp
@@ -891,7 +891,7 @@ namespace {
return true;
}
- bool CanParse(const TExprNode& node) override {
+ bool CanParse(const TExprNode& node) override {
return ResultProviderFunctions().contains(node.Content()) || node.Content() == ConfigureName;
}
diff --git a/ydb/library/yql/providers/solomon/expr_nodes/ya.make b/ydb/library/yql/providers/solomon/expr_nodes/ya.make
index ea3a97d4492..d893ae13b00 100644
--- a/ydb/library/yql/providers/solomon/expr_nodes/ya.make
+++ b/ydb/library/yql/providers/solomon/expr_nodes/ya.make
@@ -1,38 +1,38 @@
-LIBRARY()
-
-OWNER(
- monster
- g:yql
-)
-
-SRCS(
- yql_solomon_expr_nodes.cpp
-)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(
+ monster
+ g:yql
+)
+
+SRCS(
+ yql_solomon_expr_nodes.cpp
+)
+
+PEERDIR(
ydb/library/yql/core/expr_nodes
ydb/library/yql/providers/common/provider
-)
-
-SRCDIR(
+)
+
+SRCDIR(
ydb/library/yql/core/expr_nodes_gen
-)
-
-RUN_PROGRAM(
+)
+
+RUN_PROGRAM(
ydb/library/yql/core/expr_nodes_gen/gen
- yql_expr_nodes_gen.jnj
- yql_solomon_expr_nodes.json
- yql_solomon_expr_nodes.gen.h
- yql_solomon_expr_nodes.decl.inl.h
- yql_solomon_expr_nodes.defs.inl.h
- IN yql_expr_nodes_gen.jnj
- IN yql_solomon_expr_nodes.json
- OUT yql_solomon_expr_nodes.gen.h
- OUT yql_solomon_expr_nodes.decl.inl.h
- OUT yql_solomon_expr_nodes.defs.inl.h
- OUTPUT_INCLUDES
+ yql_expr_nodes_gen.jnj
+ yql_solomon_expr_nodes.json
+ yql_solomon_expr_nodes.gen.h
+ yql_solomon_expr_nodes.decl.inl.h
+ yql_solomon_expr_nodes.defs.inl.h
+ IN yql_expr_nodes_gen.jnj
+ IN yql_solomon_expr_nodes.json
+ OUT yql_solomon_expr_nodes.gen.h
+ OUT yql_solomon_expr_nodes.decl.inl.h
+ OUT yql_solomon_expr_nodes.defs.inl.h
+ OUTPUT_INCLUDES
${ARCADIA_ROOT}/ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.h
- ${ARCADIA_ROOT}/util/generic/hash_set.h
-)
-
-END()
+ ${ARCADIA_ROOT}/util/generic/hash_set.h
+)
+
+END()
diff --git a/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.cpp b/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.cpp
index e5b4a6ec050..8aa5adc03a2 100644
--- a/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.cpp
+++ b/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.cpp
@@ -1 +1 @@
-#include "yql_solomon_expr_nodes.h"
+#include "yql_solomon_expr_nodes.h"
diff --git a/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.h b/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.h
index 7787041994b..741cf7fca45 100644
--- a/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.h
+++ b/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.h
@@ -1,66 +1,66 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h>
#include <ydb/library/yql/providers/common/provider/yql_provider_names.h>
#include <ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.gen.h>
-
-namespace NYql {
-namespace NNodes {
-
+
+namespace NYql {
+namespace NNodes {
+
#include <ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.decl.inl.h>
-
-class TSoDataSource: public NGenerated::TSoDataSourceStub<TExprBase, TCallable, TCoAtom> {
-public:
- explicit TSoDataSource(const TExprNode* node)
- : TSoDataSourceStub(node)
- {
- }
-
- explicit TSoDataSource(const TExprNode::TPtr& node)
- : TSoDataSourceStub(node)
- {
- }
-
- static bool Match(const TExprNode* node) {
- if (!TSoDataSourceStub::Match(node)) {
- return false;
- }
-
- if (node->Child(0)->Content() != SolomonProviderName) {
- return false;
- }
-
- return true;
- }
-};
-
-
-class TSoDataSink: public NGenerated::TSoDataSinkStub<TExprBase, TCallable, TCoAtom> {
-public:
- explicit TSoDataSink(const TExprNode* node)
- : TSoDataSinkStub(node)
- {
- }
-
- explicit TSoDataSink(const TExprNode::TPtr& node)
- : TSoDataSinkStub(node)
- {
- }
-
- static bool Match(const TExprNode* node) {
- if (!TSoDataSinkStub::Match(node)) {
- return false;
- }
-
- if (node->Child(0)->Content() != SolomonProviderName) {
- return false;
- }
-
- return true;
- }
-};
-
+
+class TSoDataSource: public NGenerated::TSoDataSourceStub<TExprBase, TCallable, TCoAtom> {
+public:
+ explicit TSoDataSource(const TExprNode* node)
+ : TSoDataSourceStub(node)
+ {
+ }
+
+ explicit TSoDataSource(const TExprNode::TPtr& node)
+ : TSoDataSourceStub(node)
+ {
+ }
+
+ static bool Match(const TExprNode* node) {
+ if (!TSoDataSourceStub::Match(node)) {
+ return false;
+ }
+
+ if (node->Child(0)->Content() != SolomonProviderName) {
+ return false;
+ }
+
+ return true;
+ }
+};
+
+
+class TSoDataSink: public NGenerated::TSoDataSinkStub<TExprBase, TCallable, TCoAtom> {
+public:
+ explicit TSoDataSink(const TExprNode* node)
+ : TSoDataSinkStub(node)
+ {
+ }
+
+ explicit TSoDataSink(const TExprNode::TPtr& node)
+ : TSoDataSinkStub(node)
+ {
+ }
+
+ static bool Match(const TExprNode* node) {
+ if (!TSoDataSinkStub::Match(node)) {
+ return false;
+ }
+
+ if (node->Child(0)->Content() != SolomonProviderName) {
+ return false;
+ }
+
+ return true;
+ }
+};
+
#include <ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.defs.inl.h>
-
-} // namespace NNodes
-} // namespace NYql
+
+} // namespace NNodes
+} // namespace NYql
diff --git a/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.json b/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.json
index d064daa1b40..61305c9fee1 100644
--- a/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.json
+++ b/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.json
@@ -1,68 +1,68 @@
-{
- "NodeRootType": "TExprBase",
- "NodeBuilderBase": "TNodeBuilderBase",
- "ListBuilderBase": "TListBuilderBase",
- "FreeArgCallableBase": "TFreeArgCallable",
- "FreeArgBuilderBase": "TFreeArgCallableBuilderBase",
- "Nodes": [
- {
- "Name": "TSoDataSource",
- "Base": "TCallable",
- "Definition": "Custom",
- "Match": {"Type": "Callable", "Name": "DataSource"},
- "Children": [
- {"Index": 0, "Name": "Category", "Type": "TCoAtom"},
- {"Index": 1, "Name": "Cluster", "Type": "TCoAtom"}
- ]
- },
- {
- "Name": "TSoDataSink",
- "Base": "TCallable",
- "Definition": "Custom",
- "Match": {"Type": "Callable", "Name": "DataSink"},
- "Children": [
- {"Index": 0, "Name": "Category", "Type": "TCoAtom"},
- {"Index": 1, "Name": "Cluster", "Type": "TCoAtom"}
- ]
- },
- {
- "Name": "TSoRead",
- "Base": "TFreeArgCallable",
- "Match": {"Type": "Callable", "Name": "Read!"},
- "Children": [
- {"Index": 0, "Name": "World", "Type": "TExprBase"},
- {"Index": 1, "Name": "DataSource", "Type": "TSoDataSource"}
- ]
- },
- {
- "Name": "TSoWrite",
- "Base": "TFreeArgCallable",
- "Match": {"Type": "Callable", "Name": "Write!"},
- "Children": [
- {"Index": 0, "Name": "World", "Type": "TExprBase"},
- {"Index": 1, "Name": "DataSink", "Type": "TSoDataSink"}
- ]
- },
- {
- "Name": "TSoReadShardMeta",
- "Base": "TCallable",
- "Match": {"Type": "Callable", "Name": "SoReadShardMeta!"},
- "Children": [
- {"Index": 0, "Name": "World", "Type": "TExprBase"},
- {"Index": 1, "Name": "DataSource", "Type": "TSoDataSource"},
- {"Index": 2, "Name": "Shard", "Type": "TCoAtom"}
- ]
- },
- {
- "Name": "TSoWriteToShard",
- "Base": "TCallable",
- "Match": {"Type": "Callable", "Name": "SoWriteToShard!"},
- "Children": [
- {"Index": 0, "Name": "World", "Type": "TExprBase"},
- {"Index": 1, "Name": "DataSink", "Type": "TSoDataSink"},
- {"Index": 2, "Name": "Shard", "Type": "TCoAtom"},
- {"Index": 3, "Name": "Input", "Type": "TExprBase"}
- ]
+{
+ "NodeRootType": "TExprBase",
+ "NodeBuilderBase": "TNodeBuilderBase",
+ "ListBuilderBase": "TListBuilderBase",
+ "FreeArgCallableBase": "TFreeArgCallable",
+ "FreeArgBuilderBase": "TFreeArgCallableBuilderBase",
+ "Nodes": [
+ {
+ "Name": "TSoDataSource",
+ "Base": "TCallable",
+ "Definition": "Custom",
+ "Match": {"Type": "Callable", "Name": "DataSource"},
+ "Children": [
+ {"Index": 0, "Name": "Category", "Type": "TCoAtom"},
+ {"Index": 1, "Name": "Cluster", "Type": "TCoAtom"}
+ ]
+ },
+ {
+ "Name": "TSoDataSink",
+ "Base": "TCallable",
+ "Definition": "Custom",
+ "Match": {"Type": "Callable", "Name": "DataSink"},
+ "Children": [
+ {"Index": 0, "Name": "Category", "Type": "TCoAtom"},
+ {"Index": 1, "Name": "Cluster", "Type": "TCoAtom"}
+ ]
+ },
+ {
+ "Name": "TSoRead",
+ "Base": "TFreeArgCallable",
+ "Match": {"Type": "Callable", "Name": "Read!"},
+ "Children": [
+ {"Index": 0, "Name": "World", "Type": "TExprBase"},
+ {"Index": 1, "Name": "DataSource", "Type": "TSoDataSource"}
+ ]
+ },
+ {
+ "Name": "TSoWrite",
+ "Base": "TFreeArgCallable",
+ "Match": {"Type": "Callable", "Name": "Write!"},
+ "Children": [
+ {"Index": 0, "Name": "World", "Type": "TExprBase"},
+ {"Index": 1, "Name": "DataSink", "Type": "TSoDataSink"}
+ ]
+ },
+ {
+ "Name": "TSoReadShardMeta",
+ "Base": "TCallable",
+ "Match": {"Type": "Callable", "Name": "SoReadShardMeta!"},
+ "Children": [
+ {"Index": 0, "Name": "World", "Type": "TExprBase"},
+ {"Index": 1, "Name": "DataSource", "Type": "TSoDataSource"},
+ {"Index": 2, "Name": "Shard", "Type": "TCoAtom"}
+ ]
+ },
+ {
+ "Name": "TSoWriteToShard",
+ "Base": "TCallable",
+ "Match": {"Type": "Callable", "Name": "SoWriteToShard!"},
+ "Children": [
+ {"Index": 0, "Name": "World", "Type": "TExprBase"},
+ {"Index": 1, "Name": "DataSink", "Type": "TSoDataSink"},
+ {"Index": 2, "Name": "Shard", "Type": "TCoAtom"},
+ {"Index": 3, "Name": "Input", "Type": "TExprBase"}
+ ]
},
{
"Name": "TSoShard",
@@ -85,6 +85,6 @@
{"Index": 1, "Name": "Settings", "Type": "TCoNameValueTupleList"},
{"Index": 2, "Name": "Token", "Type": "TCoSecureParam", "Optional": true}
]
- }
- ]
-}
+ }
+ ]
+}
diff --git a/ydb/library/yql/providers/solomon/gateway/ya.make b/ydb/library/yql/providers/solomon/gateway/ya.make
index a5d0b49c19e..62fb6780eb9 100644
--- a/ydb/library/yql/providers/solomon/gateway/ya.make
+++ b/ydb/library/yql/providers/solomon/gateway/ya.make
@@ -1,17 +1,17 @@
-LIBRARY()
-
-OWNER(
- monster
- g:yql
-)
-
-SRCS(
- yql_solomon_gateway.cpp
-)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(
+ monster
+ g:yql
+)
+
+SRCS(
+ yql_solomon_gateway.cpp
+)
+
+PEERDIR(
ydb/library/yql/providers/common/gateway
ydb/library/yql/providers/solomon/provider
-)
-
-END()
+)
+
+END()
diff --git a/ydb/library/yql/providers/solomon/gateway/yql_solomon_gateway.cpp b/ydb/library/yql/providers/solomon/gateway/yql_solomon_gateway.cpp
index ea9119c9fd2..7ea343a160d 100644
--- a/ydb/library/yql/providers/solomon/gateway/yql_solomon_gateway.cpp
+++ b/ydb/library/yql/providers/solomon/gateway/yql_solomon_gateway.cpp
@@ -1,58 +1,58 @@
-#include "yql_solomon_gateway.h"
-
+#include "yql_solomon_gateway.h"
+
#include <ydb/library/yql/providers/common/proto/gateways_config.pb.h>
-
-namespace NYql {
-
-using namespace NThreading;
-
-class TSolomonGateway : public ISolomonGateway {
-public:
- TSolomonGateway(const TSolomonGatewayConfig& config)
- {
- for (const auto& item: config.GetClusterMapping()) {
- Clusters_[item.GetName()] = item;
- }
- }
-
- NThreading::TFuture<TGetMetaResult> GetMeta(const TGetMetaRequest& request) const override {
- Y_UNUSED(request);
- try {
- TGetMetaResult result;
- auto meta = MakeIntrusive<TSolomonMetadata>();
- result.Meta = meta;
-
- // TODO: actual meta
- meta->DoesExist = true;
-
- result.SetSuccess();
- return MakeFuture(result);
-
- } catch (const yexception& e) {
- return MakeFuture(NCommon::ResultFromException<TGetMetaResult>(e));
-
- } catch (...) {
- return MakeFuture(NCommon::ResultFromError<TGetMetaResult>(CurrentExceptionMessage()));
- }
- }
-
- TMaybe<TSolomonClusterConfig> GetClusterConfig(const TStringBuf cluster) const override {
- if (Clusters_.contains(cluster)) {
- return Clusters_.find(cluster)->second;
- }
- return Nothing();
- }
-
- bool HasCluster(const TStringBuf cluster) const override {
- return Clusters_.contains(cluster);
- }
-
-private:
- THashMap<TString, TSolomonClusterConfig> Clusters_;
-};
-
-ISolomonGateway::TPtr CreateSolomonGateway(const TSolomonGatewayConfig& config) {
- return new TSolomonGateway(config);
-}
-
-} // namespace NYql
+
+namespace NYql {
+
+using namespace NThreading;
+
+class TSolomonGateway : public ISolomonGateway {
+public:
+ TSolomonGateway(const TSolomonGatewayConfig& config)
+ {
+ for (const auto& item: config.GetClusterMapping()) {
+ Clusters_[item.GetName()] = item;
+ }
+ }
+
+ NThreading::TFuture<TGetMetaResult> GetMeta(const TGetMetaRequest& request) const override {
+ Y_UNUSED(request);
+ try {
+ TGetMetaResult result;
+ auto meta = MakeIntrusive<TSolomonMetadata>();
+ result.Meta = meta;
+
+ // TODO: actual meta
+ meta->DoesExist = true;
+
+ result.SetSuccess();
+ return MakeFuture(result);
+
+ } catch (const yexception& e) {
+ return MakeFuture(NCommon::ResultFromException<TGetMetaResult>(e));
+
+ } catch (...) {
+ return MakeFuture(NCommon::ResultFromError<TGetMetaResult>(CurrentExceptionMessage()));
+ }
+ }
+
+ TMaybe<TSolomonClusterConfig> GetClusterConfig(const TStringBuf cluster) const override {
+ if (Clusters_.contains(cluster)) {
+ return Clusters_.find(cluster)->second;
+ }
+ return Nothing();
+ }
+
+ bool HasCluster(const TStringBuf cluster) const override {
+ return Clusters_.contains(cluster);
+ }
+
+private:
+ THashMap<TString, TSolomonClusterConfig> Clusters_;
+};
+
+ISolomonGateway::TPtr CreateSolomonGateway(const TSolomonGatewayConfig& config) {
+ return new TSolomonGateway(config);
+}
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/solomon/gateway/yql_solomon_gateway.h b/ydb/library/yql/providers/solomon/gateway/yql_solomon_gateway.h
index 2319e7d8440..4b82832e032 100644
--- a/ydb/library/yql/providers/solomon/gateway/yql_solomon_gateway.h
+++ b/ydb/library/yql/providers/solomon/gateway/yql_solomon_gateway.h
@@ -1,11 +1,11 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/providers/solomon/provider/yql_solomon_gateway.h>
-
-namespace NYql {
-
-class TSolomonGatewayConfig;
-
-ISolomonGateway::TPtr CreateSolomonGateway(const TSolomonGatewayConfig& config);
-
-} // namespace NYql
+
+namespace NYql {
+
+class TSolomonGatewayConfig;
+
+ISolomonGateway::TPtr CreateSolomonGateway(const TSolomonGatewayConfig& config);
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/solomon/provider/ya.make b/ydb/library/yql/providers/solomon/provider/ya.make
index b80c01bdb74..f81eb7ba1f5 100644
--- a/ydb/library/yql/providers/solomon/provider/ya.make
+++ b/ydb/library/yql/providers/solomon/provider/ya.make
@@ -1,12 +1,12 @@
-LIBRARY()
-
-OWNER(
- monster
- g:yql
-)
-
-SRCS(
- yql_solomon_config.cpp
+LIBRARY()
+
+OWNER(
+ monster
+ g:yql
+)
+
+SRCS(
+ yql_solomon_config.cpp
yql_solomon_datasink_execution.cpp
yql_solomon_datasink_type_ann.cpp
yql_solomon_datasink.cpp
@@ -14,13 +14,13 @@ SRCS(
yql_solomon_datasource_type_ann.cpp
yql_solomon_datasource.cpp
yql_solomon_dq_integration.cpp
- yql_solomon_io_discovery.cpp
- yql_solomon_load_meta.cpp
+ yql_solomon_io_discovery.cpp
+ yql_solomon_load_meta.cpp
yql_solomon_physical_optimize.cpp
yql_solomon_provider.cpp
-)
-
-PEERDIR(
+)
+
+PEERDIR(
library/cpp/actors/protos
ydb/library/yql/dq/expr_nodes
ydb/library/yql/providers/common/config
@@ -32,8 +32,8 @@ PEERDIR(
ydb/library/yql/providers/solomon/expr_nodes
ydb/library/yql/providers/solomon/proto
ydb/library/yql/dq/opt
-)
-
+)
+
YQL_LAST_ABI_VERSION()
-END()
+END()
diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_config.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_config.cpp
index fe00223f7cb..02760151a1f 100644
--- a/ydb/library/yql/providers/solomon/provider/yql_solomon_config.cpp
+++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_config.cpp
@@ -1,15 +1,15 @@
-#include "yql_solomon_config.h"
-
-namespace NYql {
-
-using namespace NCommon;
-
-TSolomonConfiguration::TSolomonConfiguration()
-{
-}
-
-TSolomonSettings::TConstPtr TSolomonConfiguration::Snapshot() const {
+#include "yql_solomon_config.h"
+
+namespace NYql {
+
+using namespace NCommon;
+
+TSolomonConfiguration::TSolomonConfiguration()
+{
+}
+
+TSolomonSettings::TConstPtr TSolomonConfiguration::Snapshot() const {
return std::make_shared<const TSolomonSettings>(*this);
-}
-
-} // NYql
+}
+
+} // NYql
diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_config.h b/ydb/library/yql/providers/solomon/provider/yql_solomon_config.h
index 941b2adc305..4c932f947ae 100644
--- a/ydb/library/yql/providers/solomon/provider/yql_solomon_config.h
+++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_config.h
@@ -1,50 +1,50 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/providers/common/structured_token/yql_token_builder.h>
#include <ydb/library/yql/providers/common/config/yql_dispatch.h>
#include <ydb/library/yql/providers/common/config/yql_setting.h>
#include <ydb/library/yql/providers/common/proto/gateways_config.pb.h>
-
-namespace NYql {
-
-struct TSolomonSettings {
+
+namespace NYql {
+
+struct TSolomonSettings {
using TConstPtr = std::shared_ptr<const TSolomonSettings>;
-};
-
-struct TSolomonConfiguration
- : public TSolomonSettings
- , public NCommon::TSettingDispatcher
-{
- using TPtr = TIntrusivePtr<TSolomonConfiguration>;
-
- TSolomonConfiguration();
- TSolomonConfiguration(const TSolomonConfiguration&) = delete;
-
- template <typename TProtoConfig>
+};
+
+struct TSolomonConfiguration
+ : public TSolomonSettings
+ , public NCommon::TSettingDispatcher
+{
+ using TPtr = TIntrusivePtr<TSolomonConfiguration>;
+
+ TSolomonConfiguration();
+ TSolomonConfiguration(const TSolomonConfiguration&) = delete;
+
+ template <typename TProtoConfig>
void Init(const TProtoConfig& config, TIntrusivePtr<TTypeAnnotationContext> typeCtx)
- {
- TVector<TString> clusters(Reserve(config.ClusterMappingSize()));
- for (auto& cluster: config.GetClusterMapping()) {
- clusters.push_back(cluster.GetName());
+ {
+ TVector<TString> clusters(Reserve(config.ClusterMappingSize()));
+ for (auto& cluster: config.GetClusterMapping()) {
+ clusters.push_back(cluster.GetName());
ClusterConfigs[cluster.GetName()] = cluster;
const TString authToken = typeCtx->FindCredentialContent("cluster:default_" + cluster.GetName(), "default_solomon", cluster.GetToken());
Tokens[cluster.GetName()] = ComposeStructuredTokenJsonForServiceAccount(cluster.GetServiceAccountId(), cluster.GetServiceAccountIdSignature(), authToken);
- }
-
- this->SetValidClusters(clusters);
-
- this->Dispatch(config.GetDefaultSettings());
- for (auto& cluster: config.GetClusterMapping()) {
- this->Dispatch(cluster.GetName(), cluster.GetSettings());
- }
- this->FreezeDefaults();
- }
-
- TSolomonSettings::TConstPtr Snapshot() const;
+ }
+
+ this->SetValidClusters(clusters);
+
+ this->Dispatch(config.GetDefaultSettings());
+ for (auto& cluster: config.GetClusterMapping()) {
+ this->Dispatch(cluster.GetName(), cluster.GetSettings());
+ }
+ this->FreezeDefaults();
+ }
+
+ TSolomonSettings::TConstPtr Snapshot() const;
THashMap<TString, TSolomonClusterConfig> ClusterConfigs;
THashMap<TString, TString> Tokens;
-};
-
-} // NYql
+};
+
+} // NYql
diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink.cpp
index b09fd381a07..b88154b5795 100644
--- a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink.cpp
+++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink.cpp
@@ -1,180 +1,180 @@
-#include "yql_solomon_provider_impl.h"
-
+#include "yql_solomon_provider_impl.h"
+
#include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h>
#include <ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.h>
-
+
#include <ydb/library/yql/providers/common/provider/yql_data_provider_impl.h>
#include <ydb/library/yql/providers/common/provider/yql_provider.h>
#include <ydb/library/yql/providers/common/provider/yql_provider_names.h>
-
+
#include <ydb/library/yql/utils/log/log.h>
-
-namespace NYql {
-
-using namespace NNodes;
-
-class TSolomonDataSink : public TDataProviderBase {
-public:
- TSolomonDataSink(TSolomonState::TPtr state)
- : State_ {state}
- , TypeAnnotationTransformer_(CreateSolomonDataSinkTypeAnnotationTransformer(State_))
- , ExecutionTransformer_(CreateSolomonDataSinkExecTransformer(State_))
+
+namespace NYql {
+
+using namespace NNodes;
+
+class TSolomonDataSink : public TDataProviderBase {
+public:
+ TSolomonDataSink(TSolomonState::TPtr state)
+ : State_ {state}
+ , TypeAnnotationTransformer_(CreateSolomonDataSinkTypeAnnotationTransformer(State_))
+ , ExecutionTransformer_(CreateSolomonDataSinkExecTransformer(State_))
, PhysicalOptProposalTransformer_(CreateSoPhysicalOptProposalTransformer(State_))
- {
- }
-
- TStringBuf GetName() const override {
- return SolomonProviderName;
- }
-
- TExprNode::TPtr GetClusterInfo(const TString& cluster, TExprContext& ctx) override {
- auto config = State_->Gateway->GetClusterConfig(cluster);
- if (!config) {
- return {};
- }
-
+ {
+ }
+
+ TStringBuf GetName() const override {
+ return SolomonProviderName;
+ }
+
+ TExprNode::TPtr GetClusterInfo(const TString& cluster, TExprContext& ctx) override {
+ auto config = State_->Gateway->GetClusterConfig(cluster);
+ if (!config) {
+ return {};
+ }
+
TPositionHandle pos;
- return Build<TCoAtom>(ctx, pos)
- .Value(config->GetCluster())
- .Done().Ptr();
- }
-
+ return Build<TCoAtom>(ctx, pos)
+ .Value(config->GetCluster())
+ .Done().Ptr();
+ }
+
const THashMap<TString, TString>* GetClusterTokens() override {
return &State_->Configuration->Tokens;
}
- IGraphTransformer& GetTypeAnnotationTransformer(bool instantOnly) override {
- Y_UNUSED(instantOnly);
- return *TypeAnnotationTransformer_;
- }
-
- IGraphTransformer& GetCallableExecutionTransformer() override {
- return *ExecutionTransformer_;
- }
-
+ IGraphTransformer& GetTypeAnnotationTransformer(bool instantOnly) override {
+ Y_UNUSED(instantOnly);
+ return *TypeAnnotationTransformer_;
+ }
+
+ IGraphTransformer& GetCallableExecutionTransformer() override {
+ return *ExecutionTransformer_;
+ }
+
IGraphTransformer& GetPhysicalOptProposalTransformer() override {
return *PhysicalOptProposalTransformer_;
}
- bool CanParse(const TExprNode& node) override {
- if (node.IsCallable(TCoWrite::CallableName())) {
- return TSoDataSink::Match(node.Child(1));
- }
-
- return TypeAnnotationTransformer_->CanParse(node);
- }
-
+ bool CanParse(const TExprNode& node) override {
+ if (node.IsCallable(TCoWrite::CallableName())) {
+ return TSoDataSink::Match(node.Child(1));
+ }
+
+ return TypeAnnotationTransformer_->CanParse(node);
+ }
+
void FillModifyCallables(THashSet<TStringBuf>& callables) override {
callables.insert(TSoWriteToShard::CallableName());
}
- bool CanExecute(const TExprNode& node) override {
- return ExecutionTransformer_->CanExec(node);
- }
-
- bool ValidateParameters(TExprNode& node, TExprContext& ctx, TMaybe<TString>& cluster) override {
- if (node.IsCallable(TCoDataSink::CallableName())) {
- if (node.Child(0)->Content() == SolomonProviderName) {
- auto clusterName = node.Child(1)->Content();
- if (!State_->Gateway->HasCluster(clusterName)) {
+ bool CanExecute(const TExprNode& node) override {
+ return ExecutionTransformer_->CanExec(node);
+ }
+
+ bool ValidateParameters(TExprNode& node, TExprContext& ctx, TMaybe<TString>& cluster) override {
+ if (node.IsCallable(TCoDataSink::CallableName())) {
+ if (node.Child(0)->Content() == SolomonProviderName) {
+ auto clusterName = node.Child(1)->Content();
+ if (!State_->Gateway->HasCluster(clusterName)) {
ctx.AddError(TIssue(ctx.GetPosition(node.Child(1)->Pos()), TStringBuilder() <<
- "Unknown cluster name: " << clusterName
- ));
- return false;
- }
- cluster = clusterName;
- return true;
- }
- }
+ "Unknown cluster name: " << clusterName
+ ));
+ return false;
+ }
+ cluster = clusterName;
+ return true;
+ }
+ }
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Invalid Solomon DataSink parameters"));
- return false;
- }
-
- TExprNode::TPtr RewriteIO(const TExprNode::TPtr& node, TExprContext& ctx) override {
- auto maybeWrite = TMaybeNode<TSoWrite>(node);
- YQL_ENSURE(maybeWrite);
-
- YQL_CLOG(INFO, ProviderSolomon) << "RewriteIO";
-
- auto write = maybeWrite.Cast();
- auto& key = write.Arg(2).Ref();
+ return false;
+ }
+
+ TExprNode::TPtr RewriteIO(const TExprNode::TPtr& node, TExprContext& ctx) override {
+ auto maybeWrite = TMaybeNode<TSoWrite>(node);
+ YQL_ENSURE(maybeWrite);
+
+ YQL_CLOG(INFO, ProviderSolomon) << "RewriteIO";
+
+ auto write = maybeWrite.Cast();
+ auto& key = write.Arg(2).Ref();
if (!key.IsCallable(TStringBuf("Key"))) {
ctx.AddError(TIssue(ctx.GetPosition(key.Pos()), TStringBuf("Expected key")));
- return {};
- }
- if (key.ChildrenSize() < 1) {
+ return {};
+ }
+ if (key.ChildrenSize() < 1) {
ctx.AddError(TIssue(ctx.GetPosition(key.Pos()), TStringBuf("Key must have at least one component")));
- return {};
- }
-
- auto tagName = key.Child(0)->Child(0)->Content();
+ return {};
+ }
+
+ auto tagName = key.Child(0)->Child(0)->Content();
if (tagName != TStringBuf("table")) {
ctx.AddError(TIssue(ctx.GetPosition(key.Child(0)->Pos()),
- TStringBuilder() << "Unexpected tag: " << tagName));
- return {};
- }
-
- const TExprNode* nameNode = key.Child(0)->Child(1);
- if (!nameNode->IsCallable("String")) {
+ TStringBuilder() << "Unexpected tag: " << tagName));
+ return {};
+ }
+
+ const TExprNode* nameNode = key.Child(0)->Child(1);
+ if (!nameNode->IsCallable("String")) {
ctx.AddError(TIssue(ctx.GetPosition(key.Pos()), "Expected String as table key"));
- return {};
- }
-
- if (!EnsureArgsCount(*nameNode, 1, ctx)) {
- return {};
- }
-
- const TExprNode* tablePath = nameNode->Child(0);
- if (!EnsureAtom(*tablePath, ctx)) {
- return {};
- }
- if (tablePath->Content().empty()) {
+ return {};
+ }
+
+ if (!EnsureArgsCount(*nameNode, 1, ctx)) {
+ return {};
+ }
+
+ const TExprNode* tablePath = nameNode->Child(0);
+ if (!EnsureAtom(*tablePath, ctx)) {
+ return {};
+ }
+ if (tablePath->Content().empty()) {
ctx.AddError(TIssue(ctx.GetPosition(tablePath->Pos()), "Table name must not be empty"));
- return {};
- }
-
- return Build<TSoWriteToShard>(ctx, write.Pos())
- .World(write.World())
- .DataSink(write.DataSink())
- .Shard<TCoAtom>().Value(tablePath->Content()).Build()
- .Input(write.Arg(3))
- .Done().Ptr();
- }
-
- void GetRequiredChildren(const TExprNode& node, TExprNode::TListType& children) override {
- if (CanExecute(node)) {
- children.push_back(node.ChildPtr(0));
- }
- }
-
- bool GetDependencies(const TExprNode& node, TExprNode::TListType& children, bool compact) override {
- Y_UNUSED(compact);
- if (CanExecute(node)) {
- children.push_back(node.ChildPtr(0));
- return true;
- }
- return false;
- }
-
- TString GetProviderPath(const TExprNode& node) override {
- return TStringBuilder() << SolomonProviderName << '.' << node.Child(1)->Content();
- }
-
+ return {};
+ }
+
+ return Build<TSoWriteToShard>(ctx, write.Pos())
+ .World(write.World())
+ .DataSink(write.DataSink())
+ .Shard<TCoAtom>().Value(tablePath->Content()).Build()
+ .Input(write.Arg(3))
+ .Done().Ptr();
+ }
+
+ void GetRequiredChildren(const TExprNode& node, TExprNode::TListType& children) override {
+ if (CanExecute(node)) {
+ children.push_back(node.ChildPtr(0));
+ }
+ }
+
+ bool GetDependencies(const TExprNode& node, TExprNode::TListType& children, bool compact) override {
+ Y_UNUSED(compact);
+ if (CanExecute(node)) {
+ children.push_back(node.ChildPtr(0));
+ return true;
+ }
+ return false;
+ }
+
+ TString GetProviderPath(const TExprNode& node) override {
+ return TStringBuilder() << SolomonProviderName << '.' << node.Child(1)->Content();
+ }
+
IDqIntegration* GetDqIntegration() override {
return State_->IsRtmrMode() ? nullptr : State_->DqIntegration.Get();
}
-private:
- TSolomonState::TPtr State_;
-
- THolder<TVisitorTransformerBase> TypeAnnotationTransformer_;
- THolder<TExecTransformerBase> ExecutionTransformer_;
+private:
+ TSolomonState::TPtr State_;
+
+ THolder<TVisitorTransformerBase> TypeAnnotationTransformer_;
+ THolder<TExecTransformerBase> ExecutionTransformer_;
THolder<IGraphTransformer> PhysicalOptProposalTransformer_;
-};
-
-
-TIntrusivePtr<IDataProvider> CreateSolomonDataSink(TSolomonState::TPtr state) {
- return new TSolomonDataSink(state);
-}
-
-} // namespace NYql
+};
+
+
+TIntrusivePtr<IDataProvider> CreateSolomonDataSink(TSolomonState::TPtr state) {
+ return new TSolomonDataSink(state);
+}
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_execution.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_execution.cpp
index e3048498c7b..9f6048eabd6 100644
--- a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_execution.cpp
+++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_execution.cpp
@@ -1,31 +1,31 @@
-#include "yql_solomon_provider_impl.h"
-
+#include "yql_solomon_provider_impl.h"
+
#include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h>
#include <ydb/library/yql/dq/expr_nodes/dq_expr_nodes.h>
#include <ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.h>
-
+
#include <ydb/library/yql/providers/common/provider/yql_provider.h>
#include <ydb/library/yql/providers/common/provider/yql_provider_names.h>
#include <ydb/library/yql/providers/common/provider/yql_data_provider_impl.h>
#include <ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.h>
#include <ydb/library/yql/utils/log/log.h>
-
+
#include <util/string/split.h>
-namespace NYql {
-
-using namespace NNodes;
-
-class TSolomonDataSinkExecTransformer : public TExecTransformerBase {
-public:
+namespace NYql {
+
+using namespace NNodes;
+
+class TSolomonDataSinkExecTransformer : public TExecTransformerBase {
+public:
explicit TSolomonDataSinkExecTransformer(TSolomonState::TPtr state)
- : State_(state)
- {
+ : State_(state)
+ {
AddHandler({TCoCommit::CallableName()}, RequireFirst(), Hndl(&TSolomonDataSinkExecTransformer::HandleCommit));
AddHandler({TSoWriteToShard::CallableName()}, RequireFirst(), Hndl(&TSolomonDataSinkExecTransformer::HandleSoWriteToShard));
- }
-
+ }
+
TStatusCallbackPair HandleCommit(const TExprNode::TPtr& input, TExprContext& ctx) {
if (!State_->IsRtmrMode() && TDqQuery::Match(input->Child(TCoCommit::idx_World))) {
return DelegateExecutionToDqProvider(input->ChildPtr(TCoCommit::idx_World), input, ctx);
@@ -42,7 +42,7 @@ public:
return DelegateExecutionToDqProvider(input->ChildPtr(TSoWriteToShard::idx_Input), input, ctx);
}
-private:
+private:
TStatusCallbackPair DelegateExecutionToDqProvider(
const TExprNode::TPtr& input,
const TExprNode::TPtr& originInput,
@@ -128,11 +128,11 @@ private:
}
private:
- TSolomonState::TPtr State_;
-};
-
-THolder<TExecTransformerBase> CreateSolomonDataSinkExecTransformer(TSolomonState::TPtr state) {
+ TSolomonState::TPtr State_;
+};
+
+THolder<TExecTransformerBase> CreateSolomonDataSinkExecTransformer(TSolomonState::TPtr state) {
return THolder(new TSolomonDataSinkExecTransformer(state));
-}
-
-} // namespace NYql
+}
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_type_ann.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_type_ann.cpp
index dc9ea45a8f6..cd66a622c99 100644
--- a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_type_ann.cpp
+++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_type_ann.cpp
@@ -1,42 +1,42 @@
-#include "yql_solomon_provider_impl.h"
-
+#include "yql_solomon_provider_impl.h"
+
#include <ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.h>
-
-namespace NYql {
-
-using namespace NNodes;
-
-class TSolomonDataSinkTypeAnnotationTransformer : public TVisitorTransformerBase {
-public:
- TSolomonDataSinkTypeAnnotationTransformer(TSolomonState::TPtr state)
- : TVisitorTransformerBase(true)
- , State_(state)
- {
- using TSelf = TSolomonDataSinkTypeAnnotationTransformer;
- AddHandler({TSoWriteToShard::CallableName()}, Hndl(&TSelf::HandleWriteToShard));
+
+namespace NYql {
+
+using namespace NNodes;
+
+class TSolomonDataSinkTypeAnnotationTransformer : public TVisitorTransformerBase {
+public:
+ TSolomonDataSinkTypeAnnotationTransformer(TSolomonState::TPtr state)
+ : TVisitorTransformerBase(true)
+ , State_(state)
+ {
+ using TSelf = TSolomonDataSinkTypeAnnotationTransformer;
+ AddHandler({TSoWriteToShard::CallableName()}, Hndl(&TSelf::HandleWriteToShard));
AddHandler({TSoShard::CallableName()}, Hndl(&TSelf::HandleSoShard));
- AddHandler({TCoCommit::CallableName()}, Hndl(&TSelf::HandleCommit));
- }
-
-private:
- TStatus HandleWriteToShard(TExprBase input, TExprContext& ctx) {
- if (!EnsureArgsCount(input.Ref(), 4, ctx)) {
- return TStatus::Error;
- }
- TSoWriteToShard write = input.Cast<TSoWriteToShard>();
- if (!EnsureWorldType(write.World().Ref(), ctx)) {
- return TStatus::Error;
- }
- if (!EnsureSpecificDataSink(write.DataSink().Ref(), SolomonProviderName, ctx)) {
- return TStatus::Error;
- }
- if (!EnsureAtom(write.Shard().Ref(), ctx)) {
- return TStatus::Error;
- }
- input.Ptr()->SetTypeAnn(write.World().Ref().GetTypeAnn());
- return TStatus::Ok;
- }
-
+ AddHandler({TCoCommit::CallableName()}, Hndl(&TSelf::HandleCommit));
+ }
+
+private:
+ TStatus HandleWriteToShard(TExprBase input, TExprContext& ctx) {
+ if (!EnsureArgsCount(input.Ref(), 4, ctx)) {
+ return TStatus::Error;
+ }
+ TSoWriteToShard write = input.Cast<TSoWriteToShard>();
+ if (!EnsureWorldType(write.World().Ref(), ctx)) {
+ return TStatus::Error;
+ }
+ if (!EnsureSpecificDataSink(write.DataSink().Ref(), SolomonProviderName, ctx)) {
+ return TStatus::Error;
+ }
+ if (!EnsureAtom(write.Shard().Ref(), ctx)) {
+ return TStatus::Error;
+ }
+ input.Ptr()->SetTypeAnn(write.World().Ref().GetTypeAnn());
+ return TStatus::Ok;
+ }
+
TStatus HandleSoShard(TExprBase input, TExprContext& ctx) {
YQL_ENSURE(!State_->IsRtmrMode(), "SoShard can't be used in rtmr mode");
@@ -70,18 +70,18 @@ private:
return TStatus::Ok;
}
- TStatus HandleCommit(TExprBase input, TExprContext& ctx) {
- Y_UNUSED(ctx);
- auto commit = input.Cast<TCoCommit>();
- input.Ptr()->SetTypeAnn(commit.World().Ref().GetTypeAnn());
- return TStatus::Ok;
- }
-
- TSolomonState::TPtr State_;
-};
-
-THolder<TVisitorTransformerBase> CreateSolomonDataSinkTypeAnnotationTransformer(TSolomonState::TPtr state) {
+ TStatus HandleCommit(TExprBase input, TExprContext& ctx) {
+ Y_UNUSED(ctx);
+ auto commit = input.Cast<TCoCommit>();
+ input.Ptr()->SetTypeAnn(commit.World().Ref().GetTypeAnn());
+ return TStatus::Ok;
+ }
+
+ TSolomonState::TPtr State_;
+};
+
+THolder<TVisitorTransformerBase> CreateSolomonDataSinkTypeAnnotationTransformer(TSolomonState::TPtr state) {
return THolder(new TSolomonDataSinkTypeAnnotationTransformer(state));
-}
-
-} // namespace NYql
+}
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource.cpp
index 31ceb1701c2..4952cdd68ed 100644
--- a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource.cpp
+++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource.cpp
@@ -1,110 +1,110 @@
-#include "yql_solomon_provider_impl.h"
-
+#include "yql_solomon_provider_impl.h"
+
#include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h>
#include <ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.h>
-
+
#include <ydb/library/yql/providers/common/provider/yql_provider.h>
#include <ydb/library/yql/providers/common/provider/yql_provider_names.h>
#include <ydb/library/yql/providers/common/provider/yql_data_provider_impl.h>
#include <ydb/library/yql/providers/common/config/yql_configuration_transformer.h>
-
+
#include <ydb/library/yql/utils/log/log.h>
-
-namespace NYql {
-
-using namespace NNodes;
-
-class TSolomonDataSource : public TDataProviderBase {
-public:
- TSolomonDataSource(TSolomonState::TPtr state)
- : State_(state)
- , ConfigurationTransformer_(NCommon::CreateProviderConfigurationTransformer(
+
+namespace NYql {
+
+using namespace NNodes;
+
+class TSolomonDataSource : public TDataProviderBase {
+public:
+ TSolomonDataSource(TSolomonState::TPtr state)
+ : State_(state)
+ , ConfigurationTransformer_(NCommon::CreateProviderConfigurationTransformer(
State_->Configuration, *State_->Types, TString{SolomonProviderName}))
- , IODiscoveryTransformer_(CreateSolomonIODiscoveryTransformer(State_))
- , LoadMetaDataTransformer_(CreateSolomonLoadTableMetadataTransformer(State_))
- , TypeAnnotationTransformer_(CreateSolomonDataSourceTypeAnnotationTransformer(State_))
- , ExecutionTransformer_(CreateSolomonDataSourceExecTransformer(State_))
- {
- }
-
- TStringBuf GetName() const override {
- return SolomonProviderName;
- }
-
- IGraphTransformer& GetConfigurationTransformer() override {
- return *ConfigurationTransformer_;
- }
-
-// IGraphTransformer& GetIODiscoveryTransformer() override {
-// return *IODiscoveryTransformer_;
-// }
-
-// IGraphTransformer& GetLoadTableMetadataTransformer() override {
-// return *LoadMetaDataTransformer_;
-// }
-
- IGraphTransformer& GetTypeAnnotationTransformer(bool instantOnly) override {
- Y_UNUSED(instantOnly);
- return *TypeAnnotationTransformer_;
- }
-
- IGraphTransformer& GetCallableExecutionTransformer() override {
- return *ExecutionTransformer_;
- }
-
- bool ValidateParameters(TExprNode& node, TExprContext& ctx, TMaybe<TString>& cluster) override {
- if (node.IsCallable(TCoDataSource::CallableName())) {
- if (node.Child(0)->Content() == SolomonProviderName) {
- auto clusterName = node.Child(1)->Content();
- if (!State_->Gateway->HasCluster(clusterName)) {
+ , IODiscoveryTransformer_(CreateSolomonIODiscoveryTransformer(State_))
+ , LoadMetaDataTransformer_(CreateSolomonLoadTableMetadataTransformer(State_))
+ , TypeAnnotationTransformer_(CreateSolomonDataSourceTypeAnnotationTransformer(State_))
+ , ExecutionTransformer_(CreateSolomonDataSourceExecTransformer(State_))
+ {
+ }
+
+ TStringBuf GetName() const override {
+ return SolomonProviderName;
+ }
+
+ IGraphTransformer& GetConfigurationTransformer() override {
+ return *ConfigurationTransformer_;
+ }
+
+// IGraphTransformer& GetIODiscoveryTransformer() override {
+// return *IODiscoveryTransformer_;
+// }
+
+// IGraphTransformer& GetLoadTableMetadataTransformer() override {
+// return *LoadMetaDataTransformer_;
+// }
+
+ IGraphTransformer& GetTypeAnnotationTransformer(bool instantOnly) override {
+ Y_UNUSED(instantOnly);
+ return *TypeAnnotationTransformer_;
+ }
+
+ IGraphTransformer& GetCallableExecutionTransformer() override {
+ return *ExecutionTransformer_;
+ }
+
+ bool ValidateParameters(TExprNode& node, TExprContext& ctx, TMaybe<TString>& cluster) override {
+ if (node.IsCallable(TCoDataSource::CallableName())) {
+ if (node.Child(0)->Content() == SolomonProviderName) {
+ auto clusterName = node.Child(1)->Content();
+ if (!State_->Gateway->HasCluster(clusterName)) {
ctx.AddError(TIssue(ctx.GetPosition(node.Child(1)->Pos()), TStringBuilder() <<
- "Unknown cluster name: " << clusterName));
- return false;
- }
- cluster = clusterName;
- return true;
- }
- }
+ "Unknown cluster name: " << clusterName));
+ return false;
+ }
+ cluster = clusterName;
+ return true;
+ }
+ }
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Invalid Solomon DataSource parameters"));
- return false;
- }
-
- bool CanParse(const TExprNode& node) override {
- if (node.IsCallable(TCoRead::CallableName())) {
- return TSoDataSource::Match(node.Child(1));
- }
- return TypeAnnotationTransformer_->CanParse(node);
- }
-
- bool CanExecute(const TExprNode& node) override {
- return ExecutionTransformer_->CanExec(node);
- }
-
- bool CanPullResult(const TExprNode& node, TSyncMap& syncList, bool& canRef) override {
- Y_UNUSED(node);
- Y_UNUSED(syncList);
- canRef = false;
- return false;
- }
-
- TExprNode::TPtr RewriteIO(const TExprNode::TPtr& node, TExprContext& ctx) override {
- Y_UNUSED(ctx);
- YQL_CLOG(INFO, ProviderSolomon) << "RewriteIO";
- return node;
- }
-
-private:
- TSolomonState::TPtr State_;
-
- THolder<IGraphTransformer> ConfigurationTransformer_;
- THolder<IGraphTransformer> IODiscoveryTransformer_;
- THolder<IGraphTransformer> LoadMetaDataTransformer_;
- THolder<TVisitorTransformerBase> TypeAnnotationTransformer_;
- THolder<TExecTransformerBase> ExecutionTransformer_;
-};
-
-TIntrusivePtr<IDataProvider> CreateSolomonDataSource(TSolomonState::TPtr state) {
- return new TSolomonDataSource(state);
-}
-
-} // namespace NYql
+ return false;
+ }
+
+ bool CanParse(const TExprNode& node) override {
+ if (node.IsCallable(TCoRead::CallableName())) {
+ return TSoDataSource::Match(node.Child(1));
+ }
+ return TypeAnnotationTransformer_->CanParse(node);
+ }
+
+ bool CanExecute(const TExprNode& node) override {
+ return ExecutionTransformer_->CanExec(node);
+ }
+
+ bool CanPullResult(const TExprNode& node, TSyncMap& syncList, bool& canRef) override {
+ Y_UNUSED(node);
+ Y_UNUSED(syncList);
+ canRef = false;
+ return false;
+ }
+
+ TExprNode::TPtr RewriteIO(const TExprNode::TPtr& node, TExprContext& ctx) override {
+ Y_UNUSED(ctx);
+ YQL_CLOG(INFO, ProviderSolomon) << "RewriteIO";
+ return node;
+ }
+
+private:
+ TSolomonState::TPtr State_;
+
+ THolder<IGraphTransformer> ConfigurationTransformer_;
+ THolder<IGraphTransformer> IODiscoveryTransformer_;
+ THolder<IGraphTransformer> LoadMetaDataTransformer_;
+ THolder<TVisitorTransformerBase> TypeAnnotationTransformer_;
+ THolder<TExecTransformerBase> ExecutionTransformer_;
+};
+
+TIntrusivePtr<IDataProvider> CreateSolomonDataSource(TSolomonState::TPtr state) {
+ return new TSolomonDataSource(state);
+}
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_execution.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_execution.cpp
index c381baf708b..d18bba50894 100644
--- a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_execution.cpp
+++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_execution.cpp
@@ -1,22 +1,22 @@
-#include "yql_solomon_provider_impl.h"
-
-namespace NYql {
-
-using namespace NNodes;
-
-class TSolomonDataSourceExecTransformer : public TExecTransformerBase {
-public:
- TSolomonDataSourceExecTransformer(TSolomonState::TPtr state)
- : State_(state)
- {
- }
-
-private:
- TSolomonState::TPtr State_;
-};
-
-THolder<TExecTransformerBase> CreateSolomonDataSourceExecTransformer(TSolomonState::TPtr state) {
+#include "yql_solomon_provider_impl.h"
+
+namespace NYql {
+
+using namespace NNodes;
+
+class TSolomonDataSourceExecTransformer : public TExecTransformerBase {
+public:
+ TSolomonDataSourceExecTransformer(TSolomonState::TPtr state)
+ : State_(state)
+ {
+ }
+
+private:
+ TSolomonState::TPtr State_;
+};
+
+THolder<TExecTransformerBase> CreateSolomonDataSourceExecTransformer(TSolomonState::TPtr state) {
return THolder(new TSolomonDataSourceExecTransformer(state));
-}
-
-} // namespace NYql
+}
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_type_ann.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_type_ann.cpp
index f6a0ce80034..f7b49f3cdbf 100644
--- a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_type_ann.cpp
+++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_type_ann.cpp
@@ -1,25 +1,25 @@
-#include "yql_solomon_provider_impl.h"
-
+#include "yql_solomon_provider_impl.h"
+
#include <ydb/library/yql/providers/common/provider/yql_provider.h>
-
-namespace NYql {
-
-using namespace NNodes;
-
-class TSolomonDataSourceTypeAnnotationTransformer : public TVisitorTransformerBase {
-public:
- TSolomonDataSourceTypeAnnotationTransformer(TSolomonState::TPtr state)
- : TVisitorTransformerBase(true)
- , State_(state)
- {
- }
-
-private:
- TSolomonState::TPtr State_;
-};
-
-THolder<TVisitorTransformerBase> CreateSolomonDataSourceTypeAnnotationTransformer(TSolomonState::TPtr state) {
+
+namespace NYql {
+
+using namespace NNodes;
+
+class TSolomonDataSourceTypeAnnotationTransformer : public TVisitorTransformerBase {
+public:
+ TSolomonDataSourceTypeAnnotationTransformer(TSolomonState::TPtr state)
+ : TVisitorTransformerBase(true)
+ , State_(state)
+ {
+ }
+
+private:
+ TSolomonState::TPtr State_;
+};
+
+THolder<TVisitorTransformerBase> CreateSolomonDataSourceTypeAnnotationTransformer(TSolomonState::TPtr state) {
return THolder(new TSolomonDataSourceTypeAnnotationTransformer(state));
-}
-
-} // namespace NYql
+}
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_gateway.h b/ydb/library/yql/providers/solomon/provider/yql_solomon_gateway.h
index 574bf792354..f775c4a9096 100644
--- a/ydb/library/yql/providers/solomon/provider/yql_solomon_gateway.h
+++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_gateway.h
@@ -1,39 +1,39 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/providers/common/gateway/yql_provider_gateway.h>
#include <ydb/library/yql/providers/common/proto/gateways_config.pb.h>
-
+
#include <library/cpp/threading/future/future.h>
-
-namespace NYql {
-
-struct TSolomonMetadata : public TThrRefBase
-{
- using TPtr = TIntrusivePtr<TSolomonMetadata>;
-
- bool DoesExist = false;
-};
-
-class ISolomonGateway : public TThrRefBase {
-public:
- using TPtr = TIntrusivePtr<ISolomonGateway>;
-
- struct TGetMetaRequest {
- TString Cluster;
- TString Name;
- };
-
- struct TGetMetaResult : public NCommon::TOperationResult {
- TSolomonMetadata::TPtr Meta;
- };
-
- virtual ~ISolomonGateway() = default;
-
- virtual NThreading::TFuture<TGetMetaResult> GetMeta(
- const TGetMetaRequest& request) const = 0;
-
- virtual TMaybe<TSolomonClusterConfig> GetClusterConfig(const TStringBuf cluster) const = 0;
- virtual bool HasCluster(const TStringBuf cluster) const = 0;
-};
-
-} // namespace NYql
+
+namespace NYql {
+
+struct TSolomonMetadata : public TThrRefBase
+{
+ using TPtr = TIntrusivePtr<TSolomonMetadata>;
+
+ bool DoesExist = false;
+};
+
+class ISolomonGateway : public TThrRefBase {
+public:
+ using TPtr = TIntrusivePtr<ISolomonGateway>;
+
+ struct TGetMetaRequest {
+ TString Cluster;
+ TString Name;
+ };
+
+ struct TGetMetaResult : public NCommon::TOperationResult {
+ TSolomonMetadata::TPtr Meta;
+ };
+
+ virtual ~ISolomonGateway() = default;
+
+ virtual NThreading::TFuture<TGetMetaResult> GetMeta(
+ const TGetMetaRequest& request) const = 0;
+
+ virtual TMaybe<TSolomonClusterConfig> GetClusterConfig(const TStringBuf cluster) const = 0;
+ virtual bool HasCluster(const TStringBuf cluster) const = 0;
+};
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_io_discovery.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_io_discovery.cpp
index 880811beee2..15445b916eb 100644
--- a/ydb/library/yql/providers/solomon/provider/yql_solomon_io_discovery.cpp
+++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_io_discovery.cpp
@@ -1,79 +1,79 @@
-#include "yql_solomon_provider_impl.h"
-
+#include "yql_solomon_provider_impl.h"
+
#include <ydb/library/yql/providers/common/provider/yql_provider.h>
#include <ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.h>
-
-/*
+
+/*
#include <ydb/library/yql/core/yql_expr_optimize.h>
-#include <yql/library/statface_client/client.h>
-
-#include <util/generic/hash_set.h>
-*/
-
-namespace NYql {
-
-using namespace NNodes;
-
-class TSolomonIODiscoveryTransformer : public TSyncTransformerBase {
-public:
- TSolomonIODiscoveryTransformer(TSolomonState::TPtr state)
- : State_(state)
- {
- }
-
- TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) override {
- output = input;
-
- if (ctx.Step.IsDone(ctx.Step.DiscoveryIO)) {
- return TStatus::Ok;
- }
-
- TVector<TIssue> issues;
-
- auto status = OptimizeExpr(input, output, [] (const TExprNode::TPtr& node, TExprContext& ctx) -> TExprNode::TPtr {
- if (auto maybeWrite = TMaybeNode<TSoWrite>(node)) {
- if (!maybeWrite.DataSink()) {
- return node;
- }
- auto write = maybeWrite.Cast();
-
- if (!EnsureArgsCount(write.Ref(), 5, ctx)) {
- return {};
- }
-
- return Build<TSoWrite>(ctx, write.Pos())
- .World(write.World())
- .DataSink(write.DataSink())
- .FreeArgs()
- .Add<TCoAtom>()
- .Value("")
- .Build()
- .Add(write.Arg(3))
- .Build()
- .Done().Ptr();
-
- } else if (TMaybeNode<TSoRead>(node).DataSource()) {
- return node;
- }
- return node;
- }, ctx, TOptimizeExprSettings {nullptr});
-
- if (issues) {
- for (const auto& issue: issues) {
- ctx.AddError(issue);
- }
- status = status.Combine(TStatus::Error);
- }
-
- return status;
- }
-
-private:
- TSolomonState::TPtr State_;
-};
-
-THolder<IGraphTransformer> CreateSolomonIODiscoveryTransformer(TSolomonState::TPtr state) {
+#include <yql/library/statface_client/client.h>
+
+#include <util/generic/hash_set.h>
+*/
+
+namespace NYql {
+
+using namespace NNodes;
+
+class TSolomonIODiscoveryTransformer : public TSyncTransformerBase {
+public:
+ TSolomonIODiscoveryTransformer(TSolomonState::TPtr state)
+ : State_(state)
+ {
+ }
+
+ TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) override {
+ output = input;
+
+ if (ctx.Step.IsDone(ctx.Step.DiscoveryIO)) {
+ return TStatus::Ok;
+ }
+
+ TVector<TIssue> issues;
+
+ auto status = OptimizeExpr(input, output, [] (const TExprNode::TPtr& node, TExprContext& ctx) -> TExprNode::TPtr {
+ if (auto maybeWrite = TMaybeNode<TSoWrite>(node)) {
+ if (!maybeWrite.DataSink()) {
+ return node;
+ }
+ auto write = maybeWrite.Cast();
+
+ if (!EnsureArgsCount(write.Ref(), 5, ctx)) {
+ return {};
+ }
+
+ return Build<TSoWrite>(ctx, write.Pos())
+ .World(write.World())
+ .DataSink(write.DataSink())
+ .FreeArgs()
+ .Add<TCoAtom>()
+ .Value("")
+ .Build()
+ .Add(write.Arg(3))
+ .Build()
+ .Done().Ptr();
+
+ } else if (TMaybeNode<TSoRead>(node).DataSource()) {
+ return node;
+ }
+ return node;
+ }, ctx, TOptimizeExprSettings {nullptr});
+
+ if (issues) {
+ for (const auto& issue: issues) {
+ ctx.AddError(issue);
+ }
+ status = status.Combine(TStatus::Error);
+ }
+
+ return status;
+ }
+
+private:
+ TSolomonState::TPtr State_;
+};
+
+THolder<IGraphTransformer> CreateSolomonIODiscoveryTransformer(TSolomonState::TPtr state) {
return THolder(new TSolomonIODiscoveryTransformer(state));
-}
-
-} // namespace NYql
+}
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_load_meta.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_load_meta.cpp
index 42324bb3ded..8c15dc4de2c 100644
--- a/ydb/library/yql/providers/solomon/provider/yql_solomon_load_meta.cpp
+++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_load_meta.cpp
@@ -1,45 +1,45 @@
-#include "yql_solomon_provider_impl.h"
-
-namespace NYql {
-
-using namespace NNodes;
-
-class TSolomonLoadTableMetadataTransformer : public TGraphTransformerBase {
-public:
- TSolomonLoadTableMetadataTransformer(TSolomonState::TPtr state)
- : State_(state)
- {
- }
-
- TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final {
- Y_UNUSED(input);
- Y_UNUSED(output);
-
- if (ctx.Step.IsDone(TExprStep::LoadTablesMetadata)) {
- return TStatus::Ok;
- }
-
- return TStatus::Async;
- }
-
- NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode& input) final {
- Y_UNUSED(input);
- return AsyncFuture_;
- }
-
- TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext&) final {
- output = input;
-
- return TStatus::Ok;
- }
-
-private:
- TSolomonState::TPtr State_;
- NThreading::TFuture<void> AsyncFuture_;
-};
-
-THolder<IGraphTransformer> CreateSolomonLoadTableMetadataTransformer(TSolomonState::TPtr state) {
+#include "yql_solomon_provider_impl.h"
+
+namespace NYql {
+
+using namespace NNodes;
+
+class TSolomonLoadTableMetadataTransformer : public TGraphTransformerBase {
+public:
+ TSolomonLoadTableMetadataTransformer(TSolomonState::TPtr state)
+ : State_(state)
+ {
+ }
+
+ TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final {
+ Y_UNUSED(input);
+ Y_UNUSED(output);
+
+ if (ctx.Step.IsDone(TExprStep::LoadTablesMetadata)) {
+ return TStatus::Ok;
+ }
+
+ return TStatus::Async;
+ }
+
+ NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode& input) final {
+ Y_UNUSED(input);
+ return AsyncFuture_;
+ }
+
+ TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext&) final {
+ output = input;
+
+ return TStatus::Ok;
+ }
+
+private:
+ TSolomonState::TPtr State_;
+ NThreading::TFuture<void> AsyncFuture_;
+};
+
+THolder<IGraphTransformer> CreateSolomonLoadTableMetadataTransformer(TSolomonState::TPtr state) {
return THolder(new TSolomonLoadTableMetadataTransformer(state));
-}
-
-} // namespace NYql
+}
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_provider.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_provider.cpp
index d84f1502a65..1ff28bab3c6 100644
--- a/ydb/library/yql/providers/solomon/provider/yql_solomon_provider.cpp
+++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_provider.cpp
@@ -1,47 +1,47 @@
-#include "yql_solomon_provider.h"
-
+#include "yql_solomon_provider.h"
+
#include <ydb/library/yql/providers/common/proto/gateways_config.pb.h>
#include <ydb/library/yql/providers/common/provider/yql_provider_names.h>
#include <ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.h>
-
-namespace NYql {
-
+
+namespace NYql {
+
TDataProviderInitializer GetSolomonDataProviderInitializer(ISolomonGateway::TPtr gateway, bool supportRtmrMode) {
return [gateway, supportRtmrMode] (
- const TString& userName,
- const TString& sessionId,
- const TGatewaysConfig* gatewaysConfig,
- const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry,
- TIntrusivePtr<IRandomProvider> randomProvider,
- TIntrusivePtr<TTypeAnnotationContext> typeCtx,
- const TOperationProgressWriter& progressWriter,
- const TYqlOperationOptions& operationOptions)
- {
- Y_UNUSED(sessionId);
- Y_UNUSED(userName);
- Y_UNUSED(functionRegistry);
- Y_UNUSED(randomProvider);
- Y_UNUSED(progressWriter);
- Y_UNUSED(operationOptions);
-
- auto solomonState = MakeIntrusive<TSolomonState>();
-
+ const TString& userName,
+ const TString& sessionId,
+ const TGatewaysConfig* gatewaysConfig,
+ const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry,
+ TIntrusivePtr<IRandomProvider> randomProvider,
+ TIntrusivePtr<TTypeAnnotationContext> typeCtx,
+ const TOperationProgressWriter& progressWriter,
+ const TYqlOperationOptions& operationOptions)
+ {
+ Y_UNUSED(sessionId);
+ Y_UNUSED(userName);
+ Y_UNUSED(functionRegistry);
+ Y_UNUSED(randomProvider);
+ Y_UNUSED(progressWriter);
+ Y_UNUSED(operationOptions);
+
+ auto solomonState = MakeIntrusive<TSolomonState>();
+
solomonState->SupportRtmrMode = supportRtmrMode;
- solomonState->Types = typeCtx.Get();
- solomonState->Gateway = gateway;
+ solomonState->Types = typeCtx.Get();
+ solomonState->Gateway = gateway;
solomonState->DqIntegration = CreateSolomonDqIntegration(solomonState);
- if (gatewaysConfig) {
+ if (gatewaysConfig) {
solomonState->Configuration->Init(gatewaysConfig->GetSolomon(), typeCtx);
- }
-
- TDataProviderInfo info;
-
+ }
+
+ TDataProviderInfo info;
+
info.Names.insert({TString{SolomonProviderName}});
- info.Source = CreateSolomonDataSource(solomonState);
- info.Sink = CreateSolomonDataSink(solomonState);
-
- return info;
- };
-}
-
-} // namespace NYql
+ info.Source = CreateSolomonDataSource(solomonState);
+ info.Sink = CreateSolomonDataSink(solomonState);
+
+ return info;
+ };
+}
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_provider.h b/ydb/library/yql/providers/solomon/provider/yql_solomon_provider.h
index 7f4a88a1315..ef4e42b3a15 100644
--- a/ydb/library/yql/providers/solomon/provider/yql_solomon_provider.h
+++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_provider.h
@@ -1,16 +1,16 @@
-#pragma once
-
-#include "yql_solomon_gateway.h"
-#include "yql_solomon_config.h"
-
+#pragma once
+
+#include "yql_solomon_gateway.h"
+#include "yql_solomon_config.h"
+
#include <ydb/library/yql/core/yql_data_provider.h>
-
-namespace NYql {
-
-struct TSolomonState : public TThrRefBase
-{
- using TPtr = TIntrusivePtr<TSolomonState>;
-
+
+namespace NYql {
+
+struct TSolomonState : public TThrRefBase
+{
+ using TPtr = TIntrusivePtr<TSolomonState>;
+
public:
bool IsRtmrMode() const {
return SupportRtmrMode;
@@ -19,15 +19,15 @@ public:
public:
bool SupportRtmrMode = true;
- ISolomonGateway::TPtr Gateway;
- TTypeAnnotationContext* Types = nullptr;
- TSolomonConfiguration::TPtr Configuration = MakeIntrusive<TSolomonConfiguration>();
+ ISolomonGateway::TPtr Gateway;
+ TTypeAnnotationContext* Types = nullptr;
+ TSolomonConfiguration::TPtr Configuration = MakeIntrusive<TSolomonConfiguration>();
THolder<IDqIntegration> DqIntegration;
-};
-
+};
+
TDataProviderInitializer GetSolomonDataProviderInitializer(ISolomonGateway::TPtr gateway, bool supportRtmrMode = true);
-
-TIntrusivePtr<IDataProvider> CreateSolomonDataSource(TSolomonState::TPtr state);
-TIntrusivePtr<IDataProvider> CreateSolomonDataSink(TSolomonState::TPtr state);
-
-} // namespace NYql
+
+TIntrusivePtr<IDataProvider> CreateSolomonDataSource(TSolomonState::TPtr state);
+TIntrusivePtr<IDataProvider> CreateSolomonDataSink(TSolomonState::TPtr state);
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_provider_impl.h b/ydb/library/yql/providers/solomon/provider/yql_solomon_provider_impl.h
index c1f48a5b540..6517605d7b3 100644
--- a/ydb/library/yql/providers/solomon/provider/yql_solomon_provider_impl.h
+++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_provider_impl.h
@@ -1,24 +1,24 @@
-#pragma once
-
-#include "yql_solomon_provider.h"
-
+#pragma once
+
+#include "yql_solomon_provider.h"
+
#include <ydb/library/yql/core/yql_graph_transformer.h>
#include <ydb/library/yql/providers/common/transform/yql_exec.h>
#include <ydb/library/yql/providers/common/transform/yql_visit.h>
-
-#include <util/generic/ptr.h>
-
-namespace NYql {
-
-THolder<IGraphTransformer> CreateSolomonIODiscoveryTransformer(TSolomonState::TPtr state);
-THolder<IGraphTransformer> CreateSolomonLoadTableMetadataTransformer(TSolomonState::TPtr state);
-
-THolder<TVisitorTransformerBase> CreateSolomonDataSourceTypeAnnotationTransformer(TSolomonState::TPtr state);
-THolder<TExecTransformerBase> CreateSolomonDataSourceExecTransformer(TSolomonState::TPtr state);
-
-THolder<TVisitorTransformerBase> CreateSolomonDataSinkTypeAnnotationTransformer(TSolomonState::TPtr state);
-THolder<TExecTransformerBase> CreateSolomonDataSinkExecTransformer(TSolomonState::TPtr state);
-
+
+#include <util/generic/ptr.h>
+
+namespace NYql {
+
+THolder<IGraphTransformer> CreateSolomonIODiscoveryTransformer(TSolomonState::TPtr state);
+THolder<IGraphTransformer> CreateSolomonLoadTableMetadataTransformer(TSolomonState::TPtr state);
+
+THolder<TVisitorTransformerBase> CreateSolomonDataSourceTypeAnnotationTransformer(TSolomonState::TPtr state);
+THolder<TExecTransformerBase> CreateSolomonDataSourceExecTransformer(TSolomonState::TPtr state);
+
+THolder<TVisitorTransformerBase> CreateSolomonDataSinkTypeAnnotationTransformer(TSolomonState::TPtr state);
+THolder<TExecTransformerBase> CreateSolomonDataSinkExecTransformer(TSolomonState::TPtr state);
+
THolder<IGraphTransformer> CreateSoPhysicalOptProposalTransformer(TSolomonState::TPtr state);
-} // namespace NYql
+} // namespace NYql
diff --git a/ydb/library/yql/providers/solomon/ya.make b/ydb/library/yql/providers/solomon/ya.make
index 579c088007d..d5f7e727f9d 100644
--- a/ydb/library/yql/providers/solomon/ya.make
+++ b/ydb/library/yql/providers/solomon/ya.make
@@ -1,7 +1,7 @@
-RECURSE(
+RECURSE(
async_io
- expr_nodes
- gateway
+ expr_nodes
+ gateway
proto
- provider
-)
+ provider
+)
diff --git a/ydb/library/yql/public/udf/udf_helpers.h b/ydb/library/yql/public/udf/udf_helpers.h
index 2be9443559b..b1c11f78e6b 100644
--- a/ydb/library/yql/public/udf/udf_helpers.h
+++ b/ydb/library/yql/public/udf/udf_helpers.h
@@ -211,7 +211,7 @@ namespace NUdf {
namespace NYql {
namespace NUdf {
-template<bool CheckOptional, const char* TFuncName, template<class> class TFunc, typename... TUserTypes>
+template<bool CheckOptional, const char* TFuncName, template<class> class TFunc, typename... TUserTypes>
class TUserDataTypeFuncFactory : public ::NYql::NUdf::TBoxedValue {
public:
typedef bool TTypeAwareMarker;
@@ -223,37 +223,37 @@ public:
}
template<typename TUserType>
- static bool DeclareSignatureImpl(
- TDataTypeId typeId,
+ static bool DeclareSignatureImpl(
+ TDataTypeId typeId,
::NYql::NUdf::TType* userType,
::NYql::NUdf::IFunctionTypeInfoBuilder& builder,
- bool typesOnly)
- {
+ bool typesOnly)
+ {
if (TDataType<TUserType>::Id != typeId) {
return false;
}
- TFunc<TUserType>::DeclareSignature(userType, builder, typesOnly);
+ TFunc<TUserType>::DeclareSignature(userType, builder, typesOnly);
return true;
}
template<typename TUserType, typename THead, typename... TTail>
- static bool DeclareSignatureImpl(
- TDataTypeId typeId,
+ static bool DeclareSignatureImpl(
+ TDataTypeId typeId,
::NYql::NUdf::TType* userType,
::NYql::NUdf::IFunctionTypeInfoBuilder& builder,
- bool typesOnly)
- {
- if (DeclareSignatureImpl<TUserType>(typeId, userType, builder, typesOnly)) {
+ bool typesOnly)
+ {
+ if (DeclareSignatureImpl<TUserType>(typeId, userType, builder, typesOnly)) {
return true;
}
- return DeclareSignatureImpl<THead, TTail...>(typeId, userType, builder, typesOnly);
+ return DeclareSignatureImpl<THead, TTail...>(typeId, userType, builder, typesOnly);
}
static bool DeclareSignature(
const ::NYql::NUdf::TStringRef& name,
::NYql::NUdf::TType* userType,
::NYql::NUdf::IFunctionTypeInfoBuilder& builder,
- bool typesOnly)
+ bool typesOnly)
{
if (Name() != name) {
// the only case when we return false
@@ -265,37 +265,37 @@ public:
return true;
}
- auto typeHelper = builder.TypeInfoHelper();
- auto userTypeInspector = TTupleTypeInspector(*typeHelper, userType);
- if (!userTypeInspector || userTypeInspector.GetElementsCount() < 1) {
- builder.SetError("Missing or invalid user type");
- return true;
- }
-
- auto argsTypeInspector = TTupleTypeInspector(*typeHelper, userTypeInspector.GetElementType(0));
- if (!argsTypeInspector || argsTypeInspector.GetElementsCount() < 1) {
- builder.SetError("Missing or invalid user type arguments");
- return true;
- }
-
- auto argType = argsTypeInspector.GetElementType(0);
- if (CheckOptional) {
- TOptionalTypeInspector optionalTypeInspector(*typeHelper, argType);
- if (optionalTypeInspector) {
- argType = optionalTypeInspector.GetItemType();
- }
- }
-
- TDataTypeInspector dataTypeInspector(*typeHelper, argType);
- if (!dataTypeInspector) {
- builder.SetError("User type must be a data type");
+ auto typeHelper = builder.TypeInfoHelper();
+ auto userTypeInspector = TTupleTypeInspector(*typeHelper, userType);
+ if (!userTypeInspector || userTypeInspector.GetElementsCount() < 1) {
+ builder.SetError("Missing or invalid user type");
return true;
}
- builder.UserType(userType);
-
- auto typeId = dataTypeInspector.GetTypeId();
- if (!DeclareSignatureImpl<TUserTypes...>(typeId, userType, builder, typesOnly)) {
+ auto argsTypeInspector = TTupleTypeInspector(*typeHelper, userTypeInspector.GetElementType(0));
+ if (!argsTypeInspector || argsTypeInspector.GetElementsCount() < 1) {
+ builder.SetError("Missing or invalid user type arguments");
+ return true;
+ }
+
+ auto argType = argsTypeInspector.GetElementType(0);
+ if (CheckOptional) {
+ TOptionalTypeInspector optionalTypeInspector(*typeHelper, argType);
+ if (optionalTypeInspector) {
+ argType = optionalTypeInspector.GetItemType();
+ }
+ }
+
+ TDataTypeInspector dataTypeInspector(*typeHelper, argType);
+ if (!dataTypeInspector) {
+ builder.SetError("User type must be a data type");
+ return true;
+ }
+
+ builder.UserType(userType);
+
+ auto typeId = dataTypeInspector.GetTypeId();
+ if (!DeclareSignatureImpl<TUserTypes...>(typeId, userType, builder, typesOnly)) {
TStringBuilder sb;
sb << "User type " << NYql::NUdf::GetDataTypeInfo(NYql::NUdf::GetDataSlot(typeId)).Name << " is not supported";
builder.SetError(sb);
diff --git a/ydb/library/yql/public/udf/udf_type_builder.h b/ydb/library/yql/public/udf/udf_type_builder.h
index 8bfc73a449d..83f289429bf 100644
--- a/ydb/library/yql/public/udf/udf_type_builder.h
+++ b/ydb/library/yql/public/udf/udf_type_builder.h
@@ -4,7 +4,7 @@
#include "udf_types.h"
#include "udf_ptr.h"
#include "udf_string_ref.h"
-#include "udf_value.h"
+#include "udf_value.h"
#include <type_traits>
@@ -342,73 +342,73 @@ private:
UDF_ASSERT_TYPE_SIZE(IFunctionArgTypesBuilder, 16);
//////////////////////////////////////////////////////////////////////////////
-// IRefCounted
-//////////////////////////////////////////////////////////////////////////////
-class IRefCounted {
-public:
- virtual ~IRefCounted() = default;
-
- inline void Ref() noexcept {
- Refs_++;
- }
-
- inline void UnRef() noexcept {
- Y_VERIFY_DEBUG(Refs_ > 0);
- if (--Refs_ == 0) {
- delete this;
- }
- }
-
-private:
- ui32 Refs_ = 0;
- ui32 Reserved_ = 0;
-
- void Unused() {
- Y_UNUSED(Reserved_);
- }
-};
-
-UDF_ASSERT_TYPE_SIZE(IRefCounted, 16);
-
-//////////////////////////////////////////////////////////////////////////////
-// IHash
-//////////////////////////////////////////////////////////////////////////////
-class IHash : public IRefCounted {
-public:
- using TPtr = TRefCountedPtr<IHash>;
-
- virtual ui64 Hash(TUnboxedValuePod value) const = 0;
-};
-
-UDF_ASSERT_TYPE_SIZE(IHash, 16);
-
-//////////////////////////////////////////////////////////////////////////////
-// IEquate
-//////////////////////////////////////////////////////////////////////////////
-class IEquate : public IRefCounted {
-public:
- using TPtr = TRefCountedPtr<IEquate>;
-
- virtual bool Equals(TUnboxedValuePod lhs, TUnboxedValuePod rhs) const = 0;
-};
-
-UDF_ASSERT_TYPE_SIZE(IEquate, 16);
-
-//////////////////////////////////////////////////////////////////////////////
-// ICompare
-//////////////////////////////////////////////////////////////////////////////
-class ICompare : public IRefCounted {
-public:
- using TPtr = TRefCountedPtr<ICompare>;
-
- virtual bool Less(TUnboxedValuePod lhs, TUnboxedValuePod rhs) const = 0;
- // signed difference, for strings may not be just -1/0/1
- virtual int Compare(TUnboxedValuePod lhs, TUnboxedValuePod rhs) const = 0;
-};
-
-UDF_ASSERT_TYPE_SIZE(ICompare, 16);
-
-//////////////////////////////////////////////////////////////////////////////
+// IRefCounted
+//////////////////////////////////////////////////////////////////////////////
+class IRefCounted {
+public:
+ virtual ~IRefCounted() = default;
+
+ inline void Ref() noexcept {
+ Refs_++;
+ }
+
+ inline void UnRef() noexcept {
+ Y_VERIFY_DEBUG(Refs_ > 0);
+ if (--Refs_ == 0) {
+ delete this;
+ }
+ }
+
+private:
+ ui32 Refs_ = 0;
+ ui32 Reserved_ = 0;
+
+ void Unused() {
+ Y_UNUSED(Reserved_);
+ }
+};
+
+UDF_ASSERT_TYPE_SIZE(IRefCounted, 16);
+
+//////////////////////////////////////////////////////////////////////////////
+// IHash
+//////////////////////////////////////////////////////////////////////////////
+class IHash : public IRefCounted {
+public:
+ using TPtr = TRefCountedPtr<IHash>;
+
+ virtual ui64 Hash(TUnboxedValuePod value) const = 0;
+};
+
+UDF_ASSERT_TYPE_SIZE(IHash, 16);
+
+//////////////////////////////////////////////////////////////////////////////
+// IEquate
+//////////////////////////////////////////////////////////////////////////////
+class IEquate : public IRefCounted {
+public:
+ using TPtr = TRefCountedPtr<IEquate>;
+
+ virtual bool Equals(TUnboxedValuePod lhs, TUnboxedValuePod rhs) const = 0;
+};
+
+UDF_ASSERT_TYPE_SIZE(IEquate, 16);
+
+//////////////////////////////////////////////////////////////////////////////
+// ICompare
+//////////////////////////////////////////////////////////////////////////////
+class ICompare : public IRefCounted {
+public:
+ using TPtr = TRefCountedPtr<ICompare>;
+
+ virtual bool Less(TUnboxedValuePod lhs, TUnboxedValuePod rhs) const = 0;
+ // signed difference, for strings may not be just -1/0/1
+ virtual int Compare(TUnboxedValuePod lhs, TUnboxedValuePod rhs) const = 0;
+};
+
+UDF_ASSERT_TYPE_SIZE(ICompare, 16);
+
+//////////////////////////////////////////////////////////////////////////////
// IFunctionTypeInfoBuilder
//////////////////////////////////////////////////////////////////////////////
namespace NImpl {
@@ -516,15 +516,15 @@ public:
virtual TSourcePosition GetSourcePosition() = 0;
};
#endif
-
-#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 10)
+
+#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 10)
class IFunctionTypeInfoBuilder5: public IFunctionTypeInfoBuilder4 {
public:
- virtual IHash::TPtr MakeHash(const TType* type) = 0;
- virtual IEquate::TPtr MakeEquate(const TType* type) = 0;
- virtual ICompare::TPtr MakeCompare(const TType* type) = 0;
+ virtual IHash::TPtr MakeHash(const TType* type) = 0;
+ virtual IEquate::TPtr MakeEquate(const TType* type) = 0;
+ virtual ICompare::TPtr MakeCompare(const TType* type) = 0;
};
-#endif
+#endif
#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 13)
class IFunctionTypeInfoBuilder6: public IFunctionTypeInfoBuilder5 {
diff --git a/ydb/library/yql/public/udf/udf_type_ops.h b/ydb/library/yql/public/udf/udf_type_ops.h
index 4b2446c8fd9..bf7f6a310ba 100644
--- a/ydb/library/yql/public/udf/udf_type_ops.h
+++ b/ydb/library/yql/public/udf/udf_type_ops.h
@@ -1,567 +1,567 @@
-#pragma once
-
-#include "udf_types.h"
-#include "udf_value.h"
-
+#pragma once
+
+#include "udf_types.h"
+#include "udf_value.h"
+
#include <util/generic/hash.h>
-#include <util/digest/numeric.h>
-
+#include <util/digest/numeric.h>
+
namespace NYql {
-namespace NUdf {
-
+namespace NUdf {
+
using THashType = ui64;
-template <EDataSlot Type>
+template <EDataSlot Type>
inline THashType GetValueHash(const TUnboxedValuePod& value);
inline THashType GetValueHash(EDataSlot type, const TUnboxedValuePod& value);
-
-template <EDataSlot Type>
+
+template <EDataSlot Type>
inline int CompareValues(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs);
inline int CompareValues(EDataSlot type, const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs);
-
-template <EDataSlot Type>
+
+template <EDataSlot Type>
inline bool EquateValues(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs);
inline bool EquateValues(EDataSlot type, const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs);
-
-template <EDataSlot Type>
-struct TUnboxedValueHash {
+
+template <EDataSlot Type>
+struct TUnboxedValueHash {
std::size_t operator()(const TUnboxedValuePod& value) const {
- return static_cast<std::size_t>(GetValueHash<Type>(value));
- }
-};
-
-template <EDataSlot Type>
-struct TUnboxedValueEquals {
+ return static_cast<std::size_t>(GetValueHash<Type>(value));
+ }
+};
+
+template <EDataSlot Type>
+struct TUnboxedValueEquals {
bool operator()(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) const {
- return EquateValues<Type>(lhs, rhs);
- }
-};
-
-// hash
-
-template <typename T, std::enable_if_t<std::is_integral<T>::value>* = nullptr>
+ return EquateValues<Type>(lhs, rhs);
+ }
+};
+
+// hash
+
+template <typename T, std::enable_if_t<std::is_integral<T>::value>* = nullptr>
inline THashType GetIntegerHash(const TUnboxedValuePod& value) {
- return std::hash<T>()(value.Get<T>());
-}
-
-template <typename T, std::enable_if_t<std::is_floating_point<T>::value>* = nullptr>
+ return std::hash<T>()(value.Get<T>());
+}
+
+template <typename T, std::enable_if_t<std::is_floating_point<T>::value>* = nullptr>
inline THashType GetFloatHash(const TUnboxedValuePod& value) {
- const auto x = value.Get<T>();
+ const auto x = value.Get<T>();
return std::isunordered(x, x) ? ~0ULL : std::hash<T>()(x);
-}
-
+}
+
inline THashType GetStringHash(const TUnboxedValuePod& value) {
return THash<TStringBuf>{}(value.AsStringRef());
-}
-
-template <typename T, std::enable_if_t<std::is_integral<T>::value>* = nullptr>
+}
+
+template <typename T, std::enable_if_t<std::is_integral<T>::value>* = nullptr>
inline THashType GetTzIntegerHash(const TUnboxedValuePod& value) {
return CombineHashes(std::hash<T>()(value.Get<T>()), std::hash<ui16>()(value.GetTimezoneId()));
-}
-
-template <>
+}
+
+template <>
inline THashType GetValueHash<EDataSlot::Bool>(const TUnboxedValuePod& value) {
- return std::hash<bool>()(value.Get<bool>());
-}
-
-template <>
+ return std::hash<bool>()(value.Get<bool>());
+}
+
+template <>
inline THashType GetValueHash<EDataSlot::Int8>(const TUnboxedValuePod& value) {
- return GetIntegerHash<i8>(value);
-}
-
-template <>
+ return GetIntegerHash<i8>(value);
+}
+
+template <>
inline THashType GetValueHash<EDataSlot::Uint8>(const TUnboxedValuePod& value) {
- return GetIntegerHash<ui8>(value);
-}
-
-template <>
+ return GetIntegerHash<ui8>(value);
+}
+
+template <>
inline THashType GetValueHash<EDataSlot::Int16>(const TUnboxedValuePod& value) {
- return GetIntegerHash<i16>(value);
-}
-
-template <>
+ return GetIntegerHash<i16>(value);
+}
+
+template <>
inline THashType GetValueHash<EDataSlot::Uint16>(const TUnboxedValuePod& value) {
- return GetIntegerHash<ui16>(value);
-}
-
-template <>
+ return GetIntegerHash<ui16>(value);
+}
+
+template <>
inline THashType GetValueHash<EDataSlot::Int32>(const TUnboxedValuePod& value) {
- return GetIntegerHash<i32>(value);
-}
-
-template <>
+ return GetIntegerHash<i32>(value);
+}
+
+template <>
inline THashType GetValueHash<EDataSlot::Uint32>(const TUnboxedValuePod& value) {
- return GetIntegerHash<ui32>(value);
-}
-
-template <>
+ return GetIntegerHash<ui32>(value);
+}
+
+template <>
inline THashType GetValueHash<EDataSlot::Int64>(const TUnboxedValuePod& value) {
- return GetIntegerHash<i64>(value);
-}
-
-template <>
+ return GetIntegerHash<i64>(value);
+}
+
+template <>
inline THashType GetValueHash<EDataSlot::Uint64>(const TUnboxedValuePod& value) {
- return GetIntegerHash<ui64>(value);
-}
-
-template <>
+ return GetIntegerHash<ui64>(value);
+}
+
+template <>
inline THashType GetValueHash<EDataSlot::Float>(const TUnboxedValuePod& value) {
- return GetFloatHash<float>(value);
-}
-
-template <>
+ return GetFloatHash<float>(value);
+}
+
+template <>
inline THashType GetValueHash<EDataSlot::Double>(const TUnboxedValuePod& value) {
- return GetFloatHash<double>(value);
-}
-
-template <>
+ return GetFloatHash<double>(value);
+}
+
+template <>
inline THashType GetValueHash<EDataSlot::String>(const TUnboxedValuePod& value) {
- return GetStringHash(value);
-}
-
-template <>
+ return GetStringHash(value);
+}
+
+template <>
inline THashType GetValueHash<EDataSlot::Utf8>(const TUnboxedValuePod& value) {
- return GetStringHash(value);
-}
-
-template <>
+ return GetStringHash(value);
+}
+
+template <>
inline THashType GetValueHash<EDataSlot::Uuid>(const TUnboxedValuePod& value) {
- return GetStringHash(value);
-}
-
-template <>
+ return GetStringHash(value);
+}
+
+template <>
inline THashType GetValueHash<EDataSlot::Yson>(const TUnboxedValuePod&) {
- Y_FAIL("Yson isn't hashable.");
-}
-
-template <>
+ Y_FAIL("Yson isn't hashable.");
+}
+
+template <>
inline THashType GetValueHash<EDataSlot::Json>(const TUnboxedValuePod&) {
- Y_FAIL("Json isn't hashable.");
-}
-
-template <>
+ Y_FAIL("Json isn't hashable.");
+}
+
+template <>
inline THashType GetValueHash<EDataSlot::JsonDocument>(const TUnboxedValuePod&) {
Y_FAIL("JsonDocument isn't hashable.");
}
template <>
inline THashType GetValueHash<EDataSlot::Date>(const TUnboxedValuePod& value) {
- return GetIntegerHash<ui16>(value);
-}
-
-template <>
+ return GetIntegerHash<ui16>(value);
+}
+
+template <>
inline THashType GetValueHash<EDataSlot::Datetime>(const TUnboxedValuePod& value) {
- return GetIntegerHash<ui32>(value);
-}
-
-template <>
+ return GetIntegerHash<ui32>(value);
+}
+
+template <>
inline THashType GetValueHash<EDataSlot::Timestamp>(const TUnboxedValuePod& value) {
- return GetIntegerHash<ui64>(value);
-}
-
-template <>
+ return GetIntegerHash<ui64>(value);
+}
+
+template <>
inline THashType GetValueHash<EDataSlot::Interval>(const TUnboxedValuePod& value) {
- return GetIntegerHash<i64>(value);
-}
-
-template <>
+ return GetIntegerHash<i64>(value);
+}
+
+template <>
inline THashType GetValueHash<EDataSlot::TzDate>(const TUnboxedValuePod& value) {
- return GetTzIntegerHash<ui16>(value);
-}
-
-template <>
+ return GetTzIntegerHash<ui16>(value);
+}
+
+template <>
inline THashType GetValueHash<EDataSlot::TzDatetime>(const TUnboxedValuePod& value) {
- return GetTzIntegerHash<ui32>(value);
-}
-
-template <>
+ return GetTzIntegerHash<ui32>(value);
+}
+
+template <>
inline THashType GetValueHash<EDataSlot::TzTimestamp>(const TUnboxedValuePod& value) {
- return GetTzIntegerHash<ui64>(value);
-}
-
-template <>
+ return GetTzIntegerHash<ui64>(value);
+}
+
+template <>
inline THashType GetValueHash<EDataSlot::Decimal>(const TUnboxedValuePod& value) {
const auto pair = NYql::NDecimal::MakePair(value.GetInt128());
return CombineHashes(pair.first, pair.second);
-}
-
+}
+
template <>
inline THashType GetValueHash<EDataSlot::DyNumber>(const TUnboxedValuePod& value) {
return GetStringHash(value);
}
inline THashType GetValueHash(EDataSlot type, const TUnboxedValuePod& value) {
-#define HASH_TYPE(slot, ...) \
- case EDataSlot::slot: \
- return GetValueHash<EDataSlot::slot>(value);
-
- switch (type) {
- UDF_TYPE_ID_MAP(HASH_TYPE)
- }
-
-#undef HASH_TYPE
-
- Y_ENSURE(false, "Incorrect data slot: " << (ui32)type);
-}
-
-// compare
-
-template <typename T, std::enable_if_t<std::is_integral<T>::value>* = nullptr>
+#define HASH_TYPE(slot, ...) \
+ case EDataSlot::slot: \
+ return GetValueHash<EDataSlot::slot>(value);
+
+ switch (type) {
+ UDF_TYPE_ID_MAP(HASH_TYPE)
+ }
+
+#undef HASH_TYPE
+
+ Y_ENSURE(false, "Incorrect data slot: " << (ui32)type);
+}
+
+// compare
+
+template <typename T, std::enable_if_t<std::is_integral<T>::value>* = nullptr>
inline int CompareIntegers(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- const auto x = lhs.Get<T>();
- const auto y = rhs.Get<T>();
- return (x == y) ? 0 : (x < y ? -1 : 1);
-}
-
-template <typename T, std::enable_if_t<std::is_floating_point<T>::value>* = nullptr>
+ const auto x = lhs.Get<T>();
+ const auto y = rhs.Get<T>();
+ return (x == y) ? 0 : (x < y ? -1 : 1);
+}
+
+template <typename T, std::enable_if_t<std::is_floating_point<T>::value>* = nullptr>
inline int CompareFloats(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- const auto x = lhs.Get<T>();
- const auto y = rhs.Get<T>();
- if (std::isunordered(x, y)) {
- const auto xn = std::isnan(x);
- const auto yn = std::isnan(y);
- return (xn == yn) ? 0 : xn ? 1 : -1;
- }
- return (x == y) ? 0 : (x < y ? -1 : 1);
-}
-
+ const auto x = lhs.Get<T>();
+ const auto y = rhs.Get<T>();
+ if (std::isunordered(x, y)) {
+ const auto xn = std::isnan(x);
+ const auto yn = std::isnan(y);
+ return (xn == yn) ? 0 : xn ? 1 : -1;
+ }
+ return (x == y) ? 0 : (x < y ? -1 : 1);
+}
+
inline int CompareStrings(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
const TStringBuf lhsBuf = lhs.AsStringRef();
const TStringBuf rhsBuf = rhs.AsStringRef();
return lhsBuf.compare(rhsBuf);
-}
-
-template <typename T, std::enable_if_t<std::is_integral<T>::value>* = nullptr>
+}
+
+template <typename T, std::enable_if_t<std::is_integral<T>::value>* = nullptr>
inline int CompareTzIntegers(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- const auto x = lhs.Get<T>();
- const auto y = rhs.Get<T>();
- if (x < y) {
- return -1;
- }
-
- if (x > y) {
- return 1;
- }
-
- const auto tx = lhs.GetTimezoneId();
- const auto ty = rhs.GetTimezoneId();
- return (tx == ty) ? 0 : (tx < ty ? -1 : 1);
-}
-
-template <>
+ const auto x = lhs.Get<T>();
+ const auto y = rhs.Get<T>();
+ if (x < y) {
+ return -1;
+ }
+
+ if (x > y) {
+ return 1;
+ }
+
+ const auto tx = lhs.GetTimezoneId();
+ const auto ty = rhs.GetTimezoneId();
+ return (tx == ty) ? 0 : (tx < ty ? -1 : 1);
+}
+
+template <>
inline int CompareValues<EDataSlot::Bool>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- const auto x = lhs.Get<bool>();
- const auto y = rhs.Get<bool>();
- return x == y ? 0 : (!x ? -1 : 1);
-}
-
-template <>
+ const auto x = lhs.Get<bool>();
+ const auto y = rhs.Get<bool>();
+ return x == y ? 0 : (!x ? -1 : 1);
+}
+
+template <>
inline int CompareValues<EDataSlot::Int8>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return CompareIntegers<i8>(lhs, rhs);
-}
-
-template <>
+ return CompareIntegers<i8>(lhs, rhs);
+}
+
+template <>
inline int CompareValues<EDataSlot::Uint8>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return CompareIntegers<ui8>(lhs, rhs);
-}
-
-template <>
+ return CompareIntegers<ui8>(lhs, rhs);
+}
+
+template <>
inline int CompareValues<EDataSlot::Int16>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return CompareIntegers<i16>(lhs, rhs);
-}
-
-template <>
+ return CompareIntegers<i16>(lhs, rhs);
+}
+
+template <>
inline int CompareValues<EDataSlot::Uint16>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return CompareIntegers<ui16>(lhs, rhs);
-}
-
-template <>
+ return CompareIntegers<ui16>(lhs, rhs);
+}
+
+template <>
inline int CompareValues<EDataSlot::Int32>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return CompareIntegers<i32>(lhs, rhs);
-}
-
-template <>
+ return CompareIntegers<i32>(lhs, rhs);
+}
+
+template <>
inline int CompareValues<EDataSlot::Uint32>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return CompareIntegers<ui32>(lhs, rhs);
-}
-
-template <>
+ return CompareIntegers<ui32>(lhs, rhs);
+}
+
+template <>
inline int CompareValues<EDataSlot::Int64>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return CompareIntegers<i64>(lhs, rhs);
-}
-
-template <>
+ return CompareIntegers<i64>(lhs, rhs);
+}
+
+template <>
inline int CompareValues<EDataSlot::Uint64>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return CompareIntegers<ui64>(lhs, rhs);
-}
-
-template <>
+ return CompareIntegers<ui64>(lhs, rhs);
+}
+
+template <>
inline int CompareValues<EDataSlot::Float>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return CompareFloats<float>(lhs, rhs);
-}
-
-template <>
+ return CompareFloats<float>(lhs, rhs);
+}
+
+template <>
inline int CompareValues<EDataSlot::Double>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return CompareFloats<double>(lhs, rhs);
-}
-
-template <>
+ return CompareFloats<double>(lhs, rhs);
+}
+
+template <>
inline int CompareValues<EDataSlot::String>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return CompareStrings(lhs, rhs);
-}
-
-template <>
+ return CompareStrings(lhs, rhs);
+}
+
+template <>
inline int CompareValues<EDataSlot::Utf8>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return CompareStrings(lhs, rhs);
-}
-
-template <>
+ return CompareStrings(lhs, rhs);
+}
+
+template <>
inline int CompareValues<EDataSlot::Uuid>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return CompareStrings(lhs, rhs);
-}
-
-template <>
+ return CompareStrings(lhs, rhs);
+}
+
+template <>
inline int CompareValues<EDataSlot::Yson>(const TUnboxedValuePod&, const TUnboxedValuePod&) {
- Y_FAIL("Yson isn't comparable.");
-}
-
-template <>
+ Y_FAIL("Yson isn't comparable.");
+}
+
+template <>
inline int CompareValues<EDataSlot::Json>(const TUnboxedValuePod&, const TUnboxedValuePod&) {
- Y_FAIL("Json isn't comparable.");
-}
-
-template <>
+ Y_FAIL("Json isn't comparable.");
+}
+
+template <>
inline int CompareValues<EDataSlot::JsonDocument>(const TUnboxedValuePod&, const TUnboxedValuePod&) {
Y_FAIL("JsonDocument isn't comparable.");
}
template <>
inline int CompareValues<EDataSlot::Date>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return CompareIntegers<ui16>(lhs, rhs);
-}
-
-template <>
+ return CompareIntegers<ui16>(lhs, rhs);
+}
+
+template <>
inline int CompareValues<EDataSlot::Datetime>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return CompareIntegers<ui32>(lhs, rhs);
-}
-
-template <>
+ return CompareIntegers<ui32>(lhs, rhs);
+}
+
+template <>
inline int CompareValues<EDataSlot::Timestamp>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return CompareIntegers<ui64>(lhs, rhs);
-}
-
-template <>
+ return CompareIntegers<ui64>(lhs, rhs);
+}
+
+template <>
inline int CompareValues<EDataSlot::Interval>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return CompareIntegers<i64>(lhs, rhs);
-}
-
-template <>
+ return CompareIntegers<i64>(lhs, rhs);
+}
+
+template <>
inline int CompareValues<EDataSlot::TzDate>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return CompareTzIntegers<ui16>(lhs, rhs);
-}
-
-template <>
+ return CompareTzIntegers<ui16>(lhs, rhs);
+}
+
+template <>
inline int CompareValues<EDataSlot::TzDatetime>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return CompareTzIntegers<ui32>(lhs, rhs);
-}
-
-template <>
+ return CompareTzIntegers<ui32>(lhs, rhs);
+}
+
+template <>
inline int CompareValues<EDataSlot::TzTimestamp>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return CompareTzIntegers<ui64>(lhs, rhs);
-}
-
-template <>
+ return CompareTzIntegers<ui64>(lhs, rhs);
+}
+
+template <>
inline int CompareValues<EDataSlot::Decimal>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- const auto x = lhs.GetInt128();
- const auto y = rhs.GetInt128();
- return x == y ? 0 : (x < y ? -1 : 1);
-}
-
+ const auto x = lhs.GetInt128();
+ const auto y = rhs.GetInt128();
+ return x == y ? 0 : (x < y ? -1 : 1);
+}
+
template <>
inline int CompareValues<EDataSlot::DyNumber>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return CompareStrings(lhs, rhs);
}
inline int CompareValues(EDataSlot type, const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
-#define COMPARE_TYPE(slot, ...) \
- case EDataSlot::slot: \
- return CompareValues<EDataSlot::slot>(lhs, rhs);
-
- switch (type) {
- UDF_TYPE_ID_MAP(COMPARE_TYPE)
- }
-
-#undef COMPARE_TYPE
- Y_ENSURE(false, "Incorrect data slot: " << (ui32)type);
-}
-
-// equate
-
-template <typename T, std::enable_if_t<std::is_integral<T>::value>* = nullptr>
+#define COMPARE_TYPE(slot, ...) \
+ case EDataSlot::slot: \
+ return CompareValues<EDataSlot::slot>(lhs, rhs);
+
+ switch (type) {
+ UDF_TYPE_ID_MAP(COMPARE_TYPE)
+ }
+
+#undef COMPARE_TYPE
+ Y_ENSURE(false, "Incorrect data slot: " << (ui32)type);
+}
+
+// equate
+
+template <typename T, std::enable_if_t<std::is_integral<T>::value>* = nullptr>
inline bool EquateIntegers(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return lhs.Get<T>() == rhs.Get<T>();
-}
-
-template <typename T, std::enable_if_t<std::is_floating_point<T>::value>* = nullptr>
+ return lhs.Get<T>() == rhs.Get<T>();
+}
+
+template <typename T, std::enable_if_t<std::is_floating_point<T>::value>* = nullptr>
inline bool EquateFloats(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- const auto x = lhs.Get<T>();
- const auto y = rhs.Get<T>();
- return std::isunordered(x, y) ? std::isnan(x) == std::isnan(y) : x == y;
-}
-
+ const auto x = lhs.Get<T>();
+ const auto y = rhs.Get<T>();
+ return std::isunordered(x, y) ? std::isnan(x) == std::isnan(y) : x == y;
+}
+
inline bool EquateStrings(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- const auto& lhsBuf = lhs.AsStringRef();
- const auto& rhsBuf = rhs.AsStringRef();
+ const auto& lhsBuf = lhs.AsStringRef();
+ const auto& rhsBuf = rhs.AsStringRef();
return lhsBuf == rhsBuf;
-}
-
-template <typename T, std::enable_if_t<std::is_integral<T>::value>* = nullptr>
+}
+
+template <typename T, std::enable_if_t<std::is_integral<T>::value>* = nullptr>
inline bool EquateTzIntegers(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return lhs.Get<T>() == rhs.Get<T>() && lhs.GetTimezoneId() == rhs.GetTimezoneId();
-}
-
-template <>
+ return lhs.Get<T>() == rhs.Get<T>() && lhs.GetTimezoneId() == rhs.GetTimezoneId();
+}
+
+template <>
inline bool EquateValues<EDataSlot::Bool>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return EquateIntegers<bool>(lhs, rhs);
-}
-
-template <>
+ return EquateIntegers<bool>(lhs, rhs);
+}
+
+template <>
inline bool EquateValues<EDataSlot::Int8>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return EquateIntegers<i8>(lhs, rhs);
-}
-
-template <>
+ return EquateIntegers<i8>(lhs, rhs);
+}
+
+template <>
inline bool EquateValues<EDataSlot::Uint8>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return EquateIntegers<ui8>(lhs, rhs);
-}
-
-template <>
+ return EquateIntegers<ui8>(lhs, rhs);
+}
+
+template <>
inline bool EquateValues<EDataSlot::Int16>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return EquateIntegers<i16>(lhs, rhs);
-}
-
-template <>
+ return EquateIntegers<i16>(lhs, rhs);
+}
+
+template <>
inline bool EquateValues<EDataSlot::Uint16>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return EquateIntegers<ui16>(lhs, rhs);
-}
-
-template <>
+ return EquateIntegers<ui16>(lhs, rhs);
+}
+
+template <>
inline bool EquateValues<EDataSlot::Int32>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return EquateIntegers<i32>(lhs, rhs);
-}
-
-template <>
+ return EquateIntegers<i32>(lhs, rhs);
+}
+
+template <>
inline bool EquateValues<EDataSlot::Uint32>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return EquateIntegers<ui32>(lhs, rhs);
-}
-
-template <>
+ return EquateIntegers<ui32>(lhs, rhs);
+}
+
+template <>
inline bool EquateValues<EDataSlot::Int64>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return EquateIntegers<i64>(lhs, rhs);
-}
-
-template <>
+ return EquateIntegers<i64>(lhs, rhs);
+}
+
+template <>
inline bool EquateValues<EDataSlot::Uint64>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return EquateIntegers<ui64>(lhs, rhs);
-}
-
-template <>
+ return EquateIntegers<ui64>(lhs, rhs);
+}
+
+template <>
inline bool EquateValues<EDataSlot::Float>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return EquateFloats<float>(lhs, rhs);
-}
-
-template <>
+ return EquateFloats<float>(lhs, rhs);
+}
+
+template <>
inline bool EquateValues<EDataSlot::Double>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return EquateFloats<double>(lhs, rhs);
-}
-
-template <>
+ return EquateFloats<double>(lhs, rhs);
+}
+
+template <>
inline bool EquateValues<EDataSlot::String>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return EquateStrings(lhs, rhs);
-}
-
-template <>
+ return EquateStrings(lhs, rhs);
+}
+
+template <>
inline bool EquateValues<EDataSlot::Utf8>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return EquateStrings(lhs, rhs);
-}
-
-template <>
+ return EquateStrings(lhs, rhs);
+}
+
+template <>
inline bool EquateValues<EDataSlot::Uuid>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return EquateStrings(lhs, rhs);
-}
-
-template <>
+ return EquateStrings(lhs, rhs);
+}
+
+template <>
inline bool EquateValues<EDataSlot::Yson>(const TUnboxedValuePod&, const TUnboxedValuePod&) {
- Y_FAIL("Yson isn't comparable.");
-}
-
-template <>
+ Y_FAIL("Yson isn't comparable.");
+}
+
+template <>
inline bool EquateValues<EDataSlot::Json>(const TUnboxedValuePod&, const TUnboxedValuePod&) {
- Y_FAIL("Json isn't comparable.");
-}
-
-template <>
+ Y_FAIL("Json isn't comparable.");
+}
+
+template <>
inline bool EquateValues<EDataSlot::JsonDocument>(const TUnboxedValuePod&, const TUnboxedValuePod&) {
Y_FAIL("JsonDocument isn't comparable.");
}
template <>
inline bool EquateValues<EDataSlot::Date>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return EquateIntegers<ui16>(lhs, rhs);
-}
-
-template <>
+ return EquateIntegers<ui16>(lhs, rhs);
+}
+
+template <>
inline bool EquateValues<EDataSlot::Datetime>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return EquateIntegers<ui32>(lhs, rhs);
-}
-
-template <>
+ return EquateIntegers<ui32>(lhs, rhs);
+}
+
+template <>
inline bool EquateValues<EDataSlot::Timestamp>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return EquateIntegers<ui64>(lhs, rhs);
-}
-
-template <>
+ return EquateIntegers<ui64>(lhs, rhs);
+}
+
+template <>
inline bool EquateValues<EDataSlot::Interval>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return EquateIntegers<i64>(lhs, rhs);
-}
-
-template <>
+ return EquateIntegers<i64>(lhs, rhs);
+}
+
+template <>
inline bool EquateValues<EDataSlot::TzDate>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return EquateTzIntegers<ui16>(lhs, rhs);
-}
-
-template <>
+ return EquateTzIntegers<ui16>(lhs, rhs);
+}
+
+template <>
inline bool EquateValues<EDataSlot::TzDatetime>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return EquateTzIntegers<ui32>(lhs, rhs);
-}
-
-template <>
+ return EquateTzIntegers<ui32>(lhs, rhs);
+}
+
+template <>
inline bool EquateValues<EDataSlot::TzTimestamp>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return EquateTzIntegers<ui64>(lhs, rhs);
-}
-
-template <>
+ return EquateTzIntegers<ui64>(lhs, rhs);
+}
+
+template <>
inline bool EquateValues<EDataSlot::Decimal>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
- return lhs.GetInt128() == rhs.GetInt128();
-}
-
+ return lhs.GetInt128() == rhs.GetInt128();
+}
+
template <>
inline bool EquateValues<EDataSlot::DyNumber>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return EquateStrings(lhs, rhs);
}
inline bool EquateValues(EDataSlot type, const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
-#define EQUATE_TYPE(slot, ...) \
- case EDataSlot::slot: \
- return EquateValues<EDataSlot::slot>(lhs, rhs);
-
- switch (type) {
- UDF_TYPE_ID_MAP(EQUATE_TYPE)
- }
-
-#undef EQUATE_TYPE
- Y_ENSURE(false, "Incorrect data slot: " << (ui32)type);
-}
-
-} // namespace NUdf
+#define EQUATE_TYPE(slot, ...) \
+ case EDataSlot::slot: \
+ return EquateValues<EDataSlot::slot>(lhs, rhs);
+
+ switch (type) {
+ UDF_TYPE_ID_MAP(EQUATE_TYPE)
+ }
+
+#undef EQUATE_TYPE
+ Y_ENSURE(false, "Incorrect data slot: " << (ui32)type);
+}
+
+} // namespace NUdf
} // namespace NYql
diff --git a/ydb/library/yql/public/udf/udf_value.h b/ydb/library/yql/public/udf/udf_value.h
index 705ac411333..807a0b76298 100644
--- a/ydb/library/yql/public/udf/udf_value.h
+++ b/ydb/library/yql/public/udf/udf_value.h
@@ -127,7 +127,7 @@ private:
virtual bool NextPair(TUnboxedValue& key, TUnboxedValue& payload) = 0;
virtual void Apply(IApplyContext& context) const = 0;
-
+
public:
// reference counting
inline void Ref() noexcept;
@@ -147,19 +147,19 @@ private:
const ui8 Reserved_ = 0;
};
-#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 3)
+#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 3)
class IBoxedValue2 : public IBoxedValue1 {
friend struct TBoxedValueAccessor;
friend class TBlock;
private:
- // Save/Load state
- virtual ui32 GetTraverseCount() const = 0;
- virtual TUnboxedValue GetTraverseItem(ui32 index) const = 0;
- virtual TUnboxedValue Save() const = 0;
- virtual void Load(const TStringRef& state) = 0;
+ // Save/Load state
+ virtual ui32 GetTraverseCount() const = 0;
+ virtual TUnboxedValue GetTraverseItem(ui32 index) const = 0;
+ virtual TUnboxedValue Save() const = 0;
+ virtual void Load(const TStringRef& state) = 0;
};
-#endif
-
+#endif
+
#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 11)
class IBoxedValue3 : public IBoxedValue2 {
friend struct TBoxedValueAccessor;
@@ -391,7 +391,7 @@ UDF_ASSERT_TYPE_SIZE(TFlatDataBlockPtr, 8);
struct TBoxedValueAccessor
{
#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 19)
-
+
#define METHOD_MAP(xx) \
xx(HasFastListLength) \
xx(GetListLength) \
@@ -421,10 +421,10 @@ struct TBoxedValueAccessor
xx(Skip) \
xx(Next) \
xx(NextPair) \
- xx(Apply) \
- xx(GetTraverseCount) \
- xx(GetTraverseItem) \
- xx(Save) \
+ xx(Apply) \
+ xx(GetTraverseCount) \
+ xx(GetTraverseItem) \
+ xx(Save) \
xx(Load) \
xx(Push) \
xx(IsSortedDict) \
@@ -547,43 +547,43 @@ struct TBoxedValueAccessor
xx(GetTraverseCount) \
xx(GetTraverseItem) \
xx(Save) \
- xx(Load)
-
-#else
-
-#define METHOD_MAP(xx) \
- xx(HasFastListLength) \
- xx(GetListLength) \
- xx(GetEstimatedListLength) \
- xx(GetListIterator) \
- xx(GetListRepresentation) \
- xx(ReverseListImpl) \
- xx(SkipListImpl) \
- xx(TakeListImpl) \
- xx(ToIndexDictImpl) \
- xx(GetDictLength) \
- xx(GetDictIterator) \
- xx(GetKeysIterator) \
- xx(GetPayloadsIterator) \
- xx(Contains) \
- xx(Lookup) \
- xx(GetElement) \
- xx(GetElements) \
- xx(Run) \
- xx(GetResourceTag) \
- xx(GetResource) \
- xx(HasListItems) \
- xx(HasDictItems) \
- xx(GetVariantIndex) \
- xx(GetVariantItem) \
- xx(Fetch) \
- xx(Skip) \
- xx(Next) \
- xx(NextPair) \
+ xx(Load)
+
+#else
+
+#define METHOD_MAP(xx) \
+ xx(HasFastListLength) \
+ xx(GetListLength) \
+ xx(GetEstimatedListLength) \
+ xx(GetListIterator) \
+ xx(GetListRepresentation) \
+ xx(ReverseListImpl) \
+ xx(SkipListImpl) \
+ xx(TakeListImpl) \
+ xx(ToIndexDictImpl) \
+ xx(GetDictLength) \
+ xx(GetDictIterator) \
+ xx(GetKeysIterator) \
+ xx(GetPayloadsIterator) \
+ xx(Contains) \
+ xx(Lookup) \
+ xx(GetElement) \
+ xx(GetElements) \
+ xx(Run) \
+ xx(GetResourceTag) \
+ xx(GetResource) \
+ xx(HasListItems) \
+ xx(HasDictItems) \
+ xx(GetVariantIndex) \
+ xx(GetVariantItem) \
+ xx(Fetch) \
+ xx(Skip) \
+ xx(Next) \
+ xx(NextPair) \
xx(Apply)
-#endif
-
+#endif
+
enum class EMethod : ui32 {
#define MAP_HANDLER(xx) xx,
METHOD_MAP(MAP_HANDLER)
@@ -657,14 +657,14 @@ struct TBoxedValueAccessor
static inline bool NextPair(IBoxedValue& value, TUnboxedValue& key, TUnboxedValue& payload);
static inline void Apply(IBoxedValue& value, IApplyContext& context);
-
-#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 3)
- // Save/Load state
- static inline ui32 GetTraverseCount(const IBoxedValue& value);
- static inline TUnboxedValue GetTraverseItem(const IBoxedValue& value, ui32 index);
- static inline TUnboxedValue Save(const IBoxedValue& value);
- static inline void Load(IBoxedValue& value, const TStringRef& state);
-#endif
+
+#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 3)
+ // Save/Load state
+ static inline ui32 GetTraverseCount(const IBoxedValue& value);
+ static inline TUnboxedValue GetTraverseItem(const IBoxedValue& value, ui32 index);
+ static inline TUnboxedValue Save(const IBoxedValue& value);
+ static inline void Load(IBoxedValue& value, const TStringRef& state);
+#endif
#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 11)
static inline void Push(IBoxedValue& value, const TUnboxedValuePod& data);
@@ -741,15 +741,15 @@ private:
bool NextPair(TUnboxedValue& key, TUnboxedValue& payload) override;
void Apply(IApplyContext& context) const override;
-
-#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 3)
- // Save/Load state
- ui32 GetTraverseCount() const override;
- TUnboxedValue GetTraverseItem(ui32 index) const override;
- TUnboxedValue Save() const override;
- void Load(const TStringRef& state) override;
-#endif
-
+
+#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 3)
+ // Save/Load state
+ ui32 GetTraverseCount() const override;
+ TUnboxedValue GetTraverseItem(ui32 index) const override;
+ TUnboxedValue Save() const override;
+ void Load(const TStringRef& state) override;
+#endif
+
#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 11)
void Push(const TUnboxedValuePod& value) override;
#endif
@@ -919,14 +919,14 @@ public:
inline void Apply(IApplyContext& context) const;
-#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 3)
- // Save/Load state
- inline ui32 GetTraverseCount() const;
- inline TUnboxedValue GetTraverseItem(ui32 index) const;
- inline TUnboxedValue Save() const;
- inline void Load(const TStringRef& state);
-#endif
-
+#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 3)
+ // Save/Load state
+ inline ui32 GetTraverseCount() const;
+ inline TUnboxedValue GetTraverseItem(ui32 index) const;
+ inline TUnboxedValue Save() const;
+ inline void Load(const TStringRef& state);
+#endif
+
#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 12)
inline bool IsSortedDict() const;
#endif
diff --git a/ydb/library/yql/public/udf/udf_value_builder.h b/ydb/library/yql/public/udf/udf_value_builder.h
index 096f735a2b1..1a161d366c5 100644
--- a/ydb/library/yql/public/udf/udf_value_builder.h
+++ b/ydb/library/yql/public/udf/udf_value_builder.h
@@ -45,21 +45,21 @@ class IDateBuilder1
public:
virtual ~IDateBuilder1() = default;
- virtual bool MakeDate(ui32 year, ui32 month, ui32 day, ui16& value) const = 0;
- virtual bool SplitDate(ui16 value, ui32& year, ui32& month, ui32& day) const = 0;
+ virtual bool MakeDate(ui32 year, ui32 month, ui32 day, ui16& value) const = 0;
+ virtual bool SplitDate(ui16 value, ui32& year, ui32& month, ui32& day) const = 0;
- virtual bool MakeDatetime(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 minute, ui32 second, ui32& value,
- ui16 timezoneId = 0) const = 0;
- virtual bool SplitDatetime(ui32 value, ui32& year, ui32& month, ui32& day, ui32& hour, ui32& minute, ui32& second,
- ui16 timezoneId = 0) const = 0;
+ virtual bool MakeDatetime(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 minute, ui32 second, ui32& value,
+ ui16 timezoneId = 0) const = 0;
+ virtual bool SplitDatetime(ui32 value, ui32& year, ui32& month, ui32& day, ui32& hour, ui32& minute, ui32& second,
+ ui16 timezoneId = 0) const = 0;
// deprecated
- virtual bool EnrichDate(ui16 date, ui32& dayOfYear, ui32& weekOfYear, ui32& dayOfWeek) const = 0;
-
- // in minutes
- virtual bool GetTimezoneShift(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 minute, ui32 second,
- ui16 timezoneId, i32& value) const = 0;
-
+ virtual bool EnrichDate(ui16 date, ui32& dayOfYear, ui32& weekOfYear, ui32& dayOfWeek) const = 0;
+
+ // in minutes
+ virtual bool GetTimezoneShift(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 minute, ui32 second,
+ ui16 timezoneId, i32& value) const = 0;
+
#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT < UDF_ABI_COMPATIBILITY_VERSION(2, 23)
virtual void Unused7() const = 0;
virtual void Unused8() const = 0;
diff --git a/ydb/library/yql/public/udf/udf_value_inl.h b/ydb/library/yql/public/udf/udf_value_inl.h
index eb2db28e18a..028eef0cece 100644
--- a/ydb/library/yql/public/udf/udf_value_inl.h
+++ b/ydb/library/yql/public/udf/udf_value_inl.h
@@ -263,28 +263,28 @@ inline void TBoxedValueAccessor::Apply(IBoxedValue& value, IApplyContext& contex
return value.Apply(context);
}
-#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 3)
-inline ui32 TBoxedValueAccessor::GetTraverseCount(const IBoxedValue& value) {
- Y_VERIFY_DEBUG(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 3)));
- return value.GetTraverseCount();
-}
-
-inline TUnboxedValue TBoxedValueAccessor::GetTraverseItem(const IBoxedValue& value, ui32 index) {
- Y_VERIFY_DEBUG(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 3)));
- return value.GetTraverseItem(index);
-}
-
-inline TUnboxedValue TBoxedValueAccessor::Save(const IBoxedValue& value) {
- Y_VERIFY_DEBUG(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 3)));
- return value.Save();
-}
-
-inline void TBoxedValueAccessor::Load(IBoxedValue& value, const TStringRef& state) {
- Y_VERIFY_DEBUG(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 3)));
- value.Load(state);
-}
-#endif
-
+#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 3)
+inline ui32 TBoxedValueAccessor::GetTraverseCount(const IBoxedValue& value) {
+ Y_VERIFY_DEBUG(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 3)));
+ return value.GetTraverseCount();
+}
+
+inline TUnboxedValue TBoxedValueAccessor::GetTraverseItem(const IBoxedValue& value, ui32 index) {
+ Y_VERIFY_DEBUG(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 3)));
+ return value.GetTraverseItem(index);
+}
+
+inline TUnboxedValue TBoxedValueAccessor::Save(const IBoxedValue& value) {
+ Y_VERIFY_DEBUG(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 3)));
+ return value.Save();
+}
+
+inline void TBoxedValueAccessor::Load(IBoxedValue& value, const TStringRef& state) {
+ Y_VERIFY_DEBUG(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 3)));
+ value.Load(state);
+}
+#endif
+
#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 11)
inline void TBoxedValueAccessor::Push(IBoxedValue& value, const TUnboxedValuePod& data) {
Y_VERIFY_DEBUG(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 11)));
@@ -624,28 +624,28 @@ inline void TUnboxedValuePod::Apply(IApplyContext& context) const {
return TBoxedValueAccessor::Apply(*Raw.Boxed.Value, context);
}
-#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 3)
-inline ui32 TUnboxedValuePod::GetTraverseCount() const {
- UDF_VERIFY(IsBoxed(), "Value is not boxed");
- return TBoxedValueAccessor::GetTraverseCount(*Raw.Boxed.Value);
-}
-
-inline TUnboxedValue TUnboxedValuePod::GetTraverseItem(ui32 index) const {
- UDF_VERIFY(IsBoxed(), "Value is not boxed");
- return TBoxedValueAccessor::GetTraverseItem(*Raw.Boxed.Value, index);
-}
-
-inline TUnboxedValue TUnboxedValuePod::Save() const {
- UDF_VERIFY(IsBoxed(), "Value is not boxed");
- return TBoxedValueAccessor::Save(*Raw.Boxed.Value);
-}
-
-inline void TUnboxedValuePod::Load(const TStringRef& state) {
- UDF_VERIFY(IsBoxed(), "Value is not boxed");
- return TBoxedValueAccessor::Load(*Raw.Boxed.Value, state);
-}
-#endif
-
+#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 3)
+inline ui32 TUnboxedValuePod::GetTraverseCount() const {
+ UDF_VERIFY(IsBoxed(), "Value is not boxed");
+ return TBoxedValueAccessor::GetTraverseCount(*Raw.Boxed.Value);
+}
+
+inline TUnboxedValue TUnboxedValuePod::GetTraverseItem(ui32 index) const {
+ UDF_VERIFY(IsBoxed(), "Value is not boxed");
+ return TBoxedValueAccessor::GetTraverseItem(*Raw.Boxed.Value, index);
+}
+
+inline TUnboxedValue TUnboxedValuePod::Save() const {
+ UDF_VERIFY(IsBoxed(), "Value is not boxed");
+ return TBoxedValueAccessor::Save(*Raw.Boxed.Value);
+}
+
+inline void TUnboxedValuePod::Load(const TStringRef& state) {
+ UDF_VERIFY(IsBoxed(), "Value is not boxed");
+ return TBoxedValueAccessor::Load(*Raw.Boxed.Value, state);
+}
+#endif
+
#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 12)
inline bool TUnboxedValuePod::IsSortedDict() const {
UDF_VERIFY(IsBoxed(), "Value is not boxed");
diff --git a/ydb/library/yql/public/udf/udf_value_ut.cpp b/ydb/library/yql/public/udf/udf_value_ut.cpp
index 8d9708c7b19..0dc8bfee78e 100644
--- a/ydb/library/yql/public/udf/udf_value_ut.cpp
+++ b/ydb/library/yql/public/udf/udf_value_ut.cpp
@@ -144,12 +144,12 @@ Y_UNIT_TEST_SUITE(TUdfValue) {
UNIT_ASSERT_VALUES_EQUAL(28, METHOD_INDEX(Next));
UNIT_ASSERT_VALUES_EQUAL(29, METHOD_INDEX(NextPair));
UNIT_ASSERT_VALUES_EQUAL(30, METHOD_INDEX(Apply));
-#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 3)
- UNIT_ASSERT_VALUES_EQUAL(31, METHOD_INDEX(GetTraverseCount));
- UNIT_ASSERT_VALUES_EQUAL(32, METHOD_INDEX(GetTraverseItem));
- UNIT_ASSERT_VALUES_EQUAL(33, METHOD_INDEX(Save));
- UNIT_ASSERT_VALUES_EQUAL(34, METHOD_INDEX(Load));
-#endif
+#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 3)
+ UNIT_ASSERT_VALUES_EQUAL(31, METHOD_INDEX(GetTraverseCount));
+ UNIT_ASSERT_VALUES_EQUAL(32, METHOD_INDEX(GetTraverseItem));
+ UNIT_ASSERT_VALUES_EQUAL(33, METHOD_INDEX(Save));
+ UNIT_ASSERT_VALUES_EQUAL(34, METHOD_INDEX(Load));
+#endif
#undef METHOD_INDEX
}
}
diff --git a/ydb/library/yql/sql/v0/SQL.g b/ydb/library/yql/sql/v0/SQL.g
index 3b616b9c9fa..6a20a39c127 100644
--- a/ydb/library/yql/sql/v0/SQL.g
+++ b/ydb/library/yql/sql/v0/SQL.g
@@ -213,7 +213,7 @@ reduce_core:
opt_set_quantifier: (ALL | DISTINCT)?;
select_core:
- (FROM join_source)? SELECT STREAM? opt_set_quantifier result_column (COMMA result_column)* (WITHOUT column_list)? (FROM join_source)? (WHERE expr)?
+ (FROM join_source)? SELECT STREAM? opt_set_quantifier result_column (COMMA result_column)* (WITHOUT column_list)? (FROM join_source)? (WHERE expr)?
group_by_clause? (HAVING expr)? window_clause? order_by_clause?
;
@@ -229,7 +229,7 @@ grouping_element:
| cube_list
| grouping_sets_specification
//empty_grouping_set inside smart_parenthesis
- | hopping_window_specification
+ | hopping_window_specification
;
/// expect column (named column), or parenthesis list columns, or expression (named expression), or list expression
@@ -242,8 +242,8 @@ cube_list: CUBE LPAREN ordinary_grouping_set_list RPAREN;
/// SQL2003 grouping_set_list == grouping_element_list
grouping_sets_specification: GROUPING SETS LPAREN grouping_element_list RPAREN;
-hopping_window_specification: HOP LPAREN expr COMMA expr COMMA expr COMMA expr RPAREN;
-
+hopping_window_specification: HOP LPAREN expr COMMA expr COMMA expr COMMA expr RPAREN;
+
result_column:
opt_id_prefix ASTERISK
| expr (AS id_or_string)?
@@ -306,14 +306,14 @@ alter_table_add_column: ADD COLUMN? column_schema (COMMA ADD COLUMN? column_sche
alter_table_drop_column: DROP COLUMN? id;
column_schema: id_schema flex_type (NOT? NULL)?;
-column_order_by_specification: id (ASC | DESC)?;
-
-table_constraint:
- PRIMARY KEY LPAREN id (COMMA id)* RPAREN
- | PARTITION BY LPAREN id (COMMA id)* RPAREN
- | ORDER BY LPAREN column_order_by_specification (COMMA column_order_by_specification)* RPAREN
-;
-
+column_order_by_specification: id (ASC | DESC)?;
+
+table_constraint:
+ PRIMARY KEY LPAREN id (COMMA id)* RPAREN
+ | PARTITION BY LPAREN id (COMMA id)* RPAREN
+ | ORDER BY LPAREN column_order_by_specification (COMMA column_order_by_specification)* RPAREN
+;
+
drop_table_stmt: DROP TABLE (IF EXISTS)? simple_table_ref;
define_action_or_subquery_stmt: DEFINE (ACTION|SUBQUERY) bind_parameter LPAREN bind_parameter_list? RPAREN AS define_action_or_subquery_body END DEFINE;
define_action_or_subquery_body: (sql_stmt_core SEMI)* SEMI?;
@@ -435,7 +435,7 @@ keyword_expr_uncompat:
| EXISTS
| FROM
| FULL
- | HOP
+ | HOP
| NOT
| NULL
| PROCESS
@@ -772,7 +772,7 @@ GLOB: G L O B;
GROUP: G R O U P;
GROUPING: G R O U P I N G;
HAVING: H A V I N G;
-HOP: H O P;
+HOP: H O P;
IF: I F;
IGNORE: I G N O R E;
ILIKE: I L I K E;
@@ -848,7 +848,7 @@ SEMI_JOIN: S E M I;
SET: S E T;
SETS: S E T S;
SUBQUERY: S U B Q U E R Y;
-STREAM: S T R E A M;
+STREAM: S T R E A M;
SYMBOLS: S Y M B O L S;
SYSTEM: S Y S T E M;
TABLE: T A B L E;
diff --git a/ydb/library/yql/sql/v0/aggregation.cpp b/ydb/library/yql/sql/v0/aggregation.cpp
index 64406048811..ad2b3e12e52 100644
--- a/ydb/library/yql/sql/v0/aggregation.cpp
+++ b/ydb/library/yql/sql/v0/aggregation.cpp
@@ -260,8 +260,8 @@ private:
apply = L(apply, Limit);
}
- std::vector<ui32> GetFactoryColumnIndices() const final {
- return {1u, 0u};
+ std::vector<ui32> GetFactoryColumnIndices() const final {
+ return {1u, 0u};
}
bool DoInit(TContext& ctx, ISource* src) final {
@@ -338,8 +338,8 @@ private:
return Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Payload), BuildLambda(Pos, Y("row"), Predicate));
}
- std::vector<ui32> GetFactoryColumnIndices() const final {
- return {0u, 1u};
+ std::vector<ui32> GetFactoryColumnIndices() const final {
+ return {0u, 1u};
}
bool DoInit(TContext& ctx, ISource* src) final {
@@ -517,8 +517,8 @@ private:
apply = L(apply, Intervals);
}
- std::vector<ui32> GetFactoryColumnIndices() const final {
- return {0u, 1u};
+ std::vector<ui32> GetFactoryColumnIndices() const final {
+ return {0u, 1u};
}
bool DoInit(TContext& ctx, ISource* src) final {
@@ -862,118 +862,118 @@ TAggregationPtr BuildTopFreqFactoryAggregation(TPosition pos, const TString& nam
return new TTopFreqFactory(pos, name, factory, aggMode);
}
-template <bool HasKey>
-class TTopAggregationFactory final : public TAggregationFactory {
-public:
- TTopAggregationFactory(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode)
- : TAggregationFactory(pos, name, factory, aggMode)
+template <bool HasKey>
+class TTopAggregationFactory final : public TAggregationFactory {
+public:
+ TTopAggregationFactory(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode)
+ : TAggregationFactory(pos, name, factory, aggMode)
, FakeSource(BuildFakeSource(pos))
- {}
-
-private:
- bool InitAggr(TContext& ctx, bool isFactory, ISource* src, TAstListNode& node, const TVector<TNodePtr>& exprs) final {
- ui32 adjustArgsCount = isFactory ? 1 : (HasKey ? 3 : 2);
- if (exprs.size() != adjustArgsCount) {
- ctx.Error(Pos) << "Aggregation function " << (isFactory ? "factory " : "") << Name << " requires "
- << adjustArgsCount << " arguments, given: " << exprs.size();
- return false;
- }
-
- if (!isFactory) {
- Payload = exprs[0];
- if (HasKey) {
- Key = exprs[1];
- }
- }
-
- Count = exprs.back();
-
- if (!isFactory) {
+ {}
+
+private:
+ bool InitAggr(TContext& ctx, bool isFactory, ISource* src, TAstListNode& node, const TVector<TNodePtr>& exprs) final {
+ ui32 adjustArgsCount = isFactory ? 1 : (HasKey ? 3 : 2);
+ if (exprs.size() != adjustArgsCount) {
+ ctx.Error(Pos) << "Aggregation function " << (isFactory ? "factory " : "") << Name << " requires "
+ << adjustArgsCount << " arguments, given: " << exprs.size();
+ return false;
+ }
+
+ if (!isFactory) {
+ Payload = exprs[0];
+ if (HasKey) {
+ Key = exprs[1];
+ }
+ }
+
+ Count = exprs.back();
+
+ if (!isFactory) {
Name = src->MakeLocalName(Name);
- }
-
- if (!Init(ctx, src)) {
- return false;
- }
-
- if (!isFactory) {
- node.Add("Member", "row", Q(Name));
- }
-
- return true;
- }
-
- TNodePtr DoClone() const final {
- return new TTopAggregationFactory(Pos, Name, Func, AggMode);
- }
-
- TNodePtr GetApply(const TNodePtr& type) const final {
- TNodePtr apply;
- if (HasKey) {
- apply = Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Key), BuildLambda(Pos, Y("row"), Payload));
- } else {
- apply = Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Payload));
- }
- AddFactoryArguments(apply);
- return apply;
- }
-
- void AddFactoryArguments(TNodePtr& apply) const final {
- apply = L(apply, Count);
- }
-
- std::vector<ui32> GetFactoryColumnIndices() const final {
- if (HasKey) {
- return {1u, 0u};
- } else {
- return {0u};
- }
- }
-
- bool DoInit(TContext& ctx, ISource* src) final {
+ }
+
+ if (!Init(ctx, src)) {
+ return false;
+ }
+
+ if (!isFactory) {
+ node.Add("Member", "row", Q(Name));
+ }
+
+ return true;
+ }
+
+ TNodePtr DoClone() const final {
+ return new TTopAggregationFactory(Pos, Name, Func, AggMode);
+ }
+
+ TNodePtr GetApply(const TNodePtr& type) const final {
+ TNodePtr apply;
+ if (HasKey) {
+ apply = Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Key), BuildLambda(Pos, Y("row"), Payload));
+ } else {
+ apply = Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Payload));
+ }
+ AddFactoryArguments(apply);
+ return apply;
+ }
+
+ void AddFactoryArguments(TNodePtr& apply) const final {
+ apply = L(apply, Count);
+ }
+
+ std::vector<ui32> GetFactoryColumnIndices() const final {
+ if (HasKey) {
+ return {1u, 0u};
+ } else {
+ return {0u};
+ }
+ }
+
+ bool DoInit(TContext& ctx, ISource* src) final {
ctx.PushBlockShortcuts();
if (!Count->Init(ctx, FakeSource.Get())) {
- return false;
- }
+ return false;
+ }
Count = ctx.GroundBlockShortcutsForExpr(Count);
-
- if (!Payload) {
- return true;
- }
-
- if (HasKey) {
- ctx.PushBlockShortcuts();
- if (!Key->Init(ctx, src)) {
- return false;
- }
- Key = ctx.GroundBlockShortcutsForExpr(Key);
- }
-
- ctx.PushBlockShortcuts();
- if (!Payload->Init(ctx, src)) {
- return false;
- }
- Payload = ctx.GroundBlockShortcutsForExpr(Payload);
-
- if ((HasKey && Key->IsAggregated()) || (!HasKey && Payload->IsAggregated())) {
- ctx.Error(Pos) << "Aggregation of aggregated values is forbidden";
- return false;
- }
- return true;
- }
-
+
+ if (!Payload) {
+ return true;
+ }
+
+ if (HasKey) {
+ ctx.PushBlockShortcuts();
+ if (!Key->Init(ctx, src)) {
+ return false;
+ }
+ Key = ctx.GroundBlockShortcutsForExpr(Key);
+ }
+
+ ctx.PushBlockShortcuts();
+ if (!Payload->Init(ctx, src)) {
+ return false;
+ }
+ Payload = ctx.GroundBlockShortcutsForExpr(Payload);
+
+ if ((HasKey && Key->IsAggregated()) || (!HasKey && Payload->IsAggregated())) {
+ ctx.Error(Pos) << "Aggregation of aggregated values is forbidden";
+ return false;
+ }
+ return true;
+ }
+
TSourcePtr FakeSource;
- TNodePtr Key, Payload, Count;
-};
-
-template <bool HasKey>
-TAggregationPtr BuildTopFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) {
- return new TTopAggregationFactory<HasKey>(pos, name, factory, aggMode);
-}
-
-template TAggregationPtr BuildTopFactoryAggregation<false>(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode);
-template TAggregationPtr BuildTopFactoryAggregation<true >(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode);
-
+ TNodePtr Key, Payload, Count;
+};
+
+template <bool HasKey>
+TAggregationPtr BuildTopFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) {
+ return new TTopAggregationFactory<HasKey>(pos, name, factory, aggMode);
+}
+
+template TAggregationPtr BuildTopFactoryAggregation<false>(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode);
+template TAggregationPtr BuildTopFactoryAggregation<true >(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode);
+
class TCountDistinctEstimateAggregationFactory final : public TAggregationFactory {
public:
TCountDistinctEstimateAggregationFactory(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode)
diff --git a/ydb/library/yql/sql/v0/builtin.cpp b/ydb/library/yql/sql/v0/builtin.cpp
index a0945f9d05e..2b6b241a726 100644
--- a/ydb/library/yql/sql/v0/builtin.cpp
+++ b/ydb/library/yql/sql/v0/builtin.cpp
@@ -140,14 +140,14 @@ public:
auto factory = Aggr->AggregationTraitsFactory();
auto apply = Y("Apply", factory, Y("ListType", "type"));
-
- auto columnIndices = Aggr->GetFactoryColumnIndices();
- if (columnIndices.size() == 1) {
+
+ auto columnIndices = Aggr->GetFactoryColumnIndices();
+ if (columnIndices.size() == 1) {
apply = L(apply, "extractor");
} else {
// make several extractors from main that returns a tuple
- for (ui32 arg = 0; arg < columnIndices.size(); ++arg) {
- auto partial = BuildLambda(Pos, Y("row"), Y("Nth", Y("Apply", "extractor", "row"), Q(ToString(columnIndices[arg]))));
+ for (ui32 arg = 0; arg < columnIndices.size(); ++arg) {
+ auto partial = BuildLambda(Pos, Y("row"), Y("Nth", Y("Apply", "extractor", "row"), Q(ToString(columnIndices[arg]))));
apply = L(apply, partial);
}
}
@@ -778,12 +778,12 @@ public:
}
auto aliasNode = BuildFileNameArgument(Args[1]->GetPos(), Args[1]);
OpName = "Apply";
- Args[0] = Y("Udf", Q("File.ByLines"), Y("Void"),
- Y("TupleType",
- Y("TupleType", Y("DataType", dataTypeStringNode)),
- Y("StructType"),
- Y("TupleType")));
-
+ Args[0] = Y("Udf", Q("File.ByLines"), Y("Void"),
+ Y("TupleType",
+ Y("TupleType", Y("DataType", dataTypeStringNode)),
+ Y("StructType"),
+ Y("TupleType")));
+
Args[1] = Y("FilePath", aliasNode);
return TCallNode::DoInit(ctx, src);
}
@@ -1579,47 +1579,47 @@ private:
TString Mode;
};
-template <bool IsStart>
-class THoppingTime final: public TAstListNode {
-public:
- THoppingTime(TPosition pos, const TVector<TNodePtr>& args = {})
- : TAstListNode(pos)
+template <bool IsStart>
+class THoppingTime final: public TAstListNode {
+public:
+ THoppingTime(TPosition pos, const TVector<TNodePtr>& args = {})
+ : TAstListNode(pos)
{
Y_UNUSED(args);
}
-
-private:
- TNodePtr DoClone() const override {
- return new THoppingTime(GetPos());
- }
-
- bool DoInit(TContext& ctx, ISource* src) override {
- Y_UNUSED(ctx);
-
- auto window = src->GetHoppingWindowSpec();
- if (!window) {
- ctx.Error(Pos) << "No hopping window parameters in aggregation";
- return false;
- }
-
- Nodes.clear();
-
- if (!IsStart) {
- Add("Member", "row", Q("_yql_time"));
- return true;
- }
-
- Add("Sub",
- Y("Member", "row", Q("_yql_time")),
- window->Interval);
- return true;
- }
-
- void DoUpdateState() const override {
- State.Set(ENodeState::Aggregated, true);
- }
-};
-
+
+private:
+ TNodePtr DoClone() const override {
+ return new THoppingTime(GetPos());
+ }
+
+ bool DoInit(TContext& ctx, ISource* src) override {
+ Y_UNUSED(ctx);
+
+ auto window = src->GetHoppingWindowSpec();
+ if (!window) {
+ ctx.Error(Pos) << "No hopping window parameters in aggregation";
+ return false;
+ }
+
+ Nodes.clear();
+
+ if (!IsStart) {
+ Add("Member", "row", Q("_yql_time"));
+ return true;
+ }
+
+ Add("Sub",
+ Y("Member", "row", Q("_yql_time")),
+ window->Interval);
+ return true;
+ }
+
+ void DoUpdateState() const override {
+ State.Set(ENodeState::Aggregated, true);
+ }
+};
+
class TInvalidBuiltin final: public INode {
public:
TInvalidBuiltin(TPosition pos, const TString& info)
@@ -1655,8 +1655,8 @@ enum EAggrFuncTypeCallback {
LINEAR_HISTOGRAM,
PERCENTILE,
TOPFREQ,
- TOP,
- TOP_BY,
+ TOP,
+ TOP_BY,
COUNT_DISTINCT_ESTIMATE,
LIST,
UDAF
@@ -1722,12 +1722,12 @@ TAggrFuncFactoryCallback BuildAggrFuncFactoryCallback(
case TOPFREQ:
factory = BuildTopFreqFactoryAggregation(pos, realFunctionName, factoryName, aggMode);
break;
- case TOP:
- factory = BuildTopFactoryAggregation<false>(pos, realFunctionName, factoryName, aggMode);
- break;
- case TOP_BY:
- factory = BuildTopFactoryAggregation<true>(pos, realFunctionName, factoryName, aggMode);
- break;
+ case TOP:
+ factory = BuildTopFactoryAggregation<false>(pos, realFunctionName, factoryName, aggMode);
+ break;
+ case TOP_BY:
+ factory = BuildTopFactoryAggregation<true>(pos, realFunctionName, factoryName, aggMode);
+ break;
case COUNT_DISTINCT_ESTIMATE:
factory = BuildCountDistinctEstimateFactoryAggregation(pos, realFunctionName, factoryName, aggMode);
break;
@@ -2026,7 +2026,7 @@ struct TBuiltinFuncData {
{"tablerecordindex", BuildNamedBuiltinFactoryCallback<TCallDirectRow>("TableRecord") },
{"weakfield", BuildSimpleBuiltinFactoryCallback<TWeakFieldOp>()},
{"tablerow", BuildSimpleBuiltinFactoryCallback<TTableRow>() },
-
+
// Hint builtins
{"grouping", BuildSimpleBuiltinFactoryCallback<TGroupingNode>()},
@@ -2088,11 +2088,11 @@ struct TBuiltinFuncData {
{"mode", BuildAggrFuncFactoryCallback("Mode", "topfreq_traits_factory", TOPFREQ) },
{"topfreq", BuildAggrFuncFactoryCallback("TopFreq", "topfreq_traits_factory", TOPFREQ) },
- {"top", BuildAggrFuncFactoryCallback("Top", "top_traits_factory", TOP)},
- {"bottom", BuildAggrFuncFactoryCallback("Bottom", "bottom_traits_factory", TOP)},
- {"topby", BuildAggrFuncFactoryCallback("TopBy", "top_by_traits_factory", TOP_BY)},
- {"bottomby", BuildAggrFuncFactoryCallback("BottomBy", "bottom_by_traits_factory", TOP_BY)},
-
+ {"top", BuildAggrFuncFactoryCallback("Top", "top_traits_factory", TOP)},
+ {"bottom", BuildAggrFuncFactoryCallback("Bottom", "bottom_traits_factory", TOP)},
+ {"topby", BuildAggrFuncFactoryCallback("TopBy", "top_by_traits_factory", TOP_BY)},
+ {"bottomby", BuildAggrFuncFactoryCallback("BottomBy", "bottom_by_traits_factory", TOP_BY)},
+
{"histogram", BuildAggrFuncFactoryCallback("AdaptiveWardHistogram", "histogram_adaptive_ward_traits_factory", HISTOGRAM, "Histogram")},
{"adaptivewardhistogram", BuildAggrFuncFactoryCallback("AdaptiveWardHistogram", "histogram_adaptive_ward_traits_factory", HISTOGRAM)},
{"adaptiveweighthistogram", BuildAggrFuncFactoryCallback("AdaptiveWeightHistogram", "histogram_adaptive_weight_traits_factory", HISTOGRAM)},
@@ -2334,9 +2334,9 @@ TNodePtr BuildBuiltinFunc(TContext& ctx, TPosition pos, TString name, const TVec
return BuildUdf(ctx, pos, moduleName, name, newArgs);
}
- } else if (ns == "datetime2" && (name == "Format" || name == "Parse")) {
- return BuildUdf(ctx, pos, nameSpace, name, args);
-
+ } else if (ns == "datetime2" && (name == "Format" || name == "Parse")) {
+ return BuildUdf(ctx, pos, nameSpace, name, args);
+
} else if (scriptType != NKikimr::NMiniKQL::EScriptType::Unknown) {
auto scriptName = NKikimr::NMiniKQL::ScriptTypeAsStr(scriptType);
return new TScriptUdf(pos, TString(scriptName), name, args);
diff --git a/ydb/library/yql/sql/v0/node.cpp b/ydb/library/yql/sql/v0/node.cpp
index 57f0c6a561f..095e37fbf36 100644
--- a/ydb/library/yql/sql/v0/node.cpp
+++ b/ydb/library/yql/sql/v0/node.cpp
@@ -75,12 +75,12 @@ bool TTableRef::Check(TContext& ctx) const {
return true;
}
-TColumnSchema::TColumnSchema(TPosition pos, const TString& name, const TString& type, bool nullable, bool isTypeString)
+TColumnSchema::TColumnSchema(TPosition pos, const TString& name, const TString& type, bool nullable, bool isTypeString)
: Pos(pos)
, Name(name)
, Type(type)
, Nullable(nullable)
- , IsTypeString(isTypeString)
+ , IsTypeString(isTypeString)
{
}
@@ -794,9 +794,9 @@ TWindowSpecificationPtr TWindowSpecification::Clone() const {
THoppingWindowSpecPtr THoppingWindowSpec::Clone() const {
auto res = MakeIntrusive<THoppingWindowSpec>();
res->TimeExtractor = TimeExtractor->Clone();
- res->Hop = Hop->Clone();
- res->Interval = Interval->Clone();
- res->Delay = Delay->Clone();
+ res->Hop = Hop->Clone();
+ res->Interval = Interval->Clone();
+ res->Delay = Delay->Clone();
return res;
}
@@ -1018,8 +1018,8 @@ void IAggregation::AddFactoryArguments(TNodePtr& apply) const {
Y_UNUSED(apply);
}
-std::vector<ui32> IAggregation::GetFactoryColumnIndices() const {
- return {0u};
+std::vector<ui32> IAggregation::GetFactoryColumnIndices() const {
+ return {0u};
}
TNodePtr IAggregation::WindowTraits(const TNodePtr& type) const {
@@ -1190,14 +1190,14 @@ const TVector<TString>& ISource::GetTmpWindowColumns() const {
return TmpWindowColumns;
}
-void ISource::SetHoppingWindowSpec(THoppingWindowSpecPtr spec) {
- HoppingWindowSpec = spec;
-}
-
-THoppingWindowSpecPtr ISource::GetHoppingWindowSpec() const {
- return HoppingWindowSpec;
-}
-
+void ISource::SetHoppingWindowSpec(THoppingWindowSpecPtr spec) {
+ HoppingWindowSpec = spec;
+}
+
+THoppingWindowSpecPtr ISource::GetHoppingWindowSpec() const {
+ return HoppingWindowSpec;
+}
+
TWindowSpecificationPtr ISource::FindWindowSpecification(TContext& ctx, const TString& windowName) const {
auto winIter = WinSpecs.find(windowName);
if (winIter == WinSpecs.end()) {
@@ -1321,10 +1321,10 @@ bool ISource::IsOverWindowSource() const {
return !WinSpecs.empty();
}
-bool ISource::IsStream() const {
- return false;
-}
-
+bool ISource::IsStream() const {
+ return false;
+}
+
bool ISource::IsOrdered() const {
return false;
}
@@ -1452,20 +1452,20 @@ TNodePtr ISource::BuildAggregation(const TString& label) {
aggrArgs = L(aggrArgs, traits);
}
- if (HoppingWindowSpec) {
- auto hoppingTraits = Y(
- "HoppingTraits",
- Y("ListItemType", listType),
- BuildLambda(Pos, Y("row"), HoppingWindowSpec->TimeExtractor),
- HoppingWindowSpec->Hop,
- HoppingWindowSpec->Interval,
+ if (HoppingWindowSpec) {
+ auto hoppingTraits = Y(
+ "HoppingTraits",
+ Y("ListItemType", listType),
+ BuildLambda(Pos, Y("row"), HoppingWindowSpec->TimeExtractor),
+ HoppingWindowSpec->Hop,
+ HoppingWindowSpec->Interval,
HoppingWindowSpec->Delay,
Q("False"));
-
- return Y("Aggregate", label, Q(keysTuple), Q(aggrArgs),
- Q(Y(Q(Y(BuildQuotedAtom(Pos, "hopping"), hoppingTraits)))));
- }
-
+
+ return Y("Aggregate", label, Q(keysTuple), Q(aggrArgs),
+ Q(Y(Q(Y(BuildQuotedAtom(Pos, "hopping"), hoppingTraits)))));
+ }
+
return Y("Aggregate", label, Q(keysTuple), Q(aggrArgs));
}
diff --git a/ydb/library/yql/sql/v0/node.h b/ydb/library/yql/sql/v0/node.h
index 4c7cfc273b6..ec88de65cdd 100644
--- a/ydb/library/yql/sql/v0/node.h
+++ b/ydb/library/yql/sql/v0/node.h
@@ -449,9 +449,9 @@ namespace NSQLTranslationV0 {
TString Name;
TString Type;
bool Nullable;
- bool IsTypeString;
+ bool IsTypeString;
- TColumnSchema(TPosition pos, const TString& name, const TString& type, bool nullable, bool isTypeString);
+ TColumnSchema(TPosition pos, const TString& name, const TString& type, bool nullable, bool isTypeString);
};
struct TColumns: public TSimpleRefCount<TColumns> {
@@ -506,23 +506,23 @@ namespace NSQLTranslationV0 {
~TWindowSpecification() {}
};
- struct THoppingWindowSpec: public TSimpleRefCount<THoppingWindowSpec> {
- TNodePtr TimeExtractor;
- TNodePtr Hop;
- TNodePtr Interval;
- TNodePtr Delay;
+ struct THoppingWindowSpec: public TSimpleRefCount<THoppingWindowSpec> {
+ TNodePtr TimeExtractor;
+ TNodePtr Hop;
+ TNodePtr Interval;
+ TNodePtr Delay;
TIntrusivePtr<THoppingWindowSpec> Clone() const;
~THoppingWindowSpec() {}
- };
-
+ };
+
typedef TIntrusivePtr<TWindowSpecification> TWindowSpecificationPtr;
typedef TMap<TString, TWindowSpecificationPtr> TWinSpecs;
typedef TVector<TTableRef> TTableList;
- typedef TIntrusivePtr<THoppingWindowSpec> THoppingWindowSpecPtr;
-
+ typedef TIntrusivePtr<THoppingWindowSpec> THoppingWindowSpecPtr;
+
bool ValidateAllNodesForAggregation(TContext& ctx, const TVector<TNodePtr>& nodes);
class TDeferredAtom {
@@ -654,7 +654,7 @@ namespace NSQLTranslationV0 {
virtual TNodePtr AggregationTraitsFactory() const = 0;
- virtual std::vector<ui32> GetFactoryColumnIndices() const;
+ virtual std::vector<ui32> GetFactoryColumnIndices() const;
virtual void AddFactoryArguments(TNodePtr& apply) const;
@@ -718,14 +718,14 @@ namespace NSQLTranslationV0 {
virtual void AddWindowSpecs(TWinSpecs winSpecs);
virtual bool AddAggregationOverWindow(TContext& ctx, const TString& windowName, TAggregationPtr func);
virtual bool AddFuncOverWindow(TContext& ctx, const TString& windowName, TNodePtr func);
- virtual void SetHoppingWindowSpec(THoppingWindowSpecPtr spec);
- virtual THoppingWindowSpecPtr GetHoppingWindowSpec() const;
+ virtual void SetHoppingWindowSpec(THoppingWindowSpecPtr spec);
+ virtual THoppingWindowSpecPtr GetHoppingWindowSpec() const;
virtual bool IsCompositeSource() const;
virtual bool IsGroupByColumn(const TString& column) const;
virtual bool IsFlattenByColumns() const;
virtual bool IsCalcOverWindow() const;
virtual bool IsOverWindowSource() const;
- virtual bool IsStream() const;
+ virtual bool IsStream() const;
virtual bool IsOrdered() const;
virtual TWriteSettings GetWriteSettings() const;
virtual bool SetSamplingOptions(TContext& ctx, TPosition pos, ESampleMode mode, TNodePtr samplingRate, TNodePtr samplingSeed);
@@ -784,7 +784,7 @@ namespace NSQLTranslationV0 {
TMultiMap<TString, TAggregationPtr> AggregationOverWindow;
TMultiMap<TString, TNodePtr> FuncOverWindow;
TWinSpecs WinSpecs;
- THoppingWindowSpecPtr HoppingWindowSpec;
+ THoppingWindowSpecPtr HoppingWindowSpec;
TVector<ISource*> UsedSources;
TString FlattenMode;
bool FlattenColumns = false;
@@ -912,8 +912,8 @@ namespace NSQLTranslationV0 {
TAggregationPtr BuildTwoArgsFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode);
TAggregationPtr BuildHistogramFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode);
TAggregationPtr BuildLinearHistogramFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode);
- template <bool HasKey>
- TAggregationPtr BuildTopFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode);
+ template <bool HasKey>
+ TAggregationPtr BuildTopFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode);
TAggregationPtr BuildTopFreqFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode);
TAggregationPtr BuildCountDistinctEstimateFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode);
TAggregationPtr BuildListFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode);
@@ -949,7 +949,7 @@ namespace NSQLTranslationV0 {
TSourcePtr BuildMuxSource(TPosition pos, TVector<TSourcePtr>&& sources);
TSourcePtr BuildFakeSource(TPosition pos);
TSourcePtr BuildNodeSource(TPosition pos, const TNodePtr& node);
- TSourcePtr BuildTableSource(TPosition pos, const TTableRef& table, bool stream, const TString& label = TString());
+ TSourcePtr BuildTableSource(TPosition pos, const TTableRef& table, bool stream, const TString& label = TString());
TSourcePtr BuildInnerSource(TPosition pos, TNodePtr node, const TString& label = TString());
TSourcePtr BuildRefColumnSource(TPosition pos, const TString& partExpression);
TSourcePtr BuildUnionAll(TPosition pos, TVector<TSourcePtr>&& sources);
@@ -968,7 +968,7 @@ namespace NSQLTranslationV0 {
const TVector<TSortSpecificationPtr>& orderBy,
TNodePtr having,
TWinSpecs&& windowSpec,
- THoppingWindowSpecPtr hoppingWindowSpec,
+ THoppingWindowSpecPtr hoppingWindowSpec,
TVector<TNodePtr>&& terms,
bool distinct,
TVector<TNodePtr>&& without,
@@ -1006,8 +1006,8 @@ namespace NSQLTranslationV0 {
TNodePtr BuildInputOptions(TPosition pos, const TVector<TString>& hints);
TNodePtr BuildInputTables(TPosition pos, const TTableList& tables, bool inSubquery);
TNodePtr BuildCreateTable(TPosition pos, const TTableRef& table, const TVector<TColumnSchema>& columns,
- const TVector<TIdentifier>& pkColumns, const TVector<TIdentifier>& partitionByColumns,
- const TVector<std::pair<TIdentifier, bool>>& orderByColumns);
+ const TVector<TIdentifier>& pkColumns, const TVector<TIdentifier>& partitionByColumns,
+ const TVector<std::pair<TIdentifier, bool>>& orderByColumns);
TNodePtr BuildAlterTable(TPosition pos, const TTableRef& tr, const TVector<TColumnSchema>& columns, EAlterTableIntentnt mode);
TNodePtr BuildDropTable(TPosition pos, const TTableRef& table);
TNodePtr BuildWriteTable(TPosition pos, const TString& label, const TTableRef& table, EWriteColumnMode mode, TNodePtr options = nullptr);
diff --git a/ydb/library/yql/sql/v0/query.cpp b/ydb/library/yql/sql/v0/query.cpp
index 7752b28b017..fa169a2e488 100644
--- a/ydb/library/yql/sql/v0/query.cpp
+++ b/ydb/library/yql/sql/v0/query.cpp
@@ -421,14 +421,14 @@ TNodePtr BuildInputTables(TPosition pos, const TTableList& tables, bool inSubque
class TCreateTableNode final: public TAstListNode {
public:
TCreateTableNode(TPosition pos, const TTableRef& tr, const TVector<TColumnSchema>& columns,
- const TVector<TIdentifier>& pkColumns, const TVector<TIdentifier>& partitionByColumns,
- const TVector<std::pair<TIdentifier, bool>>& orderByColumns)
+ const TVector<TIdentifier>& pkColumns, const TVector<TIdentifier>& partitionByColumns,
+ const TVector<std::pair<TIdentifier, bool>>& orderByColumns)
: TAstListNode(pos)
, Table(tr)
, Columns(columns)
, PkColumns(pkColumns)
- , PartitionByColumns(partitionByColumns)
- , OrderByColumns(orderByColumns)
+ , PartitionByColumns(partitionByColumns)
+ , OrderByColumns(orderByColumns)
{}
bool DoInit(TContext& ctx, ISource* src) override {
@@ -442,7 +442,7 @@ public:
}
keys = ctx.GroundBlockShortcutsForExpr(keys);
- if (!PkColumns.empty() || !PartitionByColumns.empty() || !OrderByColumns.empty()) {
+ if (!PkColumns.empty() || !PartitionByColumns.empty() || !OrderByColumns.empty()) {
THashSet<TString> columnsSet;
for (auto& col : Columns) {
columnsSet.insert(col.Name);
@@ -454,23 +454,23 @@ public:
return false;
}
}
- for (auto& keyColumn : PartitionByColumns) {
- if (!columnsSet.contains(keyColumn.Name)) {
- ctx.Error(keyColumn.Pos) << "Undefined column: " << keyColumn.Name;
- return false;
- }
- }
- for (auto& keyColumn : OrderByColumns) {
- if (!columnsSet.contains(keyColumn.first.Name)) {
- ctx.Error(keyColumn.first.Pos) << "Undefined column: " << keyColumn.first.Name;
- return false;
- }
- }
+ for (auto& keyColumn : PartitionByColumns) {
+ if (!columnsSet.contains(keyColumn.Name)) {
+ ctx.Error(keyColumn.Pos) << "Undefined column: " << keyColumn.Name;
+ return false;
+ }
+ }
+ for (auto& keyColumn : OrderByColumns) {
+ if (!columnsSet.contains(keyColumn.first.Name)) {
+ ctx.Error(keyColumn.first.Pos) << "Undefined column: " << keyColumn.first.Name;
+ return false;
+ }
+ }
}
auto columns = Y();
for (auto& col: Columns) {
- auto type = ParseType(TypeByAlias(col.Type, !col.IsTypeString), *ctx.Pool, ctx.Issues, col.Pos);
+ auto type = ParseType(TypeByAlias(col.Type, !col.IsTypeString), *ctx.Pool, ctx.Issues, col.Pos);
if (!type) {
return false;
}
@@ -507,45 +507,45 @@ public:
opts = L(opts, Q(Y(Q("mode"), Q("create"))));
opts = L(opts, Q(Y(Q("columns"), Q(columns))));
- const auto serviceName = to_lower(Table.ServiceName(ctx));
- if (serviceName == RtmrProviderName) {
- if (!PkColumns.empty() && !PartitionByColumns.empty()) {
- ctx.Error() << "Only one of PRIMARY KEY or PARTITION BY constraints may be specified";
- return false;
- }
- } else {
- if (!PartitionByColumns.empty() || !OrderByColumns.empty()) {
- ctx.Error() << "PARTITION BY and ORDER BY are supported only for " << RtmrProviderName << " provider";
- return false;
- }
- }
-
+ const auto serviceName = to_lower(Table.ServiceName(ctx));
+ if (serviceName == RtmrProviderName) {
+ if (!PkColumns.empty() && !PartitionByColumns.empty()) {
+ ctx.Error() << "Only one of PRIMARY KEY or PARTITION BY constraints may be specified";
+ return false;
+ }
+ } else {
+ if (!PartitionByColumns.empty() || !OrderByColumns.empty()) {
+ ctx.Error() << "PARTITION BY and ORDER BY are supported only for " << RtmrProviderName << " provider";
+ return false;
+ }
+ }
+
if (!PkColumns.empty()) {
- auto primaryKey = Y();
- for (auto& col : PkColumns) {
- primaryKey = L(primaryKey, BuildQuotedAtom(col.Pos, col.Name));
- }
+ auto primaryKey = Y();
+ for (auto& col : PkColumns) {
+ primaryKey = L(primaryKey, BuildQuotedAtom(col.Pos, col.Name));
+ }
opts = L(opts, Q(Y(Q("primarykey"), Q(primaryKey))));
- if (!OrderByColumns.empty()) {
- ctx.Error() << "PRIMARY KEY cannot be used with ORDER BY, use PARTITION BY instead";
- return false;
- }
- } else {
- if (!PartitionByColumns.empty()) {
- auto partitionBy = Y();
- for (auto& col : PartitionByColumns) {
- partitionBy = L(partitionBy, BuildQuotedAtom(col.Pos, col.Name));
- }
- opts = L(opts, Q(Y(Q("partitionby"), Q(partitionBy))));
- }
- if (!OrderByColumns.empty()) {
- auto orderBy = Y();
- for (auto& col : OrderByColumns) {
- orderBy = L(orderBy, Q(Y(BuildQuotedAtom(col.first.Pos, col.first.Name), col.second ? Q("1") : Q("0"))));
- }
- opts = L(opts, Q(Y(Q("orderby"), Q(orderBy))));
- }
- }
+ if (!OrderByColumns.empty()) {
+ ctx.Error() << "PRIMARY KEY cannot be used with ORDER BY, use PARTITION BY instead";
+ return false;
+ }
+ } else {
+ if (!PartitionByColumns.empty()) {
+ auto partitionBy = Y();
+ for (auto& col : PartitionByColumns) {
+ partitionBy = L(partitionBy, BuildQuotedAtom(col.Pos, col.Name));
+ }
+ opts = L(opts, Q(Y(Q("partitionby"), Q(partitionBy))));
+ }
+ if (!OrderByColumns.empty()) {
+ auto orderBy = Y();
+ for (auto& col : OrderByColumns) {
+ orderBy = L(orderBy, Q(Y(BuildQuotedAtom(col.first.Pos, col.first.Name), col.second ? Q("1") : Q("0"))));
+ }
+ opts = L(opts, Q(Y(Q("orderby"), Q(orderBy))));
+ }
+ }
Add("block", Q(Y(
Y("let", "sink", Y("DataSink", BuildQuotedAtom(Pos, Table.ServiceName(ctx)), BuildQuotedAtom(Pos, Table.Cluster))),
@@ -564,15 +564,15 @@ private:
TTableRef Table;
TVector<TColumnSchema> Columns;
TVector<TIdentifier> PkColumns;
- TVector<TIdentifier> PartitionByColumns;
- TVector<std::pair<TIdentifier, bool>> OrderByColumns; // column, is desc?
+ TVector<TIdentifier> PartitionByColumns;
+ TVector<std::pair<TIdentifier, bool>> OrderByColumns; // column, is desc?
};
TNodePtr BuildCreateTable(TPosition pos, const TTableRef& tr, const TVector<TColumnSchema>& columns,
- const TVector<TIdentifier>& pkColumns, const TVector<TIdentifier>& partitionByColumns,
- const TVector<std::pair<TIdentifier, bool>>& orderByColumns)
+ const TVector<TIdentifier>& pkColumns, const TVector<TIdentifier>& partitionByColumns,
+ const TVector<std::pair<TIdentifier, bool>>& orderByColumns)
{
- return new TCreateTableNode(pos, tr, columns, pkColumns, partitionByColumns, orderByColumns);
+ return new TCreateTableNode(pos, tr, columns, pkColumns, partitionByColumns, orderByColumns);
}
class TAlterTableNode final: public TAstListNode {
@@ -606,7 +606,7 @@ public:
} else {
auto columns = Y();
for (auto& col: Columns) {
- auto type = ParseType(TypeByAlias(col.Type, !col.IsTypeString), *ctx.Pool, ctx.Issues, col.Pos);
+ auto type = ParseType(TypeByAlias(col.Type, !col.IsTypeString), *ctx.Pool, ctx.Issues, col.Pos);
if (!type) {
return false;
}
diff --git a/ydb/library/yql/sql/v0/select.cpp b/ydb/library/yql/sql/v0/select.cpp
index ef47023be15..07937af1c25 100644
--- a/ydb/library/yql/sql/v0/select.cpp
+++ b/ydb/library/yql/sql/v0/select.cpp
@@ -312,11 +312,11 @@ protected:
return Source->ShouldUseSourceAsColumn(source);
}
- bool IsStream() const override {
- Y_VERIFY_DEBUG(Source);
- return Source->IsStream();
- }
-
+ bool IsStream() const override {
+ Y_VERIFY_DEBUG(Source);
+ return Source->IsStream();
+ }
+
bool IsOrdered() const override {
Y_VERIFY_DEBUG(Source);
return Source->IsOrdered();
@@ -535,9 +535,9 @@ public:
return Node;
}
- bool IsStream() const override {
- return Subquery->GetSource()->IsStream();
- }
+ bool IsStream() const override {
+ return Subquery->GetSource()->IsStream();
+ }
void DoUpdateState() const override {
State.Set(ENodeState::Const, true);
@@ -565,10 +565,10 @@ TNodePtr BuildSubqueryRef(TNodePtr subquery, const TString& alias, int tupleInde
class TTableSource: public IRealSource {
public:
- TTableSource(TPosition pos, const TTableRef& table, bool stream, const TString& label)
+ TTableSource(TPosition pos, const TTableRef& table, bool stream, const TString& label)
: IRealSource(pos)
, Table(table)
- , Stream(stream)
+ , Stream(stream)
{
SetLabel(label.empty() ? Table.ShortName() : label);
}
@@ -637,10 +637,10 @@ public:
return AstNode(Table.RefName);
}
- bool IsStream() const override {
- return Stream;
- }
-
+ bool IsStream() const override {
+ return Stream;
+ }
+
TPtr DoClone() const final {
return new TTableSource(Pos, Table, Stream, GetLabel());
}
@@ -653,8 +653,8 @@ protected:
const bool Stream;
};
-TSourcePtr BuildTableSource(TPosition pos, const TTableRef& table, bool stream, const TString& label) {
- return new TTableSource(pos, table, stream, label);
+TSourcePtr BuildTableSource(TPosition pos, const TTableRef& table, bool stream, const TString& label) {
+ return new TTableSource(pos, table, stream, label);
}
class TInnerSource: public IProxySource {
@@ -1049,7 +1049,7 @@ public:
const TVector<TSortSpecificationPtr>& orderBy,
TNodePtr having,
TWinSpecs& winSpecs,
- THoppingWindowSpecPtr hoppingWindowSpec,
+ THoppingWindowSpecPtr hoppingWindowSpec,
const TVector<TNodePtr>& terms,
bool distinct,
const TVector<TNodePtr>& without,
@@ -1067,7 +1067,7 @@ public:
, Without(without)
, Distinct(distinct)
, HoppingWindowSpec(hoppingWindowSpec)
- , Stream(stream)
+ , Stream(stream)
, Settings(settings)
{
}
@@ -1104,14 +1104,14 @@ public:
if (!Source->Init(ctx, initSrc)) {
return false;
}
- if (Stream && !Source->IsStream()) {
- ctx.Error(Pos) << "SELECT STREAM is unsupported for non-streaming sources";
- return false;
- }
+ if (Stream && !Source->IsStream()) {
+ ctx.Error(Pos) << "SELECT STREAM is unsupported for non-streaming sources";
+ return false;
+ }
if (!Stream && Source->IsStream() && !ctx.PragmaDirectRead) {
- ctx.Error(Pos) << "SELECT STREAM must be used for streaming sources";
- return false;
- }
+ ctx.Error(Pos) << "SELECT STREAM must be used for streaming sources";
+ return false;
+ }
ctx.PushBlockShortcuts();
auto src = Source.Get();
@@ -1649,7 +1649,7 @@ private:
TVector<TNodePtr> Without;
const bool Distinct;
bool OrderByInit = false;
- THoppingWindowSpecPtr HoppingWindowSpec;
+ THoppingWindowSpecPtr HoppingWindowSpec;
const bool Stream;
const TWriteSettings Settings;
};
@@ -1960,7 +1960,7 @@ TSourcePtr BuildSelectCore(
const TVector<TSortSpecificationPtr>& orderBy,
TNodePtr having,
TWinSpecs&& winSpecs,
- THoppingWindowSpecPtr hoppingWindowSpec,
+ THoppingWindowSpecPtr hoppingWindowSpec,
TVector<TNodePtr>&& terms,
bool distinct,
TVector<TNodePtr>&& without,
diff --git a/ydb/library/yql/sql/v0/sql.cpp b/ydb/library/yql/sql/v0/sql.cpp
index b028fc04398..85a9b143d46 100644
--- a/ydb/library/yql/sql/v0/sql.cpp
+++ b/ydb/library/yql/sql/v0/sql.cpp
@@ -300,74 +300,74 @@ static TColumnSchema ColumnSchemaImpl(const TRule_column_schema& node, TTranslat
}
static bool CreateTableEntry(const TRule_create_table_entry& node, TTranslation& ctx,
- TVector<TColumnSchema>& columns, TVector<TIdentifier>& pkColumns,
- TVector<TIdentifier>& partitionByColumns, TVector<std::pair<TIdentifier, bool>>& orderByColumns)
-{
+ TVector<TColumnSchema>& columns, TVector<TIdentifier>& pkColumns,
+ TVector<TIdentifier>& partitionByColumns, TVector<std::pair<TIdentifier, bool>>& orderByColumns)
+{
switch (node.Alt_case()) {
case TRule_create_table_entry::kAltCreateTableEntry1:
columns.push_back(ColumnSchemaImpl(node.GetAlt_create_table_entry1().GetRule_column_schema1(), ctx));
break;
-
+
case TRule_create_table_entry::kAltCreateTableEntry2:
{
auto& constraint = node.GetAlt_create_table_entry2().GetRule_table_constraint1();
- switch (constraint.Alt_case()) {
- case TRule_table_constraint::kAltTableConstraint1: {
- auto& pkConstraint = constraint.GetAlt_table_constraint1();
- pkColumns.push_back(IdEx(pkConstraint.GetRule_id4(), ctx));
- for (auto& block : pkConstraint.GetBlock5()) {
- pkColumns.push_back(IdEx(block.GetRule_id2(), ctx));
- }
- break;
- }
- case TRule_table_constraint::kAltTableConstraint2: {
- auto& pbConstraint = constraint.GetAlt_table_constraint2();
- partitionByColumns.push_back(IdEx(pbConstraint.GetRule_id4(), ctx));
- for (auto& block : pbConstraint.GetBlock5()) {
- partitionByColumns.push_back(IdEx(block.GetRule_id2(), ctx));
- }
- break;
- }
- case TRule_table_constraint::kAltTableConstraint3: {
- auto& obConstraint = constraint.GetAlt_table_constraint3();
- auto extractDirection = [&ctx] (const TRule_column_order_by_specification& spec, bool& desc) {
- desc = false;
- if (!spec.HasBlock2()) {
- return true;
- }
-
- auto& token = spec.GetBlock2().GetToken1();
- switch (token.GetId()) {
- case SQLLexerTokens::TOKEN_ASC:
- return true;
- case SQLLexerTokens::TOKEN_DESC:
- desc = true;
- return true;
- default:
- ctx.Error() << "Unsupported direction token: " << token.GetId();
- return false;
- }
- };
-
- bool desc = false;
- auto& obSpec = obConstraint.GetRule_column_order_by_specification4();
- if (!extractDirection(obSpec, desc)) {
- return false;
- }
- orderByColumns.push_back(std::make_pair(IdEx(obSpec.GetRule_id1(), ctx), desc));
-
- for (auto& block : obConstraint.GetBlock5()) {
- auto& obSpec = block.GetRule_column_order_by_specification2();
- if (!extractDirection(obSpec, desc)) {
- return false;
- }
- orderByColumns.push_back(std::make_pair(IdEx(obSpec.GetRule_id1(), ctx), desc));
- }
- break;
- }
- default:
- ctx.AltNotImplemented("table_constraint", constraint);
- return false;
+ switch (constraint.Alt_case()) {
+ case TRule_table_constraint::kAltTableConstraint1: {
+ auto& pkConstraint = constraint.GetAlt_table_constraint1();
+ pkColumns.push_back(IdEx(pkConstraint.GetRule_id4(), ctx));
+ for (auto& block : pkConstraint.GetBlock5()) {
+ pkColumns.push_back(IdEx(block.GetRule_id2(), ctx));
+ }
+ break;
+ }
+ case TRule_table_constraint::kAltTableConstraint2: {
+ auto& pbConstraint = constraint.GetAlt_table_constraint2();
+ partitionByColumns.push_back(IdEx(pbConstraint.GetRule_id4(), ctx));
+ for (auto& block : pbConstraint.GetBlock5()) {
+ partitionByColumns.push_back(IdEx(block.GetRule_id2(), ctx));
+ }
+ break;
+ }
+ case TRule_table_constraint::kAltTableConstraint3: {
+ auto& obConstraint = constraint.GetAlt_table_constraint3();
+ auto extractDirection = [&ctx] (const TRule_column_order_by_specification& spec, bool& desc) {
+ desc = false;
+ if (!spec.HasBlock2()) {
+ return true;
+ }
+
+ auto& token = spec.GetBlock2().GetToken1();
+ switch (token.GetId()) {
+ case SQLLexerTokens::TOKEN_ASC:
+ return true;
+ case SQLLexerTokens::TOKEN_DESC:
+ desc = true;
+ return true;
+ default:
+ ctx.Error() << "Unsupported direction token: " << token.GetId();
+ return false;
+ }
+ };
+
+ bool desc = false;
+ auto& obSpec = obConstraint.GetRule_column_order_by_specification4();
+ if (!extractDirection(obSpec, desc)) {
+ return false;
+ }
+ orderByColumns.push_back(std::make_pair(IdEx(obSpec.GetRule_id1(), ctx), desc));
+
+ for (auto& block : obConstraint.GetBlock5()) {
+ auto& obSpec = block.GetRule_column_order_by_specification2();
+ if (!extractDirection(obSpec, desc)) {
+ return false;
+ }
+ orderByColumns.push_back(std::make_pair(IdEx(obSpec.GetRule_id1(), ctx), desc));
+ }
+ break;
+ }
+ default:
+ ctx.AltNotImplemented("table_constraint", constraint);
+ return false;
}
break;
}
@@ -1176,14 +1176,14 @@ public:
, GroupSetContext(groupSetContext ? groupSetContext : TGroupByClauseCtx::TPtr(new TGroupByClauseCtx()))
{}
- bool Build(const TRule_group_by_clause& node, bool stream);
+ bool Build(const TRule_group_by_clause& node, bool stream);
bool ParseList(const TRule_grouping_element_list& groupingListNode);
void SetFeatures(const TString& field) const;
TVector<TNodePtr>& Content();
TMap<TString, TNodePtr>& Aliases();
- THoppingWindowSpecPtr GetHoppingWindow();
-
+ THoppingWindowSpecPtr GetHoppingWindow();
+
private:
TVector<TNodePtr> MultiplyGroupingSets(const TVector<TNodePtr>& lhs, const TVector<TNodePtr>& rhs) const;
void ResolveGroupByAndGrouping();
@@ -1191,7 +1191,7 @@ private:
void FeedCollection(const TNodePtr& elem, TVector<TNodePtr>& collection, bool& hasEmpty) const;
bool OrdinaryGroupingSet(const TRule_ordinary_grouping_set& node);
bool OrdinaryGroupingSetList(const TRule_ordinary_grouping_set_list& node);
- bool HoppingWindow(const TRule_hopping_window_specification& node);
+ bool HoppingWindow(const TRule_hopping_window_specification& node);
bool IsNodeColumnsOrNamedExpression(const TVector<TNodePtr>& content, const TString& construction) const;
@@ -1203,7 +1203,7 @@ private:
TVector<TNodePtr> GroupBySet;
TGroupByClauseCtx::TPtr GroupSetContext;
- THoppingWindowSpecPtr HoppingWindowSpec; // stream queries
+ THoppingWindowSpecPtr HoppingWindowSpec; // stream queries
static const TString AutogenerateNamePrefix;
};
@@ -2654,7 +2654,7 @@ TSourcePtr TSqlSelect::SingleSource(const TRule_single_source& node) {
const bool stream = serviceName == RtmrProviderName;
return BuildTableSource(pos, table, stream);
- }
+ }
}
case TRule_single_source::kAltSingleSource2: {
const auto& alt = node.GetAlt_single_source2();
@@ -3084,14 +3084,14 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet
selectPos = Ctx.Pos();
}
- const bool stream = node.HasBlock3();
- const bool distinct = IsDistinctOptSet(node.GetRule_opt_set_quantifier4());
+ const bool stream = node.HasBlock3();
+ const bool distinct = IsDistinctOptSet(node.GetRule_opt_set_quantifier4());
if (distinct) {
Ctx.IncrementMonCounter("sql_features", "DistinctInSelect");
}
TSourcePtr source(BuildFakeSource(selectPos));
- if (node.HasBlock1() && node.HasBlock8()) {
+ if (node.HasBlock1() && node.HasBlock8()) {
Token(node.GetBlock8().GetToken1());
Ctx.IncrementMonCounter("sql_errors", "DoubleFrom");
Ctx.Error() << "Only one FROM clause is allowed";
@@ -3100,21 +3100,21 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet
if (node.HasBlock1()) {
source = JoinSource(node.GetBlock1().GetRule_join_source2());
Ctx.IncrementMonCounter("sql_features", "FromInFront");
- } else if (node.HasBlock8()) {
- source = JoinSource(node.GetBlock8().GetRule_join_source2());
+ } else if (node.HasBlock8()) {
+ source = JoinSource(node.GetBlock8().GetRule_join_source2());
}
if (!source) {
return nullptr;
}
-
+
TVector<TNodePtr> without;
- if (node.HasBlock7()) {
- if (!ColumnList(without, node.GetBlock7().GetRule_column_list2())) {
+ if (node.HasBlock7()) {
+ if (!ColumnList(without, node.GetBlock7().GetRule_column_list2())) {
return nullptr;
}
}
- if (node.HasBlock9()) {
- auto block = node.GetBlock9();
+ if (node.HasBlock9()) {
+ auto block = node.GetBlock9();
Token(block.GetToken1());
TPosition pos(Ctx.Pos());
TSqlExpression expr(Ctx, Mode);
@@ -3132,10 +3132,10 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet
/// \todo merge gtoupByExpr and groupBy in one
TVector<TNodePtr> groupByExpr, groupBy;
- THoppingWindowSpecPtr hoppingWindowSpec;
- if (node.HasBlock10()) {
+ THoppingWindowSpecPtr hoppingWindowSpec;
+ if (node.HasBlock10()) {
TGroupByClause clause(Ctx, Mode);
- if (!clause.Build(node.GetBlock10().GetRule_group_by_clause1(), stream)) {
+ if (!clause.Build(node.GetBlock10().GetRule_group_by_clause1(), stream)) {
return nullptr;
}
for (const auto& exprAlias: clause.Aliases()) {
@@ -3144,13 +3144,13 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet
}
groupBy = std::move(clause.Content());
clause.SetFeatures("sql_features");
- hoppingWindowSpec = clause.GetHoppingWindow();
+ hoppingWindowSpec = clause.GetHoppingWindow();
}
TNodePtr having;
- if (node.HasBlock11()) {
+ if (node.HasBlock11()) {
TSqlExpression expr(Ctx, Mode);
- having = expr.Build(node.GetBlock11().GetRule_expr2());
+ having = expr.Build(node.GetBlock11().GetRule_expr2());
if (!having) {
return nullptr;
}
@@ -3158,33 +3158,33 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet
}
TWinSpecs windowSpec;
- if (node.HasBlock12()) {
- if (stream) {
- Ctx.Error() << "WINDOW is not allowed in streaming queries";
- return nullptr;
- }
- if (!WindowClause(node.GetBlock12().GetRule_window_clause1(), windowSpec)) {
+ if (node.HasBlock12()) {
+ if (stream) {
+ Ctx.Error() << "WINDOW is not allowed in streaming queries";
return nullptr;
}
+ if (!WindowClause(node.GetBlock12().GetRule_window_clause1(), windowSpec)) {
+ return nullptr;
+ }
Ctx.IncrementMonCounter("sql_features", "WindowClause");
}
TVector<TSortSpecificationPtr> orderBy;
- if (node.HasBlock13()) {
- if (stream) {
- Ctx.Error() << "ORDER BY is not allowed in streaming queries";
- return nullptr;
- }
- if (!OrderByClause(node.GetBlock13().GetRule_order_by_clause1(), orderBy)) {
+ if (node.HasBlock13()) {
+ if (stream) {
+ Ctx.Error() << "ORDER BY is not allowed in streaming queries";
return nullptr;
}
+ if (!OrderByClause(node.GetBlock13().GetRule_order_by_clause1(), orderBy)) {
+ return nullptr;
+ }
Ctx.IncrementMonCounter("sql_features", IsColumnsOnly(orderBy) ? "OrderBy" : "OrderByExpr");
}
TVector<TNodePtr> terms;
- if (!SelectTerm(terms, node.GetRule_result_column5())) {
+ if (!SelectTerm(terms, node.GetRule_result_column5())) {
return nullptr;
}
- for (auto block: node.GetBlock6()) {
+ for (auto block: node.GetBlock6()) {
if (!SelectTerm(terms, block.GetRule_result_column2())) {
return nullptr;
}
@@ -3357,10 +3357,10 @@ bool TSqlSelect::OrderByClause(const TRule_order_by_clause& node, TVector<TSortS
return SortSpecificationList(node.GetRule_sort_specification_list3(), orderBy);
}
-bool TGroupByClause::Build(const TRule_group_by_clause& node, bool stream) {
+bool TGroupByClause::Build(const TRule_group_by_clause& node, bool stream) {
const bool distinct = IsDistinctOptSet(node.GetRule_opt_set_quantifier3());
if (distinct) {
- Ctx.Error() << "DISTINCT is not supported in GROUP BY clause yet!";
+ Ctx.Error() << "DISTINCT is not supported in GROUP BY clause yet!";
Ctx.IncrementMonCounter("sql_errors", "DistinctInGroupByNotSupported");
return false;
}
@@ -3368,14 +3368,14 @@ bool TGroupByClause::Build(const TRule_group_by_clause& node, bool stream) {
return false;
}
ResolveGroupByAndGrouping();
- if (stream && !HoppingWindowSpec) {
- Ctx.Error() << "Streaming group by query must have a hopping window specification.";
- return false;
- }
- if (!stream && HoppingWindowSpec) {
- Ctx.Error() << "Hopping window specification is not supported in a non-streaming query.";
- return false;
- }
+ if (stream && !HoppingWindowSpec) {
+ Ctx.Error() << "Streaming group by query must have a hopping window specification.";
+ return false;
+ }
+ if (!stream && HoppingWindowSpec) {
+ Ctx.Error() << "Hopping window specification is not supported in a non-streaming query.";
+ return false;
+ }
return true;
}
@@ -3422,10 +3422,10 @@ TMap<TString, TNodePtr>& TGroupByClause::Aliases() {
return GroupSetContext->NodeAliases;
}
-THoppingWindowSpecPtr TGroupByClause::GetHoppingWindow() {
- return HoppingWindowSpec;
-}
-
+THoppingWindowSpecPtr TGroupByClause::GetHoppingWindow() {
+ return HoppingWindowSpec;
+}
+
TVector<TNodePtr> TGroupByClause::MultiplyGroupingSets(const TVector<TNodePtr>& lhs, const TVector<TNodePtr>& rhs) const {
TVector<TNodePtr> content;
for (const auto& leftNode: lhs) {
@@ -3559,12 +3559,12 @@ bool TGroupByClause::GroupingElement(const TRule_grouping_element& node) {
Features().Set(EGroupByFeatures::GroupingSet);
break;
}
- case TRule_grouping_element::kAltGroupingElement5: {
- if (!HoppingWindow(node.GetAlt_grouping_element5().GetRule_hopping_window_specification1())) {
- return false;
- }
- break;
- }
+ case TRule_grouping_element::kAltGroupingElement5: {
+ if (!HoppingWindow(node.GetAlt_grouping_element5().GetRule_hopping_window_specification1())) {
+ return false;
+ }
+ break;
+ }
default:
Y_FAIL("You should change implementation according grammar changes");
}
@@ -3641,69 +3641,69 @@ bool TGroupByClause::OrdinaryGroupingSetList(const TRule_ordinary_grouping_set_l
return true;
}
-bool TGroupByClause::HoppingWindow(const TRule_hopping_window_specification& node) {
- if (HoppingWindowSpec) {
- Ctx.Error() << "Duplicate hopping window specification.";
- return false;
- }
- HoppingWindowSpec = new THoppingWindowSpec;
- {
- TSqlExpression expr(Ctx, Mode);
- HoppingWindowSpec->TimeExtractor = expr.Build(node.GetRule_expr3());
- if (!HoppingWindowSpec->TimeExtractor) {
- return false;
- }
- }
- auto processIntervalParam = [&] (const TRule_expr& rule) -> TNodePtr {
- TSqlExpression expr(Ctx, Mode);
- auto node = expr.Build(rule);
- if (!node) {
- return nullptr;
- }
-
- auto literal = node->GetLiteral("String");
- if (!literal) {
- return new TAstListNodeImpl(Ctx.Pos(), {
- new TAstAtomNodeImpl(Ctx.Pos(), "EvaluateExpr", TNodeFlags::Default),
- node
- });
- }
-
+bool TGroupByClause::HoppingWindow(const TRule_hopping_window_specification& node) {
+ if (HoppingWindowSpec) {
+ Ctx.Error() << "Duplicate hopping window specification.";
+ return false;
+ }
+ HoppingWindowSpec = new THoppingWindowSpec;
+ {
+ TSqlExpression expr(Ctx, Mode);
+ HoppingWindowSpec->TimeExtractor = expr.Build(node.GetRule_expr3());
+ if (!HoppingWindowSpec->TimeExtractor) {
+ return false;
+ }
+ }
+ auto processIntervalParam = [&] (const TRule_expr& rule) -> TNodePtr {
+ TSqlExpression expr(Ctx, Mode);
+ auto node = expr.Build(rule);
+ if (!node) {
+ return nullptr;
+ }
+
+ auto literal = node->GetLiteral("String");
+ if (!literal) {
+ return new TAstListNodeImpl(Ctx.Pos(), {
+ new TAstAtomNodeImpl(Ctx.Pos(), "EvaluateExpr", TNodeFlags::Default),
+ node
+ });
+ }
+
const auto out = NKikimr::NMiniKQL::ValueFromString(NKikimr::NUdf::EDataSlot::Interval, *literal);
if (!out) {
- Ctx.Error(node->GetPos()) << "Expected interval in ISO 8601 format";
- return nullptr;
- }
-
+ Ctx.Error(node->GetPos()) << "Expected interval in ISO 8601 format";
+ return nullptr;
+ }
+
if ('T' == literal->back()) {
Ctx.Warning(node->GetPos(), TIssuesIds::YQL_DEPRECATED_INTERVAL_CONSTANT) << "Time prefix 'T' at end of interval contant";
}
- return new TAstListNodeImpl(Ctx.Pos(), {
- new TAstAtomNodeImpl(Ctx.Pos(), "Interval", TNodeFlags::Default),
- new TAstListNodeImpl(Ctx.Pos(), {
- new TAstAtomNodeImpl(Ctx.Pos(), "quote", TNodeFlags::Default),
+ return new TAstListNodeImpl(Ctx.Pos(), {
+ new TAstAtomNodeImpl(Ctx.Pos(), "Interval", TNodeFlags::Default),
+ new TAstListNodeImpl(Ctx.Pos(), {
+ new TAstAtomNodeImpl(Ctx.Pos(), "quote", TNodeFlags::Default),
new TAstAtomNodeImpl(Ctx.Pos(), ToString(out.Get<i64>()), TNodeFlags::Default)
- })
- });
- };
-
- HoppingWindowSpec->Hop = processIntervalParam(node.GetRule_expr5());
- if (!HoppingWindowSpec->Hop) {
- return false;
- }
- HoppingWindowSpec->Interval = processIntervalParam(node.GetRule_expr7());
- if (!HoppingWindowSpec->Interval) {
- return false;
- }
- HoppingWindowSpec->Delay = processIntervalParam(node.GetRule_expr9());
- if (!HoppingWindowSpec->Delay) {
- return false;
- }
-
- return true;
-}
-
+ })
+ });
+ };
+
+ HoppingWindowSpec->Hop = processIntervalParam(node.GetRule_expr5());
+ if (!HoppingWindowSpec->Hop) {
+ return false;
+ }
+ HoppingWindowSpec->Interval = processIntervalParam(node.GetRule_expr7());
+ if (!HoppingWindowSpec->Interval) {
+ return false;
+ }
+ HoppingWindowSpec->Delay = processIntervalParam(node.GetRule_expr9());
+ if (!HoppingWindowSpec->Delay) {
+ return false;
+ }
+
+ return true;
+}
+
bool TGroupByClause::IsNodeColumnsOrNamedExpression(const TVector<TNodePtr>& content, const TString& construction) const {
for (const auto& node: content) {
if (IsAutogenerated(node->GetColumnName())) {
@@ -4153,7 +4153,7 @@ bool TSqlIntoTable::ValidateServiceName(const TRule_into_table_stmt& node, const
auto serviceName = to_lower(table.ServiceName(Ctx));
const bool isMapReduce = serviceName == YtProviderName;
const bool isKikimr = serviceName == KikimrProviderName;
- const bool isRtmr = serviceName == RtmrProviderName;
+ const bool isRtmr = serviceName == RtmrProviderName;
const bool isStat = serviceName == StatProviderName;
if (!isKikimr) {
@@ -4180,12 +4180,12 @@ bool TSqlIntoTable::ValidateServiceName(const TRule_into_table_stmt& node, const
Ctx.IncrementMonCounter("sql_errors", TStringBuilder() << SqlIntoUserModeStr << "UnsupportedFor" << serviceName);
return false;
}
- } else if (isRtmr) {
- if (mode != ESQLWriteColumnMode::InsertInto) {
- Ctx.Error(pos) << SqlIntoUserModeStr << " is unsupported for " << serviceName;
- Ctx.IncrementMonCounter("sql_errors", TStringBuilder() << SqlIntoUserModeStr << "UnsupportedFor" << serviceName);
- return false;
- }
+ } else if (isRtmr) {
+ if (mode != ESQLWriteColumnMode::InsertInto) {
+ Ctx.Error(pos) << SqlIntoUserModeStr << " is unsupported for " << serviceName;
+ Ctx.IncrementMonCounter("sql_errors", TStringBuilder() << SqlIntoUserModeStr << "UnsupportedFor" << serviceName);
+ return false;
+ }
} else if (isStat) {
if (mode != ESQLWriteColumnMode::UpsertInto) {
Ctx.Error(pos) << SqlIntoUserModeStr << " is unsupported for " << serviceName;
@@ -4338,19 +4338,19 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
TVector<TColumnSchema> columns;
TVector<TIdentifier> pkColumns;
- TVector<TIdentifier> partitionByColumns;
- TVector<std::pair<TIdentifier, bool>> orderByColumns;
-
- if (!CreateTableEntry(rule.GetRule_create_table_entry5(), *this, columns, pkColumns, partitionByColumns, orderByColumns)) {
+ TVector<TIdentifier> partitionByColumns;
+ TVector<std::pair<TIdentifier, bool>> orderByColumns;
+
+ if (!CreateTableEntry(rule.GetRule_create_table_entry5(), *this, columns, pkColumns, partitionByColumns, orderByColumns)) {
return false;
}
for (auto& block: rule.GetBlock6()) {
- if (!CreateTableEntry(block.GetRule_create_table_entry2(), *this, columns, pkColumns, partitionByColumns, orderByColumns)) {
+ if (!CreateTableEntry(block.GetRule_create_table_entry2(), *this, columns, pkColumns, partitionByColumns, orderByColumns)) {
return false;
}
}
- AddStatementToBlocks(blocks, BuildCreateTable(Ctx.Pos(), tr, columns, pkColumns, partitionByColumns, orderByColumns));
+ AddStatementToBlocks(blocks, BuildCreateTable(Ctx.Pos(), tr, columns, pkColumns, partitionByColumns, orderByColumns));
break;
}
case TRule_sql_stmt_core::kAltSqlStmtCore5: {
@@ -4561,7 +4561,7 @@ bool TSqlQuery::AlterTableAddColumns(TVector<TNodePtr>& blocks, const TRule_alte
bool TSqlQuery::AlterTableDropColumn(TVector<TNodePtr>& blocks, const TRule_alter_table_drop_column& node, const TTableRef& tr) {
TString name = Id(node.GetRule_id3(), *this);
- TColumnSchema column(Ctx.Pos(), name, "", false, false);
+ TColumnSchema column(Ctx.Pos(), name, "", false, false);
AddStatementToBlocks(blocks, BuildAlterTable(Ctx.Pos(), tr, TVector<TColumnSchema>{column}, EAlterTableIntentnt::DropColumn));
return true;
}
@@ -4885,17 +4885,17 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
TNodePtr TSqlQuery::Build(const TRule_delete_stmt& stmt) {
TTableRef table = SimpleTableRefImpl(stmt.GetRule_simple_table_ref3(), Mode, *this);
- if (!table.Check(Ctx)) {
- return nullptr;
- }
+ if (!table.Check(Ctx)) {
+ return nullptr;
+ }
auto serviceName = to_lower(table.ServiceName(Ctx));
const bool isKikimr = serviceName == KikimrProviderName;
if (!isKikimr) {
Ctx.Error(GetPos(stmt.GetToken1())) << "DELETE is unsupported for " << serviceName;
- return nullptr;
- }
+ return nullptr;
+ }
TSourcePtr source = BuildTableSource(Ctx.Pos(), table, false);
@@ -4934,17 +4934,17 @@ TNodePtr TSqlQuery::Build(const TRule_delete_stmt& stmt) {
TNodePtr TSqlQuery::Build(const TRule_update_stmt& stmt) {
TTableRef table = SimpleTableRefImpl(stmt.GetRule_simple_table_ref2(), Mode, *this);
- if (!table.Check(Ctx)) {
- return nullptr;
- }
+ if (!table.Check(Ctx)) {
+ return nullptr;
+ }
auto serviceName = to_lower(table.ServiceName(Ctx));
const bool isKikimr = serviceName == KikimrProviderName;
if (!isKikimr) {
Ctx.Error(GetPos(stmt.GetToken1())) << "UPDATE is unsupported for " << serviceName;
- return nullptr;
- }
+ return nullptr;
+ }
switch (stmt.GetBlock3().Alt_case()) {
case TRule_update_stmt_TBlock3::kAlt1: {
diff --git a/ydb/library/yql/sql/v0/sql_ut.cpp b/ydb/library/yql/sql/v0/sql_ut.cpp
index 3a70aef93cc..64a372af7f5 100644
--- a/ydb/library/yql/sql/v0/sql_ut.cpp
+++ b/ydb/library/yql/sql/v0/sql_ut.cpp
@@ -788,7 +788,7 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) {
VerifyProgram(res, elementStat, verifyLine);
UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Kikimr.PushData"]);
}
-
+
Y_UNIT_TEST(ProcessUserTypeAuth) {
NYql::TAstParseResult res = SqlToYql("process plato.Input using YDB::PushData($ROWS, AsTuple('oauth', SecureParam('api:oauth')));", 1, TString(NYql::KikimrProviderName));
UNIT_ASSERT(res.Root);
@@ -808,19 +808,19 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) {
}
Y_UNIT_TEST(SelectStreamRtmr) {
- NYql::TAstParseResult res = SqlToYql(
- "USE plato; INSERT INTO Output SELECT STREAM key FROM Input;",
- 10, TString(NYql::RtmrProviderName));
- UNIT_ASSERT(res.Root);
- }
-
+ NYql::TAstParseResult res = SqlToYql(
+ "USE plato; INSERT INTO Output SELECT STREAM key FROM Input;",
+ 10, TString(NYql::RtmrProviderName));
+ UNIT_ASSERT(res.Root);
+ }
+
Y_UNIT_TEST(GroupByHopRtmr) {
- NYql::TAstParseResult res = SqlToYql(R"(
- USE plato; INSERT INTO Output SELECT STREAM key, SUM(value) AS value FROM Input
- GROUP BY key, HOP(subkey, "PT10S", "PT30S", "PT20S");
- )", 10, TString(NYql::RtmrProviderName));
- UNIT_ASSERT(res.Root);
- }
+ NYql::TAstParseResult res = SqlToYql(R"(
+ USE plato; INSERT INTO Output SELECT STREAM key, SUM(value) AS value FROM Input
+ GROUP BY key, HOP(subkey, "PT10S", "PT30S", "PT20S");
+ )", 10, TString(NYql::RtmrProviderName));
+ UNIT_ASSERT(res.Root);
+ }
Y_UNIT_TEST(KikimrInserts) {
NYql::TAstParseResult res = SqlToYql(R"(
diff --git a/ydb/library/yql/sql/v1/SQLv1.g.in b/ydb/library/yql/sql/v1/SQLv1.g.in
index ef394c7a009..2a0d9b32e61 100644
--- a/ydb/library/yql/sql/v1/SQLv1.g.in
+++ b/ydb/library/yql/sql/v1/SQLv1.g.in
@@ -46,8 +46,8 @@ sql_stmt_core:
| drop_role_stmt
;
-expr:
- or_subexpr (OR or_subexpr)*
+expr:
+ or_subexpr (OR or_subexpr)*
| type_name_composite;
or_subexpr: and_subexpr (AND and_subexpr)*;
@@ -70,15 +70,15 @@ match_op: LIKE | ILIKE | GLOB | REGEXP | RLIKE | MATCH;
eq_subexpr: neq_subexpr ((LESS | LESS_OR_EQ | GREATER | GREATER_OR_EQ) neq_subexpr)*;
-// workaround for recursive types, '??' and closing '>>'
-shift_right: GREATER GREATER;
-rot_right: GREATER GREATER PIPE;
-double_question: QUESTION QUESTION;
+// workaround for recursive types, '??' and closing '>>'
+shift_right: GREATER GREATER;
+rot_right: GREATER GREATER PIPE;
+double_question: QUESTION QUESTION;
neq_subexpr: bit_subexpr ((SHIFT_LEFT | shift_right | ROT_LEFT | rot_right | AMPERSAND | PIPE | CARET) bit_subexpr)*
// trailing QUESTIONs are used in optional simple types (String?) and optional lambda args: ($x, $y?) -> ($x)
((double_question neq_subexpr) => double_question neq_subexpr | QUESTION+)?;
-
+
bit_subexpr: add_subexpr ((PLUS | MINUS) add_subexpr)*;
add_subexpr: mul_subexpr ((ASTERISK | SLASH | PERCENT) mul_subexpr)*;
@@ -117,7 +117,7 @@ atom_expr:
| exists_expr
| case_expr
| an_id_or_type NAMESPACE (id_or_type | STRING_VALUE)
- | value_constructor
+ | value_constructor
| bitcast_expr
| list_literal
| dict_literal
@@ -134,7 +134,7 @@ in_atom_expr:
| LPAREN select_stmt RPAREN
// TODO: resolve ANTLR error: rule in_atom_expr has non-LL(*) decision due to recursive rule invocations reachable from alts 3,8
// | LPAREN values_stmt RPAREN
- | value_constructor
+ | value_constructor
| bitcast_expr
| list_literal
| dict_literal
@@ -228,7 +228,7 @@ when_expr: WHEN expr THEN expr;
literal_value:
integer
| real
- | STRING_VALUE
+ | STRING_VALUE
| BLOB // it's unused right now
| NULL
| CURRENT_TIME // it's unused right now
@@ -249,61 +249,61 @@ signed_number: (PLUS | MINUS)? (integer | real);
type_name_simple: an_id_pure;
-integer_or_bind: integer | bind_parameter;
-type_name_tag: id | STRING_VALUE | bind_parameter;
-
-struct_arg: type_name_tag COLON type_name_or_bind;
+integer_or_bind: integer | bind_parameter;
+type_name_tag: id | STRING_VALUE | bind_parameter;
+
+struct_arg: type_name_tag COLON type_name_or_bind;
struct_arg_as: type_name_or_bind AS type_name_tag;
-variant_arg: (type_name_tag COLON)? type_name_or_bind;
-callable_arg: variant_arg (LBRACE_CURLY AUTOMAP RBRACE_CURLY)?;
-callable_arg_list: callable_arg (COMMA callable_arg)*;
-
-type_name_decimal: DECIMAL LPAREN integer_or_bind COMMA integer_or_bind RPAREN;
-type_name_optional: OPTIONAL LESS type_name_or_bind GREATER;
+variant_arg: (type_name_tag COLON)? type_name_or_bind;
+callable_arg: variant_arg (LBRACE_CURLY AUTOMAP RBRACE_CURLY)?;
+callable_arg_list: callable_arg (COMMA callable_arg)*;
+
+type_name_decimal: DECIMAL LPAREN integer_or_bind COMMA integer_or_bind RPAREN;
+type_name_optional: OPTIONAL LESS type_name_or_bind GREATER;
type_name_tuple: TUPLE (LESS (type_name_or_bind (COMMA type_name_or_bind)* COMMA?)? GREATER | NOT_EQUALS2);
type_name_struct: STRUCT (LESS (struct_arg (COMMA struct_arg)* COMMA?)? GREATER | NOT_EQUALS2);
type_name_variant: VARIANT LESS variant_arg (COMMA variant_arg)* COMMA? GREATER;
-type_name_list: LIST LESS type_name_or_bind GREATER;
-type_name_stream: STREAM LESS type_name_or_bind GREATER;
-type_name_flow: FLOW LESS type_name_or_bind GREATER;
-type_name_dict: DICT LESS type_name_or_bind COMMA type_name_or_bind GREATER;
-type_name_set: SET LESS type_name_or_bind GREATER;
+type_name_list: LIST LESS type_name_or_bind GREATER;
+type_name_stream: STREAM LESS type_name_or_bind GREATER;
+type_name_flow: FLOW LESS type_name_or_bind GREATER;
+type_name_dict: DICT LESS type_name_or_bind COMMA type_name_or_bind GREATER;
+type_name_set: SET LESS type_name_or_bind GREATER;
type_name_enum: ENUM LESS type_name_tag (COMMA type_name_tag)* COMMA? GREATER;
-type_name_resource: RESOURCE LESS type_name_tag GREATER;
-type_name_tagged: TAGGED LESS type_name_or_bind COMMA type_name_tag GREATER;
-type_name_callable: CALLABLE LESS LPAREN callable_arg_list? COMMA? (LBRACE_SQUARE callable_arg_list RBRACE_SQUARE)? RPAREN ARROW type_name_or_bind GREATER;
-
+type_name_resource: RESOURCE LESS type_name_tag GREATER;
+type_name_tagged: TAGGED LESS type_name_or_bind COMMA type_name_tag GREATER;
+type_name_callable: CALLABLE LESS LPAREN callable_arg_list? COMMA? (LBRACE_SQUARE callable_arg_list RBRACE_SQUARE)? RPAREN ARROW type_name_or_bind GREATER;
+
type_name_composite:
( type_name_optional
- | type_name_tuple
- | type_name_struct
- | type_name_variant
- | type_name_list
- | type_name_stream
- | type_name_flow
- | type_name_dict
- | type_name_set
- | type_name_enum
- | type_name_resource
- | type_name_tagged
- | type_name_callable
- ) QUESTION*;
-
+ | type_name_tuple
+ | type_name_struct
+ | type_name_variant
+ | type_name_list
+ | type_name_stream
+ | type_name_flow
+ | type_name_dict
+ | type_name_set
+ | type_name_enum
+ | type_name_resource
+ | type_name_tagged
+ | type_name_callable
+ ) QUESTION*;
+
type_name:
type_name_composite
| (type_name_decimal | type_name_simple) QUESTION*;
-type_name_or_bind: type_name | bind_parameter;
-
+type_name_or_bind: type_name | bind_parameter;
+
value_constructor_literal: STRING_VALUE;
-value_constructor:
+value_constructor:
VARIANT LPAREN expr COMMA expr COMMA expr RPAREN
| ENUM LPAREN expr COMMA expr RPAREN
| CALLABLE LPAREN expr COMMA expr RPAREN
;
-
-declare_stmt: DECLARE bind_parameter AS type_name (EQUALS literal_value)?;
-
+
+declare_stmt: DECLARE bind_parameter AS type_name (EQUALS literal_value)?;
+
module_path: DOT? an_id (DOT an_id)*;
import_stmt: IMPORT module_path SYMBOLS named_bind_parameter_list;
export_stmt: EXPORT bind_parameter_list;
@@ -316,7 +316,7 @@ pragma_stmt: PRAGMA opt_id_prefix_or_type an_id (EQUALS pragma_value | LPAREN pr
pragma_value:
signed_number
| id
- | STRING_VALUE
+ | STRING_VALUE
| bool_value
| bind_parameter
;
@@ -376,7 +376,7 @@ grouping_element:
| cube_list
| grouping_sets_specification
//empty_grouping_set inside smart_parenthesis
- | hopping_window_specification
+ | hopping_window_specification
;
/// expect column (named column), or parenthesis list columns, or expression (named expression), or list expression
@@ -389,8 +389,8 @@ cube_list: CUBE LPAREN ordinary_grouping_set_list RPAREN;
/// SQL2003 grouping_set_list == grouping_element_list
grouping_sets_specification: GROUPING SETS LPAREN grouping_element_list RPAREN;
-hopping_window_specification: HOP LPAREN expr COMMA expr COMMA expr COMMA expr RPAREN;
-
+hopping_window_specification: HOP LPAREN expr COMMA expr COMMA expr COMMA expr RPAREN;
+
result_column:
opt_id_prefix ASTERISK
| expr ((AS an_id_or_type) | an_id_pure)?
@@ -498,11 +498,11 @@ column_schema: an_id_schema type_name_or_bind family_relation? (NOT? NULL)?;
family_relation: FAMILY an_id;
column_order_by_specification: an_id (ASC | DESC)?;
-table_constraint:
+table_constraint:
PRIMARY KEY LPAREN an_id (COMMA an_id)* RPAREN
| PARTITION BY LPAREN an_id (COMMA an_id)* RPAREN
- | ORDER BY LPAREN column_order_by_specification (COMMA column_order_by_specification)* RPAREN
-;
+ | ORDER BY LPAREN column_order_by_specification (COMMA column_order_by_specification)* RPAREN
+;
table_index: INDEX an_id table_index_type
(WITH LPAREN an_id EQUALS an_id COMMA (an_id EQUALS an_id)* RPAREN)?
@@ -763,12 +763,12 @@ an_id_pure: identifier | STRING_VALUE;
opt_id_prefix: (an_id DOT)?;
cluster_expr: (an_id COLON)? (pure_column_or_named | ASTERISK);
-id_or_type: id | type_id;
+id_or_type: id | type_id;
opt_id_prefix_or_type: (an_id_or_type DOT)?;
id_or_at: AT? an_id_or_type;
id_table_or_type: an_id_table | type_id;
-id_table_or_at: AT? id_table_or_type;
-
+id_table_or_at: AT? id_table_or_type;
+
keyword:
keyword_compat
| keyword_expr_uncompat
@@ -794,7 +794,7 @@ keyword_expr_uncompat:
| EXISTS
| FROM
| FULL
- | HOP
+ | HOP
| JSON_EXISTS
| JSON_VALUE
| JSON_QUERY
@@ -1016,22 +1016,22 @@ keyword_compat: (
| XOR
);
-type_id:
+type_id:
OPTIONAL
- | TUPLE
- | STRUCT
- | VARIANT
- | LIST
-// | STREAM
- | FLOW
- | DICT
- | SET
- | ENUM
- | RESOURCE
- | TAGGED
- | CALLABLE
-;
-
+ | TUPLE
+ | STRUCT
+ | VARIANT
+ | LIST
+// | STREAM
+ | FLOW
+ | DICT
+ | SET
+ | ENUM
+ | RESOURCE
+ | TAGGED
+ | CALLABLE
+;
+
bool_value: (TRUE | FALSE);
real: REAL;
integer: DIGITS | INTEGER_VALUE;
@@ -1128,14 +1128,14 @@ ASSUME: A S S U M E;
ASYNC: A S Y N C;
ATTACH: A T T A C H;
AUTOINCREMENT: A U T O I N C R E M E N T;
-AUTOMAP: A U T O M A P;
+AUTOMAP: A U T O M A P;
BEFORE: B E F O R E;
BEGIN: B E G I N;
BERNOULLI: B E R N O U L L I;
BETWEEN: B E T W E E N;
BITCAST: B I T C A S T;
BY: B Y;
-CALLABLE: C A L L A B L E;
+CALLABLE: C A L L A B L E;
CASCADE: C A S C A D E;
CASE: C A S E;
CAST: C A S T;
@@ -1158,7 +1158,7 @@ CURRENT_TIME: C U R R E N T '_' T I M E;
CURRENT_DATE: C U R R E N T '_' D A T E;
CURRENT_TIMESTAMP: C U R R E N T '_' T I M E S T A M P;
DATABASE: D A T A B A S E;
-DECIMAL: D E C I M A L;
+DECIMAL: D E C I M A L;
DECLARE: D E C L A R E;
DEFAULT: D E F A U L T;
DEFERRABLE: D E F E R R A B L E;
@@ -1180,7 +1180,7 @@ EMPTY: E M P T Y;
EMPTY_ACTION: E M P T Y '_' A C T I O N;
ENCRYPTED: E N C R Y P T E D;
END: E N D;
-ENUM: E N U M;
+ENUM: E N U M;
ERASE: E R A S E;
ESCAPE: E S C A P E;
EVALUATE: E V A L U A T E;
@@ -1196,7 +1196,7 @@ FAIL: F A I L;
FAMILY: F A M I L Y;
FILTER: F I L T E R;
FLATTEN: F L A T T E N;
-FLOW: F L O W;
+FLOW: F L O W;
FOLLOWING: F O L L O W I N G;
FOR: F O R;
FOREIGN: F O R E I G N;
@@ -1209,7 +1209,7 @@ GROUP: G R O U P;
GROUPING: G R O U P I N G;
GROUPS: G R O U P S;
HAVING: H A V I N G;
-HOP: H O P;
+HOP: H O P;
IF: I F;
IGNORE: I G N O R E;
ILIKE: I L I K E;
@@ -1275,7 +1275,7 @@ RENAME: R E N A M E;
REPEATABLE: R E P E A T A B L E;
REPLACE: R E P L A C E;
RESET: R E S E T;
-RESOURCE: R E S O U R C E;
+RESOURCE: R E S O U R C E;
RESPECT: R E S P E C T;
RESTRICT: R E S T R I C T;
RESULT: R E S U L T;
@@ -1296,15 +1296,15 @@ SEMI: S E M I;
SET: S E T;
SETS: S E T S;
SORTED: S O R T E D;
-STREAM: S T R E A M;
-STRUCT: S T R U C T;
+STREAM: S T R E A M;
+STRUCT: S T R U C T;
SUBQUERY: S U B Q U E R Y;
SYMBOLS: S Y M B O L S;
SYNC: S Y N C;
SYSTEM: S Y S T E M;
TABLE: T A B L E;
TABLESAMPLE: T A B L E S A M P L E;
-TAGGED: T A G G E D;
+TAGGED: T A G G E D;
TEMP: T E M P;
TEMPORARY: T E M P O R A R Y;
THEN: T H E N;
@@ -1312,7 +1312,7 @@ TIES: T I E S;
TO: T O;
TRANSACTION: T R A N S A C T I O N;
TRIGGER: T R I G G E R;
-TUPLE: T U P L E;
+TUPLE: T U P L E;
UNBOUNDED: U N B O U N D E D;
UNCONDITIONAL: U N C O N D I T I O N A L;
UNION: U N I O N;
@@ -1325,7 +1325,7 @@ USER: U S E R;
USING: U S I N G;
VACUUM: V A C U U M;
VALUES: V A L U E S;
-VARIANT: V A R I A N T;
+VARIANT: V A R I A N T;
VIEW: V I E W;
VIRTUAL: V I R T U A L;
WHEN: W H E N;
@@ -1368,7 +1368,7 @@ fragment BINDIGITS: '0' B ('0' | '1')+;
fragment DECDIGITS: DIGIT+;
DIGITS: DECDIGITS | HEXDIGITS | OCTDIGITS | BINDIGITS;
-INTEGER_VALUE: DIGITS (U? (L | S | T)?);
+INTEGER_VALUE: DIGITS (U? (L | S | T)?);
fragment FLOAT_EXP : E (PLUS | MINUS)? DECDIGITS ;
REAL:
diff --git a/ydb/library/yql/sql/v1/aggregation.cpp b/ydb/library/yql/sql/v1/aggregation.cpp
index 850e0e5056e..99ddd785728 100644
--- a/ydb/library/yql/sql/v1/aggregation.cpp
+++ b/ydb/library/yql/sql/v1/aggregation.cpp
@@ -248,8 +248,8 @@ private:
}
}
- std::vector<ui32> GetFactoryColumnIndices() const final {
- return {1u, 0u};
+ std::vector<ui32> GetFactoryColumnIndices() const final {
+ return {1u, 0u};
}
bool DoInit(TContext& ctx, ISource* src) final {
@@ -330,8 +330,8 @@ private:
return Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Payload), BuildLambda(Pos, Y("row"), Predicate));
}
- std::vector<ui32> GetFactoryColumnIndices() const final {
- return {0u, 1u};
+ std::vector<ui32> GetFactoryColumnIndices() const final {
+ return {0u, 1u};
}
bool DoInit(TContext& ctx, ISource* src) final {
@@ -507,8 +507,8 @@ private:
apply = L(apply, Intervals);
}
- std::vector<ui32> GetFactoryColumnIndices() const final {
- return {0u, 1u};
+ std::vector<ui32> GetFactoryColumnIndices() const final {
+ return {0u, 1u};
}
bool DoInit(TContext& ctx, ISource* src) final {
@@ -849,119 +849,119 @@ TAggregationPtr BuildTopFreqFactoryAggregation(TPosition pos, const TString& nam
return new TTopFreqFactory(pos, name, factory, aggMode);
}
-template <bool HasKey>
-class TTopAggregationFactory final : public TAggregationFactory {
-public:
- TTopAggregationFactory(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode)
- : TAggregationFactory(pos, name, factory, aggMode)
+template <bool HasKey>
+class TTopAggregationFactory final : public TAggregationFactory {
+public:
+ TTopAggregationFactory(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode)
+ : TAggregationFactory(pos, name, factory, aggMode)
, FakeSource(BuildFakeSource(pos))
- {}
-
-private:
- bool InitAggr(TContext& ctx, bool isFactory, ISource* src, TAstListNode& node, const TVector<TNodePtr>& exprs) final {
- ui32 adjustArgsCount = isFactory ? 1 : (HasKey ? 3 : 2);
- if (exprs.size() != adjustArgsCount) {
- ctx.Error(Pos) << "Aggregation function " << (isFactory ? "factory " : "") << Name << " requires "
- << adjustArgsCount << " arguments, given: " << exprs.size();
- return false;
- }
-
+ {}
+
+private:
+ bool InitAggr(TContext& ctx, bool isFactory, ISource* src, TAstListNode& node, const TVector<TNodePtr>& exprs) final {
+ ui32 adjustArgsCount = isFactory ? 1 : (HasKey ? 3 : 2);
+ if (exprs.size() != adjustArgsCount) {
+ ctx.Error(Pos) << "Aggregation function " << (isFactory ? "factory " : "") << Name << " requires "
+ << adjustArgsCount << " arguments, given: " << exprs.size();
+ return false;
+ }
+
if (BlockWindowAggregationWithoutFrameSpec(Pos, GetName(), src, ctx)) {
return false;
}
- if (!isFactory) {
- Payload = exprs[0];
- if (HasKey) {
- Key = exprs[1];
- }
- }
-
- Count = exprs.back();
-
- if (!isFactory) {
+ if (!isFactory) {
+ Payload = exprs[0];
+ if (HasKey) {
+ Key = exprs[1];
+ }
+ }
+
+ Count = exprs.back();
+
+ if (!isFactory) {
Name = src->MakeLocalName(Name);
- }
-
- if (!Init(ctx, src)) {
- return false;
- }
-
- if (!isFactory) {
- node.Add("Member", "row", Q(Name));
+ }
+
+ if (!Init(ctx, src)) {
+ return false;
+ }
+
+ if (!isFactory) {
+ node.Add("Member", "row", Q(Name));
if (IsOverWindow()) {
src->AddTmpWindowColumn(Name);
}
- }
-
- return true;
- }
-
- TNodePtr DoClone() const final {
- return new TTopAggregationFactory(Pos, Name, Func, AggMode);
- }
-
- TNodePtr GetApply(const TNodePtr& type) const final {
- TNodePtr apply;
- if (HasKey) {
- apply = Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Key), BuildLambda(Pos, Y("row"), Payload));
- } else {
- apply = Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Payload));
- }
- AddFactoryArguments(apply);
- return apply;
- }
-
- void AddFactoryArguments(TNodePtr& apply) const final {
- apply = L(apply, Count);
- }
-
- std::vector<ui32> GetFactoryColumnIndices() const final {
- if (HasKey) {
- return {1u, 0u};
- } else {
- return {0u};
- }
- }
-
- bool DoInit(TContext& ctx, ISource* src) final {
+ }
+
+ return true;
+ }
+
+ TNodePtr DoClone() const final {
+ return new TTopAggregationFactory(Pos, Name, Func, AggMode);
+ }
+
+ TNodePtr GetApply(const TNodePtr& type) const final {
+ TNodePtr apply;
+ if (HasKey) {
+ apply = Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Key), BuildLambda(Pos, Y("row"), Payload));
+ } else {
+ apply = Y("Apply", Factory, type, BuildLambda(Pos, Y("row"), Payload));
+ }
+ AddFactoryArguments(apply);
+ return apply;
+ }
+
+ void AddFactoryArguments(TNodePtr& apply) const final {
+ apply = L(apply, Count);
+ }
+
+ std::vector<ui32> GetFactoryColumnIndices() const final {
+ if (HasKey) {
+ return {1u, 0u};
+ } else {
+ return {0u};
+ }
+ }
+
+ bool DoInit(TContext& ctx, ISource* src) final {
if (!Count->Init(ctx, FakeSource.Get())) {
- return false;
- }
-
- if (!Payload) {
- return true;
- }
-
- if (HasKey) {
- if (!Key->Init(ctx, src)) {
- return false;
- }
- }
-
- if (!Payload->Init(ctx, src)) {
- return false;
- }
-
- if ((HasKey && Key->IsAggregated()) || (!HasKey && Payload->IsAggregated())) {
- ctx.Error(Pos) << "Aggregation of aggregated values is forbidden";
- return false;
- }
- return true;
- }
-
+ return false;
+ }
+
+ if (!Payload) {
+ return true;
+ }
+
+ if (HasKey) {
+ if (!Key->Init(ctx, src)) {
+ return false;
+ }
+ }
+
+ if (!Payload->Init(ctx, src)) {
+ return false;
+ }
+
+ if ((HasKey && Key->IsAggregated()) || (!HasKey && Payload->IsAggregated())) {
+ ctx.Error(Pos) << "Aggregation of aggregated values is forbidden";
+ return false;
+ }
+ return true;
+ }
+
TSourcePtr FakeSource;
- TNodePtr Key, Payload, Count;
-};
-
-template <bool HasKey>
-TAggregationPtr BuildTopFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) {
- return new TTopAggregationFactory<HasKey>(pos, name, factory, aggMode);
-}
-
-template TAggregationPtr BuildTopFactoryAggregation<false>(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode);
-template TAggregationPtr BuildTopFactoryAggregation<true >(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode);
-
+ TNodePtr Key, Payload, Count;
+};
+
+template <bool HasKey>
+TAggregationPtr BuildTopFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode) {
+ return new TTopAggregationFactory<HasKey>(pos, name, factory, aggMode);
+}
+
+template TAggregationPtr BuildTopFactoryAggregation<false>(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode);
+template TAggregationPtr BuildTopFactoryAggregation<true >(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode);
+
class TCountDistinctEstimateAggregationFactory final : public TAggregationFactory {
public:
TCountDistinctEstimateAggregationFactory(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode)
diff --git a/ydb/library/yql/sql/v1/builtin.cpp b/ydb/library/yql/sql/v1/builtin.cpp
index 43ef1243f10..4e24ab7b28d 100644
--- a/ydb/library/yql/sql/v1/builtin.cpp
+++ b/ydb/library/yql/sql/v1/builtin.cpp
@@ -215,14 +215,14 @@ public:
auto factory = Aggr->AggregationTraitsFactory();
auto apply = Y("Apply", factory, Y("ListType", "type"));
-
- auto columnIndices = Aggr->GetFactoryColumnIndices();
- if (columnIndices.size() == 1) {
+
+ auto columnIndices = Aggr->GetFactoryColumnIndices();
+ if (columnIndices.size() == 1) {
apply = L(apply, "extractor");
} else {
// make several extractors from main that returns a tuple
- for (ui32 arg = 0; arg < columnIndices.size(); ++arg) {
- auto partial = BuildLambda(Pos, Y("row"), Y("Nth", Y("Apply", "extractor", "row"), Q(ToString(columnIndices[arg]))));
+ for (ui32 arg = 0; arg < columnIndices.size(); ++arg) {
+ auto partial = BuildLambda(Pos, Y("row"), Y("Nth", Y("Apply", "extractor", "row"), Q(ToString(columnIndices[arg]))));
apply = L(apply, partial);
}
}
@@ -1037,12 +1037,12 @@ public:
}
auto aliasNode = BuildFileNameArgument(Args[1]->GetPos(), Args[1]);
OpName = "Apply";
- Args[0] = Y("Udf", Q("File.ByLines"), Y("Void"),
- Y("TupleType",
- Y("TupleType", Y("DataType", dataTypeStringNode)),
- Y("StructType"),
- Y("TupleType")));
-
+ Args[0] = Y("Udf", Q("File.ByLines"), Y("Void"),
+ Y("TupleType",
+ Y("TupleType", Y("DataType", dataTypeStringNode)),
+ Y("StructType"),
+ Y("TupleType")));
+
Args[1] = Y("FilePath", aliasNode);
return TCallNode::DoInit(ctx, src);
}
@@ -2218,47 +2218,47 @@ private:
TString Mode;
};
-template <bool IsStart>
-class THoppingTime final: public TAstListNode {
-public:
- THoppingTime(TPosition pos, const TVector<TNodePtr>& args = {})
- : TAstListNode(pos)
+template <bool IsStart>
+class THoppingTime final: public TAstListNode {
+public:
+ THoppingTime(TPosition pos, const TVector<TNodePtr>& args = {})
+ : TAstListNode(pos)
{
Y_UNUSED(args);
}
-
-private:
- TNodePtr DoClone() const override {
- return new THoppingTime(GetPos());
- }
-
- bool DoInit(TContext& ctx, ISource* src) override {
- Y_UNUSED(ctx);
-
- auto window = src->GetHoppingWindowSpec();
- if (!window) {
- ctx.Error(Pos) << "No hopping window parameters in aggregation";
- return false;
- }
-
- Nodes.clear();
-
- if (!IsStart) {
- Add("Member", "row", Q("_yql_time"));
- return true;
- }
-
- Add("Sub",
- Y("Member", "row", Q("_yql_time")),
- window->Interval);
- return true;
- }
-
- void DoUpdateState() const override {
- State.Set(ENodeState::Aggregated, true);
- }
-};
-
+
+private:
+ TNodePtr DoClone() const override {
+ return new THoppingTime(GetPos());
+ }
+
+ bool DoInit(TContext& ctx, ISource* src) override {
+ Y_UNUSED(ctx);
+
+ auto window = src->GetHoppingWindowSpec();
+ if (!window) {
+ ctx.Error(Pos) << "No hopping window parameters in aggregation";
+ return false;
+ }
+
+ Nodes.clear();
+
+ if (!IsStart) {
+ Add("Member", "row", Q("_yql_time"));
+ return true;
+ }
+
+ Add("Sub",
+ Y("Member", "row", Q("_yql_time")),
+ window->Interval);
+ return true;
+ }
+
+ void DoUpdateState() const override {
+ State.Set(ENodeState::Aggregated, true);
+ }
+};
+
class TInvalidBuiltin final: public INode {
public:
TInvalidBuiltin(TPosition pos, const TString& info)
@@ -2293,8 +2293,8 @@ enum EAggrFuncTypeCallback {
LINEAR_HISTOGRAM,
PERCENTILE,
TOPFREQ,
- TOP,
- TOP_BY,
+ TOP,
+ TOP_BY,
COUNT_DISTINCT_ESTIMATE,
LIST,
UDAF
@@ -2363,12 +2363,12 @@ TAggrFuncFactoryCallback BuildAggrFuncFactoryCallback(
case TOPFREQ:
factory = BuildTopFreqFactoryAggregation(pos, realFunctionName, factoryName, aggMode);
break;
- case TOP:
- factory = BuildTopFactoryAggregation<false>(pos, realFunctionName, factoryName, aggMode);
- break;
- case TOP_BY:
- factory = BuildTopFactoryAggregation<true>(pos, realFunctionName, factoryName, aggMode);
- break;
+ case TOP:
+ factory = BuildTopFactoryAggregation<false>(pos, realFunctionName, factoryName, aggMode);
+ break;
+ case TOP_BY:
+ factory = BuildTopFactoryAggregation<true>(pos, realFunctionName, factoryName, aggMode);
+ break;
case COUNT_DISTINCT_ESTIMATE:
factory = BuildCountDistinctEstimateFactoryAggregation(pos, realFunctionName, factoryName, aggMode);
break;
@@ -2712,7 +2712,7 @@ struct TBuiltinFuncData {
{"jointablerow", BuildSimpleBuiltinFactoryCallback<TTableRow<true>>() },
{"tablerows", BuildSimpleBuiltinFactoryCallback<TTableRows>() },
{"weakfield", BuildSimpleBuiltinFactoryCallback<TWeakFieldOp>()},
-
+
// Hint builtins
{"grouping", BuildSimpleBuiltinFactoryCallback<TGroupingNode>()},
@@ -2779,11 +2779,11 @@ struct TBuiltinFuncData {
{"mode", BuildAggrFuncFactoryCallback("Mode", "topfreq_traits_factory", TOPFREQ) },
{"topfreq", BuildAggrFuncFactoryCallback("TopFreq", "topfreq_traits_factory", TOPFREQ) },
- {"top", BuildAggrFuncFactoryCallback("Top", "top_traits_factory", TOP)},
- {"bottom", BuildAggrFuncFactoryCallback("Bottom", "bottom_traits_factory", TOP)},
- {"topby", BuildAggrFuncFactoryCallback("TopBy", "top_by_traits_factory", TOP_BY)},
- {"bottomby", BuildAggrFuncFactoryCallback("BottomBy", "bottom_by_traits_factory", TOP_BY)},
-
+ {"top", BuildAggrFuncFactoryCallback("Top", "top_traits_factory", TOP)},
+ {"bottom", BuildAggrFuncFactoryCallback("Bottom", "bottom_traits_factory", TOP)},
+ {"topby", BuildAggrFuncFactoryCallback("TopBy", "top_by_traits_factory", TOP_BY)},
+ {"bottomby", BuildAggrFuncFactoryCallback("BottomBy", "bottom_by_traits_factory", TOP_BY)},
+
{"histogram", BuildAggrFuncFactoryCallback("AdaptiveWardHistogram", "histogram_adaptive_ward_traits_factory", HISTOGRAM, "Histogram")},
{"adaptivewardhistogram", BuildAggrFuncFactoryCallback("AdaptiveWardHistogram", "histogram_adaptive_ward_traits_factory", HISTOGRAM)},
{"adaptiveweighthistogram", BuildAggrFuncFactoryCallback("AdaptiveWeightHistogram", "histogram_adaptive_weight_traits_factory", HISTOGRAM)},
@@ -2990,13 +2990,13 @@ TNodePtr BuildBuiltinFunc(TContext& ctx, TPosition pos, TString name, const TVec
return BuildUdf(ctx, pos, moduleName, name, newArgs);
}
- } else if (ns == "datetime2" && (name == "Format" || name == "Parse")) {
- return BuildUdf(ctx, pos, nameSpace, name, args);
+ } else if (ns == "datetime2" && (name == "Format" || name == "Parse")) {
+ return BuildUdf(ctx, pos, nameSpace, name, args);
} else if (name == "MakeLibraPreprocessor") {
if (args.size() != 1) {
return new TInvalidBuiltin(pos, TStringBuilder() << name << " requires exactly one argument");
}
-
+
auto settings = NYT::TNode::CreateMap();
auto makeUdfArgs = [&args, &pos, &settings]() {
diff --git a/ydb/library/yql/sql/v1/node.cpp b/ydb/library/yql/sql/v1/node.cpp
index 870bbd9cc7f..5a3622bd46d 100644
--- a/ydb/library/yql/sql/v1/node.cpp
+++ b/ydb/library/yql/sql/v1/node.cpp
@@ -996,9 +996,9 @@ TWindowSpecificationPtr TWindowSpecification::Clone() const {
THoppingWindowSpecPtr THoppingWindowSpec::Clone() const {
auto res = MakeIntrusive<THoppingWindowSpec>();
res->TimeExtractor = TimeExtractor->Clone();
- res->Hop = Hop->Clone();
- res->Interval = Interval->Clone();
- res->Delay = Delay->Clone();
+ res->Hop = Hop->Clone();
+ res->Interval = Interval->Clone();
+ res->Delay = Delay->Clone();
res->DataWatermarks = DataWatermarks;
return res;
}
@@ -1247,8 +1247,8 @@ void IAggregation::AddFactoryArguments(TNodePtr& apply) const {
Y_UNUSED(apply);
}
-std::vector<ui32> IAggregation::GetFactoryColumnIndices() const {
- return {0u};
+std::vector<ui32> IAggregation::GetFactoryColumnIndices() const {
+ return {0u};
}
TNodePtr IAggregation::WindowTraits(const TNodePtr& type) const {
@@ -1443,14 +1443,14 @@ const TVector<TString>& ISource::GetTmpWindowColumns() const {
return TmpWindowColumns;
}
-void ISource::SetHoppingWindowSpec(THoppingWindowSpecPtr spec) {
- HoppingWindowSpec = spec;
-}
-
-THoppingWindowSpecPtr ISource::GetHoppingWindowSpec() const {
- return HoppingWindowSpec;
-}
-
+void ISource::SetHoppingWindowSpec(THoppingWindowSpecPtr spec) {
+ HoppingWindowSpec = spec;
+}
+
+THoppingWindowSpecPtr ISource::GetHoppingWindowSpec() const {
+ return HoppingWindowSpec;
+}
+
TNodePtr ISource::GetSessionWindowSpec() const {
return SessionWindow;
}
@@ -1581,10 +1581,10 @@ bool ISource::IsOverWindowSource() const {
return !WinSpecs.empty();
}
-bool ISource::IsStream() const {
- return false;
-}
-
+bool ISource::IsStream() const {
+ return false;
+}
+
EOrderKind ISource::GetOrderKind() const {
return EOrderKind::None;
}
@@ -1773,19 +1773,19 @@ TNodePtr ISource::BuildAggregation(const TString& label) {
options = L(options, Q(Y(Q("compact"))));
}
- if (HoppingWindowSpec) {
- auto hoppingTraits = Y(
- "HoppingTraits",
- Y("ListItemType", listType),
- BuildLambda(Pos, Y("row"), HoppingWindowSpec->TimeExtractor),
- HoppingWindowSpec->Hop,
- HoppingWindowSpec->Interval,
+ if (HoppingWindowSpec) {
+ auto hoppingTraits = Y(
+ "HoppingTraits",
+ Y("ListItemType", listType),
+ BuildLambda(Pos, Y("row"), HoppingWindowSpec->TimeExtractor),
+ HoppingWindowSpec->Hop,
+ HoppingWindowSpec->Interval,
HoppingWindowSpec->Delay,
HoppingWindowSpec->DataWatermarks ? Q("true") : Q("false"));
-
+
options = L(options, Q(Y(Q("hopping"), hoppingTraits)));
- }
-
+ }
+
if (SessionWindow) {
YQL_ENSURE(SessionWindow->GetLabel());
auto sessionWindow = dynamic_cast<TSessionWindow*>(SessionWindow.Get());
diff --git a/ydb/library/yql/sql/v1/node.h b/ydb/library/yql/sql/v1/node.h
index de82b45a238..e912e20c182 100644
--- a/ydb/library/yql/sql/v1/node.h
+++ b/ydb/library/yql/sql/v1/node.h
@@ -531,7 +531,7 @@ namespace NSQLTranslationV1 {
struct TColumnSchema {
TPosition Pos;
TString Name;
- TNodePtr Type;
+ TNodePtr Type;
bool Nullable;
TVector<TIdentifier> Families;
@@ -604,18 +604,18 @@ namespace NSQLTranslationV1 {
};
typedef TIntrusivePtr<TFrameSpecification> TFrameSpecificationPtr;
- struct THoppingWindowSpec: public TSimpleRefCount<THoppingWindowSpec> {
- TNodePtr TimeExtractor;
- TNodePtr Hop;
- TNodePtr Interval;
- TNodePtr Delay;
+ struct THoppingWindowSpec: public TSimpleRefCount<THoppingWindowSpec> {
+ TNodePtr TimeExtractor;
+ TNodePtr Hop;
+ TNodePtr Interval;
+ TNodePtr Delay;
bool DataWatermarks;
TIntrusivePtr<THoppingWindowSpec> Clone() const;
~THoppingWindowSpec() {}
- };
+ };
typedef TIntrusivePtr<THoppingWindowSpec> THoppingWindowSpecPtr;
-
+
struct TWindowSpecification: public TSimpleRefCount<TWindowSpecification> {
TMaybe<TString> ExistingWindowName;
TVector<TNodePtr> Partitions;
@@ -756,7 +756,7 @@ namespace NSQLTranslationV1 {
virtual TNodePtr AggregationTraitsFactory() const = 0;
- virtual std::vector<ui32> GetFactoryColumnIndices() const;
+ virtual std::vector<ui32> GetFactoryColumnIndices() const;
virtual void AddFactoryArguments(TNodePtr& apply) const;
@@ -833,8 +833,8 @@ namespace NSQLTranslationV1 {
virtual void AddWindowSpecs(TWinSpecs winSpecs);
virtual bool AddAggregationOverWindow(TContext& ctx, const TString& windowName, TAggregationPtr func);
virtual bool AddFuncOverWindow(TContext& ctx, const TString& windowName, TNodePtr func);
- virtual void SetHoppingWindowSpec(THoppingWindowSpecPtr spec);
- virtual THoppingWindowSpecPtr GetHoppingWindowSpec() const;
+ virtual void SetHoppingWindowSpec(THoppingWindowSpecPtr spec);
+ virtual THoppingWindowSpecPtr GetHoppingWindowSpec() const;
virtual TNodePtr GetSessionWindowSpec() const;
virtual bool IsCompositeSource() const;
virtual bool IsGroupByColumn(const TString& column) const;
@@ -842,7 +842,7 @@ namespace NSQLTranslationV1 {
virtual bool IsFlattenByExprs() const;
virtual bool IsCalcOverWindow() const;
virtual bool IsOverWindowSource() const;
- virtual bool IsStream() const;
+ virtual bool IsStream() const;
virtual EOrderKind GetOrderKind() const;
virtual TWriteSettings GetWriteSettings() const;
virtual bool SetSamplingOptions(TContext& ctx, TPosition pos, ESampleMode mode, TNodePtr samplingRate, TNodePtr samplingSeed);
@@ -910,7 +910,7 @@ namespace NSQLTranslationV1 {
TMap<TString, TVector<TAggregationPtr>> AggregationOverWindow;
TMap<TString, TVector<TNodePtr>> FuncOverWindow;
TWinSpecs WinSpecs;
- THoppingWindowSpecPtr HoppingWindowSpec;
+ THoppingWindowSpecPtr HoppingWindowSpec;
TNodePtr SessionWindow;
TVector<ISource*> UsedSources;
TString FlattenMode;
@@ -1227,8 +1227,8 @@ namespace NSQLTranslationV1 {
TAggregationPtr BuildTwoArgsFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode);
TAggregationPtr BuildHistogramFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode);
TAggregationPtr BuildLinearHistogramFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode);
- template <bool HasKey>
- TAggregationPtr BuildTopFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode);
+ template <bool HasKey>
+ TAggregationPtr BuildTopFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode);
TAggregationPtr BuildTopFreqFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode);
TAggregationPtr BuildCountDistinctEstimateFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode);
TAggregationPtr BuildListFactoryAggregation(TPosition pos, const TString& name, const TString& factory, EAggregateMode aggMode);
@@ -1283,7 +1283,7 @@ namespace NSQLTranslationV1 {
const TVector<TSortSpecificationPtr>& orderBy,
TNodePtr having,
TWinSpecs&& windowSpec,
- THoppingWindowSpecPtr hoppingWindowSpec,
+ THoppingWindowSpecPtr hoppingWindowSpec,
TVector<TNodePtr>&& terms,
bool distinct,
TVector<TNodePtr>&& without,
diff --git a/ydb/library/yql/sql/v1/query.cpp b/ydb/library/yql/sql/v1/query.cpp
index 725356b9f57..2fe459d1e70 100644
--- a/ydb/library/yql/sql/v1/query.cpp
+++ b/ydb/library/yql/sql/v1/query.cpp
@@ -595,17 +595,17 @@ public:
}
}
for (auto& keyColumn : Params.PartitionByColumns) {
- if (!columnsSet.contains(keyColumn.Name)) {
- ctx.Error(keyColumn.Pos) << "Undefined column: " << keyColumn.Name;
- return false;
- }
- }
+ if (!columnsSet.contains(keyColumn.Name)) {
+ ctx.Error(keyColumn.Pos) << "Undefined column: " << keyColumn.Name;
+ return false;
+ }
+ }
for (auto& keyColumn : Params.OrderByColumns) {
- if (!columnsSet.contains(keyColumn.first.Name)) {
- ctx.Error(keyColumn.first.Pos) << "Undefined column: " << keyColumn.first.Name;
- return false;
- }
- }
+ if (!columnsSet.contains(keyColumn.first.Name)) {
+ ctx.Error(keyColumn.first.Pos) << "Undefined column: " << keyColumn.first.Name;
+ return false;
+ }
+ }
THashSet<TString> indexNames;
for (const auto& index : Params.Indexes) {
@@ -641,9 +641,9 @@ public:
for (auto& col : Params.Columns) {
auto columnDesc = Y();
columnDesc = L(columnDesc, BuildQuotedAtom(Pos, col.Name));
- auto type = col.Type;
+ auto type = col.Type;
if (col.Nullable) {
- type = Y("OptionalType", type);
+ type = Y("OptionalType", type);
}
columnDesc = L(columnDesc, type);
if (col.Families) {
@@ -668,33 +668,33 @@ public:
if (Table.Service == RtmrProviderName) {
if (!Params.PkColumns.empty() && !Params.PartitionByColumns.empty()) {
- ctx.Error() << "Only one of PRIMARY KEY or PARTITION BY constraints may be specified";
- return false;
- }
- } else {
+ ctx.Error() << "Only one of PRIMARY KEY or PARTITION BY constraints may be specified";
+ return false;
+ }
+ } else {
if (!Params.OrderByColumns.empty()) {
ctx.Error() << "ORDER BY is supported only for " << RtmrProviderName << " provider";
- return false;
- }
- }
-
+ return false;
+ }
+ }
+
if (!Params.PkColumns.empty()) {
- auto primaryKey = Y();
+ auto primaryKey = Y();
for (auto& col : Params.PkColumns) {
- primaryKey = L(primaryKey, BuildQuotedAtom(col.Pos, col.Name));
- }
+ primaryKey = L(primaryKey, BuildQuotedAtom(col.Pos, col.Name));
+ }
opts = L(opts, Q(Y(Q("primarykey"), Q(primaryKey))));
if (!Params.OrderByColumns.empty()) {
- ctx.Error() << "PRIMARY KEY cannot be used with ORDER BY, use PARTITION BY instead";
- return false;
- }
+ ctx.Error() << "PRIMARY KEY cannot be used with ORDER BY, use PARTITION BY instead";
+ return false;
+ }
}
if (!Params.PartitionByColumns.empty()) {
auto partitionBy = Y();
for (auto& col : Params.PartitionByColumns) {
partitionBy = L(partitionBy, BuildQuotedAtom(col.Pos, col.Name));
- }
+ }
opts = L(opts, Q(Y(Q("partitionby"), Q(partitionBy))));
}
@@ -702,9 +702,9 @@ public:
auto orderBy = Y();
for (auto& col : Params.OrderByColumns) {
orderBy = L(orderBy, Q(Y(BuildQuotedAtom(col.first.Pos, col.first.Name), col.second ? Q("1") : Q("0"))));
- }
+ }
opts = L(opts, Q(Y(Q("orderby"), Q(orderBy))));
- }
+ }
for (const auto& index : Params.Indexes) {
const auto& desc = CreateIndexDesc(index, *this);
@@ -840,9 +840,9 @@ public:
for (auto& col : Params.AddColumns) {
auto columnDesc = Y();
columnDesc = L(columnDesc, BuildQuotedAtom(Pos, col.Name));
- auto type = col.Type;
+ auto type = col.Type;
if (col.Nullable) {
- type = Y("OptionalType", type);
+ type = Y("OptionalType", type);
}
columnDesc = L(columnDesc, type);
if (col.Families) {
diff --git a/ydb/library/yql/sql/v1/select.cpp b/ydb/library/yql/sql/v1/select.cpp
index 2841f05a5b5..f1f6f2d3ff2 100644
--- a/ydb/library/yql/sql/v1/select.cpp
+++ b/ydb/library/yql/sql/v1/select.cpp
@@ -366,11 +366,11 @@ protected:
return Source->ShouldUseSourceAsColumn(source);
}
- bool IsStream() const override {
- Y_VERIFY_DEBUG(Source);
- return Source->IsStream();
- }
-
+ bool IsStream() const override {
+ Y_VERIFY_DEBUG(Source);
+ return Source->IsStream();
+ }
+
EOrderKind GetOrderKind() const override {
Y_VERIFY_DEBUG(Source);
return Source->GetOrderKind();
@@ -614,9 +614,9 @@ public:
return SetSamplingRate(ctx, samplingRate);
}
- bool IsStream() const override {
- return Subquery->GetSource()->IsStream();
- }
+ bool IsStream() const override {
+ return Subquery->GetSource()->IsStream();
+ }
void DoUpdateState() const override {
State.Set(ENodeState::Const, true);
@@ -739,10 +739,10 @@ public:
return AstNode(Table.RefName);
}
- bool IsStream() const override {
+ bool IsStream() const override {
return IsStreamingService(Table.Service);
- }
-
+ }
+
TPtr DoClone() const final {
return new TTableSource(Pos, Table, GetLabel());
}
@@ -1400,7 +1400,7 @@ public:
const TVector<TSortSpecificationPtr>& orderBy,
TNodePtr having,
TWinSpecs& winSpecs,
- THoppingWindowSpecPtr hoppingWindowSpec,
+ THoppingWindowSpecPtr hoppingWindowSpec,
const TVector<TNodePtr>& terms,
bool distinct,
const TVector<TNodePtr>& without,
@@ -1445,9 +1445,9 @@ public:
return false;
}
if (SelectStream && !Source->IsStream()) {
- ctx.Error(Pos) << "SELECT STREAM is unsupported for non-streaming sources";
- return false;
- }
+ ctx.Error(Pos) << "SELECT STREAM is unsupported for non-streaming sources";
+ return false;
+ }
auto src = Source.Get();
bool hasError = false;
@@ -2091,7 +2091,7 @@ private:
TVector<TNodePtr> Without;
const bool Distinct;
bool OrderByInit = false;
- THoppingWindowSpecPtr HoppingWindowSpec;
+ THoppingWindowSpecPtr HoppingWindowSpec;
const bool SelectStream;
const TWriteSettings Settings;
};
@@ -2461,7 +2461,7 @@ TSourcePtr DoBuildSelectCore(
const TVector<TSortSpecificationPtr>& orderBy,
TNodePtr having,
TWinSpecs&& winSpecs,
- THoppingWindowSpecPtr hoppingWindowSpec,
+ THoppingWindowSpecPtr hoppingWindowSpec,
TVector<TNodePtr>&& terms,
bool distinct,
TVector<TNodePtr>&& without,
diff --git a/ydb/library/yql/sql/v1/sql.cpp b/ydb/library/yql/sql/v1/sql.cpp
index ab5a8647885..624c31c5d00 100644
--- a/ydb/library/yql/sql/v1/sql.cpp
+++ b/ydb/library/yql/sql/v1/sql.cpp
@@ -92,7 +92,7 @@ inline TIdentifier GetKeywordId(TTranslation& ctx, const TRule_keyword& node) {
case TRule_keyword::kAltKeyword9:
return GetIdentifier(ctx, node.GetAlt_keyword9().GetRule_keyword_schema_uncompat1());
default:
- Y_FAIL("You should change implementation according to grammar changes");
+ Y_FAIL("You should change implementation according to grammar changes");
}
}
@@ -118,21 +118,21 @@ static TString Id(const TRule_id& node, TTranslation& ctx) {
case TRule_id::kAltId2:
return GetKeyword(ctx, node.GetAlt_id2().GetRule_keyword1());
default:
- Y_FAIL("You should change implementation according to grammar changes");
- }
-}
-
-static TString Id(const TRule_id_or_type& node, TTranslation& ctx) {
- switch (node.Alt_case()) {
- case TRule_id_or_type::kAltIdOrType1:
- return Id(node.GetAlt_id_or_type1().GetRule_id1(), ctx);
- case TRule_id_or_type::kAltIdOrType2:
- return ctx.Identifier(node.GetAlt_id_or_type2().GetRule_type_id1().GetToken1());
- default:
- Y_FAIL("You should change implementation according to grammar changes");
+ Y_FAIL("You should change implementation according to grammar changes");
}
}
+static TString Id(const TRule_id_or_type& node, TTranslation& ctx) {
+ switch (node.Alt_case()) {
+ case TRule_id_or_type::kAltIdOrType1:
+ return Id(node.GetAlt_id_or_type1().GetRule_id1(), ctx);
+ case TRule_id_or_type::kAltIdOrType2:
+ return ctx.Identifier(node.GetAlt_id_or_type2().GetRule_type_id1().GetToken1());
+ default:
+ Y_FAIL("You should change implementation according to grammar changes");
+ }
+}
+
static TString Id(const TRule_id_schema& node, TTranslation& ctx) {
//id_schema:
// identifier
@@ -162,7 +162,7 @@ static TString Id(const TRule_id_schema& node, TTranslation& ctx) {
case TRule_id_schema::kAltIdSchema7:
return GetKeyword(ctx, node.GetAlt_id_schema7().GetRule_keyword_hint_uncompat1());
default:
- Y_FAIL("You should change implementation according to grammar changes");
+ Y_FAIL("You should change implementation according to grammar changes");
}
}
@@ -230,17 +230,17 @@ static TString Id(const TRule_an_id_table& node, TTranslation& ctx) {
}
}
-static TString Id(const TRule_id_table_or_type& node, TTranslation& ctx) {
- switch (node.Alt_case()) {
- case TRule_id_table_or_type::kAltIdTableOrType1:
+static TString Id(const TRule_id_table_or_type& node, TTranslation& ctx) {
+ switch (node.Alt_case()) {
+ case TRule_id_table_or_type::kAltIdTableOrType1:
return Id(node.GetAlt_id_table_or_type1().GetRule_an_id_table1(), ctx);
- case TRule_id_table_or_type::kAltIdTableOrType2:
- return ctx.Identifier(node.GetAlt_id_table_or_type2().GetRule_type_id1().GetToken1());
- default:
- Y_FAIL("You should change implementation according to grammar changes");
- }
-}
-
+ case TRule_id_table_or_type::kAltIdTableOrType2:
+ return ctx.Identifier(node.GetAlt_id_table_or_type2().GetRule_type_id1().GetToken1());
+ default:
+ Y_FAIL("You should change implementation according to grammar changes");
+ }
+}
+
static TString Id(const TRule_id_expr& node, TTranslation& ctx) {
//id_expr:
// identifier
@@ -270,7 +270,7 @@ static TString Id(const TRule_id_expr& node, TTranslation& ctx) {
case TRule_id_expr::kAltIdExpr7:
return GetKeyword(ctx, node.GetAlt_id_expr7().GetRule_keyword_schema_uncompat1());
default:
- Y_FAIL("You should change implementation according to grammar changes");
+ Y_FAIL("You should change implementation according to grammar changes");
}
}
@@ -310,7 +310,7 @@ static TString Id(const TRule_id_expr_in& node, TTranslation& ctx) {
case TRule_id_expr_in::kAltIdExprIn6:
return GetKeyword(ctx, node.GetAlt_id_expr_in6().GetRule_keyword_schema_uncompat1());
default:
- Y_FAIL("You should change implementation according to grammar changes");
+ Y_FAIL("You should change implementation according to grammar changes");
}
}
@@ -431,7 +431,7 @@ static TString Id(const TRule_an_id& node, TTranslation& ctx) {
case TRule_an_id::kAltAnId2:
return IdContentFromString(ctx.Context(), ctx.Token(node.GetAlt_an_id2().GetToken1()));
default:
- Y_FAIL("You should change implementation according to grammar changes");
+ Y_FAIL("You should change implementation according to grammar changes");
}
}
@@ -521,13 +521,13 @@ static TString OptIdPrefixAsStr(const TRule_opt_id_prefix& node, TTranslation& c
return Id(node.GetBlock1().GetRule_an_id1(), ctx);
}
-static TString OptIdPrefixAsStr(const TRule_opt_id_prefix_or_type& node, TTranslation& ctx, const TString& defaultStr = {}) {
- if (!node.HasBlock1()) {
- return defaultStr;
- }
+static TString OptIdPrefixAsStr(const TRule_opt_id_prefix_or_type& node, TTranslation& ctx, const TString& defaultStr = {}) {
+ if (!node.HasBlock1()) {
+ return defaultStr;
+ }
return Id(node.GetBlock1().GetRule_an_id_or_type1(), ctx);
-}
-
+}
+
static void PureColumnListStr(const TRule_pure_column_list& node, TTranslation& ctx, TVector<TString>& outList) {
outList.push_back(Id(node.GetRule_an_id2(), ctx));
for (auto& block: node.GetBlock3()) {
@@ -576,7 +576,7 @@ static TDeferredAtom PureColumnOrNamed(const TRule_pure_column_or_named& node, T
case TRule_pure_column_or_named::kAltPureColumnOrNamed2:
return TDeferredAtom(ctx.Context().Pos(), Id(node.GetAlt_pure_column_or_named2().GetRule_an_id1(), ctx));
default:
- Y_FAIL("You should change implementation according to grammar changes");
+ Y_FAIL("You should change implementation according to grammar changes");
}
}
@@ -623,7 +623,7 @@ static bool CreateTableIndex(const TRule_table_index& node, TTranslation& ctx, T
ctx.AltNotImplemented("local", indexType);
return false;
default:
- Y_FAIL("You should change implementation according to grammar changes");
+ Y_FAIL("You should change implementation according to grammar changes");
}
if (node.HasBlock4()) {
@@ -839,9 +839,9 @@ protected:
bool ClusterExpr(const TRule_cluster_expr& node, bool allowWildcard, TString& service, TDeferredAtom& cluster);
bool ClusterExprOrBinding(const TRule_cluster_expr& node, TString& service, TDeferredAtom& cluster, bool& isBinding);
- TMaybe<TColumnSchema> ColumnSchemaImpl(const TRule_column_schema& node);
+ TMaybe<TColumnSchema> ColumnSchemaImpl(const TRule_column_schema& node);
bool CreateTableEntry(const TRule_create_table_entry& node, TCreateTableParameters& params);
-
+
bool FillFamilySettingsEntry(const TRule_family_settings_entry& settingNode, TFamilyEntry& family);
bool FillFamilySettings(const TRule_family_settings& settingsNode, TFamilyEntry& family);
bool CreateTableSettings(const TRule_with_table_settings& settingsNode, TTableSettings& settings);
@@ -854,15 +854,15 @@ protected:
TNodePtr TypeSimple(const TRule_type_name_simple& node, bool onlyDataAllowed);
TNodePtr TypeDecimal(const TRule_type_name_decimal& node);
TNodePtr AddOptionals(const TNodePtr& node, size_t optionalCount);
- TMaybe<std::pair<TVector<TNodePtr>, bool>> CallableArgList(const TRule_callable_arg_list& argList, bool namedArgsStarted);
-
- TNodePtr IntegerOrBind(const TRule_integer_or_bind& node);
- TNodePtr TypeNameTag(const TRule_type_name_tag& node);
- TNodePtr TypeNodeOrBind(const TRule_type_name_or_bind& node);
+ TMaybe<std::pair<TVector<TNodePtr>, bool>> CallableArgList(const TRule_callable_arg_list& argList, bool namedArgsStarted);
+
+ TNodePtr IntegerOrBind(const TRule_integer_or_bind& node);
+ TNodePtr TypeNameTag(const TRule_type_name_tag& node);
+ TNodePtr TypeNodeOrBind(const TRule_type_name_or_bind& node);
TNodePtr TypeNode(const TRule_type_name& node);
TNodePtr TypeNode(const TRule_type_name_composite& node);
- TNodePtr ValueConstructorLiteral(const TRule_value_constructor_literal& node);
- TNodePtr ValueConstructor(const TRule_value_constructor& node);
+ TNodePtr ValueConstructorLiteral(const TRule_value_constructor_literal& node);
+ TNodePtr ValueConstructor(const TRule_value_constructor& node);
TNodePtr ListLiteral(const TRule_list_literal& node);
TNodePtr DictLiteral(const TRule_dict_literal& node);
TNodePtr StructLiteral(const TRule_struct_literal& node);
@@ -872,7 +872,7 @@ protected:
TWindowSpecificationPtr WindowSpecification(const TRule_window_specification_details& rule);
bool OrderByClause(const TRule_order_by_clause& node, TVector<TSortSpecificationPtr>& orderBy);
bool SortSpecificationList(const TRule_sort_specification_list& node, TVector<TSortSpecificationPtr>& sortSpecs);
-
+
bool IsDistinctOptSet(const TRule_opt_set_quantifier& node) const;
bool IsDistinctOptSet(const TRule_opt_set_quantifier& node, TPosition& distinctPos) const;
@@ -908,21 +908,21 @@ public:
}
TNodePtr Build(const TRule_expr& node) {
- // expr:
- // or_subexpr (OR or_subexpr)*
+ // expr:
+ // or_subexpr (OR or_subexpr)*
// | type_name_composite
- switch (node.Alt_case()) {
- case TRule_expr::kAltExpr1: {
- auto getNode = [](const TRule_expr_TAlt1_TBlock2& b) -> const TRule_or_subexpr& { return b.GetRule_or_subexpr2(); };
- return BinOper("Or", node.GetAlt_expr1().GetRule_or_subexpr1(), getNode,
+ switch (node.Alt_case()) {
+ case TRule_expr::kAltExpr1: {
+ auto getNode = [](const TRule_expr_TAlt1_TBlock2& b) -> const TRule_or_subexpr& { return b.GetRule_or_subexpr2(); };
+ return BinOper("Or", node.GetAlt_expr1().GetRule_or_subexpr1(), getNode,
node.GetAlt_expr1().GetBlock2().begin(), node.GetAlt_expr1().GetBlock2().end(), {});
- }
- case TRule_expr::kAltExpr2: {
+ }
+ case TRule_expr::kAltExpr2: {
return TypeNode(node.GetAlt_expr2().GetRule_type_name_composite1());
- }
- default:
- Y_FAIL("You should change implementation according to grammar changes");
- }
+ }
+ default:
+ Y_FAIL("You should change implementation according to grammar changes");
+ }
}
void SetSmartParenthesisMode(ESmartParenthesis mode) {
@@ -1015,7 +1015,7 @@ private:
auto getNode = [](const TRule_neq_subexpr::TBlock2& b) -> const TRule_bit_subexpr& { return b.GetRule_bit_subexpr2(); };
auto result = BinOpList(node.GetRule_bit_subexpr1(), getNode, node.GetBlock2().begin(), node.GetBlock2().end(), tail);
if (!result) {
- return {};
+ return {};
}
if (node.HasBlock3()) {
auto& block = node.GetBlock3();
@@ -1054,9 +1054,9 @@ private:
template <typename TNode, typename TGetNode, typename TIter>
TNodePtr BinOpList(const TNode& node, TGetNode getNode, TIter begin, TIter end, const TTrailingQuestions& tail);
- template <typename TGetNode, typename TIter>
+ template <typename TGetNode, typename TIter>
TNodePtr BinOpList(const TRule_bit_subexpr& node, TGetNode getNode, TIter begin, TIter end, const TTrailingQuestions& tail);
-
+
template <typename TGetNode, typename TIter>
TNodePtr BinOpList(const TRule_eq_subexpr& node, TGetNode getNode, TIter begin, TIter end, const TTrailingQuestions& tail);
@@ -1115,7 +1115,7 @@ public:
void InitExpr(const TNodePtr& expr);
bool Init(const TRule_using_call_expr& node);
- bool Init(const TRule_value_constructor& node);
+ bool Init(const TRule_value_constructor& node);
bool Init(const TRule_invoke_expr& node);
bool ConfigureExternalCall(const TRule_external_call_settings& node);
void IncCounters();
@@ -1789,22 +1789,22 @@ TMaybe<TSourcePtr> TSqlTranslation::AsTableImpl(const TRule_table_ref& node) {
return Nothing();
}
-TMaybe<TColumnSchema> TSqlTranslation::ColumnSchemaImpl(const TRule_column_schema& node) {
+TMaybe<TColumnSchema> TSqlTranslation::ColumnSchemaImpl(const TRule_column_schema& node) {
const bool nullable = !node.HasBlock4() || !node.GetBlock4().HasBlock1();
const TString name(Id(node.GetRule_an_id_schema1(), *this));
- const TPosition pos(Context().Pos());
- const auto type = TypeNodeOrBind(node.GetRule_type_name_or_bind2());
- if (!type) {
- return {};
- }
+ const TPosition pos(Context().Pos());
+ const auto type = TypeNodeOrBind(node.GetRule_type_name_or_bind2());
+ if (!type) {
+ return {};
+ }
TVector<TIdentifier> families;
if (node.HasBlock3()) {
const auto& familyRelation = node.GetBlock3().GetRule_family_relation1();
families.push_back(IdEx(familyRelation.GetRule_an_id2(), *this));
}
return TColumnSchema(pos, name, type, nullable, families);
-}
-
+}
+
bool TSqlTranslation::FillFamilySettingsEntry(const TRule_family_settings_entry& settingNode, TFamilyEntry& family) {
TIdentifier id = IdEx(settingNode.GetRule_an_id1(), *this);
const TRule_family_setting_value& value = settingNode.GetRule_family_setting_value3();
@@ -1834,107 +1834,107 @@ bool TSqlTranslation::FillFamilySettings(const TRule_family_settings& settingsNo
}
bool TSqlTranslation::CreateTableEntry(const TRule_create_table_entry& node, TCreateTableParameters& params)
-{
- switch (node.Alt_case()) {
- case TRule_create_table_entry::kAltCreateTableEntry1:
- {
+{
+ switch (node.Alt_case()) {
+ case TRule_create_table_entry::kAltCreateTableEntry1:
+ {
// column_schema
- auto columnSchema = ColumnSchemaImpl(node.GetAlt_create_table_entry1().GetRule_column_schema1());
- if (!columnSchema) {
- return false;
- }
+ auto columnSchema = ColumnSchemaImpl(node.GetAlt_create_table_entry1().GetRule_column_schema1());
+ if (!columnSchema) {
+ return false;
+ }
if (columnSchema->Families.size() > 1) {
Ctx.Error() << "Several column families for a single column are not yet supported";
return false;
}
params.Columns.push_back(*columnSchema);
- break;
- }
- case TRule_create_table_entry::kAltCreateTableEntry2:
- {
+ break;
+ }
+ case TRule_create_table_entry::kAltCreateTableEntry2:
+ {
// table_constraint
- auto& constraint = node.GetAlt_create_table_entry2().GetRule_table_constraint1();
- switch (constraint.Alt_case()) {
- case TRule_table_constraint::kAltTableConstraint1: {
+ auto& constraint = node.GetAlt_create_table_entry2().GetRule_table_constraint1();
+ switch (constraint.Alt_case()) {
+ case TRule_table_constraint::kAltTableConstraint1: {
if (!params.PkColumns.empty()) {
Ctx.Error() << "PRIMARY KEY statement must be specified only once";
return false;
}
- auto& pkConstraint = constraint.GetAlt_table_constraint1();
+ auto& pkConstraint = constraint.GetAlt_table_constraint1();
params.PkColumns.push_back(IdEx(pkConstraint.GetRule_an_id4(), *this));
- for (auto& block : pkConstraint.GetBlock5()) {
+ for (auto& block : pkConstraint.GetBlock5()) {
params.PkColumns.push_back(IdEx(block.GetRule_an_id2(), *this));
- }
- break;
- }
- case TRule_table_constraint::kAltTableConstraint2: {
+ }
+ break;
+ }
+ case TRule_table_constraint::kAltTableConstraint2: {
if (!params.PartitionByColumns.empty()) {
Ctx.Error() << "PARTITION BY statement must be specified only once";
return false;
}
- auto& pbConstraint = constraint.GetAlt_table_constraint2();
+ auto& pbConstraint = constraint.GetAlt_table_constraint2();
params.PartitionByColumns.push_back(IdEx(pbConstraint.GetRule_an_id4(), *this));
- for (auto& block : pbConstraint.GetBlock5()) {
+ for (auto& block : pbConstraint.GetBlock5()) {
params.PartitionByColumns.push_back(IdEx(block.GetRule_an_id2(), *this));
- }
- break;
- }
- case TRule_table_constraint::kAltTableConstraint3: {
+ }
+ break;
+ }
+ case TRule_table_constraint::kAltTableConstraint3: {
if (!params.OrderByColumns.empty()) {
Ctx.Error() << "ORDER BY statement must be specified only once";
return false;
}
- auto& obConstraint = constraint.GetAlt_table_constraint3();
- auto extractDirection = [this] (const TRule_column_order_by_specification& spec, bool& desc) {
- desc = false;
- if (!spec.HasBlock2()) {
- return true;
- }
-
- auto& token = spec.GetBlock2().GetToken1();
- switch (token.GetId()) {
- case SQLv1LexerTokens::TOKEN_ASC:
- return true;
- case SQLv1LexerTokens::TOKEN_DESC:
- desc = true;
- return true;
- default:
- Ctx.Error() << "Unsupported direction token: " << token.GetId();
- return false;
- }
- };
-
- bool desc = false;
- auto& obSpec = obConstraint.GetRule_column_order_by_specification4();
- if (!extractDirection(obSpec, desc)) {
- return false;
- }
+ auto& obConstraint = constraint.GetAlt_table_constraint3();
+ auto extractDirection = [this] (const TRule_column_order_by_specification& spec, bool& desc) {
+ desc = false;
+ if (!spec.HasBlock2()) {
+ return true;
+ }
+
+ auto& token = spec.GetBlock2().GetToken1();
+ switch (token.GetId()) {
+ case SQLv1LexerTokens::TOKEN_ASC:
+ return true;
+ case SQLv1LexerTokens::TOKEN_DESC:
+ desc = true;
+ return true;
+ default:
+ Ctx.Error() << "Unsupported direction token: " << token.GetId();
+ return false;
+ }
+ };
+
+ bool desc = false;
+ auto& obSpec = obConstraint.GetRule_column_order_by_specification4();
+ if (!extractDirection(obSpec, desc)) {
+ return false;
+ }
params.OrderByColumns.push_back(std::make_pair(IdEx(obSpec.GetRule_an_id1(), *this), desc));
-
- for (auto& block : obConstraint.GetBlock5()) {
- auto& obSpec = block.GetRule_column_order_by_specification2();
- if (!extractDirection(obSpec, desc)) {
- return false;
- }
+
+ for (auto& block : obConstraint.GetBlock5()) {
+ auto& obSpec = block.GetRule_column_order_by_specification2();
+ if (!extractDirection(obSpec, desc)) {
+ return false;
+ }
params.OrderByColumns.push_back(std::make_pair(IdEx(obSpec.GetRule_an_id1(), *this), desc));
- }
- break;
- }
- default:
- AltNotImplemented("table_constraint", constraint);
- return false;
- }
- break;
- }
- case TRule_create_table_entry::kAltCreateTableEntry3:
- {
+ }
+ break;
+ }
+ default:
+ AltNotImplemented("table_constraint", constraint);
+ return false;
+ }
+ break;
+ }
+ case TRule_create_table_entry::kAltCreateTableEntry3:
+ {
// table_index
- auto& table_index = node.GetAlt_create_table_entry3().GetRule_table_index1();
+ auto& table_index = node.GetAlt_create_table_entry3().GetRule_table_index1();
if (!CreateTableIndex(table_index, *this, params.Indexes)) {
- return false;
- }
- break;
- }
+ return false;
+ }
+ break;
+ }
case TRule_create_table_entry::kAltCreateTableEntry4:
{
// family_entry
@@ -1957,13 +1957,13 @@ bool TSqlTranslation::CreateTableEntry(const TRule_create_table_entry& node, TCr
}
break;
}
- default:
- AltNotImplemented("create_table_entry", node);
- return false;
- }
- return true;
-}
-
+ default:
+ AltNotImplemented("create_table_entry", node);
+ return false;
+ }
+ return true;
+}
+
TNodePtr LiteralNumber(TContext& ctx, const TRule_integer& node);
namespace {
@@ -2308,70 +2308,70 @@ bool TSqlTranslation::CreateTableSettings(const TRule_with_table_settings& setti
return true;
}
-bool ParseNumbers(TContext& ctx, const TString& strOrig, ui64& value, TString& suffix);
-
-TNodePtr TSqlTranslation::IntegerOrBind(const TRule_integer_or_bind& node) {
- switch (node.Alt_case()) {
- case TRule_integer_or_bind::kAltIntegerOrBind1: {
- const TString intString = Ctx.Token(node.GetAlt_integer_or_bind1().GetRule_integer1().GetToken1());
- ui64 value;
- TString suffix;
- if (!ParseNumbers(Ctx, intString, value, suffix)) {
- return {};
- }
- return BuildQuotedAtom(Ctx.Pos(), ToString(value), TNodeFlags::ArbitraryContent);
- }
- case TRule_integer_or_bind::kAltIntegerOrBind2: {
+bool ParseNumbers(TContext& ctx, const TString& strOrig, ui64& value, TString& suffix);
+
+TNodePtr TSqlTranslation::IntegerOrBind(const TRule_integer_or_bind& node) {
+ switch (node.Alt_case()) {
+ case TRule_integer_or_bind::kAltIntegerOrBind1: {
+ const TString intString = Ctx.Token(node.GetAlt_integer_or_bind1().GetRule_integer1().GetToken1());
+ ui64 value;
+ TString suffix;
+ if (!ParseNumbers(Ctx, intString, value, suffix)) {
+ return {};
+ }
+ return BuildQuotedAtom(Ctx.Pos(), ToString(value), TNodeFlags::ArbitraryContent);
+ }
+ case TRule_integer_or_bind::kAltIntegerOrBind2: {
TString bindName;
if (!NamedNodeImpl(node.GetAlt_integer_or_bind2().GetRule_bind_parameter1(), bindName, *this)) {
return {};
}
- auto namedNode = GetNamedNode(bindName);
- if (!namedNode) {
- return {};
- }
- auto atom = MakeAtomFromExpression(Ctx, namedNode);
- return atom.Build();
- }
- default:
- Y_FAIL("You should change implementation according to grammar changes");
- }
-}
-
-TNodePtr TSqlTranslation::TypeNameTag(const TRule_type_name_tag& node) {
- switch (node.Alt_case()) {
- case TRule_type_name_tag::kAltTypeNameTag1: {
- auto content = Id(node.GetAlt_type_name_tag1().GetRule_id1(), *this);
- auto atom = TDeferredAtom(Ctx.Pos(), content);
- return atom.Build();
- }
- case TRule_type_name_tag::kAltTypeNameTag2: {
+ auto namedNode = GetNamedNode(bindName);
+ if (!namedNode) {
+ return {};
+ }
+ auto atom = MakeAtomFromExpression(Ctx, namedNode);
+ return atom.Build();
+ }
+ default:
+ Y_FAIL("You should change implementation according to grammar changes");
+ }
+}
+
+TNodePtr TSqlTranslation::TypeNameTag(const TRule_type_name_tag& node) {
+ switch (node.Alt_case()) {
+ case TRule_type_name_tag::kAltTypeNameTag1: {
+ auto content = Id(node.GetAlt_type_name_tag1().GetRule_id1(), *this);
+ auto atom = TDeferredAtom(Ctx.Pos(), content);
+ return atom.Build();
+ }
+ case TRule_type_name_tag::kAltTypeNameTag2: {
auto value = Token(node.GetAlt_type_name_tag2().GetToken1());
auto parsed = StringContentOrIdContent(Ctx, Ctx.Pos(), value);
if (!parsed) {
- return {};
- }
+ return {};
+ }
auto atom = TDeferredAtom(Ctx.Pos(), parsed->Content);
- return atom.Build();
- }
- case TRule_type_name_tag::kAltTypeNameTag3: {
+ return atom.Build();
+ }
+ case TRule_type_name_tag::kAltTypeNameTag3: {
TString bindName;
if (!NamedNodeImpl(node.GetAlt_type_name_tag3().GetRule_bind_parameter1(), bindName, *this)) {
return {};
}
- auto namedNode = GetNamedNode(bindName);
- if (!namedNode) {
- return {};
- }
- TDeferredAtom atom;
- MakeTableFromExpression(Ctx, namedNode, atom);
- return atom.Build();
- }
- default:
- Y_FAIL("You should change implementation according to grammar changes");
- }
-}
-
+ auto namedNode = GetNamedNode(bindName);
+ if (!namedNode) {
+ return {};
+ }
+ TDeferredAtom atom;
+ MakeTableFromExpression(Ctx, namedNode, atom);
+ return atom.Build();
+ }
+ default:
+ Y_FAIL("You should change implementation according to grammar changes");
+ }
+}
+
TNodePtr TSqlTranslation::TypeSimple(const TRule_type_name_simple& node, bool onlyDataAllowed) {
const TString origName = Id(node.GetRule_an_id_pure1(), *this);
if (origName.empty()) {
@@ -2382,19 +2382,19 @@ TNodePtr TSqlTranslation::TypeSimple(const TRule_type_name_simple& node, bool on
TNodePtr TSqlTranslation::TypeDecimal(const TRule_type_name_decimal& node) {
auto pos = Ctx.Pos();
- auto flags = TNodeFlags::Default;
-
- auto paramOne = IntegerOrBind(node.GetRule_integer_or_bind3());
- if (!paramOne) {
- return {};
- }
- auto paramTwo = IntegerOrBind(node.GetRule_integer_or_bind5());
- if (!paramTwo) {
- return {};
- }
+ auto flags = TNodeFlags::Default;
+
+ auto paramOne = IntegerOrBind(node.GetRule_integer_or_bind3());
+ if (!paramOne) {
+ return {};
+ }
+ auto paramTwo = IntegerOrBind(node.GetRule_integer_or_bind5());
+ if (!paramTwo) {
+ return {};
+ }
return new TCallNodeImpl(pos, "DataType", { BuildQuotedAtom(pos, "Decimal", flags), paramOne, paramTwo });
-}
-
+}
+
TNodePtr TSqlTranslation::AddOptionals(const TNodePtr& node, size_t optionalCount) {
TNodePtr result = node;
if (node) {
@@ -2407,83 +2407,83 @@ TNodePtr TSqlTranslation::AddOptionals(const TNodePtr& node, size_t optionalCoun
}
-TMaybe<std::pair<TVector<TNodePtr>, bool>> TSqlTranslation::CallableArgList(const TRule_callable_arg_list& argList, bool namedArgsStarted) {
- auto pos = Ctx.Pos();
- auto flags = TNodeFlags::Default;
- auto& arg1 = argList.GetRule_callable_arg1();
- auto& varArg = arg1.GetRule_variant_arg1();
- TVector<TNodePtr> result;
- TVector<TNodePtr> items;
- auto typeNode = TypeNodeOrBind(varArg.GetRule_type_name_or_bind2());
- if (!typeNode) {
- return {};
- }
- items.push_back(typeNode);
- if (varArg.HasBlock1()) {
- namedArgsStarted = true;
- auto tag = TypeNameTag(varArg.GetBlock1().GetRule_type_name_tag1());
- if (!tag) {
- return {};
- }
- items.push_back(tag);
- }
- if (arg1.HasBlock2()) {
- if (!varArg.HasBlock1()) {
- items.push_back(BuildQuotedAtom(pos, "", flags));
- }
- items.push_back(BuildQuotedAtom(pos, "1", flags));
- }
- result.push_back(new TAstListNodeImpl(pos, items));
-
- for (auto& arg : argList.GetBlock2()) {
- auto& varArg = arg.GetRule_callable_arg2().GetRule_variant_arg1();
- TVector<TNodePtr> items;
- auto typeNode = TypeNodeOrBind(varArg.GetRule_type_name_or_bind2());
- if (!typeNode) {
- return {};
- }
- items.push_back(typeNode);
- if (varArg.HasBlock1()) {
- auto tag = TypeNameTag(varArg.GetBlock1().GetRule_type_name_tag1());
- if (!tag) {
- return {};
- }
- items.push_back(tag);
- } else {
- if (namedArgsStarted) {
- Ctx.Error() << "Expected named argument, previous argument was named";
- return {};
- }
- items.push_back(BuildQuotedAtom(pos, "", flags));
- }
- if (arg.GetRule_callable_arg2().HasBlock2()) {
- if (!varArg.HasBlock1()) {
- items.push_back(BuildQuotedAtom(pos, "", flags));
- }
- items.push_back(BuildQuotedAtom(pos, "1", flags));
- }
- result.push_back(new TAstListNodeImpl(pos, items));
- }
- return std::make_pair(result, namedArgsStarted);
-}
-
-TNodePtr TSqlTranslation::TypeNodeOrBind(const TRule_type_name_or_bind& node) {
- switch (node.Alt_case()) {
- case TRule_type_name_or_bind::kAltTypeNameOrBind1: {
- return TypeNode(node.GetAlt_type_name_or_bind1().GetRule_type_name1());
- }
- case TRule_type_name_or_bind::kAltTypeNameOrBind2: {
+TMaybe<std::pair<TVector<TNodePtr>, bool>> TSqlTranslation::CallableArgList(const TRule_callable_arg_list& argList, bool namedArgsStarted) {
+ auto pos = Ctx.Pos();
+ auto flags = TNodeFlags::Default;
+ auto& arg1 = argList.GetRule_callable_arg1();
+ auto& varArg = arg1.GetRule_variant_arg1();
+ TVector<TNodePtr> result;
+ TVector<TNodePtr> items;
+ auto typeNode = TypeNodeOrBind(varArg.GetRule_type_name_or_bind2());
+ if (!typeNode) {
+ return {};
+ }
+ items.push_back(typeNode);
+ if (varArg.HasBlock1()) {
+ namedArgsStarted = true;
+ auto tag = TypeNameTag(varArg.GetBlock1().GetRule_type_name_tag1());
+ if (!tag) {
+ return {};
+ }
+ items.push_back(tag);
+ }
+ if (arg1.HasBlock2()) {
+ if (!varArg.HasBlock1()) {
+ items.push_back(BuildQuotedAtom(pos, "", flags));
+ }
+ items.push_back(BuildQuotedAtom(pos, "1", flags));
+ }
+ result.push_back(new TAstListNodeImpl(pos, items));
+
+ for (auto& arg : argList.GetBlock2()) {
+ auto& varArg = arg.GetRule_callable_arg2().GetRule_variant_arg1();
+ TVector<TNodePtr> items;
+ auto typeNode = TypeNodeOrBind(varArg.GetRule_type_name_or_bind2());
+ if (!typeNode) {
+ return {};
+ }
+ items.push_back(typeNode);
+ if (varArg.HasBlock1()) {
+ auto tag = TypeNameTag(varArg.GetBlock1().GetRule_type_name_tag1());
+ if (!tag) {
+ return {};
+ }
+ items.push_back(tag);
+ } else {
+ if (namedArgsStarted) {
+ Ctx.Error() << "Expected named argument, previous argument was named";
+ return {};
+ }
+ items.push_back(BuildQuotedAtom(pos, "", flags));
+ }
+ if (arg.GetRule_callable_arg2().HasBlock2()) {
+ if (!varArg.HasBlock1()) {
+ items.push_back(BuildQuotedAtom(pos, "", flags));
+ }
+ items.push_back(BuildQuotedAtom(pos, "1", flags));
+ }
+ result.push_back(new TAstListNodeImpl(pos, items));
+ }
+ return std::make_pair(result, namedArgsStarted);
+}
+
+TNodePtr TSqlTranslation::TypeNodeOrBind(const TRule_type_name_or_bind& node) {
+ switch (node.Alt_case()) {
+ case TRule_type_name_or_bind::kAltTypeNameOrBind1: {
+ return TypeNode(node.GetAlt_type_name_or_bind1().GetRule_type_name1());
+ }
+ case TRule_type_name_or_bind::kAltTypeNameOrBind2: {
TString bindName;
if (!NamedNodeImpl(node.GetAlt_type_name_or_bind2().GetRule_bind_parameter1(), bindName, *this)) {
return {};
}
return GetNamedNode(bindName);
- }
+ }
default:
Y_FAIL("You should change implementation according to grammar changes");
- }
-}
-
+ }
+}
+
TNodePtr TSqlTranslation::TypeNode(const TRule_type_name& node) {
//type_name:
// type_name_composite
@@ -2492,8 +2492,8 @@ TNodePtr TSqlTranslation::TypeNode(const TRule_type_name& node) {
return TypeNode(node.GetAlt_type_name1().GetRule_type_name_composite1());
}
- TNodePtr result;
- TPosition pos = Ctx.Pos();
+ TNodePtr result;
+ TPosition pos = Ctx.Pos();
auto& alt = node.GetAlt_type_name2();
auto& block = alt.GetBlock1();
@@ -2533,39 +2533,39 @@ TNodePtr TSqlTranslation::TypeNode(const TRule_type_name_composite& node) {
// ) QUESTION*;
TNodePtr result;
TPosition pos = Ctx.Pos();
- auto flags = TNodeFlags::Default;
-
- auto wrapOneParamType = [&] (const TRule_type_name_or_bind& param, const char* type) -> TNodePtr {
- auto node = TypeNodeOrBind(param);
- return node ? new TAstListNodeImpl(pos, { BuildAtom(pos, type, flags), node }) : nullptr;
- };
- auto makeVoid = [&] () -> TNodePtr {
- return new TAstListNodeImpl(pos, { BuildAtom(pos, "VoidType", flags) });
- };
- auto makeQuote = [&] (const TNodePtr& node) -> TNodePtr {
- return new TAstListNodeImpl(pos, { new TAstAtomNodeImpl(pos, "quote", 0), node });
- };
-
- auto& block = node.GetBlock1();
- switch (block.Alt_case()) {
+ auto flags = TNodeFlags::Default;
+
+ auto wrapOneParamType = [&] (const TRule_type_name_or_bind& param, const char* type) -> TNodePtr {
+ auto node = TypeNodeOrBind(param);
+ return node ? new TAstListNodeImpl(pos, { BuildAtom(pos, type, flags), node }) : nullptr;
+ };
+ auto makeVoid = [&] () -> TNodePtr {
+ return new TAstListNodeImpl(pos, { BuildAtom(pos, "VoidType", flags) });
+ };
+ auto makeQuote = [&] (const TNodePtr& node) -> TNodePtr {
+ return new TAstListNodeImpl(pos, { new TAstAtomNodeImpl(pos, "quote", 0), node });
+ };
+
+ auto& block = node.GetBlock1();
+ switch (block.Alt_case()) {
case TRule_type_name_composite_TBlock1::kAlt1: {
auto& optionalType = block.GetAlt1().GetRule_type_name_optional1();
- result = wrapOneParamType(optionalType.GetRule_type_name_or_bind3(), "OptionalType");
- break;
- }
+ result = wrapOneParamType(optionalType.GetRule_type_name_or_bind3(), "OptionalType");
+ break;
+ }
case TRule_type_name_composite_TBlock1::kAlt2: {
auto& tupleType = block.GetAlt2().GetRule_type_name_tuple1();
- TVector<TNodePtr> items;
- items.push_back(BuildAtom(pos, "TupleType", flags));
+ TVector<TNodePtr> items;
+ items.push_back(BuildAtom(pos, "TupleType", flags));
switch (tupleType.GetBlock2().Alt_case()) {
case TRule_type_name_tuple::TBlock2::kAlt1: {
if (tupleType.GetBlock2().GetAlt1().HasBlock2()) {
auto typeNode = TypeNodeOrBind(tupleType.GetBlock2().GetAlt1().GetBlock2().GetRule_type_name_or_bind1());
- if (!typeNode) {
- return {};
- }
- items.push_back(typeNode);
+ if (!typeNode) {
+ return {};
+ }
+ items.push_back(typeNode);
for (auto& arg : tupleType.GetBlock2().GetAlt1().GetBlock2().GetBlock2()) {
auto typeNode = TypeNodeOrBind(arg.GetRule_type_name_or_bind2());
if (!typeNode) {
@@ -2573,22 +2573,22 @@ TNodePtr TSqlTranslation::TypeNode(const TRule_type_name_composite& node) {
}
items.push_back(typeNode);
}
- }
+ }
[[fallthrough]]; // AUTOGENERATED_FALLTHROUGH_FIXME
- }
+ }
case TRule_type_name_tuple::TBlock2::kAlt2:
break;
default:
Y_FAIL("You should change implementation according to grammar changes");
}
- result = new TAstListNodeImpl(pos, items);
- break;
- }
+ result = new TAstListNodeImpl(pos, items);
+ break;
+ }
case TRule_type_name_composite_TBlock1::kAlt3: {
auto& structType = block.GetAlt3().GetRule_type_name_struct1();
- TVector<TNodePtr> items;
- items.push_back(BuildAtom(pos, "StructType", flags));
+ TVector<TNodePtr> items;
+ items.push_back(BuildAtom(pos, "StructType", flags));
switch (structType.GetBlock2().Alt_case()) {
case TRule_type_name_struct::TBlock2::kAlt1: {
@@ -2617,110 +2617,110 @@ TNodePtr TSqlTranslation::TypeNode(const TRule_type_name_composite& node) {
}
}
[[fallthrough]]; // AUTOGENERATED_FALLTHROUGH_FIXME
- }
+ }
case TRule_type_name_struct::TBlock2::kAlt2:
break;
default:
Y_FAIL("You should change implementation according to grammar changes");
- }
+ }
- result = new TAstListNodeImpl(pos, items);
- break;
- }
+ result = new TAstListNodeImpl(pos, items);
+ break;
+ }
case TRule_type_name_composite_TBlock1::kAlt4: {
auto& variantType = block.GetAlt4().GetRule_type_name_variant1();
- TVector<TNodePtr> items;
- bool overStruct = false;
- auto& variantArg = variantType.GetRule_variant_arg3();
- auto typeNode = TypeNodeOrBind(variantArg.GetRule_type_name_or_bind2());
- if (!typeNode) {
- return {};
- }
- if (variantArg.HasBlock1()) {
- items.push_back(BuildAtom(pos, "StructType", flags));
- overStruct = true;
- auto tag = TypeNameTag(variantArg.GetBlock1().GetRule_type_name_tag1());
- if (!tag) {
- return {};
- }
- items.push_back(makeQuote(new TAstListNodeImpl(pos, { tag, typeNode })));
- } else {
- items.push_back(BuildAtom(pos, "TupleType", flags));
- items.push_back(typeNode);
- }
-
- for (auto& arg : variantType.GetBlock4()) {
- auto typeNode = TypeNodeOrBind(arg.GetRule_variant_arg2().GetRule_type_name_or_bind2());
- if (!typeNode) {
- return {};
- }
- if (overStruct) {
- if (!arg.GetRule_variant_arg2().HasBlock1()) {
- Ctx.Error() << "Variant over struct and tuple mixture";
- return {};
- }
- auto tag = TypeNameTag(arg.GetRule_variant_arg2().GetBlock1().GetRule_type_name_tag1());
- if (!tag) {
- return {};
- }
- items.push_back(makeQuote(new TAstListNodeImpl(pos, { tag, typeNode })));
- } else {
- if (arg.GetRule_variant_arg2().HasBlock1()) {
- Ctx.Error() << "Variant over struct and tuple mixture";
- return {};
- }
- items.push_back(typeNode);
- }
- }
- typeNode = new TAstListNodeImpl(pos, items);
- result = new TAstListNodeImpl(pos, { BuildAtom(pos, "VariantType", flags), typeNode });
- break;
- }
+ TVector<TNodePtr> items;
+ bool overStruct = false;
+ auto& variantArg = variantType.GetRule_variant_arg3();
+ auto typeNode = TypeNodeOrBind(variantArg.GetRule_type_name_or_bind2());
+ if (!typeNode) {
+ return {};
+ }
+ if (variantArg.HasBlock1()) {
+ items.push_back(BuildAtom(pos, "StructType", flags));
+ overStruct = true;
+ auto tag = TypeNameTag(variantArg.GetBlock1().GetRule_type_name_tag1());
+ if (!tag) {
+ return {};
+ }
+ items.push_back(makeQuote(new TAstListNodeImpl(pos, { tag, typeNode })));
+ } else {
+ items.push_back(BuildAtom(pos, "TupleType", flags));
+ items.push_back(typeNode);
+ }
+
+ for (auto& arg : variantType.GetBlock4()) {
+ auto typeNode = TypeNodeOrBind(arg.GetRule_variant_arg2().GetRule_type_name_or_bind2());
+ if (!typeNode) {
+ return {};
+ }
+ if (overStruct) {
+ if (!arg.GetRule_variant_arg2().HasBlock1()) {
+ Ctx.Error() << "Variant over struct and tuple mixture";
+ return {};
+ }
+ auto tag = TypeNameTag(arg.GetRule_variant_arg2().GetBlock1().GetRule_type_name_tag1());
+ if (!tag) {
+ return {};
+ }
+ items.push_back(makeQuote(new TAstListNodeImpl(pos, { tag, typeNode })));
+ } else {
+ if (arg.GetRule_variant_arg2().HasBlock1()) {
+ Ctx.Error() << "Variant over struct and tuple mixture";
+ return {};
+ }
+ items.push_back(typeNode);
+ }
+ }
+ typeNode = new TAstListNodeImpl(pos, items);
+ result = new TAstListNodeImpl(pos, { BuildAtom(pos, "VariantType", flags), typeNode });
+ break;
+ }
case TRule_type_name_composite_TBlock1::kAlt5: {
auto& listType = block.GetAlt5().GetRule_type_name_list1();
- result = wrapOneParamType(listType.GetRule_type_name_or_bind3(), "ListType");
- break;
- }
+ result = wrapOneParamType(listType.GetRule_type_name_or_bind3(), "ListType");
+ break;
+ }
case TRule_type_name_composite_TBlock1::kAlt6: {
auto& streamType = block.GetAlt6().GetRule_type_name_stream1();
- result = wrapOneParamType(streamType.GetRule_type_name_or_bind3(), "StreamType");
- break;
- }
+ result = wrapOneParamType(streamType.GetRule_type_name_or_bind3(), "StreamType");
+ break;
+ }
case TRule_type_name_composite_TBlock1::kAlt7: {
auto& flowType = block.GetAlt7().GetRule_type_name_flow1();
- result = wrapOneParamType(flowType.GetRule_type_name_or_bind3(), "FlowType");
- break;
- }
+ result = wrapOneParamType(flowType.GetRule_type_name_or_bind3(), "FlowType");
+ break;
+ }
case TRule_type_name_composite_TBlock1::kAlt8: {
auto& dictType = block.GetAlt8().GetRule_type_name_dict1();
- TVector<TNodePtr> items;
- items.push_back(BuildAtom(pos, "DictType", flags));
- auto typeNode = TypeNodeOrBind(dictType.GetRule_type_name_or_bind3());
- if (!typeNode) {
- return {};
- }
- items.push_back(typeNode);
- typeNode = TypeNodeOrBind(dictType.GetRule_type_name_or_bind5());
- if (!typeNode) {
- return {};
- }
- items.push_back(typeNode);
- result = new TAstListNodeImpl(pos, items);
- break;
- }
+ TVector<TNodePtr> items;
+ items.push_back(BuildAtom(pos, "DictType", flags));
+ auto typeNode = TypeNodeOrBind(dictType.GetRule_type_name_or_bind3());
+ if (!typeNode) {
+ return {};
+ }
+ items.push_back(typeNode);
+ typeNode = TypeNodeOrBind(dictType.GetRule_type_name_or_bind5());
+ if (!typeNode) {
+ return {};
+ }
+ items.push_back(typeNode);
+ result = new TAstListNodeImpl(pos, items);
+ break;
+ }
case TRule_type_name_composite_TBlock1::kAlt9: {
auto& setType = block.GetAlt9().GetRule_type_name_set1();
- auto typeNode = TypeNodeOrBind(setType.GetRule_type_name_or_bind3());
- if (!typeNode) {
- return {};
- }
- result = new TAstListNodeImpl(pos, { BuildAtom(pos, "DictType", flags), typeNode, makeVoid() });
- break;
- }
+ auto typeNode = TypeNodeOrBind(setType.GetRule_type_name_or_bind3());
+ if (!typeNode) {
+ return {};
+ }
+ result = new TAstListNodeImpl(pos, { BuildAtom(pos, "DictType", flags), typeNode, makeVoid() });
+ break;
+ }
case TRule_type_name_composite_TBlock1::kAlt10: {
auto& enumType = block.GetAlt10().GetRule_type_name_enum1();
- TVector<TNodePtr> items;
- items.push_back(BuildAtom(pos, "StructType", flags));
+ TVector<TNodePtr> items;
+ items.push_back(BuildAtom(pos, "StructType", flags));
auto tag = TypeNameTag(enumType.GetRule_type_name_tag3());
if (!tag) {
return {};
@@ -2728,91 +2728,91 @@ TNodePtr TSqlTranslation::TypeNode(const TRule_type_name_composite& node) {
items.push_back(makeQuote(new TAstListNodeImpl(pos, { tag, makeVoid() })));
for (auto& arg : enumType.GetBlock4()) {
auto tag = TypeNameTag(arg.GetRule_type_name_tag2());
- if (!tag) {
- return {};
- }
- items.push_back(makeQuote(new TAstListNodeImpl(pos, { tag, makeVoid() })));
- }
- auto typeNode = new TAstListNodeImpl(pos, items);
- result = new TAstListNodeImpl(pos, { BuildAtom(pos, "VariantType", flags), typeNode });
- break;
- }
+ if (!tag) {
+ return {};
+ }
+ items.push_back(makeQuote(new TAstListNodeImpl(pos, { tag, makeVoid() })));
+ }
+ auto typeNode = new TAstListNodeImpl(pos, items);
+ result = new TAstListNodeImpl(pos, { BuildAtom(pos, "VariantType", flags), typeNode });
+ break;
+ }
case TRule_type_name_composite_TBlock1::kAlt11: {
auto& resourceType = block.GetAlt11().GetRule_type_name_resource1();
- auto tag = TypeNameTag(resourceType.GetRule_type_name_tag3());
- if (!tag) {
- return {};
- }
- result = new TAstListNodeImpl(pos, { BuildAtom(pos, "ResourceType", flags), tag });
- break;
- }
+ auto tag = TypeNameTag(resourceType.GetRule_type_name_tag3());
+ if (!tag) {
+ return {};
+ }
+ result = new TAstListNodeImpl(pos, { BuildAtom(pos, "ResourceType", flags), tag });
+ break;
+ }
case TRule_type_name_composite_TBlock1::kAlt12: {
auto& taggedType = block.GetAlt12().GetRule_type_name_tagged1();
- auto typeNode = TypeNodeOrBind(taggedType.GetRule_type_name_or_bind3());
- if (!typeNode) {
- return {};
- }
- auto tag = TypeNameTag(taggedType.GetRule_type_name_tag5());
- if (!tag) {
- return {};
- }
- result = new TAstListNodeImpl(pos, { BuildAtom(pos, "TaggedType", flags), typeNode, tag });
- break;
- }
+ auto typeNode = TypeNodeOrBind(taggedType.GetRule_type_name_or_bind3());
+ if (!typeNode) {
+ return {};
+ }
+ auto tag = TypeNameTag(taggedType.GetRule_type_name_tag5());
+ if (!tag) {
+ return {};
+ }
+ result = new TAstListNodeImpl(pos, { BuildAtom(pos, "TaggedType", flags), typeNode, tag });
+ break;
+ }
case TRule_type_name_composite_TBlock1::kAlt13: {
auto& callableType = block.GetAlt13().GetRule_type_name_callable1();
- TMaybe<std::pair<TVector<TNodePtr>, bool>> requiredArgs, optionalArgs;
- bool namedArgsStarted = false;
- size_t optionalArgsCount = 0;
- if (callableType.HasBlock4()) {
- auto& argList = callableType.GetBlock4().GetRule_callable_arg_list1();
- requiredArgs = CallableArgList(argList, namedArgsStarted);
- if (!requiredArgs) {
- return {};
- }
- namedArgsStarted = requiredArgs->second;
- }
- if (callableType.HasBlock6()) {
- auto& argList = callableType.GetBlock6().GetRule_callable_arg_list2();
- optionalArgs = CallableArgList(argList, namedArgsStarted);
- if (!optionalArgs) {
- return {};
- }
- optionalArgsCount = optionalArgs->first.size();
- }
- auto returnType = TypeNodeOrBind(callableType.GetRule_type_name_or_bind9());
- if (!returnType) {
- return {};
- }
- TVector<TNodePtr> items;
- items.push_back(BuildAtom(pos, "CallableType", flags));
- if (optionalArgsCount) {
- items.push_back(makeQuote(new TAstListNodeImpl(pos,
- { BuildQuotedAtom(pos, ToString(optionalArgsCount), flags) })));
- } else {
- items.push_back(makeQuote(new TAstListNodeImpl(pos, {})));
- }
- items.push_back(makeQuote(new TAstListNodeImpl(pos, { returnType })));
- if (requiredArgs) {
- for (auto& arg: requiredArgs->first) {
- items.push_back(makeQuote(arg));
- }
- }
- if (optionalArgs) {
- for (auto& arg: optionalArgs->first) {
- items.push_back(makeQuote(arg));
- }
- }
- result = new TAstListNodeImpl(pos, items);
- break;
- }
- default:
- Y_FAIL("You should change implementation according to grammar changes");
- }
-
+ TMaybe<std::pair<TVector<TNodePtr>, bool>> requiredArgs, optionalArgs;
+ bool namedArgsStarted = false;
+ size_t optionalArgsCount = 0;
+ if (callableType.HasBlock4()) {
+ auto& argList = callableType.GetBlock4().GetRule_callable_arg_list1();
+ requiredArgs = CallableArgList(argList, namedArgsStarted);
+ if (!requiredArgs) {
+ return {};
+ }
+ namedArgsStarted = requiredArgs->second;
+ }
+ if (callableType.HasBlock6()) {
+ auto& argList = callableType.GetBlock6().GetRule_callable_arg_list2();
+ optionalArgs = CallableArgList(argList, namedArgsStarted);
+ if (!optionalArgs) {
+ return {};
+ }
+ optionalArgsCount = optionalArgs->first.size();
+ }
+ auto returnType = TypeNodeOrBind(callableType.GetRule_type_name_or_bind9());
+ if (!returnType) {
+ return {};
+ }
+ TVector<TNodePtr> items;
+ items.push_back(BuildAtom(pos, "CallableType", flags));
+ if (optionalArgsCount) {
+ items.push_back(makeQuote(new TAstListNodeImpl(pos,
+ { BuildQuotedAtom(pos, ToString(optionalArgsCount), flags) })));
+ } else {
+ items.push_back(makeQuote(new TAstListNodeImpl(pos, {})));
+ }
+ items.push_back(makeQuote(new TAstListNodeImpl(pos, { returnType })));
+ if (requiredArgs) {
+ for (auto& arg: requiredArgs->first) {
+ items.push_back(makeQuote(arg));
+ }
+ }
+ if (optionalArgs) {
+ for (auto& arg: optionalArgs->first) {
+ items.push_back(makeQuote(arg));
+ }
+ }
+ result = new TAstListNodeImpl(pos, items);
+ break;
+ }
+ default:
+ Y_FAIL("You should change implementation according to grammar changes");
+ }
+
return AddOptionals(result, node.GetBlock2().size());
-}
-
+}
+
bool Expr(TSqlExpression& sqlExpr, TVector<TNodePtr>& exprNodes, const TRule_expr& node) {
TNodePtr exprNode = sqlExpr.Build(node);
if (!exprNode) {
@@ -2835,18 +2835,18 @@ bool ExprList(TSqlExpression& sqlExpr, TVector<TNodePtr>& exprNodes, const TRule
return true;
}
-TNodePtr TSqlTranslation::ValueConstructorLiteral(const TRule_value_constructor_literal& node) {
+TNodePtr TSqlTranslation::ValueConstructorLiteral(const TRule_value_constructor_literal& node) {
return BuildLiteralSmartString(Ctx, Token(node.GetToken1()));
-}
-
-TNodePtr TSqlTranslation::ValueConstructor(const TRule_value_constructor& node) {
- TSqlCallExpr call(Ctx, Mode);
- if (!call.Init(node)) {
- return {};
- }
- return call.BuildCall();
-}
-
+}
+
+TNodePtr TSqlTranslation::ValueConstructor(const TRule_value_constructor& node) {
+ TSqlCallExpr call(Ctx, Mode);
+ if (!call.Init(node)) {
+ return {};
+ }
+ return call.BuildCall();
+}
+
TNodePtr TSqlTranslation::ListLiteral(const TRule_list_literal& node) {
TVector<TNodePtr> values;
values.push_back(new TAstAtomNodeImpl(Ctx.Pos(), "AsList", TNodeFlags::Default));
@@ -3143,23 +3143,23 @@ bool TSqlTranslation::SimpleTableRefImpl(const TRule_simple_table_ref& node, TTa
return true;
}
-bool TSqlCallExpr::Init(const TRule_value_constructor& node) {
- switch (node.Alt_case()) {
- case TRule_value_constructor::kAltValueConstructor1: {
- auto& ctor = node.GetAlt_value_constructor1();
- Func = "Variant";
- TSqlExpression expr(Ctx, Mode);
- if (!Expr(expr, Args, ctor.GetRule_expr3())) {
- return false;
- }
- if (!Expr(expr, Args, ctor.GetRule_expr5())) {
- return false;
- }
+bool TSqlCallExpr::Init(const TRule_value_constructor& node) {
+ switch (node.Alt_case()) {
+ case TRule_value_constructor::kAltValueConstructor1: {
+ auto& ctor = node.GetAlt_value_constructor1();
+ Func = "Variant";
+ TSqlExpression expr(Ctx, Mode);
+ if (!Expr(expr, Args, ctor.GetRule_expr3())) {
+ return false;
+ }
+ if (!Expr(expr, Args, ctor.GetRule_expr5())) {
+ return false;
+ }
if (!Expr(expr, Args, ctor.GetRule_expr7())) {
- return false;
- }
- break;
- }
+ return false;
+ }
+ break;
+ }
case TRule_value_constructor::kAltValueConstructor2: {
auto& ctor = node.GetAlt_value_constructor2();
Func = "Enum";
@@ -3184,13 +3184,13 @@ bool TSqlCallExpr::Init(const TRule_value_constructor& node) {
}
break;
}
- default:
- Y_FAIL("You should change implementation according to grammar changes");
- }
- PositionalArgs = Args;
- return true;
-}
-
+ default:
+ Y_FAIL("You should change implementation according to grammar changes");
+ }
+ PositionalArgs = Args;
+ return true;
+}
+
bool TSqlCallExpr::ExtractCallParam(const TRule_external_call_param& node) {
TString paramName = Id(node.GetRule_an_id1(), *this);
paramName = to_lower(paramName);
@@ -3264,7 +3264,7 @@ bool TSqlCallExpr::Init(const TRule_using_call_expr& node) {
break;
}
default:
- Y_FAIL("You should change implementation according to grammar changes");
+ Y_FAIL("You should change implementation according to grammar changes");
}
YQL_ENSURE(!DistinctAllowed);
UsingCallExpr = true;
@@ -3581,7 +3581,7 @@ public:
, CompactGroupBy(false)
{}
- bool Build(const TRule_group_by_clause& node, bool stream);
+ bool Build(const TRule_group_by_clause& node, bool stream);
bool ParseList(const TRule_grouping_element_list& groupingListNode, EGroupByFeatures featureContext);
void SetFeatures(const TString& field) const;
@@ -3589,7 +3589,7 @@ public:
TMap<TString, TNodePtr>& Aliases();
THoppingWindowSpecPtr GetHoppingWindow() const;
bool IsCompactGroupBy() const;
-
+
private:
TMaybe<TVector<TNodePtr>> MultiplyGroupingSets(const TVector<TNodePtr>& lhs, const TVector<TNodePtr>& rhs) const;
bool ResolveGroupByAndGrouping();
@@ -3597,7 +3597,7 @@ private:
void FeedCollection(const TNodePtr& elem, TVector<TNodePtr>& collection, bool& hasEmpty) const;
bool OrdinaryGroupingSet(const TRule_ordinary_grouping_set& node, EGroupByFeatures featureContext);
bool OrdinaryGroupingSetList(const TRule_ordinary_grouping_set_list& node, EGroupByFeatures featureContext);
- bool HoppingWindow(const TRule_hopping_window_specification& node);
+ bool HoppingWindow(const TRule_hopping_window_specification& node);
bool AllowUnnamed(TPosition pos, EGroupByFeatures featureContext);
@@ -3609,7 +3609,7 @@ private:
TVector<TNodePtr> GroupBySet;
TGroupByClauseCtx::TPtr GroupSetContext;
- THoppingWindowSpecPtr HoppingWindowSpec; // stream queries
+ THoppingWindowSpecPtr HoppingWindowSpec; // stream queries
static const TString AutogenerateNamePrefix;
bool CompactGroupBy;
};
@@ -4589,7 +4589,7 @@ TMaybe<TExprOrIdent> TSqlExpression::AtomExpr(const TRule_atom_expr& node, const
// | exists_expr
// | case_expr
// | an_id_or_type NAMESPACE (id_or_type | STRING_VALUE)
- // | value_constructor
+ // | value_constructor
// | bitcast_expr
// | list_literal
// | dict_literal
@@ -4647,7 +4647,7 @@ TMaybe<TExprOrIdent> TSqlExpression::AtomExpr(const TRule_atom_expr& node, const
case TRule_atom_expr::kAltAtomExpr8: {
result.Expr = ValueConstructor(node.GetAlt_atom_expr8().GetRule_value_constructor1());
break;
- }
+ }
case TRule_atom_expr::kAltAtomExpr9:
result.Expr = BitCastRule(node.GetAlt_atom_expr9().GetRule_bitcast_expr1());
break;
@@ -4678,7 +4678,7 @@ TMaybe<TExprOrIdent> TSqlExpression::InAtomExpr(const TRule_in_atom_expr& node,
// | case_expr
// | an_id_or_type NAMESPACE (id_or_type | STRING_VALUE)
// | LPAREN select_stmt RPAREN
- // | value_constructor
+ // | value_constructor
// | bitcast_expr
// | list_literal
// | dict_literal
@@ -4745,10 +4745,10 @@ TMaybe<TExprOrIdent> TSqlExpression::InAtomExpr(const TRule_in_atom_expr& node,
result.Expr = BuildSelectResult(pos, std::move(source), false, Mode == NSQLTranslation::ESqlMode::SUBQUERY, Ctx.Scoped);
break;
}
- case TRule_in_atom_expr::kAltInAtomExpr8: {
+ case TRule_in_atom_expr::kAltInAtomExpr8: {
result.Expr = ValueConstructor(node.GetAlt_in_atom_expr8().GetRule_value_constructor1());
break;
- }
+ }
case TRule_in_atom_expr::kAltInAtomExpr9:
result.Expr = BitCastRule(node.GetAlt_in_atom_expr9().GetRule_bitcast_expr1());
break;
@@ -5238,91 +5238,91 @@ TNodePtr TSqlExpression::BinOpList(const TNode& node, TGetNode getNode, TIter be
opName = "%";
Ctx.IncrementMonCounter("sql_binary_operations", "Mod");
break;
- default:
- Ctx.IncrementMonCounter("sql_errors", "UnsupportedBinaryOperation");
- Error() << "Unsupported binary operation token: " << tokenId;
- return nullptr;
- }
-
+ default:
+ Ctx.IncrementMonCounter("sql_errors", "UnsupportedBinaryOperation");
+ Error() << "Unsupported binary operation token: " << tokenId;
+ return nullptr;
+ }
+
partialResult = BuildBinaryOp(Ctx, pos, opName, partialResult, SubExpr(getNode(*begin), (begin + 1 == end) ? tail : TTrailingQuestions{}));
- ++begin;
- }
-
- return partialResult;
-}
-
-template <typename TGetNode, typename TIter>
+ ++begin;
+ }
+
+ return partialResult;
+}
+
+template <typename TGetNode, typename TIter>
TNodePtr TSqlExpression::BinOpList(const TRule_bit_subexpr& node, TGetNode getNode, TIter begin, TIter end, const TTrailingQuestions& tail) {
TNodePtr partialResult = SubExpr(node, (begin == end) ? tail : TTrailingQuestions{});
- while (begin != end) {
- Ctx.IncrementMonCounter("sql_features", "BinaryOperation");
- TString opName;
- switch (begin->GetBlock1().Alt_case()) {
- case TRule_neq_subexpr_TBlock2_TBlock1::kAlt1: {
- Token(begin->GetBlock1().GetAlt1().GetToken1());
- auto tokenId = begin->GetBlock1().GetAlt1().GetToken1().GetId();
- if (tokenId != SQLv1LexerTokens::TOKEN_SHIFT_LEFT) {
- Error() << "Unsupported binary operation token: " << tokenId;
- return {};
- }
+ while (begin != end) {
+ Ctx.IncrementMonCounter("sql_features", "BinaryOperation");
+ TString opName;
+ switch (begin->GetBlock1().Alt_case()) {
+ case TRule_neq_subexpr_TBlock2_TBlock1::kAlt1: {
+ Token(begin->GetBlock1().GetAlt1().GetToken1());
+ auto tokenId = begin->GetBlock1().GetAlt1().GetToken1().GetId();
+ if (tokenId != SQLv1LexerTokens::TOKEN_SHIFT_LEFT) {
+ Error() << "Unsupported binary operation token: " << tokenId;
+ return {};
+ }
opName = "ShiftLeft";
Ctx.IncrementMonCounter("sql_binary_operations", "ShiftLeft");
break;
- }
- case TRule_neq_subexpr_TBlock2_TBlock1::kAlt2: {
+ }
+ case TRule_neq_subexpr_TBlock2_TBlock1::kAlt2: {
opName = "ShiftRight";
Ctx.IncrementMonCounter("sql_binary_operations", "ShiftRight");
break;
- }
- case TRule_neq_subexpr_TBlock2_TBlock1::kAlt3: {
- Token(begin->GetBlock1().GetAlt3().GetToken1());
- auto tokenId = begin->GetBlock1().GetAlt3().GetToken1().GetId();
- if (tokenId != SQLv1LexerTokens::TOKEN_ROT_LEFT) {
- Error() << "Unsupported binary operation token: " << tokenId;
- return {};
- }
+ }
+ case TRule_neq_subexpr_TBlock2_TBlock1::kAlt3: {
+ Token(begin->GetBlock1().GetAlt3().GetToken1());
+ auto tokenId = begin->GetBlock1().GetAlt3().GetToken1().GetId();
+ if (tokenId != SQLv1LexerTokens::TOKEN_ROT_LEFT) {
+ Error() << "Unsupported binary operation token: " << tokenId;
+ return {};
+ }
opName = "RotLeft";
Ctx.IncrementMonCounter("sql_binary_operations", "RotLeft");
break;
- }
- case TRule_neq_subexpr_TBlock2_TBlock1::kAlt4: {
+ }
+ case TRule_neq_subexpr_TBlock2_TBlock1::kAlt4: {
opName = "RotRight";
Ctx.IncrementMonCounter("sql_binary_operations", "RotRight");
break;
- }
- case TRule_neq_subexpr_TBlock2_TBlock1::kAlt5: {
- Token(begin->GetBlock1().GetAlt5().GetToken1());
- auto tokenId = begin->GetBlock1().GetAlt5().GetToken1().GetId();
- if (tokenId != SQLv1LexerTokens::TOKEN_AMPERSAND) {
- Error() << "Unsupported binary operation token: " << tokenId;
- return {};
- }
- opName = "BitAnd";
- Ctx.IncrementMonCounter("sql_binary_operations", "BitAnd");
- break;
- }
- case TRule_neq_subexpr_TBlock2_TBlock1::kAlt6: {
- Token(begin->GetBlock1().GetAlt6().GetToken1());
- auto tokenId = begin->GetBlock1().GetAlt6().GetToken1().GetId();
- if (tokenId != SQLv1LexerTokens::TOKEN_PIPE) {
- Error() << "Unsupported binary operation token: " << tokenId;
- return {};
- }
- opName = "BitOr";
- Ctx.IncrementMonCounter("sql_binary_operations", "BitOr");
- break;
- }
- case TRule_neq_subexpr_TBlock2_TBlock1::kAlt7: {
- Token(begin->GetBlock1().GetAlt7().GetToken1());
- auto tokenId = begin->GetBlock1().GetAlt7().GetToken1().GetId();
- if (tokenId != SQLv1LexerTokens::TOKEN_CARET) {
- Error() << "Unsupported binary operation token: " << tokenId;
- return {};
- }
- opName = "BitXor";
- Ctx.IncrementMonCounter("sql_binary_operations", "BitXor");
- break;
- }
+ }
+ case TRule_neq_subexpr_TBlock2_TBlock1::kAlt5: {
+ Token(begin->GetBlock1().GetAlt5().GetToken1());
+ auto tokenId = begin->GetBlock1().GetAlt5().GetToken1().GetId();
+ if (tokenId != SQLv1LexerTokens::TOKEN_AMPERSAND) {
+ Error() << "Unsupported binary operation token: " << tokenId;
+ return {};
+ }
+ opName = "BitAnd";
+ Ctx.IncrementMonCounter("sql_binary_operations", "BitAnd");
+ break;
+ }
+ case TRule_neq_subexpr_TBlock2_TBlock1::kAlt6: {
+ Token(begin->GetBlock1().GetAlt6().GetToken1());
+ auto tokenId = begin->GetBlock1().GetAlt6().GetToken1().GetId();
+ if (tokenId != SQLv1LexerTokens::TOKEN_PIPE) {
+ Error() << "Unsupported binary operation token: " << tokenId;
+ return {};
+ }
+ opName = "BitOr";
+ Ctx.IncrementMonCounter("sql_binary_operations", "BitOr");
+ break;
+ }
+ case TRule_neq_subexpr_TBlock2_TBlock1::kAlt7: {
+ Token(begin->GetBlock1().GetAlt7().GetToken1());
+ auto tokenId = begin->GetBlock1().GetAlt7().GetToken1().GetId();
+ if (tokenId != SQLv1LexerTokens::TOKEN_CARET) {
+ Error() << "Unsupported binary operation token: " << tokenId;
+ return {};
+ }
+ opName = "BitXor";
+ Ctx.IncrementMonCounter("sql_binary_operations", "BitXor");
+ break;
+ }
default:
Y_FAIL("You should change implementation according to grammar changes");
}
@@ -6001,7 +6001,7 @@ TSourcePtr TSqlSelect::SingleSource(const TRule_single_source& node, const TVect
TPosition pos(Ctx.Pos());
Ctx.IncrementMonCounter("sql_select_clusters", table.Cluster.GetLiteral() ? *table.Cluster.GetLiteral() : "unknown");
return BuildTableSource(pos, table);
- }
+ }
}
case TRule_single_source::kAltSingleSource2: {
const auto& alt = node.GetAlt_single_source2();
@@ -6561,7 +6561,7 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet
selectPos = Ctx.Pos();
}
- const bool distinct = IsDistinctOptSet(node.GetRule_opt_set_quantifier4());
+ const bool distinct = IsDistinctOptSet(node.GetRule_opt_set_quantifier4());
if (distinct) {
Ctx.IncrementMonCounter("sql_features", "DistinctInSelect");
}
@@ -6582,7 +6582,7 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet
if (!source) {
return nullptr;
}
-
+
const bool selectStream = node.HasBlock3();
TVector<TNodePtr> without;
if (node.HasBlock8()) {
@@ -6613,7 +6613,7 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet
/// \todo merge gtoupByExpr and groupBy in one
TVector<TNodePtr> groupByExpr, groupBy;
- THoppingWindowSpecPtr hoppingWindowSpec;
+ THoppingWindowSpecPtr hoppingWindowSpec;
bool compactGroupBy = false;
if (node.HasBlock11()) {
TGroupByClause clause(Ctx, Mode);
@@ -6626,7 +6626,7 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet
}
groupBy = std::move(clause.Content());
clause.SetFeatures("sql_features");
- hoppingWindowSpec = clause.GetHoppingWindow();
+ hoppingWindowSpec = clause.GetHoppingWindow();
compactGroupBy = clause.IsCompactGroupBy();
}
@@ -6644,12 +6644,12 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet
TWinSpecs windowSpec;
if (node.HasBlock13()) {
if (source->IsStream()) {
- Ctx.Error() << "WINDOW is not allowed in streaming queries";
+ Ctx.Error() << "WINDOW is not allowed in streaming queries";
return nullptr;
}
if (!WindowClause(node.GetBlock13().GetRule_window_clause1(), windowSpec)) {
- return nullptr;
- }
+ return nullptr;
+ }
Ctx.IncrementMonCounter("sql_features", "WindowClause");
}
@@ -6662,7 +6662,7 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet
Token(orderBlock.GetRule_order_by_clause2().GetToken1());
if (source->IsStream()) {
- Ctx.Error() << "ORDER BY is not allowed in streaming queries";
+ Ctx.Error() << "ORDER BY is not allowed in streaming queries";
return nullptr;
}
@@ -6671,8 +6671,8 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet
}
if (!OrderByClause(orderBlock.GetRule_order_by_clause2(), orderBy)) {
- return nullptr;
- }
+ return nullptr;
+ }
Ctx.IncrementMonCounter("sql_features", IsColumnsOnly(orderBy)
? (assumeSorted ? "AssumeOrderBy" : "OrderBy")
: (assumeSorted ? "AssumeOrderByExpr" : "OrderByExpr")
@@ -7030,7 +7030,7 @@ bool TSqlTranslation::OrderByClause(const TRule_order_by_clause& node, TVector<T
return SortSpecificationList(node.GetRule_sort_specification_list3(), orderBy);
}
-bool TGroupByClause::Build(const TRule_group_by_clause& node, bool stream) {
+bool TGroupByClause::Build(const TRule_group_by_clause& node, bool stream) {
CompactGroupBy = node.HasBlock2();
TPosition distinctPos;
if (IsDistinctOptSet(node.GetRule_opt_set_quantifier4(), distinctPos)) {
@@ -7049,7 +7049,7 @@ bool TGroupByClause::Build(const TRule_group_by_clause& node, bool stream) {
Ctx.Error() << "Streaming group by query must have a hopping window specification.";
return false;
}
- }
+ }
return true;
}
@@ -7097,9 +7097,9 @@ TMap<TString, TNodePtr>& TGroupByClause::Aliases() {
}
THoppingWindowSpecPtr TGroupByClause::GetHoppingWindow() const {
- return HoppingWindowSpec;
-}
-
+ return HoppingWindowSpec;
+}
+
bool TGroupByClause::IsCompactGroupBy() const {
return CompactGroupBy;
}
@@ -7253,12 +7253,12 @@ bool TGroupByClause::GroupingElement(const TRule_grouping_element& node, EGroupB
Features().Set(EGroupByFeatures::GroupingSet);
break;
}
- case TRule_grouping_element::kAltGroupingElement5: {
- if (!HoppingWindow(node.GetAlt_grouping_element5().GetRule_hopping_window_specification1())) {
- return false;
- }
- break;
- }
+ case TRule_grouping_element::kAltGroupingElement5: {
+ if (!HoppingWindow(node.GetAlt_grouping_element5().GetRule_hopping_window_specification1())) {
+ return false;
+ }
+ break;
+ }
default:
Y_FAIL("You should change implementation according to grammar changes");
}
@@ -7346,72 +7346,72 @@ bool TGroupByClause::OrdinaryGroupingSetList(const TRule_ordinary_grouping_set_l
return true;
}
-bool TGroupByClause::HoppingWindow(const TRule_hopping_window_specification& node) {
- if (HoppingWindowSpec) {
- Ctx.Error() << "Duplicate hopping window specification.";
- return false;
- }
- HoppingWindowSpec = new THoppingWindowSpec;
- {
+bool TGroupByClause::HoppingWindow(const TRule_hopping_window_specification& node) {
+ if (HoppingWindowSpec) {
+ Ctx.Error() << "Duplicate hopping window specification.";
+ return false;
+ }
+ HoppingWindowSpec = new THoppingWindowSpec;
+ {
TColumnRefScope scope(Ctx, EColumnRefState::Allow);
- TSqlExpression expr(Ctx, Mode);
- HoppingWindowSpec->TimeExtractor = expr.Build(node.GetRule_expr3());
- if (!HoppingWindowSpec->TimeExtractor) {
- return false;
- }
- }
- auto processIntervalParam = [&] (const TRule_expr& rule) -> TNodePtr {
- TSqlExpression expr(Ctx, Mode);
- auto node = expr.Build(rule);
- if (!node) {
- return nullptr;
- }
-
- auto literal = node->GetLiteral("String");
- if (!literal) {
- return new TAstListNodeImpl(Ctx.Pos(), {
- new TAstAtomNodeImpl(Ctx.Pos(), "EvaluateExpr", TNodeFlags::Default),
- node
- });
- }
-
+ TSqlExpression expr(Ctx, Mode);
+ HoppingWindowSpec->TimeExtractor = expr.Build(node.GetRule_expr3());
+ if (!HoppingWindowSpec->TimeExtractor) {
+ return false;
+ }
+ }
+ auto processIntervalParam = [&] (const TRule_expr& rule) -> TNodePtr {
+ TSqlExpression expr(Ctx, Mode);
+ auto node = expr.Build(rule);
+ if (!node) {
+ return nullptr;
+ }
+
+ auto literal = node->GetLiteral("String");
+ if (!literal) {
+ return new TAstListNodeImpl(Ctx.Pos(), {
+ new TAstAtomNodeImpl(Ctx.Pos(), "EvaluateExpr", TNodeFlags::Default),
+ node
+ });
+ }
+
const auto out = NKikimr::NMiniKQL::ValueFromString(NKikimr::NUdf::EDataSlot::Interval, *literal);
if (!out) {
- Ctx.Error(node->GetPos()) << "Expected interval in ISO 8601 format";
- return nullptr;
- }
-
+ Ctx.Error(node->GetPos()) << "Expected interval in ISO 8601 format";
+ return nullptr;
+ }
+
if ('T' == literal->back()) {
Ctx.Error(node->GetPos()) << "Time prefix 'T' at end of interval constant. The designator 'T' shall be absent if all of the time components are absent.";
return nullptr;
}
- return new TAstListNodeImpl(Ctx.Pos(), {
- new TAstAtomNodeImpl(Ctx.Pos(), "Interval", TNodeFlags::Default),
- new TAstListNodeImpl(Ctx.Pos(), {
- new TAstAtomNodeImpl(Ctx.Pos(), "quote", TNodeFlags::Default),
+ return new TAstListNodeImpl(Ctx.Pos(), {
+ new TAstAtomNodeImpl(Ctx.Pos(), "Interval", TNodeFlags::Default),
+ new TAstListNodeImpl(Ctx.Pos(), {
+ new TAstAtomNodeImpl(Ctx.Pos(), "quote", TNodeFlags::Default),
new TAstAtomNodeImpl(Ctx.Pos(), ToString(out.Get<i64>()), TNodeFlags::Default)
- })
- });
- };
-
- HoppingWindowSpec->Hop = processIntervalParam(node.GetRule_expr5());
- if (!HoppingWindowSpec->Hop) {
- return false;
- }
- HoppingWindowSpec->Interval = processIntervalParam(node.GetRule_expr7());
- if (!HoppingWindowSpec->Interval) {
- return false;
- }
- HoppingWindowSpec->Delay = processIntervalParam(node.GetRule_expr9());
- if (!HoppingWindowSpec->Delay) {
- return false;
- }
+ })
+ });
+ };
+
+ HoppingWindowSpec->Hop = processIntervalParam(node.GetRule_expr5());
+ if (!HoppingWindowSpec->Hop) {
+ return false;
+ }
+ HoppingWindowSpec->Interval = processIntervalParam(node.GetRule_expr7());
+ if (!HoppingWindowSpec->Interval) {
+ return false;
+ }
+ HoppingWindowSpec->Delay = processIntervalParam(node.GetRule_expr9());
+ if (!HoppingWindowSpec->Delay) {
+ return false;
+ }
HoppingWindowSpec->DataWatermarks = Ctx.PragmaDataWatermarks;
-
- return true;
-}
-
+
+ return true;
+}
+
bool TGroupByClause::AllowUnnamed(TPosition pos, EGroupByFeatures featureContext) {
TStringBuf feature;
switch (featureContext) {
@@ -8015,7 +8015,7 @@ TNodePtr TSqlIntoTable::Build(const TRule_into_table_stmt& node) {
break;
}
default:
- Y_FAIL("You should change implementation according to grammar changes");
+ Y_FAIL("You should change implementation according to grammar changes");
}
}
@@ -8090,7 +8090,7 @@ bool TSqlIntoTable::ValidateServiceName(const TRule_into_table_stmt& node, const
auto serviceName = table.Service;
const bool isMapReduce = serviceName == YtProviderName;
const bool isKikimr = serviceName == KikimrProviderName || serviceName == YdbProviderName;
- const bool isRtmr = serviceName == RtmrProviderName;
+ const bool isRtmr = serviceName == RtmrProviderName;
const bool isStat = serviceName == StatProviderName;
if (!isKikimr) {
@@ -8117,12 +8117,12 @@ bool TSqlIntoTable::ValidateServiceName(const TRule_into_table_stmt& node, const
Ctx.IncrementMonCounter("sql_errors", TStringBuilder() << SqlIntoUserModeStr << "UnsupportedFor" << serviceName);
return false;
}
- } else if (isRtmr) {
- if (mode != ESQLWriteColumnMode::InsertInto) {
- Ctx.Error(pos) << SqlIntoUserModeStr << " is unsupported for " << serviceName;
- Ctx.IncrementMonCounter("sql_errors", TStringBuilder() << SqlIntoUserModeStr << "UnsupportedFor" << serviceName);
- return false;
- }
+ } else if (isRtmr) {
+ if (mode != ESQLWriteColumnMode::InsertInto) {
+ Ctx.Error(pos) << SqlIntoUserModeStr << " is unsupported for " << serviceName;
+ Ctx.IncrementMonCounter("sql_errors", TStringBuilder() << SqlIntoUserModeStr << "UnsupportedFor" << serviceName);
+ return false;
+ }
} else if (isStat) {
if (mode != ESQLWriteColumnMode::UpsertInto) {
Ctx.Error(pos) << SqlIntoUserModeStr << " is unsupported for " << serviceName;
@@ -8382,7 +8382,7 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
}
TCreateTableParameters params;
-
+
if (!CreateTableEntry(rule.GetRule_create_table_entry5(), params)) {
return false;
}
@@ -8795,7 +8795,7 @@ bool TSqlQuery::DeclareStatement(const TRule_declare_stmt& stmt) {
return false;
}
const auto varPos = Ctx.Pos();
- const auto typeNode = TypeNode(stmt.GetRule_type_name4());
+ const auto typeNode = TypeNode(stmt.GetRule_type_name4());
if (!typeNode) {
return false;
}
@@ -8958,9 +8958,9 @@ bool TSqlQuery::AlterTableAction(const TRule_alter_table_action& node, TAlterTab
bool TSqlQuery::AlterTableAddColumn(const TRule_alter_table_add_column& node, TAlterTableParameters& params) {
auto columnSchema = ColumnSchemaImpl(node.GetRule_column_schema3());
- if (!columnSchema) {
- return false;
- }
+ if (!columnSchema) {
+ return false;
+ }
if (columnSchema->Families.size() > 1) {
Ctx.Error() << "Several column families for a single column are not yet supported";
return false;
@@ -8968,7 +8968,7 @@ bool TSqlQuery::AlterTableAddColumn(const TRule_alter_table_add_column& node, TA
params.AddColumns.push_back(*columnSchema);
return true;
}
-
+
bool TSqlQuery::AlterTableDropColumn(const TRule_alter_table_drop_column& node, TAlterTableParameters& params) {
TString name = Id(node.GetRule_an_id3(), *this);
params.DropColumns.push_back(name);
@@ -8991,8 +8991,8 @@ bool TSqlQuery::AlterTableAddFamily(const TRule_family_entry& node, TAlterTableP
TFamilyEntry family(IdEx(node.GetRule_an_id2(), *this));
if (node.HasBlock3()) {
if (!FillFamilySettings(node.GetBlock3().GetRule_family_settings1(), family)) {
- return false;
- }
+ return false;
+ }
}
params.AddColumnFamilies.push_back(family);
return true;
@@ -9133,7 +9133,7 @@ void TSqlQuery::AlterTableDropChangefeed(const TRule_alter_table_drop_changefeed
TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success) {
success = false;
- const TString& prefix = OptIdPrefixAsStr(stmt.GetRule_opt_id_prefix_or_type2(), *this);
+ const TString& prefix = OptIdPrefixAsStr(stmt.GetRule_opt_id_prefix_or_type2(), *this);
const TString& lowerPrefix = to_lower(prefix);
const TString pragma(Id(stmt.GetRule_an_id3(), *this));
TString normalizedPragma(pragma);
@@ -9683,14 +9683,14 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
TNodePtr TSqlQuery::Build(const TRule_delete_stmt& stmt) {
TTableRef table;
if (!SimpleTableRefImpl(stmt.GetRule_simple_table_ref3(), table)) {
- return nullptr;
- }
+ return nullptr;
+ }
const bool isKikimr = table.Service == KikimrProviderName;
if (!isKikimr) {
Ctx.Error(GetPos(stmt.GetToken1())) << "DELETE is unsupported for " << table.Service;
- return nullptr;
- }
+ return nullptr;
+ }
TSourcePtr source = BuildTableSource(Ctx.Pos(), table);
@@ -9731,15 +9731,15 @@ TNodePtr TSqlQuery::Build(const TRule_delete_stmt& stmt) {
TNodePtr TSqlQuery::Build(const TRule_update_stmt& stmt) {
TTableRef table;
if (!SimpleTableRefImpl(stmt.GetRule_simple_table_ref2(), table)) {
- return nullptr;
- }
+ return nullptr;
+ }
const bool isKikimr = table.Service == KikimrProviderName;
if (!isKikimr) {
Ctx.Error(GetPos(stmt.GetToken1())) << "UPDATE is unsupported for " << table.Service;
- return nullptr;
- }
+ return nullptr;
+ }
switch (stmt.GetBlock3().Alt_case()) {
case TRule_update_stmt_TBlock3::kAlt1: {
diff --git a/ydb/library/yql/sql/v1/sql_ut.cpp b/ydb/library/yql/sql/v1/sql_ut.cpp
index 85b6e0dfcd3..553f92c7acb 100644
--- a/ydb/library/yql/sql/v1/sql_ut.cpp
+++ b/ydb/library/yql/sql/v1/sql_ut.cpp
@@ -860,7 +860,7 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) {
VerifyProgram(res, elementStat, verifyLine);
UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Kikimr.PushData"]);
}
-
+
Y_UNIT_TEST(ProcessUserTypeAuth) {
NYql::TAstParseResult res = SqlToYql("process plato.Input using YDB::PushData(TableRows(), AsTuple('oauth', SecureParam('api:oauth')));", 1, TString(NYql::KikimrProviderName));
UNIT_ASSERT(res.Root);
@@ -879,17 +879,17 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) {
}
Y_UNIT_TEST(SelectStreamRtmr) {
- NYql::TAstParseResult res = SqlToYql(
- "USE plato; INSERT INTO Output SELECT STREAM key FROM Input;",
- 10, TString(NYql::RtmrProviderName));
- UNIT_ASSERT(res.Root);
+ NYql::TAstParseResult res = SqlToYql(
+ "USE plato; INSERT INTO Output SELECT STREAM key FROM Input;",
+ 10, TString(NYql::RtmrProviderName));
+ UNIT_ASSERT(res.Root);
res = SqlToYql(
"USE plato; INSERT INTO Output SELECT key FROM Input;",
10, TString(NYql::RtmrProviderName));
UNIT_ASSERT(res.Root);
- }
-
+ }
+
Y_UNIT_TEST(SelectStreamRtmrJoinWithYt) {
NYql::TAstParseResult res = SqlToYql(
"USE plato; INSERT INTO Output SELECT STREAM key FROM Input LEFT JOIN hahn.ttt as t ON Input.key = t.Name;",
@@ -906,12 +906,12 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) {
}
Y_UNIT_TEST(GroupByHopRtmr) {
- NYql::TAstParseResult res = SqlToYql(R"(
+ NYql::TAstParseResult res = SqlToYql(R"(
USE plato; INSERT INTO Output SELECT key, SUM(value) AS value FROM Input
- GROUP BY key, HOP(subkey, "PT10S", "PT30S", "PT20S");
- )", 10, TString(NYql::RtmrProviderName));
- UNIT_ASSERT(res.Root);
- }
+ GROUP BY key, HOP(subkey, "PT10S", "PT30S", "PT20S");
+ )", 10, TString(NYql::RtmrProviderName));
+ UNIT_ASSERT(res.Root);
+ }
Y_UNIT_TEST(GroupByHopRtmrSubquery) {
// 'use plato' intentially avoided
diff --git a/ydb/library/yql/udfs/common/datetime2/datetime_udf.cpp b/ydb/library/yql/udfs/common/datetime2/datetime_udf.cpp
index 763fea8024f..eef69271851 100644
--- a/ydb/library/yql/udfs/common/datetime2/datetime_udf.cpp
+++ b/ydb/library/yql/udfs/common/datetime2/datetime_udf.cpp
@@ -1,225 +1,225 @@
#include <ydb/library/yql/public/udf/udf_helpers.h>
#include <ydb/library/yql/public/udf/tz/udf_tz.h>
#include <ydb/library/yql/minikql/mkql_type_ops.h>
-
-#include <util/datetime/base.h>
-
-using namespace NKikimr;
-using namespace NUdf;
-
-extern const char SplitName[] = "Split";
-extern const char ToSecondsName[] = "ToSeconds";
-extern const char ToMillisecondsName[] = "ToMilliseconds";
-extern const char ToMicrosecondsName[] = "ToMicroseconds";
-
-extern const char TMResourceName[] = "DateTime2.TM";
-
-namespace {
-
- constexpr size_t MAX_TIMEZONE_NAME_LEN = 64;
-
- struct TTMStorage {
- unsigned int Year : 12;
- unsigned int DayOfYear : 9;
- unsigned int WeekOfYear : 6;
+
+#include <util/datetime/base.h>
+
+using namespace NKikimr;
+using namespace NUdf;
+
+extern const char SplitName[] = "Split";
+extern const char ToSecondsName[] = "ToSeconds";
+extern const char ToMillisecondsName[] = "ToMilliseconds";
+extern const char ToMicrosecondsName[] = "ToMicroseconds";
+
+extern const char TMResourceName[] = "DateTime2.TM";
+
+namespace {
+
+ constexpr size_t MAX_TIMEZONE_NAME_LEN = 64;
+
+ struct TTMStorage {
+ unsigned int Year : 12;
+ unsigned int DayOfYear : 9;
+ unsigned int WeekOfYear : 6;
unsigned int WeekOfYearIso8601 : 6;
- unsigned int DayOfWeek : 3;
- unsigned int Month : 4;
- unsigned int Day : 5;
- unsigned int Hour : 5;
- unsigned int Minute : 6;
- unsigned int Second : 6;
- unsigned int Microsecond : 20;
- unsigned int TimezoneId : 16;
-
- inline static bool IsUniversal(ui16 timezoneId) {
- return timezoneId == 0;
- }
-
- inline void MakeDefault() {
- Year = 1970;
- Month = 1;
- Day = 1;
- Hour = 0;
- Minute = 0;
- Second = 0;
- Microsecond = 0;
- TimezoneId = 0;
- }
-
- inline void FromDate(const IDateBuilder& builder, ui16 value, ui16 timezoneId = 0) {
+ unsigned int DayOfWeek : 3;
+ unsigned int Month : 4;
+ unsigned int Day : 5;
+ unsigned int Hour : 5;
+ unsigned int Minute : 6;
+ unsigned int Second : 6;
+ unsigned int Microsecond : 20;
+ unsigned int TimezoneId : 16;
+
+ inline static bool IsUniversal(ui16 timezoneId) {
+ return timezoneId == 0;
+ }
+
+ inline void MakeDefault() {
+ Year = 1970;
+ Month = 1;
+ Day = 1;
+ Hour = 0;
+ Minute = 0;
+ Second = 0;
+ Microsecond = 0;
+ TimezoneId = 0;
+ }
+
+ inline void FromDate(const IDateBuilder& builder, ui16 value, ui16 timezoneId = 0) {
ui32 year, month, day, dayOfYear, weekOfYear, weekOfYearIso8601, dayOfWeek;
-
+
if (!builder.FullSplitDate2(value, year, month, day, dayOfYear, weekOfYear, weekOfYearIso8601, dayOfWeek, timezoneId)) {
ythrow yexception() << "Error in FullSplitDate";
- }
-
+ }
+
TimezoneId = timezoneId;
- Year = year;
- Month = month;
- Day = day;
-
- DayOfYear = dayOfYear;
- WeekOfYear = weekOfYear;
+ Year = year;
+ Month = month;
+ Day = day;
+
+ DayOfYear = dayOfYear;
+ WeekOfYear = weekOfYear;
WeekOfYearIso8601 = weekOfYearIso8601;
- DayOfWeek = dayOfWeek;
- }
-
+ DayOfWeek = dayOfWeek;
+ }
+
inline ui16 ToDate(const IDateBuilder& builder, bool local) const {
- if (!IsUniversal(TimezoneId)) {
- ui32 datetime;
+ if (!IsUniversal(TimezoneId)) {
+ ui32 datetime;
if (!builder.MakeDatetime(Year, Month, Day, local ? 0 : Hour, local ? 0 : Minute, local ? 0 : Second, datetime, TimezoneId)) {
ythrow yexception() << "Error in MakeDatetime";
}
- return datetime / 86400u;
- } else {
- ui16 date;
+ return datetime / 86400u;
+ } else {
+ ui16 date;
if (!builder.MakeDate(Year, Month, Day, date)) {
ythrow yexception() << "Error in MakeDate";
}
- return date;
- }
- }
-
- inline void FromDatetime(const IDateBuilder& builder, ui32 value, ui16 timezoneId = 0) {
+ return date;
+ }
+ }
+
+ inline void FromDatetime(const IDateBuilder& builder, ui32 value, ui16 timezoneId = 0) {
ui32 year, month, day, hour, minute, second, dayOfYear, weekOfYear, weekOfYearIso8601, dayOfWeek;
-
+
if (!builder.FullSplitDatetime2(value, year, month, day, hour, minute, second, dayOfYear, weekOfYear, weekOfYearIso8601, dayOfWeek, timezoneId)) {
ythrow yexception() << "Error in FullSplitDatetime";
}
-
+
TimezoneId = timezoneId;
- Year = year;
- Month = month;
- Day = day;
- Hour = hour;
- Minute = minute;
- Second = second;
-
- DayOfYear = dayOfYear;
- WeekOfYear = weekOfYear;
+ Year = year;
+ Month = month;
+ Day = day;
+ Hour = hour;
+ Minute = minute;
+ Second = second;
+
+ DayOfYear = dayOfYear;
+ WeekOfYear = weekOfYear;
WeekOfYearIso8601 = weekOfYearIso8601;
- DayOfWeek = dayOfWeek;
- }
-
- inline ui32 ToDatetime(const IDateBuilder& builder) const {
+ DayOfWeek = dayOfWeek;
+ }
+
+ inline ui32 ToDatetime(const IDateBuilder& builder) const {
ui32 datetime = 0;
if (!builder.MakeDatetime(Year, Month, Day, Hour, Minute, Second, datetime, TimezoneId)) {
ythrow yexception() << "Error in MakeDatetime";
}
- return datetime;
- }
-
- inline void FromTimestamp(const IDateBuilder& builder, ui64 value, ui16 timezoneId = 0) {
+ return datetime;
+ }
+
+ inline void FromTimestamp(const IDateBuilder& builder, ui64 value, ui16 timezoneId = 0) {
const ui32 seconds = value / 1000000ull;
- FromDatetime(builder, seconds, timezoneId);
- Microsecond = value - seconds * 1000000ull;
- }
-
- inline ui64 ToTimestamp(const IDateBuilder& builder) const {
- return ToDatetime(builder) * 1000000ull + Microsecond;
- }
-
- inline bool Validate(const IDateBuilder& builder) {
- ui32 datetime;
- if (!builder.MakeDatetime(Year, Month, Day, Hour, Minute, Second, datetime, TimezoneId)) {
- return false;
- }
-
+ FromDatetime(builder, seconds, timezoneId);
+ Microsecond = value - seconds * 1000000ull;
+ }
+
+ inline ui64 ToTimestamp(const IDateBuilder& builder) const {
+ return ToDatetime(builder) * 1000000ull + Microsecond;
+ }
+
+ inline bool Validate(const IDateBuilder& builder) {
+ ui32 datetime;
+ if (!builder.MakeDatetime(Year, Month, Day, Hour, Minute, Second, datetime, TimezoneId)) {
+ return false;
+ }
+
ui32 year, month, day, hour, minute, second, dayOfYear, weekOfYear, weekOfYearIso8601, dayOfWeek;
if (!builder.FullSplitDatetime2(datetime, year, month, day, hour, minute, second, dayOfYear, weekOfYear, weekOfYearIso8601, dayOfWeek, TimezoneId)) {
ythrow yexception() << "Error in FullSplitDatetime.";
}
-
- DayOfYear = dayOfYear;
- WeekOfYear = weekOfYear;
+
+ DayOfYear = dayOfYear;
+ WeekOfYear = weekOfYear;
WeekOfYearIso8601 = weekOfYearIso8601;
- DayOfWeek = dayOfWeek;
-
- return true;
- }
-
- inline void FromTimeOfDay(ui64 value) {
- Hour = value / 3600000000ull;
- value -= Hour * 3600000000ull;
- Minute = value / 60000000ull;
- value -= Minute * 60000000ull;
- Second = value / 1000000ull;
- Microsecond = value - Second * 1000000ull;
- }
-
- inline ui64 ToTimeOfDay() const {
- return ((Hour * 60ull + Minute) * 60ull + Second) * 1000000ull + Microsecond;
- }
-
- static const TTMStorage& Reference(const TUnboxedValuePod& value) {
- return *reinterpret_cast<const TTMStorage*>(value.GetRawPtr());
- }
-
- static TTMStorage& Reference(TUnboxedValuePod& value) {
- return *reinterpret_cast<TTMStorage*>(value.GetRawPtr());
- }
- };
-
- static_assert(sizeof(TTMStorage) == 16, "TTMStorage size must be equal to TUnboxedValuePod size");
-
-#define ACCESSORS(field, type) \
- inline type Get##field(const TUnboxedValuePod& tm) { \
- return (type)TTMStorage::Reference(tm).field; \
- } \
- Y_DECLARE_UNUSED inline void Set##field(TUnboxedValuePod& tm, type value) { \
- TTMStorage::Reference(tm).field = value; \
- }
-
- ACCESSORS(Year, ui16)
- ACCESSORS(DayOfYear, ui16)
- ACCESSORS(WeekOfYear, ui8)
+ DayOfWeek = dayOfWeek;
+
+ return true;
+ }
+
+ inline void FromTimeOfDay(ui64 value) {
+ Hour = value / 3600000000ull;
+ value -= Hour * 3600000000ull;
+ Minute = value / 60000000ull;
+ value -= Minute * 60000000ull;
+ Second = value / 1000000ull;
+ Microsecond = value - Second * 1000000ull;
+ }
+
+ inline ui64 ToTimeOfDay() const {
+ return ((Hour * 60ull + Minute) * 60ull + Second) * 1000000ull + Microsecond;
+ }
+
+ static const TTMStorage& Reference(const TUnboxedValuePod& value) {
+ return *reinterpret_cast<const TTMStorage*>(value.GetRawPtr());
+ }
+
+ static TTMStorage& Reference(TUnboxedValuePod& value) {
+ return *reinterpret_cast<TTMStorage*>(value.GetRawPtr());
+ }
+ };
+
+ static_assert(sizeof(TTMStorage) == 16, "TTMStorage size must be equal to TUnboxedValuePod size");
+
+#define ACCESSORS(field, type) \
+ inline type Get##field(const TUnboxedValuePod& tm) { \
+ return (type)TTMStorage::Reference(tm).field; \
+ } \
+ Y_DECLARE_UNUSED inline void Set##field(TUnboxedValuePod& tm, type value) { \
+ TTMStorage::Reference(tm).field = value; \
+ }
+
+ ACCESSORS(Year, ui16)
+ ACCESSORS(DayOfYear, ui16)
+ ACCESSORS(WeekOfYear, ui8)
ACCESSORS(WeekOfYearIso8601, ui8)
- ACCESSORS(DayOfWeek, ui8)
- ACCESSORS(Month, ui8)
- ACCESSORS(Day, ui8)
- ACCESSORS(Hour, ui8)
- ACCESSORS(Minute, ui8)
- ACCESSORS(Second, ui8)
- ACCESSORS(Microsecond, ui32)
- ACCESSORS(TimezoneId, ui16)
-
-#undef ACCESSORS
-
- inline bool ValidateYear(ui16 year) {
- return year >= NUdf::MIN_YEAR - 1 || year <= NUdf::MAX_YEAR + 1;
- }
-
- inline bool ValidateMonth(ui8 month) {
- return month >= 1 && month <= 12;
- }
-
- inline bool ValidateDay(ui8 day) {
- return day >= 1 && day <= 31;
- }
-
- inline bool ValidateHour(ui8 hour) {
- return hour < 24;
- }
-
- inline bool ValidateMinute(ui8 minute) {
- return minute < 60;
- }
-
- inline bool ValidateSecond(ui8 second) {
- return second < 60;
- }
-
- inline bool ValidateMicrosecond(ui32 microsecond) {
- return microsecond < 1000000;
- }
-
- inline bool ValidateTimezoneId(ui16 timezoneId) {
+ ACCESSORS(DayOfWeek, ui8)
+ ACCESSORS(Month, ui8)
+ ACCESSORS(Day, ui8)
+ ACCESSORS(Hour, ui8)
+ ACCESSORS(Minute, ui8)
+ ACCESSORS(Second, ui8)
+ ACCESSORS(Microsecond, ui32)
+ ACCESSORS(TimezoneId, ui16)
+
+#undef ACCESSORS
+
+ inline bool ValidateYear(ui16 year) {
+ return year >= NUdf::MIN_YEAR - 1 || year <= NUdf::MAX_YEAR + 1;
+ }
+
+ inline bool ValidateMonth(ui8 month) {
+ return month >= 1 && month <= 12;
+ }
+
+ inline bool ValidateDay(ui8 day) {
+ return day >= 1 && day <= 31;
+ }
+
+ inline bool ValidateHour(ui8 hour) {
+ return hour < 24;
+ }
+
+ inline bool ValidateMinute(ui8 minute) {
+ return minute < 60;
+ }
+
+ inline bool ValidateSecond(ui8 second) {
+ return second < 60;
+ }
+
+ inline bool ValidateMicrosecond(ui32 microsecond) {
+ return microsecond < 1000000;
+ }
+
+ inline bool ValidateTimezoneId(ui16 timezoneId) {
const auto& zones = NUdf::GetTimezones();
return timezoneId < zones.size() && !zones[timezoneId].empty();
- }
-
+ }
+
inline bool ValidateMonthShortName(const std::string_view& monthName, ui8& month) {
static constexpr auto cmp = [](const std::string_view& a, const std::string_view& b) {
int cmp = strnicmp(a.data(), b.data(), std::min(a.size(), b.size()));
@@ -290,44 +290,44 @@ namespace {
return interval > -i64(MAX_TIMESTAMP) && interval < i64(MAX_TIMESTAMP);
}
- // Split
-
- template <typename TUserDataType>
- class TSplit : public TBoxedValue {
+ // Split
+
+ template <typename TUserDataType>
+ class TSplit : public TBoxedValue {
const TSourcePosition Pos_;
- public:
+ public:
explicit TSplit(TSourcePosition pos)
: Pos_(pos)
{}
- TUnboxedValue Run(
- const IValueBuilder* valueBuilder,
- const TUnboxedValuePod* args) const override;
-
- static void DeclareSignature(
- TType* userType,
- IFunctionTypeInfoBuilder& builder,
- bool typesOnly)
- {
- builder.UserType(userType);
+ TUnboxedValue Run(
+ const IValueBuilder* valueBuilder,
+ const TUnboxedValuePod* args) const override;
+
+ static void DeclareSignature(
+ TType* userType,
+ IFunctionTypeInfoBuilder& builder,
+ bool typesOnly)
+ {
+ builder.UserType(userType);
builder.Args()->Add<TUserDataType>().Flags(ICallablePayload::TArgumentFlags::AutoMap);
- builder.Returns(builder.Resource(TMResourceName));
-
- if (!typesOnly) {
+ builder.Returns(builder.Resource(TMResourceName));
+
+ if (!typesOnly) {
builder.Implementation(new TSplit<TUserDataType>(builder.GetSourcePosition()));
- }
- }
- };
-
- template <>
- TUnboxedValue TSplit<TDate>::Run(
- const IValueBuilder* valueBuilder,
- const TUnboxedValuePod* args) const
- {
+ }
+ }
+ };
+
+ template <>
+ TUnboxedValue TSplit<TDate>::Run(
+ const IValueBuilder* valueBuilder,
+ const TUnboxedValuePod* args) const
+ {
try {
EMPTY_RESULT_ON_EMPTY_ARG(0);
-
+
auto& builder = valueBuilder->GetDateBuilder();
TUnboxedValuePod result(0);
auto& storage = TTMStorage::Reference(result);
@@ -336,16 +336,16 @@ namespace {
} catch (const std::exception& e) {
UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data());
}
- }
-
- template <>
- TUnboxedValue TSplit<TDatetime>::Run(
- const IValueBuilder* valueBuilder,
- const TUnboxedValuePod* args) const
- {
+ }
+
+ template <>
+ TUnboxedValue TSplit<TDatetime>::Run(
+ const IValueBuilder* valueBuilder,
+ const TUnboxedValuePod* args) const
+ {
try {
EMPTY_RESULT_ON_EMPTY_ARG(0);
-
+
auto& builder = valueBuilder->GetDateBuilder();
TUnboxedValuePod result(0);
auto& storage = TTMStorage::Reference(result);
@@ -354,16 +354,16 @@ namespace {
} catch (const std::exception& e) {
UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data());
}
- }
-
- template <>
- TUnboxedValue TSplit<TTimestamp>::Run(
- const IValueBuilder* valueBuilder,
- const TUnboxedValuePod* args) const
- {
+ }
+
+ template <>
+ TUnboxedValue TSplit<TTimestamp>::Run(
+ const IValueBuilder* valueBuilder,
+ const TUnboxedValuePod* args) const
+ {
try {
EMPTY_RESULT_ON_EMPTY_ARG(0);
-
+
auto& builder = valueBuilder->GetDateBuilder();
TUnboxedValuePod result(0);
auto& storage = TTMStorage::Reference(result);
@@ -372,16 +372,16 @@ namespace {
} catch (const std::exception& e) {
UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data());
}
- }
-
- template <>
- TUnboxedValue TSplit<TTzDate>::Run(
- const IValueBuilder* valueBuilder,
- const TUnboxedValuePod* args) const
- {
+ }
+
+ template <>
+ TUnboxedValue TSplit<TTzDate>::Run(
+ const IValueBuilder* valueBuilder,
+ const TUnboxedValuePod* args) const
+ {
try {
EMPTY_RESULT_ON_EMPTY_ARG(0);
-
+
auto& builder = valueBuilder->GetDateBuilder();
TUnboxedValuePod result(0);
auto& storage = TTMStorage::Reference(result);
@@ -390,16 +390,16 @@ namespace {
} catch (const std::exception& e) {
UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data());
}
- }
-
- template <>
- TUnboxedValue TSplit<TTzDatetime>::Run(
- const IValueBuilder* valueBuilder,
- const TUnboxedValuePod* args) const
- {
+ }
+
+ template <>
+ TUnboxedValue TSplit<TTzDatetime>::Run(
+ const IValueBuilder* valueBuilder,
+ const TUnboxedValuePod* args) const
+ {
try {
EMPTY_RESULT_ON_EMPTY_ARG(0);
-
+
auto& builder = valueBuilder->GetDateBuilder();
TUnboxedValuePod result(0);
auto& storage = TTMStorage::Reference(result);
@@ -408,16 +408,16 @@ namespace {
} catch (const std::exception& e) {
UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data());
}
- }
-
- template <>
- TUnboxedValue TSplit<TTzTimestamp>::Run(
- const IValueBuilder* valueBuilder,
- const TUnboxedValuePod* args) const
- {
+ }
+
+ template <>
+ TUnboxedValue TSplit<TTzTimestamp>::Run(
+ const IValueBuilder* valueBuilder,
+ const TUnboxedValuePod* args) const
+ {
try {
EMPTY_RESULT_ON_EMPTY_ARG(0);
-
+
auto& builder = valueBuilder->GetDateBuilder();
TUnboxedValuePod result(0);
auto& storage = TTMStorage::Reference(result);
@@ -426,65 +426,65 @@ namespace {
} catch (const std::exception& e) {
UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data());
}
- }
-
- // Make*
-
- SIMPLE_UDF(TMakeDate, TDate(TAutoMap<TResource<TMResourceName>>)) {
- auto& builder = valueBuilder->GetDateBuilder();
- auto& storage = TTMStorage::Reference(args[0]);
+ }
+
+ // Make*
+
+ SIMPLE_UDF(TMakeDate, TDate(TAutoMap<TResource<TMResourceName>>)) {
+ auto& builder = valueBuilder->GetDateBuilder();
+ auto& storage = TTMStorage::Reference(args[0]);
return TUnboxedValuePod(storage.ToDate(builder, false));
- }
-
- SIMPLE_UDF(TMakeDatetime, TDatetime(TAutoMap<TResource<TMResourceName>>)) {
- auto& builder = valueBuilder->GetDateBuilder();
- auto& storage = TTMStorage::Reference(args[0]);
- return TUnboxedValuePod(storage.ToDatetime(builder));
- }
-
- SIMPLE_UDF(TMakeTimestamp, TTimestamp(TAutoMap<TResource<TMResourceName>>)) {
- auto& builder = valueBuilder->GetDateBuilder();
- auto& storage = TTMStorage::Reference(args[0]);
- return TUnboxedValuePod(storage.ToTimestamp(builder));
- }
-
- SIMPLE_UDF(TMakeTzDate, TTzDate(TAutoMap<TResource<TMResourceName>>)) {
- auto& builder = valueBuilder->GetDateBuilder();
- auto& storage = TTMStorage::Reference(args[0]);
+ }
+
+ SIMPLE_UDF(TMakeDatetime, TDatetime(TAutoMap<TResource<TMResourceName>>)) {
+ auto& builder = valueBuilder->GetDateBuilder();
+ auto& storage = TTMStorage::Reference(args[0]);
+ return TUnboxedValuePod(storage.ToDatetime(builder));
+ }
+
+ SIMPLE_UDF(TMakeTimestamp, TTimestamp(TAutoMap<TResource<TMResourceName>>)) {
+ auto& builder = valueBuilder->GetDateBuilder();
+ auto& storage = TTMStorage::Reference(args[0]);
+ return TUnboxedValuePod(storage.ToTimestamp(builder));
+ }
+
+ SIMPLE_UDF(TMakeTzDate, TTzDate(TAutoMap<TResource<TMResourceName>>)) {
+ auto& builder = valueBuilder->GetDateBuilder();
+ auto& storage = TTMStorage::Reference(args[0]);
TUnboxedValuePod result(storage.ToDate(builder, true));
- result.SetTimezoneId(storage.TimezoneId);
- return result;
- }
-
- SIMPLE_UDF(TMakeTzDatetime, TTzDatetime(TAutoMap<TResource<TMResourceName>>)) {
- auto& builder = valueBuilder->GetDateBuilder();
- auto& storage = TTMStorage::Reference(args[0]);
- TUnboxedValuePod result(storage.ToDatetime(builder));
- result.SetTimezoneId(storage.TimezoneId);
- return result;
- }
-
- SIMPLE_UDF(TMakeTzTimestamp, TTzTimestamp(TAutoMap<TResource<TMResourceName>>)) {
- auto& builder = valueBuilder->GetDateBuilder();
- auto& storage = TTMStorage::Reference(args[0]);
- TUnboxedValuePod result(storage.ToTimestamp(builder));
- result.SetTimezoneId(storage.TimezoneId);
- return result;
- }
-
- // Get*
-
-#define GET_METHOD(field, type) \
- SIMPLE_UDF(TGet##field, type(TAutoMap<TResource<TMResourceName>>)) { \
- Y_UNUSED(valueBuilder); \
- return TUnboxedValuePod(Get##field(args[0])); \
- }
-
- GET_METHOD(Year, ui16)
- GET_METHOD(DayOfYear, ui16)
- GET_METHOD(Month, ui8)
-
- SIMPLE_UDF(TGetMonthName, char*(TAutoMap<TResource<TMResourceName>>)) {
+ result.SetTimezoneId(storage.TimezoneId);
+ return result;
+ }
+
+ SIMPLE_UDF(TMakeTzDatetime, TTzDatetime(TAutoMap<TResource<TMResourceName>>)) {
+ auto& builder = valueBuilder->GetDateBuilder();
+ auto& storage = TTMStorage::Reference(args[0]);
+ TUnboxedValuePod result(storage.ToDatetime(builder));
+ result.SetTimezoneId(storage.TimezoneId);
+ return result;
+ }
+
+ SIMPLE_UDF(TMakeTzTimestamp, TTzTimestamp(TAutoMap<TResource<TMResourceName>>)) {
+ auto& builder = valueBuilder->GetDateBuilder();
+ auto& storage = TTMStorage::Reference(args[0]);
+ TUnboxedValuePod result(storage.ToTimestamp(builder));
+ result.SetTimezoneId(storage.TimezoneId);
+ return result;
+ }
+
+ // Get*
+
+#define GET_METHOD(field, type) \
+ SIMPLE_UDF(TGet##field, type(TAutoMap<TResource<TMResourceName>>)) { \
+ Y_UNUSED(valueBuilder); \
+ return TUnboxedValuePod(Get##field(args[0])); \
+ }
+
+ GET_METHOD(Year, ui16)
+ GET_METHOD(DayOfYear, ui16)
+ GET_METHOD(Month, ui8)
+
+ SIMPLE_UDF(TGetMonthName, char*(TAutoMap<TResource<TMResourceName>>)) {
Y_UNUSED(valueBuilder);
static const std::array<TUnboxedValue, 12U> monthNames = {{
TUnboxedValuePod::Embedded(TStringRef::Of("January")),
@@ -501,19 +501,19 @@ namespace {
TUnboxedValuePod::Embedded(TStringRef::Of("December"))
}};
return monthNames.at(GetMonth(*args) - 1U);
- }
-
- GET_METHOD(WeekOfYear, ui8)
+ }
+
+ GET_METHOD(WeekOfYear, ui8)
GET_METHOD(WeekOfYearIso8601, ui8)
-
- SIMPLE_UDF(TGetDayOfMonth, ui8(TAutoMap<TResource<TMResourceName>>)) {
- Y_UNUSED(valueBuilder);
- return TUnboxedValuePod(GetDay(args[0]));
- }
-
- GET_METHOD(DayOfWeek, ui8)
-
- SIMPLE_UDF(TGetDayOfWeekName, char*(TAutoMap<TResource<TMResourceName>>)) {
+
+ SIMPLE_UDF(TGetDayOfMonth, ui8(TAutoMap<TResource<TMResourceName>>)) {
+ Y_UNUSED(valueBuilder);
+ return TUnboxedValuePod(GetDay(args[0]));
+ }
+
+ GET_METHOD(DayOfWeek, ui8)
+
+ SIMPLE_UDF(TGetDayOfWeekName, char*(TAutoMap<TResource<TMResourceName>>)) {
Y_UNUSED(valueBuilder);
static const std::array<TUnboxedValue, 7U> dayNames = {{
TUnboxedValuePod::Embedded(TStringRef::Of("Monday")),
@@ -525,98 +525,98 @@ namespace {
TUnboxedValuePod::Embedded(TStringRef::Of("Sunday"))
}};
return dayNames.at(GetDayOfWeek(*args) - 1U);
- }
-
- GET_METHOD(Hour, ui8)
- GET_METHOD(Minute, ui8)
- GET_METHOD(Second, ui8)
-
- SIMPLE_UDF(TGetMillisecondOfSecond, ui32(TAutoMap<TResource<TMResourceName>>)) {
- Y_UNUSED(valueBuilder);
- return TUnboxedValuePod(GetMicrosecond(args[0]) / 1000u);
- }
-
- SIMPLE_UDF(TGetMicrosecondOfSecond, ui32(TAutoMap<TResource<TMResourceName>>)) {
- Y_UNUSED(valueBuilder);
- return TUnboxedValuePod(GetMicrosecond(args[0]));
- }
-
- GET_METHOD(TimezoneId, ui16)
-
- SIMPLE_UDF(TGetTimezoneName, char*(TAutoMap<TResource<TMResourceName>>)) {
- auto timezoneId = GetTimezoneId(args[0]);
+ }
+
+ GET_METHOD(Hour, ui8)
+ GET_METHOD(Minute, ui8)
+ GET_METHOD(Second, ui8)
+
+ SIMPLE_UDF(TGetMillisecondOfSecond, ui32(TAutoMap<TResource<TMResourceName>>)) {
+ Y_UNUSED(valueBuilder);
+ return TUnboxedValuePod(GetMicrosecond(args[0]) / 1000u);
+ }
+
+ SIMPLE_UDF(TGetMicrosecondOfSecond, ui32(TAutoMap<TResource<TMResourceName>>)) {
+ Y_UNUSED(valueBuilder);
+ return TUnboxedValuePod(GetMicrosecond(args[0]));
+ }
+
+ GET_METHOD(TimezoneId, ui16)
+
+ SIMPLE_UDF(TGetTimezoneName, char*(TAutoMap<TResource<TMResourceName>>)) {
+ auto timezoneId = GetTimezoneId(args[0]);
if (timezoneId >= NUdf::GetTimezones().size()) {
- return TUnboxedValuePod();
- }
- return valueBuilder->NewString(NUdf::GetTimezones()[timezoneId]);
- }
-
- // Update
-
- class TUpdate : public TBoxedValue {
+ return TUnboxedValuePod();
+ }
+ return valueBuilder->NewString(NUdf::GetTimezones()[timezoneId]);
+ }
+
+ // Update
+
+ class TUpdate : public TBoxedValue {
const TSourcePosition Pos_;
public:
explicit TUpdate(TSourcePosition pos)
: Pos_(pos)
{}
- TUnboxedValue Run(
- const IValueBuilder* valueBuilder,
- const TUnboxedValuePod* args) const override
- {
+ TUnboxedValue Run(
+ const IValueBuilder* valueBuilder,
+ const TUnboxedValuePod* args) const override
+ {
try {
EMPTY_RESULT_ON_EMPTY_ARG(0);
auto result = args[0];
-
+
if (args[1]) {
auto year = args[1].Get<ui16>();
if (!ValidateYear(year)) {
return TUnboxedValuePod();
}
SetYear(result, year);
- }
+ }
if (args[2]) {
auto month = args[2].Get<ui8>();
if (!ValidateMonth(month)) {
return TUnboxedValuePod();
}
SetMonth(result, month);
- }
+ }
if (args[3]) {
auto day = args[3].Get<ui8>();
if (!ValidateDay(day)) {
return TUnboxedValuePod();
}
SetDay(result, day);
- }
+ }
if (args[4]) {
auto hour = args[4].Get<ui8>();
if (!ValidateHour(hour)) {
return TUnboxedValuePod();
}
SetHour(result, hour);
- }
+ }
if (args[5]) {
auto minute = args[5].Get<ui8>();
if (!ValidateMinute(minute)) {
return TUnboxedValuePod();
}
SetMinute(result, minute);
- }
+ }
if (args[6]) {
auto second = args[6].Get<ui8>();
if (!ValidateSecond(second)) {
return TUnboxedValuePod();
}
SetSecond(result, second);
- }
+ }
if (args[7]) {
auto microsecond = args[7].Get<ui32>();
if (!ValidateMicrosecond(microsecond)) {
return TUnboxedValuePod();
}
SetMicrosecond(result, microsecond);
- }
+ }
if (args[8]) {
auto timezoneId = args[8].Get<ui16>();
if (!ValidateTimezoneId(timezoneId)) {
@@ -628,320 +628,320 @@ namespace {
auto& builder = valueBuilder->GetDateBuilder();
auto& storage = TTMStorage::Reference(result);
if (!storage.Validate(builder)) {
- return TUnboxedValuePod();
- }
+ return TUnboxedValuePod();
+ }
return result;
} catch (const std::exception& e) {
UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data());
- }
- }
-
- static const TStringRef& Name() {
- static auto name = TStringRef::Of("Update");
- return name;
- }
-
- static bool DeclareSignature(
- const TStringRef& name,
- TType*,
- IFunctionTypeInfoBuilder& builder,
- bool typesOnly)
- {
- if (Name() != name) {
- return false;
- }
-
- auto resourceType = builder.Resource(TMResourceName);
- auto optionalResourceType = builder.Optional()->Item(resourceType).Build();
-
- builder.OptionalArgs(8).Args()->Add(resourceType).Flags(ICallablePayload::TArgumentFlags::AutoMap)
- .Add(builder.Optional()->Item<ui16>().Build()).Name("Year")
- .Add(builder.Optional()->Item<ui8>().Build()).Name("Month")
- .Add(builder.Optional()->Item<ui8>().Build()).Name("Day")
- .Add(builder.Optional()->Item<ui8>().Build()).Name("Hour")
- .Add(builder.Optional()->Item<ui8>().Build()).Name("Minute")
- .Add(builder.Optional()->Item<ui8>().Build()).Name("Second")
- .Add(builder.Optional()->Item<ui32>().Build()).Name("Microsecond")
+ }
+ }
+
+ static const TStringRef& Name() {
+ static auto name = TStringRef::Of("Update");
+ return name;
+ }
+
+ static bool DeclareSignature(
+ const TStringRef& name,
+ TType*,
+ IFunctionTypeInfoBuilder& builder,
+ bool typesOnly)
+ {
+ if (Name() != name) {
+ return false;
+ }
+
+ auto resourceType = builder.Resource(TMResourceName);
+ auto optionalResourceType = builder.Optional()->Item(resourceType).Build();
+
+ builder.OptionalArgs(8).Args()->Add(resourceType).Flags(ICallablePayload::TArgumentFlags::AutoMap)
+ .Add(builder.Optional()->Item<ui16>().Build()).Name("Year")
+ .Add(builder.Optional()->Item<ui8>().Build()).Name("Month")
+ .Add(builder.Optional()->Item<ui8>().Build()).Name("Day")
+ .Add(builder.Optional()->Item<ui8>().Build()).Name("Hour")
+ .Add(builder.Optional()->Item<ui8>().Build()).Name("Minute")
+ .Add(builder.Optional()->Item<ui8>().Build()).Name("Second")
+ .Add(builder.Optional()->Item<ui32>().Build()).Name("Microsecond")
.Add(builder.Optional()->Item<ui16>().Build()).Name("TimezoneId");
-
- builder.Returns(optionalResourceType);
-
- if (!typesOnly) {
+
+ builder.Returns(optionalResourceType);
+
+ if (!typesOnly) {
builder.Implementation(new TUpdate(builder.GetSourcePosition()));
- }
-
- return true;
- }
- };
-
- // From*
-
+ }
+
+ return true;
+ }
+ };
+
+ // From*
+
SIMPLE_UDF(TFromSeconds, TOptional<TTimestamp>(TAutoMap<ui32>)) {
- Y_UNUSED(valueBuilder);
+ Y_UNUSED(valueBuilder);
auto res = args[0].Get<ui32>();
if (!ValidateDatetime(res)) {
return TUnboxedValuePod();
}
return TUnboxedValuePod((ui64)(res * 1000000ull));
- }
-
+ }
+
SIMPLE_UDF(TFromMilliseconds, TOptional<TTimestamp>(TAutoMap<ui64>)) {
- Y_UNUSED(valueBuilder);
+ Y_UNUSED(valueBuilder);
auto res = args[0].Get<ui64>();
if (res >= MAX_TIMESTAMP / 1000u) {
return TUnboxedValuePod();
}
return TUnboxedValuePod(res * 1000u);
- }
-
+ }
+
SIMPLE_UDF(TFromMicroseconds, TOptional<TTimestamp>(TAutoMap<ui64>)) {
- Y_UNUSED(valueBuilder);
+ Y_UNUSED(valueBuilder);
auto res = args[0].Get<ui64>();
if (!ValidateTimestamp(res)) {
return TUnboxedValuePod();
}
return TUnboxedValuePod(res);
- }
-
+ }
+
SIMPLE_UDF(TIntervalFromDays, TOptional<TInterval>(TAutoMap<i32>)) {
- Y_UNUSED(valueBuilder);
+ Y_UNUSED(valueBuilder);
const i64 res = i64(args[0].Get<i32>()) * 86400000000ll;
return ValidateInterval(res) ? TUnboxedValuePod(res) : TUnboxedValuePod();
- }
-
+ }
+
SIMPLE_UDF(TIntervalFromHours, TOptional<TInterval>(TAutoMap<i32>)) {
- Y_UNUSED(valueBuilder);
+ Y_UNUSED(valueBuilder);
const i64 res = i64(args[0].Get<i32>()) * 3600000000ll;
return ValidateInterval(res) ? TUnboxedValuePod(res) : TUnboxedValuePod();
- }
-
+ }
+
SIMPLE_UDF(TIntervalFromMinutes, TOptional<TInterval>(TAutoMap<i32>)) {
- Y_UNUSED(valueBuilder);
+ Y_UNUSED(valueBuilder);
const i64 res = i64(args[0].Get<i32>()) * 60000000ll;
return ValidateInterval(res) ? TUnboxedValuePod(res) : TUnboxedValuePod();
- }
-
+ }
+
SIMPLE_UDF(TIntervalFromSeconds, TOptional<TInterval>(TAutoMap<i32>)) {
- Y_UNUSED(valueBuilder);
+ Y_UNUSED(valueBuilder);
const i64 res = i64(args[0].Get<i32>()) * 1000000ll;
return ValidateInterval(res) ? TUnboxedValuePod(res) : TUnboxedValuePod();
- }
-
+ }
+
SIMPLE_UDF(TIntervalFromMilliseconds, TOptional<TInterval>(TAutoMap<i64>)) {
- Y_UNUSED(valueBuilder);
+ Y_UNUSED(valueBuilder);
const i64 res = i64(args[0].Get<i64>()) * 1000ll;
return ValidateInterval(res) ? TUnboxedValuePod(res) : TUnboxedValuePod();
- }
-
+ }
+
SIMPLE_UDF(TIntervalFromMicroseconds, TOptional<TInterval>(TAutoMap<i64>)) {
- Y_UNUSED(valueBuilder);
+ Y_UNUSED(valueBuilder);
const i64 res = args[0].Get<i64>();
return ValidateInterval(res) ? TUnboxedValuePod(res) : TUnboxedValuePod();
- }
-
- // To*
-
+ }
+
+ // To*
+
SIMPLE_UDF(TToDays, i32(TAutoMap<TInterval>)) {
- Y_UNUSED(valueBuilder);
+ Y_UNUSED(valueBuilder);
return TUnboxedValuePod(i32(args[0].Get<i64>() / 86400000000ll));
- }
-
- SIMPLE_UDF(TToHours, i32(TAutoMap<TInterval>)) {
- Y_UNUSED(valueBuilder);
+ }
+
+ SIMPLE_UDF(TToHours, i32(TAutoMap<TInterval>)) {
+ Y_UNUSED(valueBuilder);
return TUnboxedValuePod(i32(args[0].Get<i64>() / 3600000000ll));
- }
-
- SIMPLE_UDF(TToMinutes, i32(TAutoMap<TInterval>)) {
- Y_UNUSED(valueBuilder);
+ }
+
+ SIMPLE_UDF(TToMinutes, i32(TAutoMap<TInterval>)) {
+ Y_UNUSED(valueBuilder);
return TUnboxedValuePod(i32(args[0].Get<i64>() / 60000000ll));
- }
-
-#define DECLARE_TO_VALUE(units, TSigned, TUnsigned) \
- template <typename TUserDataType> \
- class TTo##units : public TBoxedValue { \
- private: \
- TUnboxedValue Run( \
- const IValueBuilder* valueBuilder, \
- const TUnboxedValuePod* args) const override; \
- public: \
- static void DeclareSignature( \
- TType* userType, \
- IFunctionTypeInfoBuilder& builder, \
- bool typesOnly) \
- { \
- builder.UserType(userType); \
- builder.Args()->Add<TUserDataType>() \
- .Flags(ICallablePayload::TArgumentFlags::AutoMap).Done(); \
- \
- if (TDataType<TUserDataType>::Id == TDataType<TInterval>::Id) { \
- builder.Returns<TSigned>(); \
- } else { \
- builder.Returns<TUnsigned>(); \
- } \
- \
- if (!typesOnly) { \
- builder.Implementation(new TTo##units<TUserDataType>); \
- } \
- } \
- };
-
- DECLARE_TO_VALUE(Seconds, i32, ui32);
- DECLARE_TO_VALUE(Milliseconds, i64, ui64);
- DECLARE_TO_VALUE(Microseconds, i64, ui64);
-
-#define TO_METHOD(units, type, expr) \
- template <> \
- TUnboxedValue TTo##units<type>::Run( \
- const IValueBuilder* valueBuilder, \
- const TUnboxedValuePod* args) const \
- { \
- Y_UNUSED(valueBuilder); \
- EMPTY_RESULT_ON_EMPTY_ARG(0); \
- return TUnboxedValuePod(expr); \
- }
-
- TO_METHOD(Seconds, TDate, (ui32)(args[0].Get<ui16>() * 86400u));
- TO_METHOD(Seconds, TDatetime, args[0].Get<ui32>());
- TO_METHOD(Seconds, TTimestamp, (ui32)(args[0].Get<ui64>() / 1000000ull));
- TO_METHOD(Seconds, TInterval, (i64)(args[0].Get<i64>() / 1000000ll));
- TO_METHOD(Seconds, TTzDate, (ui32)(args[0].Get<ui16>() * 86400u));
- TO_METHOD(Seconds, TTzDatetime, args[0].Get<ui32>());
- TO_METHOD(Seconds, TTzTimestamp, (ui32)(args[0].Get<ui64>() / 1000000u));
-
- TO_METHOD(Milliseconds, TDate, (ui64)(args[0].Get<ui16>() * 86400000ull));
- TO_METHOD(Milliseconds, TDatetime, (ui64)(args[0].Get<ui32>() * 1000ull));
- TO_METHOD(Milliseconds, TTimestamp, (ui64)(args[0].Get<ui64>() / 1000ull));
- TO_METHOD(Milliseconds, TInterval, (i64)(args[0].Get<i64>() / 1000ll));
- TO_METHOD(Milliseconds, TTzDate, (ui64)(args[0].Get<ui16>() * 86400000ull));
- TO_METHOD(Milliseconds, TTzDatetime, (ui64)(args[0].Get<ui32>() * 1000ull));
- TO_METHOD(Milliseconds, TTzTimestamp, (ui64)(args[0].Get<ui64>() / 1000ull));
-
- TO_METHOD(Microseconds, TDate, (ui64)(args[0].Get<ui16>() * 86400000000ull));
- TO_METHOD(Microseconds, TDatetime, (ui64)(args[0].Get<ui32>() * 1000000ull));
- TO_METHOD(Microseconds, TTimestamp, args[0].Get<ui64>());
- TO_METHOD(Microseconds, TInterval, args[0].Get<i64>());
- TO_METHOD(Microseconds, TTzDate, (ui64)(args[0].Get<ui16>() * 86400000000ull));
- TO_METHOD(Microseconds, TTzDatetime, (ui64)(args[0].Get<ui32>() * 1000000ull));
- TO_METHOD(Microseconds, TTzTimestamp, args[0].Get<ui64>());
-
- // StartOf*
-
- SIMPLE_UDF(TStartOfYear, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>)) {
- auto result = args[0];
- auto& storage = TTMStorage::Reference(result);
- storage.Month = 1;
- storage.Day = 1;
- storage.Hour = 0;
- storage.Minute = 0;
- storage.Second = 0;
- storage.Microsecond = 0;
-
- auto& builder = valueBuilder->GetDateBuilder();
- if (!storage.Validate(builder)) {
- return TUnboxedValuePod();
- }
- return result;
- }
-
- SIMPLE_UDF(TStartOfQuarter, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>)) {
- auto result = args[0];
- auto& storage = TTMStorage::Reference(result);
- storage.Month = (storage.Month - 1) / 3 * 3 + 1;
- storage.Day = 1;
- storage.Hour = 0;
- storage.Minute = 0;
- storage.Second = 0;
- storage.Microsecond = 0;
-
- auto& builder = valueBuilder->GetDateBuilder();
- if (!storage.Validate(builder)) {
- return TUnboxedValuePod();
- }
- return result;
- }
-
- SIMPLE_UDF(TStartOfMonth, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>)) {
- auto result = args[0];
- auto& storage = TTMStorage::Reference(result);
- storage.Day = 1;
- storage.Hour = 0;
- storage.Minute = 0;
- storage.Second = 0;
- storage.Microsecond = 0;
-
- auto& builder = valueBuilder->GetDateBuilder();
- if (!storage.Validate(builder)) {
- return TUnboxedValuePod();
- }
- return result;
- }
-
- SIMPLE_UDF(TStartOfWeek, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>)) {
- auto result = args[0];
- auto& storage = TTMStorage::Reference(result);
- auto& builder = valueBuilder->GetDateBuilder();
-
+ }
+
+#define DECLARE_TO_VALUE(units, TSigned, TUnsigned) \
+ template <typename TUserDataType> \
+ class TTo##units : public TBoxedValue { \
+ private: \
+ TUnboxedValue Run( \
+ const IValueBuilder* valueBuilder, \
+ const TUnboxedValuePod* args) const override; \
+ public: \
+ static void DeclareSignature( \
+ TType* userType, \
+ IFunctionTypeInfoBuilder& builder, \
+ bool typesOnly) \
+ { \
+ builder.UserType(userType); \
+ builder.Args()->Add<TUserDataType>() \
+ .Flags(ICallablePayload::TArgumentFlags::AutoMap).Done(); \
+ \
+ if (TDataType<TUserDataType>::Id == TDataType<TInterval>::Id) { \
+ builder.Returns<TSigned>(); \
+ } else { \
+ builder.Returns<TUnsigned>(); \
+ } \
+ \
+ if (!typesOnly) { \
+ builder.Implementation(new TTo##units<TUserDataType>); \
+ } \
+ } \
+ };
+
+ DECLARE_TO_VALUE(Seconds, i32, ui32);
+ DECLARE_TO_VALUE(Milliseconds, i64, ui64);
+ DECLARE_TO_VALUE(Microseconds, i64, ui64);
+
+#define TO_METHOD(units, type, expr) \
+ template <> \
+ TUnboxedValue TTo##units<type>::Run( \
+ const IValueBuilder* valueBuilder, \
+ const TUnboxedValuePod* args) const \
+ { \
+ Y_UNUSED(valueBuilder); \
+ EMPTY_RESULT_ON_EMPTY_ARG(0); \
+ return TUnboxedValuePod(expr); \
+ }
+
+ TO_METHOD(Seconds, TDate, (ui32)(args[0].Get<ui16>() * 86400u));
+ TO_METHOD(Seconds, TDatetime, args[0].Get<ui32>());
+ TO_METHOD(Seconds, TTimestamp, (ui32)(args[0].Get<ui64>() / 1000000ull));
+ TO_METHOD(Seconds, TInterval, (i64)(args[0].Get<i64>() / 1000000ll));
+ TO_METHOD(Seconds, TTzDate, (ui32)(args[0].Get<ui16>() * 86400u));
+ TO_METHOD(Seconds, TTzDatetime, args[0].Get<ui32>());
+ TO_METHOD(Seconds, TTzTimestamp, (ui32)(args[0].Get<ui64>() / 1000000u));
+
+ TO_METHOD(Milliseconds, TDate, (ui64)(args[0].Get<ui16>() * 86400000ull));
+ TO_METHOD(Milliseconds, TDatetime, (ui64)(args[0].Get<ui32>() * 1000ull));
+ TO_METHOD(Milliseconds, TTimestamp, (ui64)(args[0].Get<ui64>() / 1000ull));
+ TO_METHOD(Milliseconds, TInterval, (i64)(args[0].Get<i64>() / 1000ll));
+ TO_METHOD(Milliseconds, TTzDate, (ui64)(args[0].Get<ui16>() * 86400000ull));
+ TO_METHOD(Milliseconds, TTzDatetime, (ui64)(args[0].Get<ui32>() * 1000ull));
+ TO_METHOD(Milliseconds, TTzTimestamp, (ui64)(args[0].Get<ui64>() / 1000ull));
+
+ TO_METHOD(Microseconds, TDate, (ui64)(args[0].Get<ui16>() * 86400000000ull));
+ TO_METHOD(Microseconds, TDatetime, (ui64)(args[0].Get<ui32>() * 1000000ull));
+ TO_METHOD(Microseconds, TTimestamp, args[0].Get<ui64>());
+ TO_METHOD(Microseconds, TInterval, args[0].Get<i64>());
+ TO_METHOD(Microseconds, TTzDate, (ui64)(args[0].Get<ui16>() * 86400000000ull));
+ TO_METHOD(Microseconds, TTzDatetime, (ui64)(args[0].Get<ui32>() * 1000000ull));
+ TO_METHOD(Microseconds, TTzTimestamp, args[0].Get<ui64>());
+
+ // StartOf*
+
+ SIMPLE_UDF(TStartOfYear, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>)) {
+ auto result = args[0];
+ auto& storage = TTMStorage::Reference(result);
+ storage.Month = 1;
+ storage.Day = 1;
+ storage.Hour = 0;
+ storage.Minute = 0;
+ storage.Second = 0;
+ storage.Microsecond = 0;
+
+ auto& builder = valueBuilder->GetDateBuilder();
+ if (!storage.Validate(builder)) {
+ return TUnboxedValuePod();
+ }
+ return result;
+ }
+
+ SIMPLE_UDF(TStartOfQuarter, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>)) {
+ auto result = args[0];
+ auto& storage = TTMStorage::Reference(result);
+ storage.Month = (storage.Month - 1) / 3 * 3 + 1;
+ storage.Day = 1;
+ storage.Hour = 0;
+ storage.Minute = 0;
+ storage.Second = 0;
+ storage.Microsecond = 0;
+
+ auto& builder = valueBuilder->GetDateBuilder();
+ if (!storage.Validate(builder)) {
+ return TUnboxedValuePod();
+ }
+ return result;
+ }
+
+ SIMPLE_UDF(TStartOfMonth, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>)) {
+ auto result = args[0];
+ auto& storage = TTMStorage::Reference(result);
+ storage.Day = 1;
+ storage.Hour = 0;
+ storage.Minute = 0;
+ storage.Second = 0;
+ storage.Microsecond = 0;
+
+ auto& builder = valueBuilder->GetDateBuilder();
+ if (!storage.Validate(builder)) {
+ return TUnboxedValuePod();
+ }
+ return result;
+ }
+
+ SIMPLE_UDF(TStartOfWeek, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>)) {
+ auto result = args[0];
+ auto& storage = TTMStorage::Reference(result);
+ auto& builder = valueBuilder->GetDateBuilder();
+
const auto date = storage.ToDatetime(builder);
const ui32 shift = 86400u * (storage.DayOfWeek - 1u);
- if (shift > date) {
- return TUnboxedValuePod();
- }
+ if (shift > date) {
+ return TUnboxedValuePod();
+ }
storage.FromDatetime(builder, date - shift, storage.TimezoneId);
-
- storage.Hour = 0;
- storage.Minute = 0;
- storage.Second = 0;
- storage.Microsecond = 0;
- return result;
- }
-
- SIMPLE_UDF(TStartOfDay, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>)) {
- auto result = args[0];
- auto& storage = TTMStorage::Reference(result);
- storage.Hour = 0;
- storage.Minute = 0;
- storage.Second = 0;
- storage.Microsecond = 0;
-
- auto& builder = valueBuilder->GetDateBuilder();
- if (!storage.Validate(builder)) {
- return TUnboxedValuePod();
- }
- return result;
- }
-
+
+ storage.Hour = 0;
+ storage.Minute = 0;
+ storage.Second = 0;
+ storage.Microsecond = 0;
+ return result;
+ }
+
+ SIMPLE_UDF(TStartOfDay, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>)) {
+ auto result = args[0];
+ auto& storage = TTMStorage::Reference(result);
+ storage.Hour = 0;
+ storage.Minute = 0;
+ storage.Second = 0;
+ storage.Microsecond = 0;
+
+ auto& builder = valueBuilder->GetDateBuilder();
+ if (!storage.Validate(builder)) {
+ return TUnboxedValuePod();
+ }
+ return result;
+ }
+
SIMPLE_UDF(TStartOf, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>, TAutoMap<TInterval>)) {
- auto result = args[0];
- ui64 interval = std::abs(args[1].Get<i64>());
- if (interval == 0) {
- return result;
- }
- auto& storage = TTMStorage::Reference(result);
- if (interval >= 86400000000ull) {
- // treat as StartOfDay
- storage.Hour = 0;
- storage.Minute = 0;
- storage.Second = 0;
- storage.Microsecond = 0;
- } else {
- auto current = storage.ToTimeOfDay();
- auto rounded = current / interval * interval;
- storage.FromTimeOfDay(rounded);
- }
-
- auto& builder = valueBuilder->GetDateBuilder();
- if (!storage.Validate(builder)) {
- return TUnboxedValuePod();
- }
- return result;
- }
-
- SIMPLE_UDF(TTimeOfDay, TInterval(TAutoMap<TResource<TMResourceName>>)) {
- Y_UNUSED(valueBuilder);
- auto& storage = TTMStorage::Reference(args[0]);
- return TUnboxedValuePod((i64)storage.ToTimeOfDay());
- }
-
+ auto result = args[0];
+ ui64 interval = std::abs(args[1].Get<i64>());
+ if (interval == 0) {
+ return result;
+ }
+ auto& storage = TTMStorage::Reference(result);
+ if (interval >= 86400000000ull) {
+ // treat as StartOfDay
+ storage.Hour = 0;
+ storage.Minute = 0;
+ storage.Second = 0;
+ storage.Microsecond = 0;
+ } else {
+ auto current = storage.ToTimeOfDay();
+ auto rounded = current / interval * interval;
+ storage.FromTimeOfDay(rounded);
+ }
+
+ auto& builder = valueBuilder->GetDateBuilder();
+ if (!storage.Validate(builder)) {
+ return TUnboxedValuePod();
+ }
+ return result;
+ }
+
+ SIMPLE_UDF(TTimeOfDay, TInterval(TAutoMap<TResource<TMResourceName>>)) {
+ Y_UNUSED(valueBuilder);
+ auto& storage = TTMStorage::Reference(args[0]);
+ return TUnboxedValuePod((i64)storage.ToTimeOfDay());
+ }
+
// Add ...
SIMPLE_UDF(TShiftYears, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>, i32)) {
@@ -1016,79 +1016,79 @@ namespace {
}
};
- // Format
-
- class TFormat : public TBoxedValue {
- public:
- class TFactory : public TBoxedValue {
- public:
- explicit TFactory(TSourcePosition pos)
- : Pos_(pos)
- {}
-
- private:
+ // Format
+
+ class TFormat : public TBoxedValue {
+ public:
+ class TFactory : public TBoxedValue {
+ public:
+ explicit TFactory(TSourcePosition pos)
+ : Pos_(pos)
+ {}
+
+ private:
TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const final try {
return TUnboxedValuePod(new TFormat(args[0], Pos_));
} catch (const std::exception& e) {
UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data());
- }
+ }
const TSourcePosition Pos_;
- };
-
- static const TStringRef& Name() {
- static auto name = TStringRef::Of("Format");
- return name;
- }
-
- static bool DeclareSignature(
- const TStringRef& name,
- TType*,
- IFunctionTypeInfoBuilder& builder,
- bool typesOnly)
- {
- if (Name() != name) {
- return false;
- }
-
- auto resourceType = builder.Resource(TMResourceName);
-
+ };
+
+ static const TStringRef& Name() {
+ static auto name = TStringRef::Of("Format");
+ return name;
+ }
+
+ static bool DeclareSignature(
+ const TStringRef& name,
+ TType*,
+ IFunctionTypeInfoBuilder& builder,
+ bool typesOnly)
+ {
+ if (Name() != name) {
+ return false;
+ }
+
+ auto resourceType = builder.Resource(TMResourceName);
+
builder.Args()->Add(resourceType).Flags(ICallablePayload::TArgumentFlags::AutoMap);
- builder.RunConfig<char*>().Returns<char*>();
-
- if (!typesOnly) {
- builder.Implementation(new TFormat::TFactory(builder.GetSourcePosition()));
- }
-
- return true;
- }
-
- private:
+ builder.RunConfig<char*>().Returns<char*>();
+
+ if (!typesOnly) {
+ builder.Implementation(new TFormat::TFactory(builder.GetSourcePosition()));
+ }
+
+ return true;
+ }
+
+ private:
const TSourcePosition Pos_;
const TUnboxedValue Format_;
std::vector<std::function<size_t(char*, const TUnboxedValuePod&, const IDateBuilder&)>> Printers_;
-
- size_t ReservedSize_;
-
- struct TDataPrinter {
+
+ size_t ReservedSize_;
+
+ struct TDataPrinter {
const std::string_view Data;
-
- size_t operator()(char* out, const TUnboxedValuePod&, const IDateBuilder&) const {
+
+ size_t operator()(char* out, const TUnboxedValuePod&, const IDateBuilder&) const {
std::memcpy(out, Data.data(), Data.size());
return Data.size();
- }
- };
-
- private:
- TUnboxedValue Run(
- const IValueBuilder* valueBuilder,
- const TUnboxedValuePod* args) const override
- {
+ }
+ };
+
+ private:
+ TUnboxedValue Run(
+ const IValueBuilder* valueBuilder,
+ const TUnboxedValuePod* args) const override
+ {
try {
EMPTY_RESULT_ON_EMPTY_ARG(0);
const auto value = args[0];
-
+
auto& builder = valueBuilder->GetDateBuilder();
-
+
auto result = valueBuilder->NewStringNotFilled(ReservedSize_);
auto pos = result.AsStringRef().Data();
ui32 size = 0U;
@@ -1099,7 +1099,7 @@ namespace {
pos += plus;
}
}
-
+
if (size < ReservedSize_) {
result = valueBuilder->SubString(result.Release(), 0U, size);
}
@@ -1108,84 +1108,84 @@ namespace {
} catch (const std::exception& e) {
UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data());
}
- }
-
+ }
+
TFormat(const TUnboxedValuePod& runConfig, TSourcePosition pos)
: Pos_(pos)
, Format_(runConfig)
- {
+ {
const std::string_view formatView(Format_.AsStringRef());
auto dataStart = formatView.begin();
size_t dataSize = 0U;
ReservedSize_ = 0U;
-
+
for (auto ptr = formatView.begin(); formatView.end() != ptr; ++ptr) {
- if (*ptr != '%') {
+ if (*ptr != '%') {
++dataSize;
- continue;
- }
-
+ continue;
+ }
+
if (dataSize) {
Printers_.emplace_back(TDataPrinter{std::string_view(&*dataStart, dataSize)});
ReservedSize_ += dataSize;
dataSize = 0U;
- }
-
+ }
+
if (formatView.end() == ++ptr) {
ythrow yexception() << "format string ends with single %%";
- }
-
- switch (*ptr) {
- case '%': {
+ }
+
+ switch (*ptr) {
+ case '%': {
static constexpr size_t size = 1;
Printers_.emplace_back([](char* out, const TUnboxedValuePod&, const IDateBuilder&) {
- *out = '%';
- return size;
- });
- ReservedSize_ += size;
- break;
- }
- case 'Y': {
+ *out = '%';
+ return size;
+ });
+ ReservedSize_ += size;
+ break;
+ }
+ case 'Y': {
static constexpr size_t size = 4;
Printers_.emplace_back([](char* out, const TUnboxedValuePod& value, const IDateBuilder&) {
return PrintNDigits<size>::Do(GetYear(value), out);
- });
- ReservedSize_ += size;
- break;
- }
- case 'm': {
+ });
+ ReservedSize_ += size;
+ break;
+ }
+ case 'm': {
static constexpr size_t size = 2;
Printers_.emplace_back([](char* out, const TUnboxedValuePod& value, const IDateBuilder&) {
return PrintNDigits<size>::Do(GetMonth(value), out);
- });
- ReservedSize_ += size;
- break;
- }
- case 'd': {
+ });
+ ReservedSize_ += size;
+ break;
+ }
+ case 'd': {
static constexpr size_t size = 2;
Printers_.emplace_back([](char* out, const TUnboxedValuePod& value, const IDateBuilder&) {
return PrintNDigits<size>::Do(GetDay(value), out);
- });
- ReservedSize_ += size;
- break;
- }
- case 'H': {
+ });
+ ReservedSize_ += size;
+ break;
+ }
+ case 'H': {
static constexpr size_t size = 2;
Printers_.emplace_back([](char* out, const TUnboxedValuePod& value, const IDateBuilder&) {
return PrintNDigits<size>::Do(GetHour(value), out);
- });
- ReservedSize_ += size;
- break;
- }
- case 'M': {
+ });
+ ReservedSize_ += size;
+ break;
+ }
+ case 'M': {
static constexpr size_t size = 2;
Printers_.emplace_back([](char* out, const TUnboxedValuePod& value, const IDateBuilder&) {
return PrintNDigits<size>::Do(GetMinute(value), out);
- });
- ReservedSize_ += size;
- break;
- }
- case 'S':
+ });
+ ReservedSize_ += size;
+ break;
+ }
+ case 'S':
Printers_.emplace_back([](char* out, const TUnboxedValuePod& value, const IDateBuilder&) {
constexpr size_t size = 2;
if (const auto microsecond = GetMicrosecond(value)) {
@@ -1193,46 +1193,46 @@ namespace {
*out++ = '.';
constexpr size_t msize = 6;
return size + 1U + PrintNDigits<msize, false>::Do(microsecond, out);
- }
+ }
return PrintNDigits<size>::Do(GetSecond(value), out);
- });
- ReservedSize_ += 9;
- break;
-
- case 'z': {
+ });
+ ReservedSize_ += 9;
+ break;
+
+ case 'z': {
static constexpr size_t size = 5;
Printers_.emplace_back([](char* out, const TUnboxedValuePod& value, const IDateBuilder& builder) {
- auto timezoneId = GetTimezoneId(value);
- if (TTMStorage::IsUniversal(timezoneId)) {
+ auto timezoneId = GetTimezoneId(value);
+ if (TTMStorage::IsUniversal(timezoneId)) {
std::memcpy(out, "+0000", size);
- return size;
- }
- i32 shift;
- if (!builder.GetTimezoneShift(GetYear(value), GetMonth(value), GetDay(value),
- GetHour(value), GetMinute(value), GetSecond(value), timezoneId, shift))
- {
+ return size;
+ }
+ i32 shift;
+ if (!builder.GetTimezoneShift(GetYear(value), GetMonth(value), GetDay(value),
+ GetHour(value), GetMinute(value), GetSecond(value), timezoneId, shift))
+ {
std::memcpy(out, "+0000", size);
- return size;
- }
+ return size;
+ }
*out++ = shift > 0 ? '+' : '-';
shift = std::abs(shift);
out += PrintNDigits<2U>::Do(shift / 60U, out);
out += PrintNDigits<2U>::Do(shift % 60U, out);
- return size;
- });
- ReservedSize_ += size;
- break;
- }
- case 'Z':
+ return size;
+ });
+ ReservedSize_ += size;
+ break;
+ }
+ case 'Z':
Printers_.emplace_back([](char* out, const TUnboxedValuePod& value, const IDateBuilder&) {
const auto timezoneId = GetTimezoneId(value);
const auto tzName = NUdf::GetTimezones()[timezoneId];
std::memcpy(out, tzName.data(), std::min(tzName.size(), MAX_TIMEZONE_NAME_LEN));
return tzName.size();
- });
- ReservedSize_ += MAX_TIMEZONE_NAME_LEN;
- break;
+ });
+ ReservedSize_ += MAX_TIMEZONE_NAME_LEN;
+ break;
case 'b': {
static constexpr size_t size = 3;
Printers_.emplace_back([](char* out, const TUnboxedValuePod& value, const IDateBuilder&) {
@@ -1283,20 +1283,20 @@ namespace {
ReservedSize_ += 9U; // MAX_MONTH_FULL_NAME_LEN
break;
}
- default:
- ythrow yexception() << "invalid format character: " << *ptr;
- }
-
+ default:
+ ythrow yexception() << "invalid format character: " << *ptr;
+ }
+
dataStart = ptr + 1U;
- }
-
+ }
+
if (dataSize) {
Printers_.emplace_back(TDataPrinter{std::string_view(dataStart, dataSize)});
ReservedSize_ += dataSize;
- }
- }
- };
-
+ }
+ }
+ };
+
template<size_t Digits>
struct ParseExaclyNDigits;
@@ -1322,90 +1322,90 @@ namespace {
}
};
- // Parse
-
- class TParse : public TBoxedValue {
- public:
- class TFactory : public TBoxedValue {
- public:
- explicit TFactory(TSourcePosition pos)
- : Pos_(pos)
- {}
-
- private:
+ // Parse
+
+ class TParse : public TBoxedValue {
+ public:
+ class TFactory : public TBoxedValue {
+ public:
+ explicit TFactory(TSourcePosition pos)
+ : Pos_(pos)
+ {}
+
+ private:
TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const final try {
return TUnboxedValuePod(new TParse(args[0], Pos_));
} catch (const std::exception& e) {
UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data());
- }
-
+ }
+
const TSourcePosition Pos_;
- };
-
- static const TStringRef& Name() {
- static auto name = TStringRef::Of("Parse");
- return name;
- }
-
- static bool DeclareSignature(
- const TStringRef& name,
- TType*,
- IFunctionTypeInfoBuilder& builder,
- bool typesOnly)
- {
- if (Name() != name) {
- return false;
- }
-
- auto resourceType = builder.Resource(TMResourceName);
- auto optionalResourceType = builder.Optional()->Item(resourceType).Build();
-
- builder.Args()->Add<char*>().Flags(ICallablePayload::TArgumentFlags::AutoMap)
- .Add(builder.Optional()->Item<ui16>())
- .Done()
- .OptionalArgs(1);
- builder.RunConfig<char*>().Returns(optionalResourceType);
-
- if (!typesOnly) {
- builder.Implementation(new TParse::TFactory(builder.GetSourcePosition()));
- }
-
- return true;
- }
-
- private:
+ };
+
+ static const TStringRef& Name() {
+ static auto name = TStringRef::Of("Parse");
+ return name;
+ }
+
+ static bool DeclareSignature(
+ const TStringRef& name,
+ TType*,
+ IFunctionTypeInfoBuilder& builder,
+ bool typesOnly)
+ {
+ if (Name() != name) {
+ return false;
+ }
+
+ auto resourceType = builder.Resource(TMResourceName);
+ auto optionalResourceType = builder.Optional()->Item(resourceType).Build();
+
+ builder.Args()->Add<char*>().Flags(ICallablePayload::TArgumentFlags::AutoMap)
+ .Add(builder.Optional()->Item<ui16>())
+ .Done()
+ .OptionalArgs(1);
+ builder.RunConfig<char*>().Returns(optionalResourceType);
+
+ if (!typesOnly) {
+ builder.Implementation(new TParse::TFactory(builder.GetSourcePosition()));
+ }
+
+ return true;
+ }
+
+ private:
const TSourcePosition Pos_;
const TUnboxedValue Format_;
-
+
std::vector<std::function<bool(std::string_view::const_iterator& it, size_t, TUnboxedValuePod&, const IDateBuilder&)>> Scanners_;
-
- struct TDataScanner {
+
+ struct TDataScanner {
const std::string_view Data_;
-
+
bool operator()(std::string_view::const_iterator& it, size_t limit, TUnboxedValuePod&, const IDateBuilder&) const {
if (limit < Data_.size() || !std::equal(Data_.begin(), Data_.end(), it)) {
return false;
- }
+ }
std::advance(it, Data_.size());
return true;
- }
- };
-
- TUnboxedValue Run(
- const IValueBuilder* valueBuilder,
- const TUnboxedValuePod* args) const override
- {
+ }
+ };
+
+ TUnboxedValue Run(
+ const IValueBuilder* valueBuilder,
+ const TUnboxedValuePod* args) const override
+ {
try {
EMPTY_RESULT_ON_EMPTY_ARG(0);
-
+
const std::string_view buffer = args[0].AsStringRef();
-
+
TUnboxedValuePod result(0);
auto& storage = TTMStorage::Reference(result);
storage.MakeDefault();
-
+
auto& builder = valueBuilder->GetDateBuilder();
-
+
auto it = buffer.begin();
for (const auto& scanner : Scanners_) {
if (!scanner(it, std::distance(it, buffer.end()), result, builder)) {
@@ -1414,159 +1414,159 @@ namespace {
}
if (buffer.end() != it || !storage.Validate(builder)) {
- return TUnboxedValuePod();
- }
+ return TUnboxedValuePod();
+ }
return result;
} catch (const std::exception& e) {
UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data());
- }
- }
-
+ }
+ }
+
TParse(const TUnboxedValuePod& runConfig, TSourcePosition pos)
: Pos_(pos)
, Format_(runConfig)
- {
+ {
const std::string_view formatView(Format_.AsStringRef());
auto dataStart = formatView.begin();
size_t dataSize = 0U;
-
+
for (auto ptr = formatView.begin(); formatView.end() != ptr; ++ptr) {
- if (*ptr != '%') {
+ if (*ptr != '%') {
++dataSize;
- continue;
- }
-
+ continue;
+ }
+
if (dataSize) {
Scanners_.emplace_back(TDataScanner{std::string_view(&*dataStart, dataSize)});
dataSize = 0;
- }
-
+ }
+
if (++ptr == formatView.end()) {
ythrow yexception() << "format string ends with single %%";
- }
-
- switch (*ptr) {
- case '%':
+ }
+
+ switch (*ptr) {
+ case '%':
Scanners_.emplace_back([](std::string_view::const_iterator& it, size_t limit, TUnboxedValuePod&, const IDateBuilder&) {
return limit > 0U && *it++ == '%';
- });
- break;
-
- case 'Y': {
+ });
+ break;
+
+ case 'Y': {
static constexpr size_t size = 4;
Scanners_.emplace_back([](std::string_view::const_iterator& it, size_t limit, TUnboxedValuePod& result, const IDateBuilder&) {
ui32 year = 0U;
if (limit < size || !ParseExaclyNDigits<size>::Do(it, year) || !ValidateYear(year)) {
return false;
- }
- SetYear(result, year);
+ }
+ SetYear(result, year);
return true;
- });
- break;
- }
- case 'm': {
+ });
+ break;
+ }
+ case 'm': {
static constexpr size_t size = 2;
Scanners_.emplace_back([](std::string_view::const_iterator& it, size_t limit, TUnboxedValuePod& result, const IDateBuilder&) {
ui32 month = 0U;
if (limit < size || !ParseExaclyNDigits<size>::Do(it, month) || !ValidateMonth(month)) {
return false;
- }
- SetMonth(result, month);
+ }
+ SetMonth(result, month);
return true;
- });
- break;
- }
- case 'd': {
+ });
+ break;
+ }
+ case 'd': {
static constexpr size_t size = 2;
Scanners_.emplace_back([](std::string_view::const_iterator& it, size_t limit, TUnboxedValuePod& result, const IDateBuilder&) {
ui32 day = 0U;
if (limit < size || !ParseExaclyNDigits<size>::Do(it, day) || !ValidateDay(day)) {
return false;
- }
- SetDay(result, day);
+ }
+ SetDay(result, day);
return true;
- });
- break;
- }
- case 'H': {
+ });
+ break;
+ }
+ case 'H': {
static constexpr size_t size = 2;
Scanners_.emplace_back([](std::string_view::const_iterator& it, size_t limit, TUnboxedValuePod& result, const IDateBuilder&) {
ui32 hour = 0U;
if (limit < size || !ParseExaclyNDigits<size>::Do(it, hour) || !ValidateHour(hour)) {
return false;
- }
- SetHour(result, hour);
+ }
+ SetHour(result, hour);
return true;
- });
- break;
- }
- case 'M': {
+ });
+ break;
+ }
+ case 'M': {
static constexpr size_t size = 2;
Scanners_.emplace_back([](std::string_view::const_iterator& it, size_t limit, TUnboxedValuePod& result, const IDateBuilder&) {
ui32 minute = 0U;
if (limit < size || !ParseExaclyNDigits<size>::Do(it, minute) || !ValidateMinute(minute)) {
return false;
- }
- SetMinute(result, minute);
+ }
+ SetMinute(result, minute);
return true;
- });
- break;
- }
- case 'S': {
+ });
+ break;
+ }
+ case 'S': {
static constexpr size_t size = 2;
Scanners_.emplace_back([](std::string_view::const_iterator& it, size_t limit, TUnboxedValuePod& result, const IDateBuilder&) {
ui32 second = 0U;
if (limit < size || !ParseExaclyNDigits<size>::Do(it, second) || !ValidateSecond(second)) {
return false;
- }
- SetSecond(result, second);
+ }
+ SetSecond(result, second);
limit -= size;
if (!limit || *it != '.') {
return true;
- }
-
+ }
+
++it;
--limit;
ui32 usec = 0U;
-
+
size_t digits = 6U;
for (; limit; --limit) {
const auto c = *it;
if (!digits || !std::isdigit(c)) {
- break;
- }
+ break;
+ }
usec *= 10U;
usec += c - '0';
++it;
--digits;
- }
+ }
while (digits--) {
usec *= 10U;
- }
- SetMicrosecond(result, usec);
+ }
+ SetMicrosecond(result, usec);
return true;
- });
- break;
- }
- case 'Z':
+ });
+ break;
+ }
+ case 'Z':
Scanners_.emplace_back([](std::string_view::const_iterator& it, size_t limit, TUnboxedValuePod& result, const IDateBuilder& builder) {
const auto start = it;
while (limit > 0 && (std::isalnum(*it) || *it == '/' || *it == '_' || *it == '-' || *it == '+')) {
++it;
- --limit;
- }
+ --limit;
+ }
const auto size = std::distance(start, it);
-
- ui32 timezoneId;
+
+ ui32 timezoneId;
if (!builder.FindTimezoneId(TStringRef(&*start, size), timezoneId)) {
return false;
- }
- SetTimezoneId(result, timezoneId);
+ }
+ SetTimezoneId(result, timezoneId);
return true;
- });
- break;
+ });
+ break;
case 'b': {
static constexpr size_t size = 3;
Scanners_.emplace_back([](std::string_view::const_iterator& it, size_t limit, TUnboxedValuePod& result, const IDateBuilder&) {
@@ -1607,135 +1607,135 @@ namespace {
});
break;
}
- default:
- ythrow yexception() << "invalid format character: " << *ptr;
- }
-
+ default:
+ ythrow yexception() << "invalid format character: " << *ptr;
+ }
+
dataStart = ptr + 1U;
- }
-
+ }
+
if (dataSize) {
Scanners_.emplace_back(TDataScanner{std::string_view(&*dataStart, dataSize)});
- }
- }
- };
-
-#define PARSE_SPECIFIC_FORMAT(format) \
- SIMPLE_UDF(TParse##format, TOptional<TResource<TMResourceName>>(TAutoMap<char*>)) { \
- auto str = args[0].AsStringRef(); \
- TInstant instant; \
- if (!TInstant::TryParse##format(TStringBuf(str.Data(), str.Size()), instant)) { \
- return TUnboxedValuePod(); \
- } \
- auto& builder = valueBuilder->GetDateBuilder(); \
- TUnboxedValuePod result(0); \
- auto& storage = TTMStorage::Reference(result); \
- storage.FromTimestamp(builder, instant.MicroSeconds()); \
- return result; \
- }
-
- PARSE_SPECIFIC_FORMAT(Rfc822);
- PARSE_SPECIFIC_FORMAT(Iso8601);
- PARSE_SPECIFIC_FORMAT(Http);
- PARSE_SPECIFIC_FORMAT(X509);
-
- SIMPLE_MODULE(TDateTime2Module,
- TUserDataTypeFuncFactory<true, SplitName, TSplit,
- TDate,
- TDatetime,
- TTimestamp,
- TTzDate,
- TTzDatetime,
- TTzTimestamp>,
-
- TMakeDate,
- TMakeDatetime,
- TMakeTimestamp,
- TMakeTzDate,
- TMakeTzDatetime,
- TMakeTzTimestamp,
-
- TGetYear,
- TGetDayOfYear,
- TGetMonth,
- TGetMonthName,
- TGetWeekOfYear,
+ }
+ }
+ };
+
+#define PARSE_SPECIFIC_FORMAT(format) \
+ SIMPLE_UDF(TParse##format, TOptional<TResource<TMResourceName>>(TAutoMap<char*>)) { \
+ auto str = args[0].AsStringRef(); \
+ TInstant instant; \
+ if (!TInstant::TryParse##format(TStringBuf(str.Data(), str.Size()), instant)) { \
+ return TUnboxedValuePod(); \
+ } \
+ auto& builder = valueBuilder->GetDateBuilder(); \
+ TUnboxedValuePod result(0); \
+ auto& storage = TTMStorage::Reference(result); \
+ storage.FromTimestamp(builder, instant.MicroSeconds()); \
+ return result; \
+ }
+
+ PARSE_SPECIFIC_FORMAT(Rfc822);
+ PARSE_SPECIFIC_FORMAT(Iso8601);
+ PARSE_SPECIFIC_FORMAT(Http);
+ PARSE_SPECIFIC_FORMAT(X509);
+
+ SIMPLE_MODULE(TDateTime2Module,
+ TUserDataTypeFuncFactory<true, SplitName, TSplit,
+ TDate,
+ TDatetime,
+ TTimestamp,
+ TTzDate,
+ TTzDatetime,
+ TTzTimestamp>,
+
+ TMakeDate,
+ TMakeDatetime,
+ TMakeTimestamp,
+ TMakeTzDate,
+ TMakeTzDatetime,
+ TMakeTzTimestamp,
+
+ TGetYear,
+ TGetDayOfYear,
+ TGetMonth,
+ TGetMonthName,
+ TGetWeekOfYear,
TGetWeekOfYearIso8601,
- TGetDayOfMonth,
- TGetDayOfWeek,
- TGetDayOfWeekName,
- TGetHour,
- TGetMinute,
- TGetSecond,
- TGetMillisecondOfSecond,
- TGetMicrosecondOfSecond,
- TGetTimezoneId,
- TGetTimezoneName,
-
- TUpdate,
-
- TFromSeconds,
- TFromMilliseconds,
- TFromMicroseconds,
-
- TIntervalFromDays,
- TIntervalFromHours,
- TIntervalFromMinutes,
- TIntervalFromSeconds,
- TIntervalFromMilliseconds,
- TIntervalFromMicroseconds,
-
- TToDays,
- TToHours,
- TToMinutes,
-
- TStartOfYear,
- TStartOfQuarter,
- TStartOfMonth,
- TStartOfWeek,
- TStartOfDay,
- TStartOf,
- TTimeOfDay,
-
+ TGetDayOfMonth,
+ TGetDayOfWeek,
+ TGetDayOfWeekName,
+ TGetHour,
+ TGetMinute,
+ TGetSecond,
+ TGetMillisecondOfSecond,
+ TGetMicrosecondOfSecond,
+ TGetTimezoneId,
+ TGetTimezoneName,
+
+ TUpdate,
+
+ TFromSeconds,
+ TFromMilliseconds,
+ TFromMicroseconds,
+
+ TIntervalFromDays,
+ TIntervalFromHours,
+ TIntervalFromMinutes,
+ TIntervalFromSeconds,
+ TIntervalFromMilliseconds,
+ TIntervalFromMicroseconds,
+
+ TToDays,
+ TToHours,
+ TToMinutes,
+
+ TStartOfYear,
+ TStartOfQuarter,
+ TStartOfMonth,
+ TStartOfWeek,
+ TStartOfDay,
+ TStartOf,
+ TTimeOfDay,
+
TShiftYears,
TShiftQuarters,
TShiftMonths,
- TUserDataTypeFuncFactory<true, ToSecondsName, TToSeconds,
- TDate,
- TDatetime,
- TTimestamp,
- TInterval,
- TTzDate,
- TTzDatetime,
- TTzTimestamp>,
-
- TUserDataTypeFuncFactory<true, ToMillisecondsName, TToMilliseconds,
- TDate,
- TDatetime,
- TTimestamp,
- TInterval,
- TTzDate,
- TTzDatetime,
- TTzTimestamp>,
-
- TUserDataTypeFuncFactory<true, ToMicrosecondsName, TToMicroseconds,
- TDate,
- TDatetime,
- TTimestamp,
- TInterval,
- TTzDate,
- TTzDatetime,
- TTzTimestamp>,
-
- TFormat,
- TParse,
-
- TParseRfc822,
- TParseIso8601,
- TParseHttp,
- TParseX509
- )
-}
-
-REGISTER_MODULES(TDateTime2Module)
+ TUserDataTypeFuncFactory<true, ToSecondsName, TToSeconds,
+ TDate,
+ TDatetime,
+ TTimestamp,
+ TInterval,
+ TTzDate,
+ TTzDatetime,
+ TTzTimestamp>,
+
+ TUserDataTypeFuncFactory<true, ToMillisecondsName, TToMilliseconds,
+ TDate,
+ TDatetime,
+ TTimestamp,
+ TInterval,
+ TTzDate,
+ TTzDatetime,
+ TTzTimestamp>,
+
+ TUserDataTypeFuncFactory<true, ToMicrosecondsName, TToMicroseconds,
+ TDate,
+ TDatetime,
+ TTimestamp,
+ TInterval,
+ TTzDate,
+ TTzDatetime,
+ TTzTimestamp>,
+
+ TFormat,
+ TParse,
+
+ TParseRfc822,
+ TParseIso8601,
+ TParseHttp,
+ TParseX509
+ )
+}
+
+REGISTER_MODULES(TDateTime2Module)
diff --git a/ydb/library/yql/udfs/common/datetime2/ya.make b/ydb/library/yql/udfs/common/datetime2/ya.make
index 3d0db781edc..faa95811b4d 100644
--- a/ydb/library/yql/udfs/common/datetime2/ya.make
+++ b/ydb/library/yql/udfs/common/datetime2/ya.make
@@ -1,24 +1,24 @@
-YQL_UDF(datetime2_udf)
-
-YQL_ABI_VERSION(
- 2
+YQL_UDF(datetime2_udf)
+
+YQL_ABI_VERSION(
+ 2
24
- 0
-)
-
+ 0
+)
+
OWNER(
g:yql
g:yql_ydb_core
)
-
-SRCS(
- datetime_udf.cpp
-)
-
-PEERDIR(
- util/draft
+
+SRCS(
+ datetime_udf.cpp
+)
+
+PEERDIR(
+ util/draft
ydb/library/yql/minikql
ydb/library/yql/public/udf/tz
-)
-
-END()
+)
+
+END()
diff --git a/ydb/library/yql/udfs/common/set/set_udf.cpp b/ydb/library/yql/udfs/common/set/set_udf.cpp
index f282c22842c..ac009ed71db 100644
--- a/ydb/library/yql/udfs/common/set/set_udf.cpp
+++ b/ydb/library/yql/udfs/common/set/set_udf.cpp
@@ -1,52 +1,52 @@
-
+
#include <ydb/library/yql/public/udf/udf_type_ops.h>
#include <ydb/library/yql/public/udf/udf_helpers.h>
-
-#include <unordered_set>
-
-using namespace NKikimr;
-using namespace NUdf;
-
-namespace {
-
-template <typename THash, typename TEquals>
-class TSetBase {
-private:
+
+#include <unordered_set>
+
+using namespace NKikimr;
+using namespace NUdf;
+
+namespace {
+
+template <typename THash, typename TEquals>
+class TSetBase {
+private:
std::unordered_set<TUnboxedValue, THash, TEquals, TUnboxedValue::TAllocator> Set;
- ui32 MaxSize = 0;
+ ui32 MaxSize = 0;
bool WasChanged = false;
-
-protected:
- TSetBase(THash hash, TEquals equals)
- : Set(1, hash, equals)
- {}
-
- void Init(const TUnboxedValuePod& value, ui32 maxSize) {
- MaxSize = maxSize ? maxSize : std::numeric_limits<ui32>::max();
- AddValue(value);
- }
-
- void Merge(const TSetBase& left, const TSetBase& right) {
- MaxSize = std::max(left.MaxSize, right.MaxSize);
- for (const auto& item : left.Set) {
- AddValue(item);
- }
- for (const auto& item : right.Set) {
- AddValue(item);
- }
- }
-
- void Deserialize(const TUnboxedValuePod& serialized) {
- MaxSize = serialized.GetElement(0).Get<ui32>();
- auto list = serialized.GetElement(1);
-
- const auto listIter = list.GetListIterator();
- for (TUnboxedValue current; listIter.Next(current);) {
- AddValue(current);
- }
- }
-
-public:
+
+protected:
+ TSetBase(THash hash, TEquals equals)
+ : Set(1, hash, equals)
+ {}
+
+ void Init(const TUnboxedValuePod& value, ui32 maxSize) {
+ MaxSize = maxSize ? maxSize : std::numeric_limits<ui32>::max();
+ AddValue(value);
+ }
+
+ void Merge(const TSetBase& left, const TSetBase& right) {
+ MaxSize = std::max(left.MaxSize, right.MaxSize);
+ for (const auto& item : left.Set) {
+ AddValue(item);
+ }
+ for (const auto& item : right.Set) {
+ AddValue(item);
+ }
+ }
+
+ void Deserialize(const TUnboxedValuePod& serialized) {
+ MaxSize = serialized.GetElement(0).Get<ui32>();
+ auto list = serialized.GetElement(1);
+
+ const auto listIter = list.GetListIterator();
+ for (TUnboxedValue current; listIter.Next(current);) {
+ AddValue(current);
+ }
+ }
+
+public:
void ResetChanged() {
WasChanged = false;
}
@@ -55,182 +55,182 @@ public:
return WasChanged;
}
- TUnboxedValue Serialize(const IValueBuilder* builder) {
- TUnboxedValue* values = nullptr;
- auto list = builder->NewArray(Set.size(), values);
-
- for (const auto& item : Set) {
- *values++ = item;
- }
-
- TUnboxedValue* items = nullptr;
- auto result = builder->NewArray(2U, items);
- items[0] = TUnboxedValuePod(MaxSize);
- items[1] = list;
-
- return result;
- }
-
- TUnboxedValue GetResult(const IValueBuilder* builder) {
- TUnboxedValue* values = nullptr;
- auto result = builder->NewArray(Set.size(), values);
-
- for (const auto& item : Set) {
- *values++ = item;
- }
- return result;
- }
-
- void AddValue(const TUnboxedValuePod& value) {
- if (Set.size() < MaxSize) {
+ TUnboxedValue Serialize(const IValueBuilder* builder) {
+ TUnboxedValue* values = nullptr;
+ auto list = builder->NewArray(Set.size(), values);
+
+ for (const auto& item : Set) {
+ *values++ = item;
+ }
+
+ TUnboxedValue* items = nullptr;
+ auto result = builder->NewArray(2U, items);
+ items[0] = TUnboxedValuePod(MaxSize);
+ items[1] = list;
+
+ return result;
+ }
+
+ TUnboxedValue GetResult(const IValueBuilder* builder) {
+ TUnboxedValue* values = nullptr;
+ auto result = builder->NewArray(Set.size(), values);
+
+ for (const auto& item : Set) {
+ *values++ = item;
+ }
+ return result;
+ }
+
+ void AddValue(const TUnboxedValuePod& value) {
+ if (Set.size() < MaxSize) {
WasChanged = Set.insert(TUnboxedValuePod(value)).second;
- }
- }
-};
-
-template <EDataSlot Slot>
-class TSetData
- : public TSetBase<TUnboxedValueHash<Slot>, TUnboxedValueEquals<Slot>>
-{
-public:
- using TBase = TSetBase<TUnboxedValueHash<Slot>, TUnboxedValueEquals<Slot>>;
-
- TSetData(const TUnboxedValuePod& value, ui32 maxSize)
- : TBase(TUnboxedValueHash<Slot>(), TUnboxedValueEquals<Slot>())
- {
- TBase::Init(value, maxSize);
- }
-
- TSetData(const TSetData& left, const TSetData& right)
- : TBase(TUnboxedValueHash<Slot>(), TUnboxedValueEquals<Slot>())
- {
- TBase::Merge(left, right);
- }
-
- explicit TSetData(const TUnboxedValuePod& serialized)
- : TBase(TUnboxedValueHash<Slot>(), TUnboxedValueEquals<Slot>())
- {
- TBase::Deserialize(serialized);
- }
-};
-
-struct TGenericHash {
- IHash::TPtr Hash;
-
- std::size_t operator()(const TUnboxedValuePod& value) const {
- return Hash->Hash(value);
- }
-};
-
-struct TGenericEquals {
- IEquate::TPtr Equate;
-
- bool operator()(const TUnboxedValuePod& left, const TUnboxedValuePod& right) const {
- return Equate->Equals(left, right);
- }
-};
-
-class TSetGeneric
- : public TSetBase<TGenericHash, TGenericEquals>
-{
-public:
- using TBase = TSetBase<TGenericHash, TGenericEquals>;
-
- TSetGeneric(const TUnboxedValuePod& value, ui32 maxSize,
- IHash::TPtr hash, IEquate::TPtr equate)
- : TBase(TGenericHash{hash}, TGenericEquals{equate})
- {
- TBase::Init(value, maxSize);
- }
-
- TSetGeneric(const TSetGeneric& left, const TSetGeneric& right,
- IHash::TPtr hash, IEquate::TPtr equate)
- : TBase(TGenericHash{hash}, TGenericEquals{equate})
- {
- TBase::Merge(left, right);
- }
-
- TSetGeneric(const TUnboxedValuePod& serialized,
- IHash::TPtr hash, IEquate::TPtr equate)
- : TBase(TGenericHash{hash}, TGenericEquals{equate})
- {
- TBase::Deserialize(serialized);
- }
-};
-
-extern const char SetResourceNameGeneric[] = "Set.SetResource.Generic";
-class TSetResource:
- public TBoxedResource<TSetGeneric, SetResourceNameGeneric>
-{
-public:
- template <typename... Args>
- inline TSetResource(Args&&... args)
- : TBoxedResource(std::forward<Args>(args)...)
- {}
-};
-
-template <EDataSlot Slot>
-class TSetResourceData;
-
-template <EDataSlot Slot>
-TSetResourceData<Slot>* GetSetResourceData(const TUnboxedValuePod& arg) {
- TSetResourceData<Slot>::Validate(arg);
- return static_cast<TSetResourceData<Slot>*>(arg.AsBoxed().Get());
-}
-
-TSetResource* GetSetResource(const TUnboxedValuePod& arg) {
- TSetResource::Validate(arg);
- return static_cast<TSetResource*>(arg.AsBoxed().Get());
-}
-
-
-template <EDataSlot Slot>
-class TSetCreateData: public TBoxedValue {
-private:
+ }
+ }
+};
+
+template <EDataSlot Slot>
+class TSetData
+ : public TSetBase<TUnboxedValueHash<Slot>, TUnboxedValueEquals<Slot>>
+{
+public:
+ using TBase = TSetBase<TUnboxedValueHash<Slot>, TUnboxedValueEquals<Slot>>;
+
+ TSetData(const TUnboxedValuePod& value, ui32 maxSize)
+ : TBase(TUnboxedValueHash<Slot>(), TUnboxedValueEquals<Slot>())
+ {
+ TBase::Init(value, maxSize);
+ }
+
+ TSetData(const TSetData& left, const TSetData& right)
+ : TBase(TUnboxedValueHash<Slot>(), TUnboxedValueEquals<Slot>())
+ {
+ TBase::Merge(left, right);
+ }
+
+ explicit TSetData(const TUnboxedValuePod& serialized)
+ : TBase(TUnboxedValueHash<Slot>(), TUnboxedValueEquals<Slot>())
+ {
+ TBase::Deserialize(serialized);
+ }
+};
+
+struct TGenericHash {
+ IHash::TPtr Hash;
+
+ std::size_t operator()(const TUnboxedValuePod& value) const {
+ return Hash->Hash(value);
+ }
+};
+
+struct TGenericEquals {
+ IEquate::TPtr Equate;
+
+ bool operator()(const TUnboxedValuePod& left, const TUnboxedValuePod& right) const {
+ return Equate->Equals(left, right);
+ }
+};
+
+class TSetGeneric
+ : public TSetBase<TGenericHash, TGenericEquals>
+{
+public:
+ using TBase = TSetBase<TGenericHash, TGenericEquals>;
+
+ TSetGeneric(const TUnboxedValuePod& value, ui32 maxSize,
+ IHash::TPtr hash, IEquate::TPtr equate)
+ : TBase(TGenericHash{hash}, TGenericEquals{equate})
+ {
+ TBase::Init(value, maxSize);
+ }
+
+ TSetGeneric(const TSetGeneric& left, const TSetGeneric& right,
+ IHash::TPtr hash, IEquate::TPtr equate)
+ : TBase(TGenericHash{hash}, TGenericEquals{equate})
+ {
+ TBase::Merge(left, right);
+ }
+
+ TSetGeneric(const TUnboxedValuePod& serialized,
+ IHash::TPtr hash, IEquate::TPtr equate)
+ : TBase(TGenericHash{hash}, TGenericEquals{equate})
+ {
+ TBase::Deserialize(serialized);
+ }
+};
+
+extern const char SetResourceNameGeneric[] = "Set.SetResource.Generic";
+class TSetResource:
+ public TBoxedResource<TSetGeneric, SetResourceNameGeneric>
+{
+public:
+ template <typename... Args>
+ inline TSetResource(Args&&... args)
+ : TBoxedResource(std::forward<Args>(args)...)
+ {}
+};
+
+template <EDataSlot Slot>
+class TSetResourceData;
+
+template <EDataSlot Slot>
+TSetResourceData<Slot>* GetSetResourceData(const TUnboxedValuePod& arg) {
+ TSetResourceData<Slot>::Validate(arg);
+ return static_cast<TSetResourceData<Slot>*>(arg.AsBoxed().Get());
+}
+
+TSetResource* GetSetResource(const TUnboxedValuePod& arg) {
+ TSetResource::Validate(arg);
+ return static_cast<TSetResource*>(arg.AsBoxed().Get());
+}
+
+
+template <EDataSlot Slot>
+class TSetCreateData: public TBoxedValue {
+private:
TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override {
- return TUnboxedValuePod(new TSetResourceData<Slot>(args[0], args[1].Get<ui32>()));
- }
-};
-
-class TSetCreate: public TBoxedValue {
-private:
+ return TUnboxedValuePod(new TSetResourceData<Slot>(args[0], args[1].Get<ui32>()));
+ }
+};
+
+class TSetCreate: public TBoxedValue {
+private:
TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override {
- return TUnboxedValuePod(new TSetResource(args[0], args[1].Get<ui32>(), Hash_, Equate_));
- }
-
-public:
- TSetCreate(IHash::TPtr hash, IEquate::TPtr equate)
- : Hash_(hash)
- , Equate_(equate)
- {}
-
-private:
- IHash::TPtr Hash_;
- IEquate::TPtr Equate_;
-};
-
-template <EDataSlot Slot>
-class TSetAddValueData: public TBoxedValue {
-private:
+ return TUnboxedValuePod(new TSetResource(args[0], args[1].Get<ui32>(), Hash_, Equate_));
+ }
+
+public:
+ TSetCreate(IHash::TPtr hash, IEquate::TPtr equate)
+ : Hash_(hash)
+ , Equate_(equate)
+ {}
+
+private:
+ IHash::TPtr Hash_;
+ IEquate::TPtr Equate_;
+};
+
+template <EDataSlot Slot>
+class TSetAddValueData: public TBoxedValue {
+private:
TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override {
- auto resource = GetSetResourceData<Slot>(args[0]);
+ auto resource = GetSetResourceData<Slot>(args[0]);
resource->Get()->ResetChanged();
- resource->Get()->AddValue(args[1]);
- return TUnboxedValuePod(resource);
- }
-};
-
-class TSetAddValue: public TBoxedValue {
-private:
+ resource->Get()->AddValue(args[1]);
+ return TUnboxedValuePod(resource);
+ }
+};
+
+class TSetAddValue: public TBoxedValue {
+private:
TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override {
- auto resource = GetSetResource(args[0]);
+ auto resource = GetSetResource(args[0]);
resource->Get()->ResetChanged();
- resource->Get()->AddValue(args[1]);
- return TUnboxedValuePod(resource);
- }
-};
-
-template <EDataSlot Slot>
+ resource->Get()->AddValue(args[1]);
+ return TUnboxedValuePod(resource);
+ }
+};
+
+template <EDataSlot Slot>
class TSetWasChangedData: public TBoxedValue {
private:
TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override {
@@ -248,239 +248,239 @@ private:
};
template <EDataSlot Slot>
-class TSetSerializeData: public TBoxedValue {
-private:
+class TSetSerializeData: public TBoxedValue {
+private:
TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const override {
- return GetSetResourceData<Slot>(args[0])->Get()->Serialize(valueBuilder);
- }
-};
-
-class TSetSerialize: public TBoxedValue {
-private:
+ return GetSetResourceData<Slot>(args[0])->Get()->Serialize(valueBuilder);
+ }
+};
+
+class TSetSerialize: public TBoxedValue {
+private:
TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const override {
- return GetSetResource(args[0])->Get()->Serialize(valueBuilder);
- }
-};
-
-template <EDataSlot Slot>
-class TSetDeserializeData: public TBoxedValue {
-private:
+ return GetSetResource(args[0])->Get()->Serialize(valueBuilder);
+ }
+};
+
+template <EDataSlot Slot>
+class TSetDeserializeData: public TBoxedValue {
+private:
TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override {
- return TUnboxedValuePod(new TSetResourceData<Slot>(args[0]));
- }
-};
-
-class TSetDeserialize: public TBoxedValue {
-private:
+ return TUnboxedValuePod(new TSetResourceData<Slot>(args[0]));
+ }
+};
+
+class TSetDeserialize: public TBoxedValue {
+private:
TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override {
- return TUnboxedValuePod(new TSetResource(args[0], Hash_, Equate_));
- }
-
-public:
- TSetDeserialize(IHash::TPtr hash, IEquate::TPtr equate)
- : Hash_(hash)
- , Equate_(equate)
- {}
-
-private:
- IHash::TPtr Hash_;
- IEquate::TPtr Equate_;
-};
-
-template <EDataSlot Slot>
-class TSetMergeData: public TBoxedValue {
-private:
+ return TUnboxedValuePod(new TSetResource(args[0], Hash_, Equate_));
+ }
+
+public:
+ TSetDeserialize(IHash::TPtr hash, IEquate::TPtr equate)
+ : Hash_(hash)
+ , Equate_(equate)
+ {}
+
+private:
+ IHash::TPtr Hash_;
+ IEquate::TPtr Equate_;
+};
+
+template <EDataSlot Slot>
+class TSetMergeData: public TBoxedValue {
+private:
TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override {
- auto left = GetSetResourceData<Slot>(args[0]);
- auto right = GetSetResourceData<Slot>(args[1]);
- return TUnboxedValuePod(new TSetResourceData<Slot>(*left->Get(), *right->Get()));
- }
-};
-
-class TSetMerge: public TBoxedValue {
-private:
+ auto left = GetSetResourceData<Slot>(args[0]);
+ auto right = GetSetResourceData<Slot>(args[1]);
+ return TUnboxedValuePod(new TSetResourceData<Slot>(*left->Get(), *right->Get()));
+ }
+};
+
+class TSetMerge: public TBoxedValue {
+private:
TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override {
- auto left = GetSetResource(args[0]);
- auto right = GetSetResource(args[1]);
- return TUnboxedValuePod(new TSetResource(*left->Get(), *right->Get(), Hash_, Equate_));
- }
-
-public:
- TSetMerge(IHash::TPtr hash, IEquate::TPtr equate)
- : Hash_(hash)
- , Equate_(equate)
- {}
-
-private:
- IHash::TPtr Hash_;
- IEquate::TPtr Equate_;
-};
-
-template <EDataSlot Slot>
-class TSetGetResultData: public TBoxedValue {
-private:
+ auto left = GetSetResource(args[0]);
+ auto right = GetSetResource(args[1]);
+ return TUnboxedValuePod(new TSetResource(*left->Get(), *right->Get(), Hash_, Equate_));
+ }
+
+public:
+ TSetMerge(IHash::TPtr hash, IEquate::TPtr equate)
+ : Hash_(hash)
+ , Equate_(equate)
+ {}
+
+private:
+ IHash::TPtr Hash_;
+ IEquate::TPtr Equate_;
+};
+
+template <EDataSlot Slot>
+class TSetGetResultData: public TBoxedValue {
+private:
TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const override {
- return GetSetResourceData<Slot>(args[0])->Get()->GetResult(valueBuilder);
- }
-};
-
-class TSetGetResult: public TBoxedValue {
-private:
+ return GetSetResourceData<Slot>(args[0])->Get()->GetResult(valueBuilder);
+ }
+};
+
+class TSetGetResult: public TBoxedValue {
+private:
TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const override {
- return GetSetResource(args[0])->Get()->GetResult(valueBuilder);
- }
-};
-
-
-#define MAKE_RESOURCE(slot, ...) \
-extern const char SetResourceName##slot[] = "Set.SetResource."#slot; \
-template <> \
-class TSetResourceData<EDataSlot::slot>: \
- public TBoxedResource<TSetData<EDataSlot::slot>, SetResourceName##slot> \
-{ \
-public: \
- template <typename... Args> \
- inline TSetResourceData(Args&&... args) \
- : TBoxedResource(std::forward<Args>(args)...) \
- {} \
-};
-
-UDF_TYPE_ID_MAP(MAKE_RESOURCE)
-
-#define MAKE_IMPL(operation, slot) \
-case EDataSlot::slot: \
- builder.Implementation(new operation<EDataSlot::slot>); \
- break;
-
-#define MAKE_CREATE(slot, ...) MAKE_IMPL(TSetCreateData, slot)
-#define MAKE_ADD_VALUE(slot, ...) MAKE_IMPL(TSetAddValueData, slot)
+ return GetSetResource(args[0])->Get()->GetResult(valueBuilder);
+ }
+};
+
+
+#define MAKE_RESOURCE(slot, ...) \
+extern const char SetResourceName##slot[] = "Set.SetResource."#slot; \
+template <> \
+class TSetResourceData<EDataSlot::slot>: \
+ public TBoxedResource<TSetData<EDataSlot::slot>, SetResourceName##slot> \
+{ \
+public: \
+ template <typename... Args> \
+ inline TSetResourceData(Args&&... args) \
+ : TBoxedResource(std::forward<Args>(args)...) \
+ {} \
+};
+
+UDF_TYPE_ID_MAP(MAKE_RESOURCE)
+
+#define MAKE_IMPL(operation, slot) \
+case EDataSlot::slot: \
+ builder.Implementation(new operation<EDataSlot::slot>); \
+ break;
+
+#define MAKE_CREATE(slot, ...) MAKE_IMPL(TSetCreateData, slot)
+#define MAKE_ADD_VALUE(slot, ...) MAKE_IMPL(TSetAddValueData, slot)
#define MAKE_WAS_CHANGED(slot, ...) MAKE_IMPL(TSetWasChangedData, slot)
-#define MAKE_SERIALIZE(slot, ...) MAKE_IMPL(TSetSerializeData, slot)
-#define MAKE_DESERIALIZE(slot, ...) MAKE_IMPL(TSetDeserializeData, slot)
-#define MAKE_MERGE(slot, ...) MAKE_IMPL(TSetMergeData, slot)
-#define MAKE_GET_RESULT(slot, ...) MAKE_IMPL(TSetGetResultData, slot)
-
-#define MAKE_TYPE(slot, ...) \
-case EDataSlot::slot: \
- setType = builder.Resource(SetResourceName##slot); \
- break;
-
-
-static const auto CreateName = TStringRef::Of("Create");
-static const auto AddValueName = TStringRef::Of("AddValue");
+#define MAKE_SERIALIZE(slot, ...) MAKE_IMPL(TSetSerializeData, slot)
+#define MAKE_DESERIALIZE(slot, ...) MAKE_IMPL(TSetDeserializeData, slot)
+#define MAKE_MERGE(slot, ...) MAKE_IMPL(TSetMergeData, slot)
+#define MAKE_GET_RESULT(slot, ...) MAKE_IMPL(TSetGetResultData, slot)
+
+#define MAKE_TYPE(slot, ...) \
+case EDataSlot::slot: \
+ setType = builder.Resource(SetResourceName##slot); \
+ break;
+
+
+static const auto CreateName = TStringRef::Of("Create");
+static const auto AddValueName = TStringRef::Of("AddValue");
static const auto WasChangedName = TStringRef::Of("WasChanged"); // must be used right after AddValue
-static const auto SerializeName = TStringRef::Of("Serialize");
-static const auto DeserializeName = TStringRef::Of("Deserialize");
-static const auto MergeName = TStringRef::Of("Merge");
-static const auto GetResultName = TStringRef::Of("GetResult");
-
-class TSetModule: public IUdfModule {
-public:
- TStringRef Name() const {
- return TStringRef::Of("Set");
- }
-
- void CleanupOnTerminate() const final {
- }
-
- void GetAllFunctions(IFunctionsSink& sink) const final {
- sink.Add(CreateName)->SetTypeAwareness();
- sink.Add(AddValueName)->SetTypeAwareness();
+static const auto SerializeName = TStringRef::Of("Serialize");
+static const auto DeserializeName = TStringRef::Of("Deserialize");
+static const auto MergeName = TStringRef::Of("Merge");
+static const auto GetResultName = TStringRef::Of("GetResult");
+
+class TSetModule: public IUdfModule {
+public:
+ TStringRef Name() const {
+ return TStringRef::Of("Set");
+ }
+
+ void CleanupOnTerminate() const final {
+ }
+
+ void GetAllFunctions(IFunctionsSink& sink) const final {
+ sink.Add(CreateName)->SetTypeAwareness();
+ sink.Add(AddValueName)->SetTypeAwareness();
sink.Add(WasChangedName)->SetTypeAwareness();
- sink.Add(SerializeName)->SetTypeAwareness();
- sink.Add(DeserializeName)->SetTypeAwareness();
- sink.Add(MergeName)->SetTypeAwareness();
- sink.Add(GetResultName)->SetTypeAwareness();
- }
-
- void BuildFunctionTypeInfo(
- const TStringRef& name,
- TType* userType,
- const TStringRef& typeConfig,
- ui32 flags,
- IFunctionTypeInfoBuilder& builder) const final
- {
- Y_UNUSED(typeConfig);
-
- try {
- const bool typesOnly = (flags & TFlags::TypesOnly);
- builder.UserType(userType);
-
- auto typeHelper = builder.TypeInfoHelper();
-
- auto userTypeInspector = TTupleTypeInspector(*typeHelper, userType);
- if (!userTypeInspector || userTypeInspector.GetElementsCount() != 3) {
- builder.SetError("User type is not a 3-tuple");
- return;
- }
-
- bool isGeneric = false;
- IHash::TPtr hash;
- IEquate::TPtr equate;
- TMaybe<EDataSlot> slot;
-
- auto valueType = userTypeInspector.GetElementType(2);
- auto valueTypeInspector = TDataTypeInspector(*typeHelper, valueType);
- if (!valueTypeInspector) {
- isGeneric = true;
- hash = builder.MakeHash(valueType);
- equate = builder.MakeEquate(valueType);
- if (!hash || !equate) {
- return;
- }
- } else {
- slot = FindDataSlot(valueTypeInspector.GetTypeId());
- if (!slot) {
- builder.SetError("Unknown data type");
- return;
- }
- const auto& features = NUdf::GetDataTypeInfo(*slot).Features;
- if (!(features & NUdf::CanHash) || !(features & NUdf::CanEquate)) {
- builder.SetError("Data type is not hashable or equatable");
- return;
- }
- }
-
- auto serializedListType = builder.List()->Item(valueType).Build();
- auto serializedType = builder.Tuple()->Add<ui32>().Add(serializedListType).Build();
-
- TType* setType = nullptr;
- if (isGeneric) {
- setType = builder.Resource(SetResourceNameGeneric);
- } else {
- switch (*slot) {
- UDF_TYPE_ID_MAP(MAKE_TYPE)
- }
- }
-
- if (name == CreateName) {
- builder.Args()->Add(valueType).Add<ui32>().Done().Returns(setType);
-
- if (!typesOnly) {
- if (isGeneric) {
- builder.Implementation(new TSetCreate(hash, equate));
- } else {
- switch (*slot) {
- UDF_TYPE_ID_MAP(MAKE_CREATE)
- }
- }
- }
- }
-
- if (name == AddValueName) {
- builder.Args()->Add(setType).Add(valueType).Done().Returns(setType);
-
- if (!typesOnly) {
- if (isGeneric) {
- builder.Implementation(new TSetAddValue);
- } else {
- switch (*slot) {
- UDF_TYPE_ID_MAP(MAKE_ADD_VALUE)
- }
- }
- }
- }
-
+ sink.Add(SerializeName)->SetTypeAwareness();
+ sink.Add(DeserializeName)->SetTypeAwareness();
+ sink.Add(MergeName)->SetTypeAwareness();
+ sink.Add(GetResultName)->SetTypeAwareness();
+ }
+
+ void BuildFunctionTypeInfo(
+ const TStringRef& name,
+ TType* userType,
+ const TStringRef& typeConfig,
+ ui32 flags,
+ IFunctionTypeInfoBuilder& builder) const final
+ {
+ Y_UNUSED(typeConfig);
+
+ try {
+ const bool typesOnly = (flags & TFlags::TypesOnly);
+ builder.UserType(userType);
+
+ auto typeHelper = builder.TypeInfoHelper();
+
+ auto userTypeInspector = TTupleTypeInspector(*typeHelper, userType);
+ if (!userTypeInspector || userTypeInspector.GetElementsCount() != 3) {
+ builder.SetError("User type is not a 3-tuple");
+ return;
+ }
+
+ bool isGeneric = false;
+ IHash::TPtr hash;
+ IEquate::TPtr equate;
+ TMaybe<EDataSlot> slot;
+
+ auto valueType = userTypeInspector.GetElementType(2);
+ auto valueTypeInspector = TDataTypeInspector(*typeHelper, valueType);
+ if (!valueTypeInspector) {
+ isGeneric = true;
+ hash = builder.MakeHash(valueType);
+ equate = builder.MakeEquate(valueType);
+ if (!hash || !equate) {
+ return;
+ }
+ } else {
+ slot = FindDataSlot(valueTypeInspector.GetTypeId());
+ if (!slot) {
+ builder.SetError("Unknown data type");
+ return;
+ }
+ const auto& features = NUdf::GetDataTypeInfo(*slot).Features;
+ if (!(features & NUdf::CanHash) || !(features & NUdf::CanEquate)) {
+ builder.SetError("Data type is not hashable or equatable");
+ return;
+ }
+ }
+
+ auto serializedListType = builder.List()->Item(valueType).Build();
+ auto serializedType = builder.Tuple()->Add<ui32>().Add(serializedListType).Build();
+
+ TType* setType = nullptr;
+ if (isGeneric) {
+ setType = builder.Resource(SetResourceNameGeneric);
+ } else {
+ switch (*slot) {
+ UDF_TYPE_ID_MAP(MAKE_TYPE)
+ }
+ }
+
+ if (name == CreateName) {
+ builder.Args()->Add(valueType).Add<ui32>().Done().Returns(setType);
+
+ if (!typesOnly) {
+ if (isGeneric) {
+ builder.Implementation(new TSetCreate(hash, equate));
+ } else {
+ switch (*slot) {
+ UDF_TYPE_ID_MAP(MAKE_CREATE)
+ }
+ }
+ }
+ }
+
+ if (name == AddValueName) {
+ builder.Args()->Add(setType).Add(valueType).Done().Returns(setType);
+
+ if (!typesOnly) {
+ if (isGeneric) {
+ builder.Implementation(new TSetAddValue);
+ } else {
+ switch (*slot) {
+ UDF_TYPE_ID_MAP(MAKE_ADD_VALUE)
+ }
+ }
+ }
+ }
+
if (name == WasChangedName) {
builder.Args()->Add(setType).Done().Returns<bool>();
@@ -495,70 +495,70 @@ public:
}
}
- if (name == MergeName) {
- builder.Args()->Add(setType).Add(setType).Done().Returns(setType);
-
- if (!typesOnly) {
- if (isGeneric) {
- builder.Implementation(new TSetMerge(hash, equate));
- } else {
- switch (*slot) {
- UDF_TYPE_ID_MAP(MAKE_MERGE)
- }
- }
- }
- }
-
- if (name == SerializeName) {
- builder.Args()->Add(setType).Done().Returns(serializedType);
-
- if (!typesOnly) {
- if (isGeneric) {
- builder.Implementation(new TSetSerialize);
- } else {
- switch (*slot) {
- UDF_TYPE_ID_MAP(MAKE_SERIALIZE)
- }
- }
- }
- }
-
- if (name == DeserializeName) {
- builder.Args()->Add(serializedType).Done().Returns(setType);
-
- if (!typesOnly) {
- if (isGeneric) {
- builder.Implementation(new TSetDeserialize(hash, equate));
- } else {
- switch (*slot) {
- UDF_TYPE_ID_MAP(MAKE_DESERIALIZE)
- }
- }
- }
- }
-
- if (name == GetResultName) {
- auto resultType = builder.List()->Item(valueType).Build();
-
- builder.Args()->Add(setType).Done().Returns(resultType);
-
- if (!typesOnly) {
- if (isGeneric) {
- builder.Implementation(new TSetGetResult);
- } else {
- switch (*slot) {
- UDF_TYPE_ID_MAP(MAKE_GET_RESULT)
- }
- }
- }
- }
-
- } catch (const std::exception& e) {
- builder.SetError(CurrentExceptionMessage());
- }
- }
-};
-
-} // namespace
-
-REGISTER_MODULES(TSetModule)
+ if (name == MergeName) {
+ builder.Args()->Add(setType).Add(setType).Done().Returns(setType);
+
+ if (!typesOnly) {
+ if (isGeneric) {
+ builder.Implementation(new TSetMerge(hash, equate));
+ } else {
+ switch (*slot) {
+ UDF_TYPE_ID_MAP(MAKE_MERGE)
+ }
+ }
+ }
+ }
+
+ if (name == SerializeName) {
+ builder.Args()->Add(setType).Done().Returns(serializedType);
+
+ if (!typesOnly) {
+ if (isGeneric) {
+ builder.Implementation(new TSetSerialize);
+ } else {
+ switch (*slot) {
+ UDF_TYPE_ID_MAP(MAKE_SERIALIZE)
+ }
+ }
+ }
+ }
+
+ if (name == DeserializeName) {
+ builder.Args()->Add(serializedType).Done().Returns(setType);
+
+ if (!typesOnly) {
+ if (isGeneric) {
+ builder.Implementation(new TSetDeserialize(hash, equate));
+ } else {
+ switch (*slot) {
+ UDF_TYPE_ID_MAP(MAKE_DESERIALIZE)
+ }
+ }
+ }
+ }
+
+ if (name == GetResultName) {
+ auto resultType = builder.List()->Item(valueType).Build();
+
+ builder.Args()->Add(setType).Done().Returns(resultType);
+
+ if (!typesOnly) {
+ if (isGeneric) {
+ builder.Implementation(new TSetGetResult);
+ } else {
+ switch (*slot) {
+ UDF_TYPE_ID_MAP(MAKE_GET_RESULT)
+ }
+ }
+ }
+ }
+
+ } catch (const std::exception& e) {
+ builder.SetError(CurrentExceptionMessage());
+ }
+ }
+};
+
+} // namespace
+
+REGISTER_MODULES(TSetModule)
diff --git a/ydb/library/yql/udfs/common/set/ya.make b/ydb/library/yql/udfs/common/set/ya.make
index bfc20204e95..1c047bb108c 100644
--- a/ydb/library/yql/udfs/common/set/ya.make
+++ b/ydb/library/yql/udfs/common/set/ya.make
@@ -1,15 +1,15 @@
-YQL_UDF(set_udf)
-
-YQL_ABI_VERSION(
- 2
+YQL_UDF(set_udf)
+
+YQL_ABI_VERSION(
+ 2
19
- 0
-)
-
+ 0
+)
+
OWNER(g:yql g:yql_ydb_core)
-
-SRCS(
- set_udf.cpp
-)
-
-END()
+
+SRCS(
+ set_udf.cpp
+)
+
+END()
diff --git a/ydb/library/yql/udfs/common/top/top_udf.cpp b/ydb/library/yql/udfs/common/top/top_udf.cpp
index 748ffe489b3..c6f78b444ae 100644
--- a/ydb/library/yql/udfs/common/top/top_udf.cpp
+++ b/ydb/library/yql/udfs/common/top/top_udf.cpp
@@ -1,68 +1,68 @@
#include <ydb/library/yql/public/udf/udf_helpers.h>
#include <ydb/library/yql/public/udf/udf_type_ops.h>
-
+
#include <library/cpp/containers/top_keeper/top_keeper.h>
-
+
#include <util/generic/set.h>
#include <algorithm>
#include <iterator>
-using namespace NKikimr;
-using namespace NUdf;
-
-namespace {
-
-using TUnboxedValuePair = std::pair<TUnboxedValue, TUnboxedValue>;
-
-template <EDataSlot Slot, bool IsTop>
-struct TDataCompare {
+using namespace NKikimr;
+using namespace NUdf;
+
+namespace {
+
+using TUnboxedValuePair = std::pair<TUnboxedValue, TUnboxedValue>;
+
+template <EDataSlot Slot, bool IsTop>
+struct TDataCompare {
bool operator()(const TUnboxedValue& left, const TUnboxedValue& right) const {
- if (IsTop) {
- return CompareValues<Slot>(left, right) > 0;
- } else {
- return CompareValues<Slot>(left, right) < 0;
- }
- }
-};
-
-template <EDataSlot Slot, bool IsTop>
-struct TDataPairCompare {
+ if (IsTop) {
+ return CompareValues<Slot>(left, right) > 0;
+ } else {
+ return CompareValues<Slot>(left, right) < 0;
+ }
+ }
+};
+
+template <EDataSlot Slot, bool IsTop>
+struct TDataPairCompare {
bool operator()(const TUnboxedValuePair& left, const TUnboxedValuePair& right) const {
- if (IsTop) {
+ if (IsTop) {
return CompareValues<Slot>(left.first, right.first) > 0;
- } else {
+ } else {
return CompareValues<Slot>(left.first, right.first) < 0;
- }
- }
-};
-
-template <bool IsTop>
-struct TGenericCompare {
- ICompare::TPtr Compare;
-
+ }
+ }
+};
+
+template <bool IsTop>
+struct TGenericCompare {
+ ICompare::TPtr Compare;
+
bool operator()(const TUnboxedValue& left, const TUnboxedValue& right) const {
- if (IsTop) {
- return Compare->Less(right, left);
- } else {
- return Compare->Less(left, right);
- }
- }
-};
-
-template <bool IsTop>
-struct TGenericPairCompare {
- ICompare::TPtr Compare;
-
+ if (IsTop) {
+ return Compare->Less(right, left);
+ } else {
+ return Compare->Less(left, right);
+ }
+ }
+};
+
+template <bool IsTop>
+struct TGenericPairCompare {
+ ICompare::TPtr Compare;
+
bool operator()(const TUnboxedValuePair& left, const TUnboxedValuePair& right) const {
- if (IsTop) {
+ if (IsTop) {
return Compare->Less(right.first, left.first);
- } else {
+ } else {
return Compare->Less(left.first, right.first);
- }
- }
-};
-
+ }
+ }
+};
+
template <typename TValue, typename TCompare, typename TAllocator>
class TTopKeeperContainer {
TTopKeeper<TValue, TCompare, true, TAllocator> Keeper;
@@ -144,806 +144,806 @@ public:
}
};
-template <typename TCompare>
-class TTopKeeperWrapperBase {
-protected:
+template <typename TCompare>
+class TTopKeeperWrapperBase {
+protected:
TTopKeeperContainer<TUnboxedValue, TCompare, TUnboxedValue::TAllocator> Keeper;
-
-protected:
- explicit TTopKeeperWrapperBase(TCompare compare)
+
+protected:
+ explicit TTopKeeperWrapperBase(TCompare compare)
: Keeper(compare)
- {}
-
- void Init(const TUnboxedValuePod& value, ui32 maxSize) {
- Keeper.SetMaxSize(maxSize);
- AddValue(value);
- }
-
- void Merge(TTopKeeperWrapperBase& left, TTopKeeperWrapperBase& right) {
- Keeper.SetMaxSize(left.Keeper.GetMaxSize());
- for (const auto& item : left.Keeper.GetInternal()) {
- AddValue(item);
- }
- for (const auto& item : right.Keeper.GetInternal()) {
- AddValue(item);
- }
- }
-
- void Deserialize(const TUnboxedValuePod& serialized) {
- auto maxSize = serialized.GetElement(0).Get<ui32>();
- auto list = serialized.GetElement(1);
-
- Keeper.SetMaxSize(maxSize);
- const auto listIter = list.GetListIterator();
- for (TUnboxedValue current; listIter.Next(current);) {
- AddValue(current);
- }
- }
-
-public:
- void AddValue(const TUnboxedValuePod& value) {
- Keeper.Insert(TUnboxedValuePod(value));
- }
-
- TUnboxedValue Serialize(const IValueBuilder* builder) {
- TUnboxedValue* values = nullptr;
- auto list = builder->NewArray(Keeper.GetSize(), values);
-
- for (const auto& item : Keeper.GetInternal()) {
- *values++ = item;
- }
-
- TUnboxedValue* items = nullptr;
- auto result = builder->NewArray(2U, items);
- items[0] = TUnboxedValuePod((ui32)Keeper.GetMaxSize());
- items[1] = list;
-
- return result;
- }
-
- TUnboxedValue GetResult(const IValueBuilder* builder) {
- TUnboxedValue* values = nullptr;
- auto list = builder->NewArray(Keeper.GetSize(), values);
-
- for (const auto& item : Keeper.GetInternal()) {
- *values++ = item;
- }
- return list;
- }
-};
-
-template <typename TCompare>
-class TTopKeeperPairWrapperBase {
-protected:
+ {}
+
+ void Init(const TUnboxedValuePod& value, ui32 maxSize) {
+ Keeper.SetMaxSize(maxSize);
+ AddValue(value);
+ }
+
+ void Merge(TTopKeeperWrapperBase& left, TTopKeeperWrapperBase& right) {
+ Keeper.SetMaxSize(left.Keeper.GetMaxSize());
+ for (const auto& item : left.Keeper.GetInternal()) {
+ AddValue(item);
+ }
+ for (const auto& item : right.Keeper.GetInternal()) {
+ AddValue(item);
+ }
+ }
+
+ void Deserialize(const TUnboxedValuePod& serialized) {
+ auto maxSize = serialized.GetElement(0).Get<ui32>();
+ auto list = serialized.GetElement(1);
+
+ Keeper.SetMaxSize(maxSize);
+ const auto listIter = list.GetListIterator();
+ for (TUnboxedValue current; listIter.Next(current);) {
+ AddValue(current);
+ }
+ }
+
+public:
+ void AddValue(const TUnboxedValuePod& value) {
+ Keeper.Insert(TUnboxedValuePod(value));
+ }
+
+ TUnboxedValue Serialize(const IValueBuilder* builder) {
+ TUnboxedValue* values = nullptr;
+ auto list = builder->NewArray(Keeper.GetSize(), values);
+
+ for (const auto& item : Keeper.GetInternal()) {
+ *values++ = item;
+ }
+
+ TUnboxedValue* items = nullptr;
+ auto result = builder->NewArray(2U, items);
+ items[0] = TUnboxedValuePod((ui32)Keeper.GetMaxSize());
+ items[1] = list;
+
+ return result;
+ }
+
+ TUnboxedValue GetResult(const IValueBuilder* builder) {
+ TUnboxedValue* values = nullptr;
+ auto list = builder->NewArray(Keeper.GetSize(), values);
+
+ for (const auto& item : Keeper.GetInternal()) {
+ *values++ = item;
+ }
+ return list;
+ }
+};
+
+template <typename TCompare>
+class TTopKeeperPairWrapperBase {
+protected:
TTopKeeperContainer<TUnboxedValuePair, TCompare, TStdAllocatorForUdf<TUnboxedValuePair>> Keeper;
-
-protected:
- explicit TTopKeeperPairWrapperBase(TCompare compare)
+
+protected:
+ explicit TTopKeeperPairWrapperBase(TCompare compare)
: Keeper(compare)
- {}
-
- void Init(const TUnboxedValuePod& key, const TUnboxedValuePod& payload, ui32 maxSize) {
- Keeper.SetMaxSize(maxSize);
- AddValue(key, payload);
- }
-
- void Merge(TTopKeeperPairWrapperBase& left, TTopKeeperPairWrapperBase& right) {
- Keeper.SetMaxSize(left.Keeper.GetMaxSize());
- for (const auto& item : left.Keeper.GetInternal()) {
- AddValue(item.first, item.second);
- }
- for (const auto& item : right.Keeper.GetInternal()) {
- AddValue(item.first, item.second);
- }
- }
-
- void Deserialize(const TUnboxedValuePod& serialized) {
- auto maxSize = serialized.GetElement(0).Get<ui32>();
- auto list = serialized.GetElement(1);
-
- Keeper.SetMaxSize(maxSize);
- const auto listIter = list.GetListIterator();
- for (TUnboxedValue current; listIter.Next(current);) {
- AddValue(current.GetElement(0), current.GetElement(1));
- }
- }
-
-public:
- void AddValue(const TUnboxedValuePod& key, const TUnboxedValuePod& payload) {
- Keeper.Insert(std::make_pair(TUnboxedValuePod(key), TUnboxedValuePod(payload)));
- }
-
- TUnboxedValue Serialize(const IValueBuilder* builder) {
- TUnboxedValue* values = nullptr;
- auto list = builder->NewArray(Keeper.GetSize(), values);
-
- for (const auto& item : Keeper.GetInternal()) {
- TUnboxedValue* items = nullptr;
- auto pair = builder->NewArray(2U, items);
- items[0] = item.first;
- items[1] = item.second;
- *values++ = pair;
- }
-
- TUnboxedValue* items = nullptr;
- auto result = builder->NewArray(2U, items);
- items[0] = TUnboxedValuePod((ui32)Keeper.GetMaxSize());
- items[1] = list;
-
- return result;
- }
-
- TUnboxedValue GetResult(const IValueBuilder* builder) {
- TUnboxedValue* values = nullptr;
- auto list = builder->NewArray(Keeper.GetSize(), values);
-
- for (const auto& item : Keeper.GetInternal()) {
- *values++ = item.second;
- }
- return list;
- }
-};
-
-
-template <EDataSlot Slot, bool HasKey, bool IsTop>
-class TTopKeeperDataWrapper;
-
-template <EDataSlot Slot, bool IsTop>
-class TTopKeeperDataWrapper<Slot, false, IsTop>
- : public TTopKeeperWrapperBase<TDataCompare<Slot, IsTop>>
-{
-public:
- using TBase = TTopKeeperWrapperBase<TDataCompare<Slot, IsTop>>;
-
- TTopKeeperDataWrapper(const TUnboxedValuePod& value, ui32 maxSize)
- : TBase(TDataCompare<Slot, IsTop>())
- {
- TBase::Init(value, maxSize);
- }
-
- TTopKeeperDataWrapper(TTopKeeperDataWrapper& left, TTopKeeperDataWrapper& right)
- : TBase(TDataCompare<Slot, IsTop>())
- {
- TBase::Merge(left, right);
- }
-
- explicit TTopKeeperDataWrapper(const TUnboxedValuePod& serialized)
- : TBase(TDataCompare<Slot, IsTop>())
- {
- TBase::Deserialize(serialized);
- }
-};
-
-template <EDataSlot Slot, bool IsTop>
-class TTopKeeperDataWrapper<Slot, true, IsTop>
- : public TTopKeeperPairWrapperBase<TDataPairCompare<Slot, IsTop>>
-{
-public:
- using TBase = TTopKeeperPairWrapperBase<TDataPairCompare<Slot, IsTop>>;
-
- TTopKeeperDataWrapper(const TUnboxedValuePod& key, const TUnboxedValuePod& payload, ui32 maxSize)
- : TBase(TDataPairCompare<Slot, IsTop>())
- {
- TBase::Init(key, payload, maxSize);
- }
-
- TTopKeeperDataWrapper(TTopKeeperDataWrapper& left, TTopKeeperDataWrapper& right)
- : TBase(TDataPairCompare<Slot, IsTop>())
- {
- TBase::Merge(left, right);
- }
-
- explicit TTopKeeperDataWrapper(const TUnboxedValuePod& serialized)
- : TBase(TDataPairCompare<Slot, IsTop>())
- {
- TBase::Deserialize(serialized);
- }
-};
-
-template <bool HasKey, bool IsTop>
-class TTopKeeperWrapper;
-
-template <bool IsTop>
-class TTopKeeperWrapper<false, IsTop>
- : public TTopKeeperWrapperBase<TGenericCompare<IsTop>>
-{
-public:
- using TBase = TTopKeeperWrapperBase<TGenericCompare<IsTop>>;
-
- TTopKeeperWrapper(const TUnboxedValuePod& value, ui32 maxSize, ICompare::TPtr compare)
- : TBase(TGenericCompare<IsTop>{compare})
- {
- TBase::Init(value, maxSize);
- }
-
- TTopKeeperWrapper(TTopKeeperWrapper& left, TTopKeeperWrapper& right, ICompare::TPtr compare)
- : TBase(TGenericCompare<IsTop>{compare})
- {
- TBase::Merge(left, right);
- }
-
- TTopKeeperWrapper(const TUnboxedValuePod& serialized, ICompare::TPtr compare)
- : TBase(TGenericCompare<IsTop>{compare})
- {
- TBase::Deserialize(serialized);
- }
-};
-
-template <bool IsTop>
-class TTopKeeperWrapper<true, IsTop>
- : public TTopKeeperPairWrapperBase<TGenericPairCompare<IsTop>>
-{
-public:
- using TBase = TTopKeeperPairWrapperBase<TGenericPairCompare<IsTop>>;
-
- TTopKeeperWrapper(const TUnboxedValuePod& key, const TUnboxedValuePod& payload, ui32 maxSize, ICompare::TPtr compare)
- : TBase(TGenericPairCompare<IsTop>{compare})
- {
- TBase::Init(key, payload, maxSize);
- }
-
- TTopKeeperWrapper(TTopKeeperWrapper& left, TTopKeeperWrapper& right, ICompare::TPtr compare)
- : TBase(TGenericPairCompare<IsTop>{compare})
- {
- TBase::Merge(left, right);
- }
-
- TTopKeeperWrapper(const TUnboxedValuePod& serialized, ICompare::TPtr compare)
- : TBase(TGenericPairCompare<IsTop>{compare})
- {
- TBase::Deserialize(serialized);
- }
-};
-
-
-template <EDataSlot Slot, bool HasKey, bool IsTop>
-class TTopResourceData;
-
-template <EDataSlot Slot, bool HasKey, bool IsTop>
-TTopResourceData<Slot, HasKey, IsTop>* GetTopResourceData(const TUnboxedValuePod& arg) {
- TTopResourceData<Slot, HasKey, IsTop>::Validate(arg);
- return static_cast<TTopResourceData<Slot, HasKey, IsTop>*>(arg.AsBoxed().Get());
-}
-
-template <bool HasKey, bool IsTop>
-class TTopResource;
-
-template <bool HasKey, bool IsTop>
-TTopResource<HasKey, IsTop>* GetTopResource(const TUnboxedValuePod& arg) {
- TTopResource<HasKey, IsTop>::Validate(arg);
- return static_cast<TTopResource<HasKey, IsTop>*>(arg.AsBoxed().Get());
-}
-
-
-template <EDataSlot Slot, bool HasKey, bool IsTop>
-class TTopCreateData : public TBoxedValue {
-private:
- template <bool HasKey_ = HasKey, typename std::enable_if_t<!HasKey_>* = nullptr>
- TUnboxedValue RunImpl(const TUnboxedValuePod* args) const {
- return TUnboxedValuePod(
- new TTopResourceData<Slot, HasKey, IsTop>(args[0], args[1].Get<ui32>()));
- }
-
- template <bool HasKey_ = HasKey, typename std::enable_if_t<HasKey_>* = nullptr>
- TUnboxedValue RunImpl(const TUnboxedValuePod* args) const {
- return TUnboxedValuePod(
- new TTopResourceData<Slot, HasKey, IsTop>(args[0], args[1], args[2].Get<ui32>()));
- }
-
- TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override {
- return RunImpl(args);
- }
-};
-
-template <bool HasKey, bool IsTop>
-class TTopCreate : public TBoxedValue {
-private:
- template <bool HasKey_ = HasKey, typename std::enable_if_t<!HasKey_>* = nullptr>
- TUnboxedValue RunImpl(const TUnboxedValuePod* args) const {
- return TUnboxedValuePod(
- new TTopResource<HasKey, IsTop>(args[0], args[1].Get<ui32>(), Compare_));
- }
-
- template <bool HasKey_ = HasKey, typename std::enable_if_t<HasKey_>* = nullptr>
- TUnboxedValue RunImpl(const TUnboxedValuePod* args) const {
- return TUnboxedValuePod(
- new TTopResource<HasKey, IsTop>(args[0], args[1], args[2].Get<ui32>(), Compare_));
- }
-
- TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override {
- return RunImpl(args);
- }
-
-public:
- explicit TTopCreate(ICompare::TPtr compare)
- : Compare_(compare)
- {}
-
-private:
- ICompare::TPtr Compare_;
-};
-
-template <EDataSlot Slot, bool HasKey, bool IsTop>
-class TTopAddValueData : public TBoxedValue {
-private:
- template <bool HasKey_ = HasKey, typename std::enable_if_t<!HasKey_>* = nullptr>
- TUnboxedValue RunImpl(const TUnboxedValuePod* args) const {
- auto resource = GetTopResourceData<Slot, HasKey, IsTop>(args[0]);
- resource->Get()->AddValue(args[1]);
- return TUnboxedValuePod(resource);
- }
-
- template <bool HasKey_ = HasKey, typename std::enable_if_t<HasKey_>* = nullptr>
- TUnboxedValue RunImpl(const TUnboxedValuePod* args) const {
- auto resource = GetTopResourceData<Slot, HasKey, IsTop>(args[0]);
- resource->Get()->AddValue(args[1], args[2]);
- return TUnboxedValuePod(resource);
- }
-
- TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override {
- return RunImpl(args);
- }
-};
-
-template <bool HasKey, bool IsTop>
-class TTopAddValue : public TBoxedValue {
-private:
- template <bool HasKey_ = HasKey, typename std::enable_if_t<!HasKey_>* = nullptr>
- TUnboxedValue RunImpl(const TUnboxedValuePod* args) const {
- auto resource = GetTopResource<HasKey, IsTop>(args[0]);
- resource->Get()->AddValue(args[1]);
- return TUnboxedValuePod(resource);
- }
-
- template <bool HasKey_ = HasKey, typename std::enable_if_t<HasKey_>* = nullptr>
- TUnboxedValue RunImpl(const TUnboxedValuePod* args) const {
- auto resource = GetTopResource<HasKey, IsTop>(args[0]);
- resource->Get()->AddValue(args[1], args[2]);
- return TUnboxedValuePod(resource);
- }
-
- TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override {
- return RunImpl(args);
- }
-
-public:
- explicit TTopAddValue(ICompare::TPtr)
- {}
-};
-
-template <EDataSlot Slot, bool HasKey, bool IsTop>
-class TTopSerializeData : public TBoxedValue {
-private:
+ {}
+
+ void Init(const TUnboxedValuePod& key, const TUnboxedValuePod& payload, ui32 maxSize) {
+ Keeper.SetMaxSize(maxSize);
+ AddValue(key, payload);
+ }
+
+ void Merge(TTopKeeperPairWrapperBase& left, TTopKeeperPairWrapperBase& right) {
+ Keeper.SetMaxSize(left.Keeper.GetMaxSize());
+ for (const auto& item : left.Keeper.GetInternal()) {
+ AddValue(item.first, item.second);
+ }
+ for (const auto& item : right.Keeper.GetInternal()) {
+ AddValue(item.first, item.second);
+ }
+ }
+
+ void Deserialize(const TUnboxedValuePod& serialized) {
+ auto maxSize = serialized.GetElement(0).Get<ui32>();
+ auto list = serialized.GetElement(1);
+
+ Keeper.SetMaxSize(maxSize);
+ const auto listIter = list.GetListIterator();
+ for (TUnboxedValue current; listIter.Next(current);) {
+ AddValue(current.GetElement(0), current.GetElement(1));
+ }
+ }
+
+public:
+ void AddValue(const TUnboxedValuePod& key, const TUnboxedValuePod& payload) {
+ Keeper.Insert(std::make_pair(TUnboxedValuePod(key), TUnboxedValuePod(payload)));
+ }
+
+ TUnboxedValue Serialize(const IValueBuilder* builder) {
+ TUnboxedValue* values = nullptr;
+ auto list = builder->NewArray(Keeper.GetSize(), values);
+
+ for (const auto& item : Keeper.GetInternal()) {
+ TUnboxedValue* items = nullptr;
+ auto pair = builder->NewArray(2U, items);
+ items[0] = item.first;
+ items[1] = item.second;
+ *values++ = pair;
+ }
+
+ TUnboxedValue* items = nullptr;
+ auto result = builder->NewArray(2U, items);
+ items[0] = TUnboxedValuePod((ui32)Keeper.GetMaxSize());
+ items[1] = list;
+
+ return result;
+ }
+
+ TUnboxedValue GetResult(const IValueBuilder* builder) {
+ TUnboxedValue* values = nullptr;
+ auto list = builder->NewArray(Keeper.GetSize(), values);
+
+ for (const auto& item : Keeper.GetInternal()) {
+ *values++ = item.second;
+ }
+ return list;
+ }
+};
+
+
+template <EDataSlot Slot, bool HasKey, bool IsTop>
+class TTopKeeperDataWrapper;
+
+template <EDataSlot Slot, bool IsTop>
+class TTopKeeperDataWrapper<Slot, false, IsTop>
+ : public TTopKeeperWrapperBase<TDataCompare<Slot, IsTop>>
+{
+public:
+ using TBase = TTopKeeperWrapperBase<TDataCompare<Slot, IsTop>>;
+
+ TTopKeeperDataWrapper(const TUnboxedValuePod& value, ui32 maxSize)
+ : TBase(TDataCompare<Slot, IsTop>())
+ {
+ TBase::Init(value, maxSize);
+ }
+
+ TTopKeeperDataWrapper(TTopKeeperDataWrapper& left, TTopKeeperDataWrapper& right)
+ : TBase(TDataCompare<Slot, IsTop>())
+ {
+ TBase::Merge(left, right);
+ }
+
+ explicit TTopKeeperDataWrapper(const TUnboxedValuePod& serialized)
+ : TBase(TDataCompare<Slot, IsTop>())
+ {
+ TBase::Deserialize(serialized);
+ }
+};
+
+template <EDataSlot Slot, bool IsTop>
+class TTopKeeperDataWrapper<Slot, true, IsTop>
+ : public TTopKeeperPairWrapperBase<TDataPairCompare<Slot, IsTop>>
+{
+public:
+ using TBase = TTopKeeperPairWrapperBase<TDataPairCompare<Slot, IsTop>>;
+
+ TTopKeeperDataWrapper(const TUnboxedValuePod& key, const TUnboxedValuePod& payload, ui32 maxSize)
+ : TBase(TDataPairCompare<Slot, IsTop>())
+ {
+ TBase::Init(key, payload, maxSize);
+ }
+
+ TTopKeeperDataWrapper(TTopKeeperDataWrapper& left, TTopKeeperDataWrapper& right)
+ : TBase(TDataPairCompare<Slot, IsTop>())
+ {
+ TBase::Merge(left, right);
+ }
+
+ explicit TTopKeeperDataWrapper(const TUnboxedValuePod& serialized)
+ : TBase(TDataPairCompare<Slot, IsTop>())
+ {
+ TBase::Deserialize(serialized);
+ }
+};
+
+template <bool HasKey, bool IsTop>
+class TTopKeeperWrapper;
+
+template <bool IsTop>
+class TTopKeeperWrapper<false, IsTop>
+ : public TTopKeeperWrapperBase<TGenericCompare<IsTop>>
+{
+public:
+ using TBase = TTopKeeperWrapperBase<TGenericCompare<IsTop>>;
+
+ TTopKeeperWrapper(const TUnboxedValuePod& value, ui32 maxSize, ICompare::TPtr compare)
+ : TBase(TGenericCompare<IsTop>{compare})
+ {
+ TBase::Init(value, maxSize);
+ }
+
+ TTopKeeperWrapper(TTopKeeperWrapper& left, TTopKeeperWrapper& right, ICompare::TPtr compare)
+ : TBase(TGenericCompare<IsTop>{compare})
+ {
+ TBase::Merge(left, right);
+ }
+
+ TTopKeeperWrapper(const TUnboxedValuePod& serialized, ICompare::TPtr compare)
+ : TBase(TGenericCompare<IsTop>{compare})
+ {
+ TBase::Deserialize(serialized);
+ }
+};
+
+template <bool IsTop>
+class TTopKeeperWrapper<true, IsTop>
+ : public TTopKeeperPairWrapperBase<TGenericPairCompare<IsTop>>
+{
+public:
+ using TBase = TTopKeeperPairWrapperBase<TGenericPairCompare<IsTop>>;
+
+ TTopKeeperWrapper(const TUnboxedValuePod& key, const TUnboxedValuePod& payload, ui32 maxSize, ICompare::TPtr compare)
+ : TBase(TGenericPairCompare<IsTop>{compare})
+ {
+ TBase::Init(key, payload, maxSize);
+ }
+
+ TTopKeeperWrapper(TTopKeeperWrapper& left, TTopKeeperWrapper& right, ICompare::TPtr compare)
+ : TBase(TGenericPairCompare<IsTop>{compare})
+ {
+ TBase::Merge(left, right);
+ }
+
+ TTopKeeperWrapper(const TUnboxedValuePod& serialized, ICompare::TPtr compare)
+ : TBase(TGenericPairCompare<IsTop>{compare})
+ {
+ TBase::Deserialize(serialized);
+ }
+};
+
+
+template <EDataSlot Slot, bool HasKey, bool IsTop>
+class TTopResourceData;
+
+template <EDataSlot Slot, bool HasKey, bool IsTop>
+TTopResourceData<Slot, HasKey, IsTop>* GetTopResourceData(const TUnboxedValuePod& arg) {
+ TTopResourceData<Slot, HasKey, IsTop>::Validate(arg);
+ return static_cast<TTopResourceData<Slot, HasKey, IsTop>*>(arg.AsBoxed().Get());
+}
+
+template <bool HasKey, bool IsTop>
+class TTopResource;
+
+template <bool HasKey, bool IsTop>
+TTopResource<HasKey, IsTop>* GetTopResource(const TUnboxedValuePod& arg) {
+ TTopResource<HasKey, IsTop>::Validate(arg);
+ return static_cast<TTopResource<HasKey, IsTop>*>(arg.AsBoxed().Get());
+}
+
+
+template <EDataSlot Slot, bool HasKey, bool IsTop>
+class TTopCreateData : public TBoxedValue {
+private:
+ template <bool HasKey_ = HasKey, typename std::enable_if_t<!HasKey_>* = nullptr>
+ TUnboxedValue RunImpl(const TUnboxedValuePod* args) const {
+ return TUnboxedValuePod(
+ new TTopResourceData<Slot, HasKey, IsTop>(args[0], args[1].Get<ui32>()));
+ }
+
+ template <bool HasKey_ = HasKey, typename std::enable_if_t<HasKey_>* = nullptr>
+ TUnboxedValue RunImpl(const TUnboxedValuePod* args) const {
+ return TUnboxedValuePod(
+ new TTopResourceData<Slot, HasKey, IsTop>(args[0], args[1], args[2].Get<ui32>()));
+ }
+
+ TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override {
+ return RunImpl(args);
+ }
+};
+
+template <bool HasKey, bool IsTop>
+class TTopCreate : public TBoxedValue {
+private:
+ template <bool HasKey_ = HasKey, typename std::enable_if_t<!HasKey_>* = nullptr>
+ TUnboxedValue RunImpl(const TUnboxedValuePod* args) const {
+ return TUnboxedValuePod(
+ new TTopResource<HasKey, IsTop>(args[0], args[1].Get<ui32>(), Compare_));
+ }
+
+ template <bool HasKey_ = HasKey, typename std::enable_if_t<HasKey_>* = nullptr>
+ TUnboxedValue RunImpl(const TUnboxedValuePod* args) const {
+ return TUnboxedValuePod(
+ new TTopResource<HasKey, IsTop>(args[0], args[1], args[2].Get<ui32>(), Compare_));
+ }
+
+ TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override {
+ return RunImpl(args);
+ }
+
+public:
+ explicit TTopCreate(ICompare::TPtr compare)
+ : Compare_(compare)
+ {}
+
+private:
+ ICompare::TPtr Compare_;
+};
+
+template <EDataSlot Slot, bool HasKey, bool IsTop>
+class TTopAddValueData : public TBoxedValue {
+private:
+ template <bool HasKey_ = HasKey, typename std::enable_if_t<!HasKey_>* = nullptr>
+ TUnboxedValue RunImpl(const TUnboxedValuePod* args) const {
+ auto resource = GetTopResourceData<Slot, HasKey, IsTop>(args[0]);
+ resource->Get()->AddValue(args[1]);
+ return TUnboxedValuePod(resource);
+ }
+
+ template <bool HasKey_ = HasKey, typename std::enable_if_t<HasKey_>* = nullptr>
+ TUnboxedValue RunImpl(const TUnboxedValuePod* args) const {
+ auto resource = GetTopResourceData<Slot, HasKey, IsTop>(args[0]);
+ resource->Get()->AddValue(args[1], args[2]);
+ return TUnboxedValuePod(resource);
+ }
+
+ TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override {
+ return RunImpl(args);
+ }
+};
+
+template <bool HasKey, bool IsTop>
+class TTopAddValue : public TBoxedValue {
+private:
+ template <bool HasKey_ = HasKey, typename std::enable_if_t<!HasKey_>* = nullptr>
+ TUnboxedValue RunImpl(const TUnboxedValuePod* args) const {
+ auto resource = GetTopResource<HasKey, IsTop>(args[0]);
+ resource->Get()->AddValue(args[1]);
+ return TUnboxedValuePod(resource);
+ }
+
+ template <bool HasKey_ = HasKey, typename std::enable_if_t<HasKey_>* = nullptr>
+ TUnboxedValue RunImpl(const TUnboxedValuePod* args) const {
+ auto resource = GetTopResource<HasKey, IsTop>(args[0]);
+ resource->Get()->AddValue(args[1], args[2]);
+ return TUnboxedValuePod(resource);
+ }
+
+ TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override {
+ return RunImpl(args);
+ }
+
+public:
+ explicit TTopAddValue(ICompare::TPtr)
+ {}
+};
+
+template <EDataSlot Slot, bool HasKey, bool IsTop>
+class TTopSerializeData : public TBoxedValue {
+private:
TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const override {
- auto resource = GetTopResourceData<Slot, HasKey, IsTop>(args[0]);
- return resource->Get()->Serialize(valueBuilder);
- }
-};
-
-template <bool HasKey, bool IsTop>
-class TTopSerialize : public TBoxedValue {
-private:
+ auto resource = GetTopResourceData<Slot, HasKey, IsTop>(args[0]);
+ return resource->Get()->Serialize(valueBuilder);
+ }
+};
+
+template <bool HasKey, bool IsTop>
+class TTopSerialize : public TBoxedValue {
+private:
TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const override {
- auto resource = GetTopResource<HasKey, IsTop>(args[0]);
- return resource->Get()->Serialize(valueBuilder);
- }
-
-public:
- explicit TTopSerialize(ICompare::TPtr)
- {}
-};
-
-template <EDataSlot Slot, bool HasKey, bool IsTop>
-class TTopDeserializeData : public TBoxedValue {
-private:
+ auto resource = GetTopResource<HasKey, IsTop>(args[0]);
+ return resource->Get()->Serialize(valueBuilder);
+ }
+
+public:
+ explicit TTopSerialize(ICompare::TPtr)
+ {}
+};
+
+template <EDataSlot Slot, bool HasKey, bool IsTop>
+class TTopDeserializeData : public TBoxedValue {
+private:
TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override {
- return TUnboxedValuePod(new TTopResourceData<Slot, HasKey, IsTop>(args[0]));
- }
-};
-
-template <bool HasKey, bool IsTop>
-class TTopDeserialize : public TBoxedValue {
-private:
+ return TUnboxedValuePod(new TTopResourceData<Slot, HasKey, IsTop>(args[0]));
+ }
+};
+
+template <bool HasKey, bool IsTop>
+class TTopDeserialize : public TBoxedValue {
+private:
TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override {
- return TUnboxedValuePod(new TTopResource<HasKey, IsTop>(args[0], Compare_));
- }
-
-public:
- explicit TTopDeserialize(ICompare::TPtr compare)
- : Compare_(compare)
- {}
-
-private:
- ICompare::TPtr Compare_;
-};
-
-template <EDataSlot Slot, bool HasKey, bool IsTop>
-class TTopMergeData : public TBoxedValue {
-private:
+ return TUnboxedValuePod(new TTopResource<HasKey, IsTop>(args[0], Compare_));
+ }
+
+public:
+ explicit TTopDeserialize(ICompare::TPtr compare)
+ : Compare_(compare)
+ {}
+
+private:
+ ICompare::TPtr Compare_;
+};
+
+template <EDataSlot Slot, bool HasKey, bool IsTop>
+class TTopMergeData : public TBoxedValue {
+private:
TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override {
- auto left = GetTopResourceData<Slot, HasKey, IsTop>(args[0]);
- auto right = GetTopResourceData<Slot, HasKey, IsTop>(args[1]);
- return TUnboxedValuePod(new TTopResourceData<Slot, HasKey, IsTop>(*left->Get(), *right->Get()));
- }
-};
-
-template <bool HasKey, bool IsTop>
-class TTopMerge : public TBoxedValue {
-private:
+ auto left = GetTopResourceData<Slot, HasKey, IsTop>(args[0]);
+ auto right = GetTopResourceData<Slot, HasKey, IsTop>(args[1]);
+ return TUnboxedValuePod(new TTopResourceData<Slot, HasKey, IsTop>(*left->Get(), *right->Get()));
+ }
+};
+
+template <bool HasKey, bool IsTop>
+class TTopMerge : public TBoxedValue {
+private:
TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const override {
- auto left = GetTopResource<HasKey, IsTop>(args[0]);
- auto right = GetTopResource<HasKey, IsTop>(args[1]);
- return TUnboxedValuePod(new TTopResource<HasKey, IsTop>(*left->Get(), *right->Get(), Compare_));
- }
-
-public:
- explicit TTopMerge(ICompare::TPtr compare)
- : Compare_(compare)
- {}
-
-private:
- ICompare::TPtr Compare_;
-};
-
-template <EDataSlot Slot, bool HasKey, bool IsTop>
-class TTopGetResultData : public TBoxedValue {
-private:
+ auto left = GetTopResource<HasKey, IsTop>(args[0]);
+ auto right = GetTopResource<HasKey, IsTop>(args[1]);
+ return TUnboxedValuePod(new TTopResource<HasKey, IsTop>(*left->Get(), *right->Get(), Compare_));
+ }
+
+public:
+ explicit TTopMerge(ICompare::TPtr compare)
+ : Compare_(compare)
+ {}
+
+private:
+ ICompare::TPtr Compare_;
+};
+
+template <EDataSlot Slot, bool HasKey, bool IsTop>
+class TTopGetResultData : public TBoxedValue {
+private:
TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const override {
- auto resource = GetTopResourceData<Slot, HasKey, IsTop>(args[0]);
- return resource->Get()->GetResult(valueBuilder);
- }
-};
-
-template <bool HasKey, bool IsTop>
-class TTopGetResult : public TBoxedValue {
-private:
+ auto resource = GetTopResourceData<Slot, HasKey, IsTop>(args[0]);
+ return resource->Get()->GetResult(valueBuilder);
+ }
+};
+
+template <bool HasKey, bool IsTop>
+class TTopGetResult : public TBoxedValue {
+private:
TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const override {
- auto resource = GetTopResource<HasKey, IsTop>(args[0]);
- return resource->Get()->GetResult(valueBuilder);
- }
-
-public:
- explicit TTopGetResult(ICompare::TPtr)
- {}
-};
-
-
-#define RESOURCE(slot, hasKey, isTop) \
-extern const char TopResourceName_##slot##_##hasKey##_##isTop[] = \
- "Top.TopResource."#slot"."#hasKey"."#isTop; \
-template <> \
-class TTopResourceData<EDataSlot::slot, hasKey, isTop>: \
- public TBoxedResource< \
- TTopKeeperDataWrapper<EDataSlot::slot, hasKey, isTop>, \
- TopResourceName_##slot##_##hasKey##_##isTop> \
-{ \
-public: \
- template <typename... Args> \
- inline TTopResourceData(Args&&... args) \
- : TBoxedResource(std::forward<Args>(args)...) \
- {} \
-};
-
-#define RESOURCE_00(slot, ...) RESOURCE(slot, false, false)
-#define RESOURCE_01(slot, ...) RESOURCE(slot, false, true)
-#define RESOURCE_10(slot, ...) RESOURCE(slot, true, false)
-#define RESOURCE_11(slot, ...) RESOURCE(slot, true, true)
-
-UDF_TYPE_ID_MAP(RESOURCE_00)
-UDF_TYPE_ID_MAP(RESOURCE_01)
-UDF_TYPE_ID_MAP(RESOURCE_10)
-UDF_TYPE_ID_MAP(RESOURCE_11)
-
-#define MAKE_IMPL(operation, slot, hasKey, isTop) \
- case EDataSlot::slot: \
- builder.Implementation(new operation<EDataSlot::slot, hasKey, isTop>); \
- break;
-
-#define CREATE_00(slot, ...) MAKE_IMPL(TTopCreateData, slot, false, false)
-#define CREATE_01(slot, ...) MAKE_IMPL(TTopCreateData, slot, false, true)
-#define CREATE_10(slot, ...) MAKE_IMPL(TTopCreateData, slot, true, false)
-#define CREATE_11(slot, ...) MAKE_IMPL(TTopCreateData, slot, true, true)
-
-#define ADD_VALUE_00(slot, ...) MAKE_IMPL(TTopAddValueData, slot, false, false)
-#define ADD_VALUE_01(slot, ...) MAKE_IMPL(TTopAddValueData, slot, false, true)
-#define ADD_VALUE_10(slot, ...) MAKE_IMPL(TTopAddValueData, slot, true, false)
-#define ADD_VALUE_11(slot, ...) MAKE_IMPL(TTopAddValueData, slot, true, true)
-
-#define MERGE_00(slot, ...) MAKE_IMPL(TTopMergeData, slot, false, false)
-#define MERGE_01(slot, ...) MAKE_IMPL(TTopMergeData, slot, false, true)
-#define MERGE_10(slot, ...) MAKE_IMPL(TTopMergeData, slot, true, false)
-#define MERGE_11(slot, ...) MAKE_IMPL(TTopMergeData, slot, true, true)
-
-#define SERIALIZE_00(slot, ...) MAKE_IMPL(TTopSerializeData, slot, false, false)
-#define SERIALIZE_01(slot, ...) MAKE_IMPL(TTopSerializeData, slot, false, true)
-#define SERIALIZE_10(slot, ...) MAKE_IMPL(TTopSerializeData, slot, true, false)
-#define SERIALIZE_11(slot, ...) MAKE_IMPL(TTopSerializeData, slot, true, true)
-
-#define DESERIALIZE_00(slot, ...) MAKE_IMPL(TTopDeserializeData, slot, false, false)
-#define DESERIALIZE_01(slot, ...) MAKE_IMPL(TTopDeserializeData, slot, false, true)
-#define DESERIALIZE_10(slot, ...) MAKE_IMPL(TTopDeserializeData, slot, true, false)
-#define DESERIALIZE_11(slot, ...) MAKE_IMPL(TTopDeserializeData, slot, true, true)
-
-#define GET_RESULT_00(slot, ...) MAKE_IMPL(TTopGetResultData, slot, false, false)
-#define GET_RESULT_01(slot, ...) MAKE_IMPL(TTopGetResultData, slot, false, true)
-#define GET_RESULT_10(slot, ...) MAKE_IMPL(TTopGetResultData, slot, true, false)
-#define GET_RESULT_11(slot, ...) MAKE_IMPL(TTopGetResultData, slot, true, true)
-
-#define MAKE_TYPE(slot, hasKey, isTop) \
- case EDataSlot::slot: \
- topType = builder.Resource(TopResourceName_##slot##_##hasKey##_##isTop); \
- break;
-
-#define TYPE_00(slot, ...) MAKE_TYPE(slot, false, false)
-#define TYPE_01(slot, ...) MAKE_TYPE(slot, false, true)
-#define TYPE_10(slot, ...) MAKE_TYPE(slot, true, false)
-#define TYPE_11(slot, ...) MAKE_TYPE(slot, true, true)
-
-#define PARAMETRIZE(action) \
- if (hasKey) { \
- if (isTop) { \
- switch (*slot) { \
- UDF_TYPE_ID_MAP(action##_11) \
- } \
- } else { \
- switch (*slot) { \
- UDF_TYPE_ID_MAP(action##_10) \
- } \
- } \
- } else { \
- if (isTop) { \
- switch (*slot) { \
- UDF_TYPE_ID_MAP(action##_01) \
- } \
- } else { \
- switch (*slot) { \
- UDF_TYPE_ID_MAP(action##_00) \
- } \
- } \
- }
-
-
-#define RESOURCE_GENERIC(hasKey, isTop) \
-extern const char TopResourceName_Generic_##hasKey##_##isTop[] = \
- "Top.TopResource.Generic."#hasKey"."#isTop; \
-template <> \
-class TTopResource<hasKey, isTop>: \
- public TBoxedResource< \
- TTopKeeperWrapper<hasKey, isTop>, \
- TopResourceName_Generic_##hasKey##_##isTop> \
-{ \
-public: \
- template <typename... Args> \
- inline TTopResource(Args&&... args) \
- : TBoxedResource(std::forward<Args>(args)...) \
- {} \
-};
-
-RESOURCE_GENERIC(false, false)
-RESOURCE_GENERIC(false, true)
-RESOURCE_GENERIC(true, false)
-RESOURCE_GENERIC(true, true)
-
-#define MAKE_IMPL_GENERIC(operation, hasKey, isTop) \
- builder.Implementation(new operation<hasKey, isTop>(compare));
-
-#define CREATE_GENERIC(hasKey, isTop) MAKE_IMPL_GENERIC(TTopCreate, hasKey, isTop)
-#define ADD_VALUE_GENERIC(hasKey, isTop) MAKE_IMPL_GENERIC(TTopAddValue, hasKey, isTop)
-#define MERGE_GENERIC(hasKey, isTop) MAKE_IMPL_GENERIC(TTopMerge, hasKey, isTop)
-#define SERIALIZE_GENERIC(hasKey, isTop) MAKE_IMPL_GENERIC(TTopSerialize, hasKey, isTop)
-#define DESERIALIZE_GENERIC(hasKey, isTop) MAKE_IMPL_GENERIC(TTopDeserialize, hasKey, isTop)
-#define GET_RESULT_GENERIC(hasKey, isTop) MAKE_IMPL_GENERIC(TTopGetResult, hasKey, isTop)
-
-#define TYPE_GENERIC(hasKey, isTop) \
- topType = builder.Resource(TopResourceName_Generic_##hasKey##_##isTop);
-
-#define PARAMETRIZE_GENERIC(action) \
- if (hasKey) { \
- if (isTop) { \
- action(true, true) \
- } else { \
- action(true, false) \
- } \
- } else { \
- if (isTop) { \
- action(false, true) \
- } else { \
- action(false, false) \
- } \
- }
-
-
-static const auto CreateName = TStringRef::Of("Create");
-static const auto AddValueName = TStringRef::Of("AddValue");
-static const auto SerializeName = TStringRef::Of("Serialize");
-static const auto DeserializeName = TStringRef::Of("Deserialize");
-static const auto MergeName = TStringRef::Of("Merge");
-static const auto GetResultName = TStringRef::Of("GetResult");
-
-class TTopModule : public IUdfModule {
-public:
- TStringRef Name() const {
- return TStringRef::Of("Top");
- }
-
- void CleanupOnTerminate() const final {
- }
-
- void GetAllFunctions(IFunctionsSink& sink) const final {
- sink.Add(CreateName)->SetTypeAwareness();
- sink.Add(AddValueName)->SetTypeAwareness();
- sink.Add(SerializeName)->SetTypeAwareness();
- sink.Add(DeserializeName)->SetTypeAwareness();
- sink.Add(MergeName)->SetTypeAwareness();
- sink.Add(GetResultName)->SetTypeAwareness();
- }
-
- void BuildFunctionTypeInfo(
- const TStringRef& name,
- TType* userType,
- const TStringRef& typeConfig,
- ui32 flags,
- IFunctionTypeInfoBuilder& builder) const final
- {
- Y_UNUSED(typeConfig);
-
- try {
- bool typesOnly = (flags & TFlags::TypesOnly);
- builder.UserType(userType);
-
- if (typeConfig.Size() != 2) {
- builder.SetError(TStringBuilder() << "Invalid type config: " << typeConfig.Data());
- return;
- }
-
- bool hasKey = (typeConfig.Data()[0] == '1');
- bool isTop = (typeConfig.Data()[1] == '1');
-
- auto typeHelper = builder.TypeInfoHelper();
-
- auto userTypeInspector = TTupleTypeInspector(*typeHelper, userType);
- if (!userTypeInspector || userTypeInspector.GetElementsCount() != 3) {
- builder.SetError("User type is not a 3-tuple");
- return;
- }
-
- auto valueType = userTypeInspector.GetElementType(2);
- auto keyType = valueType;
- auto payloadType = valueType;
-
- if (hasKey) {
- auto keyPayloadTypeInspector = TTupleTypeInspector(*typeHelper, valueType);
- if (!keyPayloadTypeInspector || keyPayloadTypeInspector.GetElementsCount() != 2) {
- builder.SetError("Key/payload type is not a 2-tuple");
- return;
- }
-
- keyType = keyPayloadTypeInspector.GetElementType(0);
- payloadType = keyPayloadTypeInspector.GetElementType(1);
- }
-
- bool isGeneric = false;
- ICompare::TPtr compare;
- TMaybe<EDataSlot> slot;
-
- auto keyTypeInspector = TDataTypeInspector(*typeHelper, keyType);
- if (!keyTypeInspector) {
- isGeneric = true;
- compare = builder.MakeCompare(keyType);
- if (!compare) {
- return;
- }
- } else {
- slot = FindDataSlot(keyTypeInspector.GetTypeId());
- if (!slot) {
- builder.SetError("Unknown data type");
- return;
- }
- if (!(GetDataTypeInfo(*slot).Features & NUdf::CanCompare)) {
- builder.SetError("Data type is not comparable");
- return;
- }
- }
-
- auto serializedListType = builder.List()->Item(valueType).Build();
- auto serializedType = builder.Tuple()->Add<ui32>().Add(serializedListType).Build();
-
- TType* topType = nullptr;
- if (isGeneric) {
- PARAMETRIZE_GENERIC(TYPE_GENERIC)
- } else {
- PARAMETRIZE(TYPE)
- }
-
- if (name == CreateName) {
- if (hasKey) {
- builder.Args()->Add(keyType).Add(payloadType).Add<ui32>().Done().Returns(topType);
- } else {
- builder.Args()->Add(valueType).Add<ui32>().Done().Returns(topType);
- }
-
- if (!typesOnly) {
- if (isGeneric) {
- PARAMETRIZE_GENERIC(CREATE_GENERIC)
- } else {
- PARAMETRIZE(CREATE)
- }
- }
- }
-
- if (name == AddValueName) {
- if (hasKey) {
- builder.Args()->Add(topType).Add(keyType).Add(payloadType).Done().Returns(topType);
- } else {
- builder.Args()->Add(topType).Add(valueType).Done().Returns(topType);
- }
-
- if (!typesOnly) {
- if (isGeneric) {
- PARAMETRIZE_GENERIC(ADD_VALUE_GENERIC)
- } else {
- PARAMETRIZE(ADD_VALUE)
- }
- }
- }
-
- if (name == SerializeName) {
- builder.Args()->Add(topType).Done().Returns(serializedType);
-
- if (!typesOnly) {
- if (isGeneric) {
- PARAMETRIZE_GENERIC(SERIALIZE_GENERIC)
- } else {
- PARAMETRIZE(SERIALIZE)
- }
- }
- }
-
- if (name == DeserializeName) {
- builder.Args()->Add(serializedType).Done().Returns(topType);
-
- if (!typesOnly) {
- if (isGeneric) {
- PARAMETRIZE_GENERIC(DESERIALIZE_GENERIC)
- } else {
- PARAMETRIZE(DESERIALIZE)
- }
- }
- }
-
- if (name == MergeName) {
- builder.Args()->Add(topType).Add(topType).Done().Returns(topType);
-
- if (!typesOnly) {
- if (isGeneric) {
- PARAMETRIZE_GENERIC(MERGE_GENERIC)
- } else {
- PARAMETRIZE(MERGE)
- }
- }
- }
-
- if (name == GetResultName) {
- auto listType = builder.List()->Item(payloadType).Build();
-
- builder.Args()->Add(topType).Done().Returns(listType);
-
- if (!typesOnly) {
- if (isGeneric) {
- PARAMETRIZE_GENERIC(GET_RESULT_GENERIC)
- } else {
- PARAMETRIZE(GET_RESULT)
- }
- }
- }
-
- } catch (const std::exception& e) {
- builder.SetError(CurrentExceptionMessage());
- }
- }
-};
-
-} // namespace
-
-REGISTER_MODULES(TTopModule)
-
+ auto resource = GetTopResource<HasKey, IsTop>(args[0]);
+ return resource->Get()->GetResult(valueBuilder);
+ }
+
+public:
+ explicit TTopGetResult(ICompare::TPtr)
+ {}
+};
+
+
+#define RESOURCE(slot, hasKey, isTop) \
+extern const char TopResourceName_##slot##_##hasKey##_##isTop[] = \
+ "Top.TopResource."#slot"."#hasKey"."#isTop; \
+template <> \
+class TTopResourceData<EDataSlot::slot, hasKey, isTop>: \
+ public TBoxedResource< \
+ TTopKeeperDataWrapper<EDataSlot::slot, hasKey, isTop>, \
+ TopResourceName_##slot##_##hasKey##_##isTop> \
+{ \
+public: \
+ template <typename... Args> \
+ inline TTopResourceData(Args&&... args) \
+ : TBoxedResource(std::forward<Args>(args)...) \
+ {} \
+};
+
+#define RESOURCE_00(slot, ...) RESOURCE(slot, false, false)
+#define RESOURCE_01(slot, ...) RESOURCE(slot, false, true)
+#define RESOURCE_10(slot, ...) RESOURCE(slot, true, false)
+#define RESOURCE_11(slot, ...) RESOURCE(slot, true, true)
+
+UDF_TYPE_ID_MAP(RESOURCE_00)
+UDF_TYPE_ID_MAP(RESOURCE_01)
+UDF_TYPE_ID_MAP(RESOURCE_10)
+UDF_TYPE_ID_MAP(RESOURCE_11)
+
+#define MAKE_IMPL(operation, slot, hasKey, isTop) \
+ case EDataSlot::slot: \
+ builder.Implementation(new operation<EDataSlot::slot, hasKey, isTop>); \
+ break;
+
+#define CREATE_00(slot, ...) MAKE_IMPL(TTopCreateData, slot, false, false)
+#define CREATE_01(slot, ...) MAKE_IMPL(TTopCreateData, slot, false, true)
+#define CREATE_10(slot, ...) MAKE_IMPL(TTopCreateData, slot, true, false)
+#define CREATE_11(slot, ...) MAKE_IMPL(TTopCreateData, slot, true, true)
+
+#define ADD_VALUE_00(slot, ...) MAKE_IMPL(TTopAddValueData, slot, false, false)
+#define ADD_VALUE_01(slot, ...) MAKE_IMPL(TTopAddValueData, slot, false, true)
+#define ADD_VALUE_10(slot, ...) MAKE_IMPL(TTopAddValueData, slot, true, false)
+#define ADD_VALUE_11(slot, ...) MAKE_IMPL(TTopAddValueData, slot, true, true)
+
+#define MERGE_00(slot, ...) MAKE_IMPL(TTopMergeData, slot, false, false)
+#define MERGE_01(slot, ...) MAKE_IMPL(TTopMergeData, slot, false, true)
+#define MERGE_10(slot, ...) MAKE_IMPL(TTopMergeData, slot, true, false)
+#define MERGE_11(slot, ...) MAKE_IMPL(TTopMergeData, slot, true, true)
+
+#define SERIALIZE_00(slot, ...) MAKE_IMPL(TTopSerializeData, slot, false, false)
+#define SERIALIZE_01(slot, ...) MAKE_IMPL(TTopSerializeData, slot, false, true)
+#define SERIALIZE_10(slot, ...) MAKE_IMPL(TTopSerializeData, slot, true, false)
+#define SERIALIZE_11(slot, ...) MAKE_IMPL(TTopSerializeData, slot, true, true)
+
+#define DESERIALIZE_00(slot, ...) MAKE_IMPL(TTopDeserializeData, slot, false, false)
+#define DESERIALIZE_01(slot, ...) MAKE_IMPL(TTopDeserializeData, slot, false, true)
+#define DESERIALIZE_10(slot, ...) MAKE_IMPL(TTopDeserializeData, slot, true, false)
+#define DESERIALIZE_11(slot, ...) MAKE_IMPL(TTopDeserializeData, slot, true, true)
+
+#define GET_RESULT_00(slot, ...) MAKE_IMPL(TTopGetResultData, slot, false, false)
+#define GET_RESULT_01(slot, ...) MAKE_IMPL(TTopGetResultData, slot, false, true)
+#define GET_RESULT_10(slot, ...) MAKE_IMPL(TTopGetResultData, slot, true, false)
+#define GET_RESULT_11(slot, ...) MAKE_IMPL(TTopGetResultData, slot, true, true)
+
+#define MAKE_TYPE(slot, hasKey, isTop) \
+ case EDataSlot::slot: \
+ topType = builder.Resource(TopResourceName_##slot##_##hasKey##_##isTop); \
+ break;
+
+#define TYPE_00(slot, ...) MAKE_TYPE(slot, false, false)
+#define TYPE_01(slot, ...) MAKE_TYPE(slot, false, true)
+#define TYPE_10(slot, ...) MAKE_TYPE(slot, true, false)
+#define TYPE_11(slot, ...) MAKE_TYPE(slot, true, true)
+
+#define PARAMETRIZE(action) \
+ if (hasKey) { \
+ if (isTop) { \
+ switch (*slot) { \
+ UDF_TYPE_ID_MAP(action##_11) \
+ } \
+ } else { \
+ switch (*slot) { \
+ UDF_TYPE_ID_MAP(action##_10) \
+ } \
+ } \
+ } else { \
+ if (isTop) { \
+ switch (*slot) { \
+ UDF_TYPE_ID_MAP(action##_01) \
+ } \
+ } else { \
+ switch (*slot) { \
+ UDF_TYPE_ID_MAP(action##_00) \
+ } \
+ } \
+ }
+
+
+#define RESOURCE_GENERIC(hasKey, isTop) \
+extern const char TopResourceName_Generic_##hasKey##_##isTop[] = \
+ "Top.TopResource.Generic."#hasKey"."#isTop; \
+template <> \
+class TTopResource<hasKey, isTop>: \
+ public TBoxedResource< \
+ TTopKeeperWrapper<hasKey, isTop>, \
+ TopResourceName_Generic_##hasKey##_##isTop> \
+{ \
+public: \
+ template <typename... Args> \
+ inline TTopResource(Args&&... args) \
+ : TBoxedResource(std::forward<Args>(args)...) \
+ {} \
+};
+
+RESOURCE_GENERIC(false, false)
+RESOURCE_GENERIC(false, true)
+RESOURCE_GENERIC(true, false)
+RESOURCE_GENERIC(true, true)
+
+#define MAKE_IMPL_GENERIC(operation, hasKey, isTop) \
+ builder.Implementation(new operation<hasKey, isTop>(compare));
+
+#define CREATE_GENERIC(hasKey, isTop) MAKE_IMPL_GENERIC(TTopCreate, hasKey, isTop)
+#define ADD_VALUE_GENERIC(hasKey, isTop) MAKE_IMPL_GENERIC(TTopAddValue, hasKey, isTop)
+#define MERGE_GENERIC(hasKey, isTop) MAKE_IMPL_GENERIC(TTopMerge, hasKey, isTop)
+#define SERIALIZE_GENERIC(hasKey, isTop) MAKE_IMPL_GENERIC(TTopSerialize, hasKey, isTop)
+#define DESERIALIZE_GENERIC(hasKey, isTop) MAKE_IMPL_GENERIC(TTopDeserialize, hasKey, isTop)
+#define GET_RESULT_GENERIC(hasKey, isTop) MAKE_IMPL_GENERIC(TTopGetResult, hasKey, isTop)
+
+#define TYPE_GENERIC(hasKey, isTop) \
+ topType = builder.Resource(TopResourceName_Generic_##hasKey##_##isTop);
+
+#define PARAMETRIZE_GENERIC(action) \
+ if (hasKey) { \
+ if (isTop) { \
+ action(true, true) \
+ } else { \
+ action(true, false) \
+ } \
+ } else { \
+ if (isTop) { \
+ action(false, true) \
+ } else { \
+ action(false, false) \
+ } \
+ }
+
+
+static const auto CreateName = TStringRef::Of("Create");
+static const auto AddValueName = TStringRef::Of("AddValue");
+static const auto SerializeName = TStringRef::Of("Serialize");
+static const auto DeserializeName = TStringRef::Of("Deserialize");
+static const auto MergeName = TStringRef::Of("Merge");
+static const auto GetResultName = TStringRef::Of("GetResult");
+
+class TTopModule : public IUdfModule {
+public:
+ TStringRef Name() const {
+ return TStringRef::Of("Top");
+ }
+
+ void CleanupOnTerminate() const final {
+ }
+
+ void GetAllFunctions(IFunctionsSink& sink) const final {
+ sink.Add(CreateName)->SetTypeAwareness();
+ sink.Add(AddValueName)->SetTypeAwareness();
+ sink.Add(SerializeName)->SetTypeAwareness();
+ sink.Add(DeserializeName)->SetTypeAwareness();
+ sink.Add(MergeName)->SetTypeAwareness();
+ sink.Add(GetResultName)->SetTypeAwareness();
+ }
+
+ void BuildFunctionTypeInfo(
+ const TStringRef& name,
+ TType* userType,
+ const TStringRef& typeConfig,
+ ui32 flags,
+ IFunctionTypeInfoBuilder& builder) const final
+ {
+ Y_UNUSED(typeConfig);
+
+ try {
+ bool typesOnly = (flags & TFlags::TypesOnly);
+ builder.UserType(userType);
+
+ if (typeConfig.Size() != 2) {
+ builder.SetError(TStringBuilder() << "Invalid type config: " << typeConfig.Data());
+ return;
+ }
+
+ bool hasKey = (typeConfig.Data()[0] == '1');
+ bool isTop = (typeConfig.Data()[1] == '1');
+
+ auto typeHelper = builder.TypeInfoHelper();
+
+ auto userTypeInspector = TTupleTypeInspector(*typeHelper, userType);
+ if (!userTypeInspector || userTypeInspector.GetElementsCount() != 3) {
+ builder.SetError("User type is not a 3-tuple");
+ return;
+ }
+
+ auto valueType = userTypeInspector.GetElementType(2);
+ auto keyType = valueType;
+ auto payloadType = valueType;
+
+ if (hasKey) {
+ auto keyPayloadTypeInspector = TTupleTypeInspector(*typeHelper, valueType);
+ if (!keyPayloadTypeInspector || keyPayloadTypeInspector.GetElementsCount() != 2) {
+ builder.SetError("Key/payload type is not a 2-tuple");
+ return;
+ }
+
+ keyType = keyPayloadTypeInspector.GetElementType(0);
+ payloadType = keyPayloadTypeInspector.GetElementType(1);
+ }
+
+ bool isGeneric = false;
+ ICompare::TPtr compare;
+ TMaybe<EDataSlot> slot;
+
+ auto keyTypeInspector = TDataTypeInspector(*typeHelper, keyType);
+ if (!keyTypeInspector) {
+ isGeneric = true;
+ compare = builder.MakeCompare(keyType);
+ if (!compare) {
+ return;
+ }
+ } else {
+ slot = FindDataSlot(keyTypeInspector.GetTypeId());
+ if (!slot) {
+ builder.SetError("Unknown data type");
+ return;
+ }
+ if (!(GetDataTypeInfo(*slot).Features & NUdf::CanCompare)) {
+ builder.SetError("Data type is not comparable");
+ return;
+ }
+ }
+
+ auto serializedListType = builder.List()->Item(valueType).Build();
+ auto serializedType = builder.Tuple()->Add<ui32>().Add(serializedListType).Build();
+
+ TType* topType = nullptr;
+ if (isGeneric) {
+ PARAMETRIZE_GENERIC(TYPE_GENERIC)
+ } else {
+ PARAMETRIZE(TYPE)
+ }
+
+ if (name == CreateName) {
+ if (hasKey) {
+ builder.Args()->Add(keyType).Add(payloadType).Add<ui32>().Done().Returns(topType);
+ } else {
+ builder.Args()->Add(valueType).Add<ui32>().Done().Returns(topType);
+ }
+
+ if (!typesOnly) {
+ if (isGeneric) {
+ PARAMETRIZE_GENERIC(CREATE_GENERIC)
+ } else {
+ PARAMETRIZE(CREATE)
+ }
+ }
+ }
+
+ if (name == AddValueName) {
+ if (hasKey) {
+ builder.Args()->Add(topType).Add(keyType).Add(payloadType).Done().Returns(topType);
+ } else {
+ builder.Args()->Add(topType).Add(valueType).Done().Returns(topType);
+ }
+
+ if (!typesOnly) {
+ if (isGeneric) {
+ PARAMETRIZE_GENERIC(ADD_VALUE_GENERIC)
+ } else {
+ PARAMETRIZE(ADD_VALUE)
+ }
+ }
+ }
+
+ if (name == SerializeName) {
+ builder.Args()->Add(topType).Done().Returns(serializedType);
+
+ if (!typesOnly) {
+ if (isGeneric) {
+ PARAMETRIZE_GENERIC(SERIALIZE_GENERIC)
+ } else {
+ PARAMETRIZE(SERIALIZE)
+ }
+ }
+ }
+
+ if (name == DeserializeName) {
+ builder.Args()->Add(serializedType).Done().Returns(topType);
+
+ if (!typesOnly) {
+ if (isGeneric) {
+ PARAMETRIZE_GENERIC(DESERIALIZE_GENERIC)
+ } else {
+ PARAMETRIZE(DESERIALIZE)
+ }
+ }
+ }
+
+ if (name == MergeName) {
+ builder.Args()->Add(topType).Add(topType).Done().Returns(topType);
+
+ if (!typesOnly) {
+ if (isGeneric) {
+ PARAMETRIZE_GENERIC(MERGE_GENERIC)
+ } else {
+ PARAMETRIZE(MERGE)
+ }
+ }
+ }
+
+ if (name == GetResultName) {
+ auto listType = builder.List()->Item(payloadType).Build();
+
+ builder.Args()->Add(topType).Done().Returns(listType);
+
+ if (!typesOnly) {
+ if (isGeneric) {
+ PARAMETRIZE_GENERIC(GET_RESULT_GENERIC)
+ } else {
+ PARAMETRIZE(GET_RESULT)
+ }
+ }
+ }
+
+ } catch (const std::exception& e) {
+ builder.SetError(CurrentExceptionMessage());
+ }
+ }
+};
+
+} // namespace
+
+REGISTER_MODULES(TTopModule)
+
diff --git a/ydb/library/yql/udfs/common/top/ya.make b/ydb/library/yql/udfs/common/top/ya.make
index e99fe77c36c..1daeb15ebad 100644
--- a/ydb/library/yql/udfs/common/top/ya.make
+++ b/ydb/library/yql/udfs/common/top/ya.make
@@ -1,19 +1,19 @@
-YQL_UDF(top_udf)
-
-YQL_ABI_VERSION(
- 2
- 10
- 0
-)
-
+YQL_UDF(top_udf)
+
+YQL_ABI_VERSION(
+ 2
+ 10
+ 0
+)
+
OWNER(g:yql g:yql_ydb_core)
-
-SRCS(
- top_udf.cpp
-)
-
-PEERDIR(
+
+SRCS(
+ top_udf.cpp
+)
+
+PEERDIR(
library/cpp/containers/top_keeper
-)
-
-END()
+)
+
+END()
diff --git a/ydb/library/yql/udfs/common/topfreq/static/topfreq.cpp b/ydb/library/yql/udfs/common/topfreq/static/topfreq.cpp
index 31fcd88b547..1621bf51626 100644
--- a/ydb/library/yql/udfs/common/topfreq/static/topfreq.cpp
+++ b/ydb/library/yql/udfs/common/topfreq/static/topfreq.cpp
@@ -5,50 +5,50 @@
using namespace NKikimr;
using namespace NUdf;
-template <typename THash, typename TEquals>
-TTopFreqBase<THash, TEquals>::TTopFreqBase(THash hash, TEquals equals)
- : Indices_(0, hash, equals)
-{}
-
-template <typename THash, typename TEquals>
-void TTopFreqBase<THash, TEquals>::Init(const TUnboxedValuePod& value, const ui32 minSize, const ui32 maxSize) {
- MinSize_ = minSize;
- MaxSize_ = maxSize;
-
+template <typename THash, typename TEquals>
+TTopFreqBase<THash, TEquals>::TTopFreqBase(THash hash, TEquals equals)
+ : Indices_(0, hash, equals)
+{}
+
+template <typename THash, typename TEquals>
+void TTopFreqBase<THash, TEquals>::Init(const TUnboxedValuePod& value, const ui32 minSize, const ui32 maxSize) {
+ MinSize_ = minSize;
+ MaxSize_ = maxSize;
+
Freqs_.reserve(MaxSize_ + 1);
Indices_.reserve(MaxSize_ + 1);
-
- AddValue(value);
-}
-
-template <typename THash, typename TEquals>
-void TTopFreqBase<THash, TEquals>::Merge(const TTopFreqBase& topFreq1, const TTopFreqBase& topFreq2) {
- MinSize_ = std::max(topFreq1.MinSize_, topFreq2.MinSize_);
- MaxSize_ = std::max(topFreq1.MaxSize_, topFreq2.MaxSize_);
-
- Freqs_.reserve(std::max(MaxSize_ + 1, ui32(topFreq1.Freqs_.size() + topFreq2.Freqs_.size())));
- Indices_.reserve(MaxSize_ + 1);
-
- Add(topFreq1);
- Add(topFreq2);
-}
-
-template <typename THash, typename TEquals>
-void TTopFreqBase<THash, TEquals>::Deserialize(const TUnboxedValuePod& serialized) {
- MinSize_ = serialized.GetElement(0).Get<ui32>();
- MaxSize_ = serialized.GetElement(1).Get<ui32>();
-
+
+ AddValue(value);
+}
+
+template <typename THash, typename TEquals>
+void TTopFreqBase<THash, TEquals>::Merge(const TTopFreqBase& topFreq1, const TTopFreqBase& topFreq2) {
+ MinSize_ = std::max(topFreq1.MinSize_, topFreq2.MinSize_);
+ MaxSize_ = std::max(topFreq1.MaxSize_, topFreq2.MaxSize_);
+
+ Freqs_.reserve(std::max(MaxSize_ + 1, ui32(topFreq1.Freqs_.size() + topFreq2.Freqs_.size())));
+ Indices_.reserve(MaxSize_ + 1);
+
+ Add(topFreq1);
+ Add(topFreq2);
+}
+
+template <typename THash, typename TEquals>
+void TTopFreqBase<THash, TEquals>::Deserialize(const TUnboxedValuePod& serialized) {
+ MinSize_ = serialized.GetElement(0).Get<ui32>();
+ MaxSize_ = serialized.GetElement(1).Get<ui32>();
+
Freqs_.reserve(MaxSize_ + 1);
Indices_.reserve(MaxSize_ + 1);
-
- const auto listIter = serialized.GetElement(2).GetListIterator();
+
+ const auto listIter = serialized.GetElement(2).GetListIterator();
for (TUnboxedValue current; listIter.Next(current);) {
Update(current.GetElement(1), current.GetElement(0).Get<ui64>());
}
}
-template <typename THash, typename TEquals>
-TUnboxedValue TTopFreqBase<THash, TEquals>::Convert(const IValueBuilder* valueBuilder) const {
+template <typename THash, typename TEquals>
+TUnboxedValue TTopFreqBase<THash, TEquals>::Convert(const IValueBuilder* valueBuilder) const {
TUnboxedValue* values = nullptr;
const auto list = valueBuilder->NewArray(Freqs_.size(), values);
for (const auto& item : Freqs_) {
@@ -60,8 +60,8 @@ TUnboxedValue TTopFreqBase<THash, TEquals>::Convert(const IValueBuilder* valueBu
return list;
}
-template <typename THash, typename TEquals>
-void TTopFreqBase<THash, TEquals>::Add(const TTopFreqBase& otherModeCalc) {
+template <typename THash, typename TEquals>
+void TTopFreqBase<THash, TEquals>::Add(const TTopFreqBase& otherModeCalc) {
for (auto& it : otherModeCalc.Freqs_) {
Update(it.first, it.second);
}
@@ -69,23 +69,23 @@ void TTopFreqBase<THash, TEquals>::Add(const TTopFreqBase& otherModeCalc) {
TryCompress();
}
-template <typename THash, typename TEquals>
-TUnboxedValue TTopFreqBase<THash, TEquals>::Get(const IValueBuilder* builder, ui32 resultSize) {
+template <typename THash, typename TEquals>
+TUnboxedValue TTopFreqBase<THash, TEquals>::Get(const IValueBuilder* builder, ui32 resultSize) {
resultSize = std::min(resultSize, ui32(Freqs_.size()));
Compress(resultSize, true);
return Convert(builder);
}
-template <typename THash, typename TEquals>
-void TTopFreqBase<THash, TEquals>::AddValue(const TUnboxedValuePod& value) {
+template <typename THash, typename TEquals>
+void TTopFreqBase<THash, TEquals>::AddValue(const TUnboxedValuePod& value) {
Update(value, 1);
TryCompress();
}
-template <typename THash, typename TEquals>
-void TTopFreqBase<THash, TEquals>::Update(const TUnboxedValuePod& value, ui64 freq) {
+template <typename THash, typename TEquals>
+void TTopFreqBase<THash, TEquals>::Update(const TUnboxedValuePod& value, ui64 freq) {
Freqs_.emplace_back(TUnboxedValuePod(value), freq);
- auto mapInsertResult = Indices_.emplace(TUnboxedValuePod(value), Freqs_.size() - 1);
+ auto mapInsertResult = Indices_.emplace(TUnboxedValuePod(value), Freqs_.size() - 1);
if (!mapInsertResult.second) {
Freqs_[mapInsertResult.first->second].second += freq;
@@ -93,16 +93,16 @@ void TTopFreqBase<THash, TEquals>::Update(const TUnboxedValuePod& value, ui64 fr
}
}
-template <typename THash, typename TEquals>
-void TTopFreqBase<THash, TEquals>::TryCompress() {
+template <typename THash, typename TEquals>
+void TTopFreqBase<THash, TEquals>::TryCompress() {
ui32 freqSize = Freqs_.size();
if (freqSize > MaxSize_) {
Compress(MinSize_);
}
}
-template <typename THash, typename TEquals>
-void TTopFreqBase<THash, TEquals>::Compress(ui32 newSize, bool sort) {
+template <typename THash, typename TEquals>
+void TTopFreqBase<THash, TEquals>::Compress(ui32 newSize, bool sort) {
auto compare = [](const TVectorElement& v1, const TVectorElement& v2) {
return v1.second > v2.second;
};
@@ -117,12 +117,12 @@ void TTopFreqBase<THash, TEquals>::Compress(ui32 newSize, bool sort) {
Freqs_.resize(newSize);
for (ui32 i = 0; i < newSize; i++) {
- Indices_[Freqs_[i].first] = i;
+ Indices_[Freqs_[i].first] = i;
}
}
-template <typename THash, typename TEquals>
-TUnboxedValue TTopFreqBase<THash, TEquals>::Serialize(const IValueBuilder* builder) {
+template <typename THash, typename TEquals>
+TUnboxedValue TTopFreqBase<THash, TEquals>::Serialize(const IValueBuilder* builder) {
if (ui32(Freqs_.size()) > MinSize_) {
Compress(MinSize_);
}
@@ -135,79 +135,79 @@ TUnboxedValue TTopFreqBase<THash, TEquals>::Serialize(const IValueBuilder* build
return tuple;
}
-template <EDataSlot Slot>
-TTopFreqData<Slot>::TTopFreqData(const TUnboxedValuePod& value, const ui32 minSize, const ui32 maxSize)
- : TBase(TUnboxedValueHash<Slot>(), TUnboxedValueEquals<Slot>())
-{
- TBase::Init(value, minSize, maxSize);
-}
-
-template <EDataSlot Slot>
-TTopFreqData<Slot>::TTopFreqData(const TTopFreqData& topFreq1, const TTopFreqData& topFreq2)
- : TBase(TUnboxedValueHash<Slot>(), TUnboxedValueEquals<Slot>())
-{
- TBase::Merge(topFreq1, topFreq2);
-}
-
-template <EDataSlot Slot>
-TTopFreqData<Slot>::TTopFreqData(const TUnboxedValuePod& serialized)
- : TBase(TUnboxedValueHash<Slot>(), TUnboxedValueEquals<Slot>())
-{
- TBase::Deserialize(serialized);
-}
-
-template <EDataSlot Slot>
-TUnboxedValue TTopFreqData<Slot>::Serialize(const IValueBuilder* builder) {
- return TBase::Serialize(builder);
-}
-
-template <EDataSlot Slot>
-TUnboxedValue TTopFreqData<Slot>::Get(const IValueBuilder* builder, ui32 resultSize) {
- return TBase::Get(builder, resultSize);
-}
-
-template <EDataSlot Slot>
-void TTopFreqData<Slot>::AddValue(const TUnboxedValuePod& value) {
- TBase::AddValue(value);
-}
-
-#define INSTANCE_FOR(slot, ...) \
- template class TTopFreqData<EDataSlot::slot>;
-
-UDF_TYPE_ID_MAP(INSTANCE_FOR)
-
+template <EDataSlot Slot>
+TTopFreqData<Slot>::TTopFreqData(const TUnboxedValuePod& value, const ui32 minSize, const ui32 maxSize)
+ : TBase(TUnboxedValueHash<Slot>(), TUnboxedValueEquals<Slot>())
+{
+ TBase::Init(value, minSize, maxSize);
+}
+
+template <EDataSlot Slot>
+TTopFreqData<Slot>::TTopFreqData(const TTopFreqData& topFreq1, const TTopFreqData& topFreq2)
+ : TBase(TUnboxedValueHash<Slot>(), TUnboxedValueEquals<Slot>())
+{
+ TBase::Merge(topFreq1, topFreq2);
+}
+
+template <EDataSlot Slot>
+TTopFreqData<Slot>::TTopFreqData(const TUnboxedValuePod& serialized)
+ : TBase(TUnboxedValueHash<Slot>(), TUnboxedValueEquals<Slot>())
+{
+ TBase::Deserialize(serialized);
+}
+
+template <EDataSlot Slot>
+TUnboxedValue TTopFreqData<Slot>::Serialize(const IValueBuilder* builder) {
+ return TBase::Serialize(builder);
+}
+
+template <EDataSlot Slot>
+TUnboxedValue TTopFreqData<Slot>::Get(const IValueBuilder* builder, ui32 resultSize) {
+ return TBase::Get(builder, resultSize);
+}
+
+template <EDataSlot Slot>
+void TTopFreqData<Slot>::AddValue(const TUnboxedValuePod& value) {
+ TBase::AddValue(value);
+}
+
+#define INSTANCE_FOR(slot, ...) \
+ template class TTopFreqData<EDataSlot::slot>;
+
+UDF_TYPE_ID_MAP(INSTANCE_FOR)
+
#undef INSTANCE_FOR
-
-TTopFreqGeneric::TTopFreqGeneric(const TUnboxedValuePod& value, const ui32 minSize, const ui32 maxSize,
- IHash::TPtr hash, IEquate::TPtr equate)
- : TBase(TGenericHash{hash}, TGenericEquals{equate})
-{
- TBase::Init(value, minSize, maxSize);
-}
-
-TTopFreqGeneric::TTopFreqGeneric(const TTopFreqGeneric& topFreq1, const TTopFreqGeneric& topFreq2,
- IHash::TPtr hash, IEquate::TPtr equate)
- : TBase(TGenericHash{hash}, TGenericEquals{equate})
-{
- TBase::Merge(topFreq1, topFreq2);
-}
-
-TTopFreqGeneric::TTopFreqGeneric(const TUnboxedValuePod& serialized,
- IHash::TPtr hash, IEquate::TPtr equate)
- : TBase(TGenericHash{hash}, TGenericEquals{equate})
-{
- TBase::Deserialize(serialized);
-}
-
-TUnboxedValue TTopFreqGeneric::Serialize(const IValueBuilder* builder) {
- return TBase::Serialize(builder);
-}
-
-TUnboxedValue TTopFreqGeneric::Get(const IValueBuilder* builder, ui32 resultSize) {
- return TBase::Get(builder, resultSize);
-}
-
-void TTopFreqGeneric::AddValue(const TUnboxedValuePod& value) {
- TBase::AddValue(value);
-}
-
+
+TTopFreqGeneric::TTopFreqGeneric(const TUnboxedValuePod& value, const ui32 minSize, const ui32 maxSize,
+ IHash::TPtr hash, IEquate::TPtr equate)
+ : TBase(TGenericHash{hash}, TGenericEquals{equate})
+{
+ TBase::Init(value, minSize, maxSize);
+}
+
+TTopFreqGeneric::TTopFreqGeneric(const TTopFreqGeneric& topFreq1, const TTopFreqGeneric& topFreq2,
+ IHash::TPtr hash, IEquate::TPtr equate)
+ : TBase(TGenericHash{hash}, TGenericEquals{equate})
+{
+ TBase::Merge(topFreq1, topFreq2);
+}
+
+TTopFreqGeneric::TTopFreqGeneric(const TUnboxedValuePod& serialized,
+ IHash::TPtr hash, IEquate::TPtr equate)
+ : TBase(TGenericHash{hash}, TGenericEquals{equate})
+{
+ TBase::Deserialize(serialized);
+}
+
+TUnboxedValue TTopFreqGeneric::Serialize(const IValueBuilder* builder) {
+ return TBase::Serialize(builder);
+}
+
+TUnboxedValue TTopFreqGeneric::Get(const IValueBuilder* builder, ui32 resultSize) {
+ return TBase::Get(builder, resultSize);
+}
+
+void TTopFreqGeneric::AddValue(const TUnboxedValuePod& value) {
+ TBase::AddValue(value);
+}
+
diff --git a/ydb/library/yql/udfs/common/topfreq/static/topfreq.h b/ydb/library/yql/udfs/common/topfreq/static/topfreq.h
index 411190d2571..f329322615b 100644
--- a/ydb/library/yql/udfs/common/topfreq/static/topfreq.h
+++ b/ydb/library/yql/udfs/common/topfreq/static/topfreq.h
@@ -6,9 +6,9 @@
#include <unordered_map>
-template <typename THash, typename TEquals>
-class TTopFreqBase {
-protected:
+template <typename THash, typename TEquals>
+class TTopFreqBase {
+protected:
using TUnboxedValuePod = NKikimr::NUdf::TUnboxedValuePod;
using TUnboxedValue = NKikimr::NUdf::TUnboxedValue;
using IValueBuilder = NKikimr::NUdf::IValueBuilder;
@@ -21,77 +21,77 @@ protected:
ui32 MinSize_ = 0;
ui32 MaxSize_ = 0;
- void Add(const TTopFreqBase& otherCalc);
+ void Add(const TTopFreqBase& otherCalc);
void Update(const TUnboxedValuePod& key, const ui64 value);
void TryCompress();
void Compress(ui32 newSize, bool sort = false);
TUnboxedValue Convert(const IValueBuilder* valueBuilder) const;
-protected:
- TTopFreqBase(THash hash, TEquals equals);
-
- void Init(const TUnboxedValuePod& value, const ui32 minSize, const ui32 maxSize);
- void Merge(const TTopFreqBase& TopFreq1, const TTopFreqBase& TopFreq2);
- void Deserialize(const TUnboxedValuePod& serialized);
+protected:
+ TTopFreqBase(THash hash, TEquals equals);
+ void Init(const TUnboxedValuePod& value, const ui32 minSize, const ui32 maxSize);
+ void Merge(const TTopFreqBase& TopFreq1, const TTopFreqBase& TopFreq2);
+ void Deserialize(const TUnboxedValuePod& serialized);
+
TUnboxedValue Serialize(const IValueBuilder* builder);
TUnboxedValue Get(const IValueBuilder* builder, ui32 resultSize);
- void AddValue(const TUnboxedValuePod& value);
-};
-
-template <NKikimr::NUdf::EDataSlot Slot>
-class TTopFreqData
- : public TTopFreqBase<
- NKikimr::NUdf::TUnboxedValueHash<Slot>,
- NKikimr::NUdf::TUnboxedValueEquals<Slot>>
-{
-public:
- using TBase = TTopFreqBase<
- NKikimr::NUdf::TUnboxedValueHash<Slot>,
- NKikimr::NUdf::TUnboxedValueEquals<Slot>>;
-
- TTopFreqData(const NKikimr::NUdf::TUnboxedValuePod& value, const ui32 minSize, const ui32 maxSize);
- TTopFreqData(const TTopFreqData& topFreq1, const TTopFreqData& topFreq2);
- TTopFreqData(const NKikimr::NUdf::TUnboxedValuePod& serialized);
-
- NKikimr::NUdf::TUnboxedValue Serialize(const NKikimr::NUdf::IValueBuilder* builder);
- NKikimr::NUdf::TUnboxedValue Get(const NKikimr::NUdf::IValueBuilder* builder, ui32 resultSize);
- void AddValue(const NKikimr::NUdf::TUnboxedValuePod& value);
-};
-
-struct TGenericHash {
- NKikimr::NUdf::IHash::TPtr Hash;
-
- std::size_t operator()(const NKikimr::NUdf::TUnboxedValuePod& value) const {
- return Hash->Hash(value);
- }
-};
-
-struct TGenericEquals {
- NKikimr::NUdf::IEquate::TPtr Equate;
-
- bool operator()(
- const NKikimr::NUdf::TUnboxedValuePod& left,
- const NKikimr::NUdf::TUnboxedValuePod& right) const
- {
- return Equate->Equals(left, right);
- }
-};
-
-class TTopFreqGeneric
- : public TTopFreqBase<TGenericHash, TGenericEquals>
-{
-public:
- using TBase = TTopFreqBase<TGenericHash, TGenericEquals>;
-
- TTopFreqGeneric(const NKikimr::NUdf::TUnboxedValuePod& value, const ui32 minSize, const ui32 maxSize,
- NKikimr::NUdf::IHash::TPtr hash, NKikimr::NUdf::IEquate::TPtr equate);
- TTopFreqGeneric(const TTopFreqGeneric& topFreq1, const TTopFreqGeneric& topFreq2,
- NKikimr::NUdf::IHash::TPtr hash, NKikimr::NUdf::IEquate::TPtr equate);
- TTopFreqGeneric(const NKikimr::NUdf::TUnboxedValuePod& serialized,
- NKikimr::NUdf::IHash::TPtr hash, NKikimr::NUdf::IEquate::TPtr equate);
-
- NKikimr::NUdf::TUnboxedValue Serialize(const NKikimr::NUdf::IValueBuilder* builder);
- NKikimr::NUdf::TUnboxedValue Get(const NKikimr::NUdf::IValueBuilder* builder, ui32 resultSize);
- void AddValue(const NKikimr::NUdf::TUnboxedValuePod& value);
+ void AddValue(const TUnboxedValuePod& value);
};
+
+template <NKikimr::NUdf::EDataSlot Slot>
+class TTopFreqData
+ : public TTopFreqBase<
+ NKikimr::NUdf::TUnboxedValueHash<Slot>,
+ NKikimr::NUdf::TUnboxedValueEquals<Slot>>
+{
+public:
+ using TBase = TTopFreqBase<
+ NKikimr::NUdf::TUnboxedValueHash<Slot>,
+ NKikimr::NUdf::TUnboxedValueEquals<Slot>>;
+
+ TTopFreqData(const NKikimr::NUdf::TUnboxedValuePod& value, const ui32 minSize, const ui32 maxSize);
+ TTopFreqData(const TTopFreqData& topFreq1, const TTopFreqData& topFreq2);
+ TTopFreqData(const NKikimr::NUdf::TUnboxedValuePod& serialized);
+
+ NKikimr::NUdf::TUnboxedValue Serialize(const NKikimr::NUdf::IValueBuilder* builder);
+ NKikimr::NUdf::TUnboxedValue Get(const NKikimr::NUdf::IValueBuilder* builder, ui32 resultSize);
+ void AddValue(const NKikimr::NUdf::TUnboxedValuePod& value);
+};
+
+struct TGenericHash {
+ NKikimr::NUdf::IHash::TPtr Hash;
+
+ std::size_t operator()(const NKikimr::NUdf::TUnboxedValuePod& value) const {
+ return Hash->Hash(value);
+ }
+};
+
+struct TGenericEquals {
+ NKikimr::NUdf::IEquate::TPtr Equate;
+
+ bool operator()(
+ const NKikimr::NUdf::TUnboxedValuePod& left,
+ const NKikimr::NUdf::TUnboxedValuePod& right) const
+ {
+ return Equate->Equals(left, right);
+ }
+};
+
+class TTopFreqGeneric
+ : public TTopFreqBase<TGenericHash, TGenericEquals>
+{
+public:
+ using TBase = TTopFreqBase<TGenericHash, TGenericEquals>;
+
+ TTopFreqGeneric(const NKikimr::NUdf::TUnboxedValuePod& value, const ui32 minSize, const ui32 maxSize,
+ NKikimr::NUdf::IHash::TPtr hash, NKikimr::NUdf::IEquate::TPtr equate);
+ TTopFreqGeneric(const TTopFreqGeneric& topFreq1, const TTopFreqGeneric& topFreq2,
+ NKikimr::NUdf::IHash::TPtr hash, NKikimr::NUdf::IEquate::TPtr equate);
+ TTopFreqGeneric(const NKikimr::NUdf::TUnboxedValuePod& serialized,
+ NKikimr::NUdf::IHash::TPtr hash, NKikimr::NUdf::IEquate::TPtr equate);
+
+ NKikimr::NUdf::TUnboxedValue Serialize(const NKikimr::NUdf::IValueBuilder* builder);
+ NKikimr::NUdf::TUnboxedValue Get(const NKikimr::NUdf::IValueBuilder* builder, ui32 resultSize);
+ void AddValue(const NKikimr::NUdf::TUnboxedValuePod& value);
+};
diff --git a/ydb/library/yql/udfs/common/topfreq/static/topfreq_udf.h b/ydb/library/yql/udfs/common/topfreq/static/topfreq_udf.h
index 5ea0a5f7294..ba483f2f4a6 100644
--- a/ydb/library/yql/udfs/common/topfreq/static/topfreq_udf.h
+++ b/ydb/library/yql/udfs/common/topfreq/static/topfreq_udf.h
@@ -15,203 +15,203 @@ using namespace NYql;
using namespace NUdf;
namespace {
- extern const char TopFreqResourceNameGeneric[] = "TopFreq.TopFreqResource.Generic";
- class TTopFreqResource:
- public TBoxedResource<TTopFreqGeneric, TopFreqResourceNameGeneric>
- {
- public:
- template <typename... Args>
- inline TTopFreqResource(Args&&... args)
- : TBoxedResource(std::forward<Args>(args)...)
- {}
- };
-
- template <EDataSlot Slot>
- class TTopFreqResourceData;
-
- template <EDataSlot Slot>
- TTopFreqResourceData<Slot>* GetTopFreqResourceData(const TUnboxedValuePod& arg) {
- TTopFreqResourceData<Slot>::Validate(arg);
- return static_cast<TTopFreqResourceData<Slot>*>(arg.AsBoxed().Get());
+ extern const char TopFreqResourceNameGeneric[] = "TopFreq.TopFreqResource.Generic";
+ class TTopFreqResource:
+ public TBoxedResource<TTopFreqGeneric, TopFreqResourceNameGeneric>
+ {
+ public:
+ template <typename... Args>
+ inline TTopFreqResource(Args&&... args)
+ : TBoxedResource(std::forward<Args>(args)...)
+ {}
+ };
+
+ template <EDataSlot Slot>
+ class TTopFreqResourceData;
+
+ template <EDataSlot Slot>
+ TTopFreqResourceData<Slot>* GetTopFreqResourceData(const TUnboxedValuePod& arg) {
+ TTopFreqResourceData<Slot>::Validate(arg);
+ return static_cast<TTopFreqResourceData<Slot>*>(arg.AsBoxed().Get());
}
- TTopFreqResource* GetTopFreqResource(const TUnboxedValuePod& arg) {
- TTopFreqResource::Validate(arg);
- return static_cast<TTopFreqResource*>(arg.AsBoxed().Get());
- }
-
-
- template <EDataSlot Slot>
- class TTopFreq_CreateData: public TBoxedValue {
- private:
- TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const {
- ui32 minSize = args[1].Get<ui32>();
- return TUnboxedValuePod(new TTopFreqResourceData<Slot>(args[0], minSize, minSize * 2));
- }
- };
-
+ TTopFreqResource* GetTopFreqResource(const TUnboxedValuePod& arg) {
+ TTopFreqResource::Validate(arg);
+ return static_cast<TTopFreqResource*>(arg.AsBoxed().Get());
+ }
+
+
+ template <EDataSlot Slot>
+ class TTopFreq_CreateData: public TBoxedValue {
+ private:
+ TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const {
+ ui32 minSize = args[1].Get<ui32>();
+ return TUnboxedValuePod(new TTopFreqResourceData<Slot>(args[0], minSize, minSize * 2));
+ }
+ };
+
class TTopFreq_Create: public TBoxedValue {
private:
- TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const {
+ TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const {
ui32 minSize = args[1].Get<ui32>();
- return TUnboxedValuePod(new TTopFreqResource(args[0], minSize, minSize * 2, Hash_, Equate_));
+ return TUnboxedValuePod(new TTopFreqResource(args[0], minSize, minSize * 2, Hash_, Equate_));
}
public:
- TTopFreq_Create(IHash::TPtr hash, IEquate::TPtr equate)
- : Hash_(hash)
- , Equate_(equate)
- {}
-
- private:
- IHash::TPtr Hash_;
- IEquate::TPtr Equate_;
- };
-
- template <EDataSlot Slot>
- class TTopFreq_AddValueData: public TBoxedValue {
- private:
- TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const {
- const auto topFreq = GetTopFreqResourceData<Slot>(args[0]);
- topFreq->Get()->AddValue(args[1]);
- return TUnboxedValuePod(topFreq);
+ TTopFreq_Create(IHash::TPtr hash, IEquate::TPtr equate)
+ : Hash_(hash)
+ , Equate_(equate)
+ {}
+
+ private:
+ IHash::TPtr Hash_;
+ IEquate::TPtr Equate_;
+ };
+
+ template <EDataSlot Slot>
+ class TTopFreq_AddValueData: public TBoxedValue {
+ private:
+ TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const {
+ const auto topFreq = GetTopFreqResourceData<Slot>(args[0]);
+ topFreq->Get()->AddValue(args[1]);
+ return TUnboxedValuePod(topFreq);
}
};
class TTopFreq_AddValue: public TBoxedValue {
private:
- TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const {
- const auto topFreq = GetTopFreqResource(args[0]);
+ TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const {
+ const auto topFreq = GetTopFreqResource(args[0]);
topFreq->Get()->AddValue(args[1]);
return TUnboxedValuePod(topFreq);
}
- };
+ };
- template <EDataSlot Slot>
- class TTopFreq_SerializeData: public TBoxedValue {
- private:
- TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const {
- return GetTopFreqResourceData<Slot>(args[0])->Get()->Serialize(valueBuilder);
+ template <EDataSlot Slot>
+ class TTopFreq_SerializeData: public TBoxedValue {
+ private:
+ TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const {
+ return GetTopFreqResourceData<Slot>(args[0])->Get()->Serialize(valueBuilder);
}
};
class TTopFreq_Serialize: public TBoxedValue {
private:
- TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const {
- return GetTopFreqResource(args[0])->Get()->Serialize(valueBuilder);
+ TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const {
+ return GetTopFreqResource(args[0])->Get()->Serialize(valueBuilder);
}
- };
+ };
- template <EDataSlot Slot>
- class TTopFreq_DeserializeData: public TBoxedValue {
- private:
- TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const {
- return TUnboxedValuePod(new TTopFreqResourceData<Slot>(args[0]));
+ template <EDataSlot Slot>
+ class TTopFreq_DeserializeData: public TBoxedValue {
+ private:
+ TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const {
+ return TUnboxedValuePod(new TTopFreqResourceData<Slot>(args[0]));
}
};
class TTopFreq_Deserialize: public TBoxedValue {
private:
- TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const {
- return TUnboxedValuePod(new TTopFreqResource(args[0], Hash_, Equate_));
+ TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const {
+ return TUnboxedValuePod(new TTopFreqResource(args[0], Hash_, Equate_));
}
public:
- TTopFreq_Deserialize(IHash::TPtr hash, IEquate::TPtr equate)
- : Hash_(hash)
- , Equate_(equate)
- {}
-
- private:
- IHash::TPtr Hash_;
- IEquate::TPtr Equate_;
- };
-
- template <EDataSlot Slot>
- class TTopFreq_MergeData: public TBoxedValue {
- private:
- TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const {
- const auto topFreq0 = GetTopFreqResourceData<Slot>(args[0]);
- const auto topFreq1 = GetTopFreqResourceData<Slot>(args[1]);
- return TUnboxedValuePod(new TTopFreqResourceData<Slot>(*topFreq0->Get(), *topFreq1->Get()));
+ TTopFreq_Deserialize(IHash::TPtr hash, IEquate::TPtr equate)
+ : Hash_(hash)
+ , Equate_(equate)
+ {}
+
+ private:
+ IHash::TPtr Hash_;
+ IEquate::TPtr Equate_;
+ };
+
+ template <EDataSlot Slot>
+ class TTopFreq_MergeData: public TBoxedValue {
+ private:
+ TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const {
+ const auto topFreq0 = GetTopFreqResourceData<Slot>(args[0]);
+ const auto topFreq1 = GetTopFreqResourceData<Slot>(args[1]);
+ return TUnboxedValuePod(new TTopFreqResourceData<Slot>(*topFreq0->Get(), *topFreq1->Get()));
}
};
class TTopFreq_Merge: public TBoxedValue {
private:
- TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const {
- const auto topFreq0 = GetTopFreqResource(args[0]);
- const auto topFreq1 = GetTopFreqResource(args[1]);
- return TUnboxedValuePod(new TTopFreqResource(*topFreq0->Get(), *topFreq1->Get(), Hash_, Equate_));
+ TUnboxedValue Run(const IValueBuilder*, const TUnboxedValuePod* args) const {
+ const auto topFreq0 = GetTopFreqResource(args[0]);
+ const auto topFreq1 = GetTopFreqResource(args[1]);
+ return TUnboxedValuePod(new TTopFreqResource(*topFreq0->Get(), *topFreq1->Get(), Hash_, Equate_));
}
public:
- TTopFreq_Merge(IHash::TPtr hash, IEquate::TPtr equate)
- : Hash_(hash)
- , Equate_(equate)
- {}
-
- private:
- IHash::TPtr Hash_;
- IEquate::TPtr Equate_;
+ TTopFreq_Merge(IHash::TPtr hash, IEquate::TPtr equate)
+ : Hash_(hash)
+ , Equate_(equate)
+ {}
+
+ private:
+ IHash::TPtr Hash_;
+ IEquate::TPtr Equate_;
};
- template <EDataSlot Slot>
- class TTopFreq_GetData: public TBoxedValue {
+ template <EDataSlot Slot>
+ class TTopFreq_GetData: public TBoxedValue {
private:
- TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const {
- return GetTopFreqResourceData<Slot>(args[0])->Get()->Get(valueBuilder, args[1].Get<ui32>());
+ TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const {
+ return GetTopFreqResourceData<Slot>(args[0])->Get()->Get(valueBuilder, args[1].Get<ui32>());
}
- };
+ };
- class TTopFreq_Get: public TBoxedValue {
- private:
- TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const {
- return GetTopFreqResource(args[0])->Get()->Get(valueBuilder, args[1].Get<ui32>());
+ class TTopFreq_Get: public TBoxedValue {
+ private:
+ TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const {
+ return GetTopFreqResource(args[0])->Get()->Get(valueBuilder, args[1].Get<ui32>());
}
};
-
-#define MAKE_RESOURCE(slot, ...) \
- extern const char TopFreqResourceName##slot[] = "TopFreq.TopFreqResource."#slot; \
- template <> \
- class TTopFreqResourceData<EDataSlot::slot>: \
- public TBoxedResource<TTopFreqData<EDataSlot::slot>, TopFreqResourceName##slot> \
- { \
- public: \
- template <typename... Args> \
- inline TTopFreqResourceData(Args&&... args) \
- : TBoxedResource(std::forward<Args>(args)...) \
- {} \
- };
-
- UDF_TYPE_ID_MAP(MAKE_RESOURCE)
-
-#define MAKE_IMPL(operation, slot) \
- case EDataSlot::slot: \
- builder.Implementation(new operation<EDataSlot::slot>); \
- break;
-
-#define MAKE_CREATE(slot, ...) MAKE_IMPL(TTopFreq_CreateData, slot)
-#define MAKE_ADD_VALUE(slot, ...) MAKE_IMPL(TTopFreq_AddValueData, slot)
-#define MAKE_SERIALIZE(slot, ...) MAKE_IMPL(TTopFreq_SerializeData, slot)
-#define MAKE_DESERIALIZE(slot, ...) MAKE_IMPL(TTopFreq_DeserializeData, slot)
-#define MAKE_MERGE(slot, ...) MAKE_IMPL(TTopFreq_MergeData, slot)
-#define MAKE_GET(slot, ...) MAKE_IMPL(TTopFreq_GetData, slot)
-
-#define MAKE_TYPE(slot, ...) \
- case EDataSlot::slot: \
- topFreqType = builder.Resource(TopFreqResourceName##slot); \
- break;
-
-
- static const auto CreateName = TStringRef::Of("TopFreq_Create");
- static const auto AddValueName = TStringRef::Of("TopFreq_AddValue");
- static const auto SerializeName = TStringRef::Of("TopFreq_Serialize");
- static const auto DeserializeName = TStringRef::Of("TopFreq_Deserialize");
- static const auto MergeName = TStringRef::Of("TopFreq_Merge");
- static const auto GetName = TStringRef::Of("TopFreq_Get");
-
+
+#define MAKE_RESOURCE(slot, ...) \
+ extern const char TopFreqResourceName##slot[] = "TopFreq.TopFreqResource."#slot; \
+ template <> \
+ class TTopFreqResourceData<EDataSlot::slot>: \
+ public TBoxedResource<TTopFreqData<EDataSlot::slot>, TopFreqResourceName##slot> \
+ { \
+ public: \
+ template <typename... Args> \
+ inline TTopFreqResourceData(Args&&... args) \
+ : TBoxedResource(std::forward<Args>(args)...) \
+ {} \
+ };
+
+ UDF_TYPE_ID_MAP(MAKE_RESOURCE)
+
+#define MAKE_IMPL(operation, slot) \
+ case EDataSlot::slot: \
+ builder.Implementation(new operation<EDataSlot::slot>); \
+ break;
+
+#define MAKE_CREATE(slot, ...) MAKE_IMPL(TTopFreq_CreateData, slot)
+#define MAKE_ADD_VALUE(slot, ...) MAKE_IMPL(TTopFreq_AddValueData, slot)
+#define MAKE_SERIALIZE(slot, ...) MAKE_IMPL(TTopFreq_SerializeData, slot)
+#define MAKE_DESERIALIZE(slot, ...) MAKE_IMPL(TTopFreq_DeserializeData, slot)
+#define MAKE_MERGE(slot, ...) MAKE_IMPL(TTopFreq_MergeData, slot)
+#define MAKE_GET(slot, ...) MAKE_IMPL(TTopFreq_GetData, slot)
+
+#define MAKE_TYPE(slot, ...) \
+ case EDataSlot::slot: \
+ topFreqType = builder.Resource(TopFreqResourceName##slot); \
+ break;
+
+
+ static const auto CreateName = TStringRef::Of("TopFreq_Create");
+ static const auto AddValueName = TStringRef::Of("TopFreq_AddValue");
+ static const auto SerializeName = TStringRef::Of("TopFreq_Serialize");
+ static const auto DeserializeName = TStringRef::Of("TopFreq_Deserialize");
+ static const auto MergeName = TStringRef::Of("TopFreq_Merge");
+ static const auto GetName = TStringRef::Of("TopFreq_Get");
+
class TTopFreqModule: public IUdfModule {
public:
TStringRef Name() const {
@@ -222,167 +222,167 @@ namespace {
}
void GetAllFunctions(IFunctionsSink& sink) const final {
- sink.Add(CreateName)->SetTypeAwareness();
- sink.Add(AddValueName)->SetTypeAwareness();
- sink.Add(SerializeName)->SetTypeAwareness();
- sink.Add(DeserializeName)->SetTypeAwareness();
- sink.Add(MergeName)->SetTypeAwareness();
- sink.Add(GetName)->SetTypeAwareness();
+ sink.Add(CreateName)->SetTypeAwareness();
+ sink.Add(AddValueName)->SetTypeAwareness();
+ sink.Add(SerializeName)->SetTypeAwareness();
+ sink.Add(DeserializeName)->SetTypeAwareness();
+ sink.Add(MergeName)->SetTypeAwareness();
+ sink.Add(GetName)->SetTypeAwareness();
}
void BuildFunctionTypeInfo(
const TStringRef& name,
- TType* userType,
+ TType* userType,
const TStringRef& typeConfig,
ui32 flags,
- IFunctionTypeInfoBuilder& builder) const final
- {
+ IFunctionTypeInfoBuilder& builder) const final
+ {
Y_UNUSED(typeConfig);
try {
const bool typesOnly = (flags & TFlags::TypesOnly);
builder.UserType(userType);
- auto typeHelper = builder.TypeInfoHelper();
-
- auto userTypeInspector = TTupleTypeInspector(*typeHelper, userType);
- if (!userTypeInspector || userTypeInspector.GetElementsCount() != 3) {
- builder.SetError("User type is not a 3-tuple");
- return;
- }
-
- bool isGeneric = false;
- IHash::TPtr hash;
- IEquate::TPtr equate;
- TMaybe<EDataSlot> slot;
-
- auto valueType = userTypeInspector.GetElementType(2);
- auto valueTypeInspector = TDataTypeInspector(*typeHelper, valueType);
- if (!valueTypeInspector) {
- isGeneric = true;
- hash = builder.MakeHash(valueType);
- equate = builder.MakeEquate(valueType);
- if (!hash || !equate) {
- return;
- }
- } else {
- slot = FindDataSlot(valueTypeInspector.GetTypeId());
- if (!slot) {
- builder.SetError("Unknown data type");
- return;
- }
- const auto& features = NUdf::GetDataTypeInfo(*slot).Features;
- if (!(features & NUdf::CanHash) || !(features & NUdf::CanEquate)) {
- builder.SetError("Data type is not hashable or equatable");
- return;
- }
- }
-
- auto serializedItemType = builder.Tuple()->Add<ui64>().Add(valueType).Build();
- auto serializedListType = builder.List()->Item(serializedItemType).Build();
- auto serializedType = builder.Tuple()->Add<ui32>().Add<ui32>().Add(serializedListType).Build();
-
- TType* topFreqType = nullptr;
- if (isGeneric) {
- topFreqType = builder.Resource(TopFreqResourceNameGeneric);
- } else {
- switch (*slot) {
- UDF_TYPE_ID_MAP(MAKE_TYPE)
- }
- }
-
- if (name == CreateName) {
- builder.Args()->Add(valueType).Add<ui32>().Done().Returns(topFreqType);
+ auto typeHelper = builder.TypeInfoHelper();
+
+ auto userTypeInspector = TTupleTypeInspector(*typeHelper, userType);
+ if (!userTypeInspector || userTypeInspector.GetElementsCount() != 3) {
+ builder.SetError("User type is not a 3-tuple");
+ return;
+ }
+
+ bool isGeneric = false;
+ IHash::TPtr hash;
+ IEquate::TPtr equate;
+ TMaybe<EDataSlot> slot;
+
+ auto valueType = userTypeInspector.GetElementType(2);
+ auto valueTypeInspector = TDataTypeInspector(*typeHelper, valueType);
+ if (!valueTypeInspector) {
+ isGeneric = true;
+ hash = builder.MakeHash(valueType);
+ equate = builder.MakeEquate(valueType);
+ if (!hash || !equate) {
+ return;
+ }
+ } else {
+ slot = FindDataSlot(valueTypeInspector.GetTypeId());
+ if (!slot) {
+ builder.SetError("Unknown data type");
+ return;
+ }
+ const auto& features = NUdf::GetDataTypeInfo(*slot).Features;
+ if (!(features & NUdf::CanHash) || !(features & NUdf::CanEquate)) {
+ builder.SetError("Data type is not hashable or equatable");
+ return;
+ }
+ }
+
+ auto serializedItemType = builder.Tuple()->Add<ui64>().Add(valueType).Build();
+ auto serializedListType = builder.List()->Item(serializedItemType).Build();
+ auto serializedType = builder.Tuple()->Add<ui32>().Add<ui32>().Add(serializedListType).Build();
+
+ TType* topFreqType = nullptr;
+ if (isGeneric) {
+ topFreqType = builder.Resource(TopFreqResourceNameGeneric);
+ } else {
+ switch (*slot) {
+ UDF_TYPE_ID_MAP(MAKE_TYPE)
+ }
+ }
+
+ if (name == CreateName) {
+ builder.Args()->Add(valueType).Add<ui32>().Done().Returns(topFreqType);
if (!typesOnly) {
- if (isGeneric) {
- builder.Implementation(new TTopFreq_Create(hash, equate));
- } else {
- switch (*slot) {
- UDF_TYPE_ID_MAP(MAKE_CREATE)
- }
+ if (isGeneric) {
+ builder.Implementation(new TTopFreq_Create(hash, equate));
+ } else {
+ switch (*slot) {
+ UDF_TYPE_ID_MAP(MAKE_CREATE)
+ }
}
}
}
- if (name == AddValueName) {
- builder.Args()->Add(topFreqType).Add(valueType).Done().Returns(topFreqType);
+ if (name == AddValueName) {
+ builder.Args()->Add(topFreqType).Add(valueType).Done().Returns(topFreqType);
if (!typesOnly) {
- if (isGeneric) {
- builder.Implementation(new TTopFreq_AddValue);
- } else {
- switch (*slot) {
- UDF_TYPE_ID_MAP(MAKE_ADD_VALUE)
- }
+ if (isGeneric) {
+ builder.Implementation(new TTopFreq_AddValue);
+ } else {
+ switch (*slot) {
+ UDF_TYPE_ID_MAP(MAKE_ADD_VALUE)
+ }
}
}
}
- if (name == MergeName) {
+ if (name == MergeName) {
builder.Args()->Add(topFreqType).Add(topFreqType).Done().Returns(topFreqType);
if (!typesOnly) {
- if (isGeneric) {
- builder.Implementation(new TTopFreq_Merge(hash, equate));
- } else {
- switch (*slot) {
- UDF_TYPE_ID_MAP(MAKE_MERGE)
- }
+ if (isGeneric) {
+ builder.Implementation(new TTopFreq_Merge(hash, equate));
+ } else {
+ switch (*slot) {
+ UDF_TYPE_ID_MAP(MAKE_MERGE)
+ }
}
}
}
- if (name == SerializeName) {
- builder.Args()->Add(topFreqType).Done().Returns(serializedType);
+ if (name == SerializeName) {
+ builder.Args()->Add(topFreqType).Done().Returns(serializedType);
if (!typesOnly) {
- if (isGeneric) {
- builder.Implementation(new TTopFreq_Serialize);
- } else {
- switch (*slot) {
- UDF_TYPE_ID_MAP(MAKE_SERIALIZE)
- }
+ if (isGeneric) {
+ builder.Implementation(new TTopFreq_Serialize);
+ } else {
+ switch (*slot) {
+ UDF_TYPE_ID_MAP(MAKE_SERIALIZE)
+ }
}
}
}
- if (name == DeserializeName) {
- builder.Args()->Add(serializedType).Done().Returns(topFreqType);
+ if (name == DeserializeName) {
+ builder.Args()->Add(serializedType).Done().Returns(topFreqType);
if (!typesOnly) {
- if (isGeneric) {
- builder.Implementation(new TTopFreq_Deserialize(hash, equate));
- } else {
- switch (*slot) {
- UDF_TYPE_ID_MAP(MAKE_DESERIALIZE)
- }
+ if (isGeneric) {
+ builder.Implementation(new TTopFreq_Deserialize(hash, equate));
+ } else {
+ switch (*slot) {
+ UDF_TYPE_ID_MAP(MAKE_DESERIALIZE)
+ }
}
}
}
- if (name == GetName) {
+ if (name == GetName) {
ui32 indexF, indexV;
- auto itemType = builder.Struct()->AddField<ui64>("Frequency", &indexF).AddField("Value", valueType, &indexV).Build();
- auto resultType = builder.List()->Item(itemType).Build();
+ auto itemType = builder.Struct()->AddField<ui64>("Frequency", &indexF).AddField("Value", valueType, &indexV).Build();
+ auto resultType = builder.List()->Item(itemType).Build();
- builder.Args()->Add(topFreqType).Add<ui32>().Done().Returns(resultType);
+ builder.Args()->Add(topFreqType).Add<ui32>().Done().Returns(resultType);
if (!typesOnly) {
- if (isGeneric) {
- builder.Implementation(new TTopFreq_Get);
- } else {
- switch (*slot) {
- UDF_TYPE_ID_MAP(MAKE_GET)
- }
+ if (isGeneric) {
+ builder.Implementation(new TTopFreq_Get);
+ } else {
+ switch (*slot) {
+ UDF_TYPE_ID_MAP(MAKE_GET)
+ }
}
}
}
-
+
} catch (const std::exception& e) {
builder.SetError(CurrentExceptionMessage());
}
}
};
-
-} // namespace
+
+} // namespace
diff --git a/ydb/library/yql/udfs/common/topfreq/static/ya.make b/ydb/library/yql/udfs/common/topfreq/static/ya.make
index ce5d98cb6b2..64f5124e414 100644
--- a/ydb/library/yql/udfs/common/topfreq/static/ya.make
+++ b/ydb/library/yql/udfs/common/topfreq/static/ya.make
@@ -2,7 +2,7 @@ LIBRARY()
YQL_ABI_VERSION(
2
- 10
+ 10
0
)
diff --git a/ydb/library/yql/udfs/common/topfreq/topfreq_udf_ut.cpp b/ydb/library/yql/udfs/common/topfreq/topfreq_udf_ut.cpp
index c6f6ecab72a..12597a7f92e 100644
--- a/ydb/library/yql/udfs/common/topfreq/topfreq_udf_ut.cpp
+++ b/ydb/library/yql/udfs/common/topfreq/topfreq_udf_ut.cpp
@@ -155,7 +155,7 @@ namespace NYql {
auto getArgsType = pgmBuilder.NewTupleType({resourceType, ui32Type});
auto getUserType = pgmBuilder.NewTupleType({getArgsType, emptyStructType, valueType});
auto udfTopFreq_Get = pgmBuilder.Udf("TopFreq.TopFreq_Get", TRuntimeNode(), getUserType);
-
+
TRuntimeNode pgmTopFreq;
{
auto value = pgmBuilder.NewDataLiteral<ui64>(1);
@@ -245,22 +245,22 @@ namespace NYql {
auto addValueArgsType = pgmBuilder.NewTupleType({resourceType, valueType});
auto addValueUserType = pgmBuilder.NewTupleType({addValueArgsType, emptyStructType, valueType});
auto udfTopFreq_AddValue = pgmBuilder.Udf("TopFreq.TopFreq_AddValue", TRuntimeNode(), addValueUserType);
-
+
auto getArgsType = pgmBuilder.NewTupleType({resourceType, ui32Type});
auto getUserType = pgmBuilder.NewTupleType({getArgsType, emptyStructType, valueType});
auto udfTopFreq_Get = pgmBuilder.Udf("TopFreq.TopFreq_Get", TRuntimeNode(), getUserType);
-
+
auto serializeArgsType = pgmBuilder.NewTupleType({resourceType});
auto serializeUserType = pgmBuilder.NewTupleType({serializeArgsType, emptyStructType, valueType});
auto udfTopFreq_Serialize = pgmBuilder.Udf("TopFreq.TopFreq_Serialize", TRuntimeNode(), serializeUserType);
-
+
auto serializedType = pgmBuilder.NewTupleType({ui32Type, ui32Type,
pgmBuilder.NewListType(pgmBuilder.NewTupleType({ui64Type, valueType}))});
-
+
auto deserializeArgsType = pgmBuilder.NewTupleType({serializedType});
auto deserializeUserType = pgmBuilder.NewTupleType({deserializeArgsType, emptyStructType, valueType});
auto udfTopFreq_Deserialize = pgmBuilder.Udf("TopFreq.TopFreq_Deserialize", TRuntimeNode(), deserializeUserType);
-
+
TRuntimeNode pgmTopFreq;
{
auto value = pgmBuilder.NewDataLiteral<bool>(true);
@@ -338,22 +338,22 @@ namespace NYql {
auto mergeArgsType = pgmBuilder.NewTupleType({resourceType, resourceType});
auto mergeUserType = pgmBuilder.NewTupleType({mergeArgsType, emptyStructType, valueType});
auto udfTopFreq_Merge = pgmBuilder.Udf("TopFreq.TopFreq_Merge", TRuntimeNode(), mergeUserType);
-
+
auto getArgsType = pgmBuilder.NewTupleType({resourceType, ui32Type});
auto getUserType = pgmBuilder.NewTupleType({getArgsType, emptyStructType, valueType});
auto udfTopFreq_Get = pgmBuilder.Udf("TopFreq.TopFreq_Get", TRuntimeNode(), getUserType);
-
+
auto serializeArgsType = pgmBuilder.NewTupleType({resourceType});
auto serializeUserType = pgmBuilder.NewTupleType({serializeArgsType, emptyStructType, valueType});
auto udfTopFreq_Serialize = pgmBuilder.Udf("TopFreq.TopFreq_Serialize", TRuntimeNode(), serializeUserType);
-
+
auto serializedType = pgmBuilder.NewTupleType({ui32Type, ui32Type,
pgmBuilder.NewListType(pgmBuilder.NewTupleType({ui64Type, valueType}))});
-
+
auto deserializeArgsType = pgmBuilder.NewTupleType({serializedType});
auto deserializeUserType = pgmBuilder.NewTupleType({deserializeArgsType, emptyStructType, valueType});
auto udfTopFreq_Deserialize = pgmBuilder.Udf("TopFreq.TopFreq_Deserialize", TRuntimeNode(), deserializeUserType);
-
+
static const ui64 BigNum = 20;
static const ui64 BigEach = 5000;
static const ui64 SmallNum = 500;
diff --git a/ydb/library/yql/udfs/common/topfreq/ya.make b/ydb/library/yql/udfs/common/topfreq/ya.make
index 63448b297d4..a2cead62aed 100644
--- a/ydb/library/yql/udfs/common/topfreq/ya.make
+++ b/ydb/library/yql/udfs/common/topfreq/ya.make
@@ -2,7 +2,7 @@ YQL_UDF(topfreq_udf)
YQL_ABI_VERSION(
2
- 10
+ 10
0
)
diff --git a/ydb/library/yql/utils/log/log_component.h b/ydb/library/yql/utils/log/log_component.h
index d7a1e10ca16..c83497f8963 100644
--- a/ydb/library/yql/utils/log/log_component.h
+++ b/ydb/library/yql/utils/log/log_component.h
@@ -23,7 +23,7 @@ enum class EComponent {
Performance, Perf = Performance,
Net,
ProviderStat,
- ProviderSolomon,
+ ProviderSolomon,
CoreEval,
CorePeepHole,
ProviderDq,
diff --git a/ydb/public/lib/deprecated/kicli/cpp_ut.cpp b/ydb/public/lib/deprecated/kicli/cpp_ut.cpp
index cb7dc3ed6d9..d7d8052cb32 100644
--- a/ydb/public/lib/deprecated/kicli/cpp_ut.cpp
+++ b/ydb/public/lib/deprecated/kicli/cpp_ut.cpp
@@ -21,7 +21,7 @@ Tests::TServer StartupKikimr(NMsgBusProxy::TMsgBusClientConfig& clientConfig,
const ui32 port = pm.GetPort(12001);
auto settings = Tests::TServerSettings(port);
settings.SetLogBackend(logBackend);
- settings.SetEnableSystemViews(false);
+ settings.SetEnableSystemViews(false);
Tests::TServer Server(settings);
Tests::TClient Client(settings);
Client.InitRootScheme();
@@ -40,7 +40,7 @@ Tests::TServer StartupKikimr(NGRpcProxy::TGRpcClientConfig& clientConfig,
auto settings = Tests::TServerSettings(msgbusPort);
settings.SetLogBackend(logBackend);
settings.AppConfig.CopyFrom(config);
- settings.SetEnableSystemViews(false);
+ settings.SetEnableSystemViews(false);
settings.SetEnableMvcc(false);
Tests::TServer Server(settings);
diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp
index 4a25c372ce4..c18e2bb9b98 100644
--- a/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp
+++ b/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp
@@ -2,7 +2,7 @@
#include <ydb/public/lib/ydb_cli/common/pretty_table.h>
#include <ydb/public/lib/ydb_cli/common/query_stats.h>
-
+
#include <util/folder/path.h>
#include <util/folder/dirut.h>
@@ -28,7 +28,7 @@ void TCommandExecuteYqlScript::Config(TConfig& config) {
config.Opts->AddLongOption('f', "file", "[Required] Script file").RequiredArgument("PATH").StoreResult(&ScriptFile);
config.Opts->AddLongOption("explain", "Explain query").Optional().StoreTrue(&Explain);
config.Opts->AddLongOption("show-response-metadata", ResponseHeadersHelp).Optional().StoreTrue(&ShowHeaders);
-
+
AddParametersOption(config);
AddInputFormats(config, {
@@ -36,14 +36,14 @@ void TCommandExecuteYqlScript::Config(TConfig& config) {
EOutputFormat::JsonBase64
});
- AddFormats(config, {
- EOutputFormat::Pretty,
- EOutputFormat::JsonUnicode,
+ AddFormats(config, {
+ EOutputFormat::Pretty,
+ EOutputFormat::JsonUnicode,
EOutputFormat::JsonUnicodeArray,
EOutputFormat::JsonBase64,
EOutputFormat::JsonBase64Array
- });
-
+ });
+
config.SetFreeArgsNum(0);
AddCommandExamples(
@@ -64,7 +64,7 @@ void TCommandExecuteYqlScript::Config(TConfig& config) {
void TCommandExecuteYqlScript::Parse(TConfig& config) {
TClientCommand::Parse(config);
- ParseFormats();
+ ParseFormats();
if (!Script && !ScriptFile) {
throw TMissUseException() << "Neither \"Text of script\" (\"--script\", \"-s\") "
<< "nor \"Path to file with script text\" (\"--file\", \"-f\") were provided.";
diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp
index 2f94b2fc033..f56db9d5205 100644
--- a/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp
+++ b/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp
@@ -344,7 +344,7 @@ void TCommandExecuteQuery::Config(TConfig& config) {
TTableCommand::Config(config);
AddExamplesOption(config);
- config.Opts->AddLongOption('t', "type", "Query type [data, scheme, scan]")
+ config.Opts->AddLongOption('t', "type", "Query type [data, scheme, scan]")
.RequiredArgument("[String]").DefaultValue("data").StoreResult(&QueryType);
config.Opts->AddLongOption("stats", "Collect statistics mode (for data & scan queries) [none, basic, full]")
.RequiredArgument("[String]").StoreResult(&CollectStatsMode);
@@ -354,7 +354,7 @@ void TCommandExecuteQuery::Config(TConfig& config) {
config.Opts->AddLongOption('q', "query", "Text of query to execute").RequiredArgument("[String]").StoreResult(&Query);
config.Opts->AddLongOption('f', "file", "Path to file with query text to execute")
.RequiredArgument("PATH").StoreResult(&QueryFile);
-
+
AddParametersOption(config, "(for data & scan queries)");
AddInputFormats(config, {
@@ -363,22 +363,22 @@ void TCommandExecuteQuery::Config(TConfig& config) {
});
- AddFormats(config, {
- EOutputFormat::Pretty,
- EOutputFormat::JsonUnicode,
+ AddFormats(config, {
+ EOutputFormat::Pretty,
+ EOutputFormat::JsonUnicode,
EOutputFormat::JsonUnicodeArray,
EOutputFormat::JsonBase64,
EOutputFormat::JsonBase64Array
- });
-
+ });
+
CheckExamples(config);
-
+
config.SetFreeArgsNum(0);
}
void TCommandExecuteQuery::Parse(TConfig& config) {
TClientCommand::Parse(config);
- ParseFormats();
+ ParseFormats();
if (BasicStats && CollectStatsMode) {
throw TMissUseException() << "Both mutually exclusive options \"--stats\" and \"-s\" are provided.";
}
@@ -398,9 +398,9 @@ int TCommandExecuteQuery::Run(TConfig& config) {
if (QueryType == "scheme") {
return ExecuteSchemeQuery(config);
}
- if (QueryType == "scan") {
- return ExecuteScanQuery(config);
- }
+ if (QueryType == "scan") {
+ return ExecuteScanQuery(config);
+ }
}
throw TMissUseException() << "Unknown query type.";
}
@@ -476,13 +476,13 @@ int TCommandExecuteQuery::ExecuteSchemeQuery(TConfig& config) {
return EXIT_SUCCESS;
}
-int TCommandExecuteQuery::ExecuteScanQuery(TConfig& config) {
- NTable::TTableClient client(CreateDriver(config));
+int TCommandExecuteQuery::ExecuteScanQuery(TConfig& config) {
+ NTable::TTableClient client(CreateDriver(config));
auto defaultStatsMode = BasicStats ? NTable::ECollectQueryStatsMode::Basic : NTable::ECollectQueryStatsMode::None;
- NTable::TStreamExecScanQuerySettings settings;
+ NTable::TStreamExecScanQuerySettings settings;
settings.CollectQueryStats(ParseQueryStatsMode(CollectStatsMode, defaultStatsMode));
-
+
NTable::TAsyncScanQueryPartIterator asyncResult;
if (Parameters.size()) {
auto validateResult = ExplainQuery(config, Query, NScripting::ExplainYqlRequestMode::Validate);
@@ -496,17 +496,17 @@ int TCommandExecuteQuery::ExecuteScanQuery(TConfig& config) {
}
auto result = asyncResult.GetValueSync();
- ThrowOnError(result);
- PrintScanQueryResponse(result);
- return EXIT_SUCCESS;
-}
-
-void TCommandExecuteQuery::PrintScanQueryResponse(NTable::TScanQueryPartIterator& result) {
- SetInterruptHandlers();
+ ThrowOnError(result);
+ PrintScanQueryResponse(result);
+ return EXIT_SUCCESS;
+}
+
+void TCommandExecuteQuery::PrintScanQueryResponse(NTable::TScanQueryPartIterator& result) {
+ SetInterruptHandlers();
TStringStream statsStr;
{
TResultSetPrinter printer(OutputFormat, &IsInterrupted);
-
+
while (!IsInterrupted()) {
auto streamPart = result.ReadNext().GetValueSync();
if (!streamPart.IsSuccess()) {
@@ -514,7 +514,7 @@ void TCommandExecuteQuery::PrintScanQueryResponse(NTable::TScanQueryPartIterator
break;
}
ThrowOnError(streamPart);
- }
+ }
if (streamPart.HasResultSet()) {
printer.Print(streamPart.GetResultSet());
@@ -534,13 +534,13 @@ void TCommandExecuteQuery::PrintScanQueryResponse(NTable::TScanQueryPartIterator
if (statsStr.Size()) {
Cout << Endl << "Statistics:" << statsStr.Str();
- }
-
+ }
+
if (IsInterrupted()) {
Cerr << "<INTERRUPTED>" << Endl;
}
-}
-
+}
+
TCommandExplain::TCommandExplain()
: TTableCommand("explain", {}, "Explain query")
{}
@@ -688,13 +688,13 @@ void TCommandReadTable::Config(TConfig& config) {
EOutputFormat::JsonBase64
});
- AddFormats(config, {
- EOutputFormat::Pretty,
- EOutputFormat::JsonUnicode,
+ AddFormats(config, {
+ EOutputFormat::Pretty,
+ EOutputFormat::JsonUnicode,
EOutputFormat::JsonUnicodeArray,
EOutputFormat::JsonBase64,
EOutputFormat::JsonBase64Array
- });
+ });
// TODO: KIKIMR-8675
// Add csv format
@@ -809,7 +809,7 @@ int TCommandReadTable::Run(TConfig& config) {
void TCommandReadTable::PrintResponse(NTable::TTablePartIterator& result) {
size_t totalRows = 0;
- SetInterruptHandlers();
+ SetInterruptHandlers();
TResultSetPrinter printer(OutputFormat, &IsInterrupted);
while (!IsInterrupted()) {
diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_table.h b/ydb/public/lib/ydb_cli/commands/ydb_service_table.h
index 5cd2514aa02..66067e96734 100644
--- a/ydb/public/lib/ydb_cli/commands/ydb_service_table.h
+++ b/ydb/public/lib/ydb_cli/commands/ydb_service_table.h
@@ -86,28 +86,28 @@ protected:
void CheckQueryOptions() const;
void CheckQueryFile();
-protected:
+protected:
TString Query;
TString QueryFile;
};
class TCommandExecuteQuery : public TTableCommand, TCommandQueryBase, TCommandWithParameters,
- public TCommandWithFormat, public TInterruptibleCommand
-{
+ public TCommandWithFormat, public TInterruptibleCommand
+{
public:
TCommandExecuteQuery();
virtual void Config(TConfig& config) override;
virtual void Parse(TConfig& config) override;
virtual int Run(TConfig& config) override;
-
+
int ExecuteDataQuery(TConfig& config);
void PrintDataQueryResponse(NTable::TDataQueryResult& result);
-
+
int ExecuteSchemeQuery(TConfig& config);
- int ExecuteScanQuery(TConfig& config);
- void PrintScanQueryResponse(NTable::TScanQueryPartIterator& result);
-
+ int ExecuteScanQuery(TConfig& config);
+ void PrintScanQueryResponse(NTable::TScanQueryPartIterator& result);
+
private:
TString CollectStatsMode;
TString TxMode;
@@ -129,9 +129,9 @@ private:
bool Analyze = false;
};
-class TCommandReadTable : public TYdbCommand, public TCommandWithPath,
- public TCommandWithFormat, public TInterruptibleCommand
-{
+class TCommandReadTable : public TYdbCommand, public TCommandWithPath,
+ public TCommandWithFormat, public TInterruptibleCommand
+{
public:
TCommandReadTable();
virtual void Config(TConfig& config) override;
diff --git a/ydb/public/lib/ydb_cli/common/format.cpp b/ydb/public/lib/ydb_cli/common/format.cpp
index 7184ab41d1f..13100b19df9 100644
--- a/ydb/public/lib/ydb_cli/common/format.cpp
+++ b/ydb/public/lib/ydb_cli/common/format.cpp
@@ -1,11 +1,11 @@
#include "format.h"
-#include "pretty_table.h"
+#include "pretty_table.h"
#include <util/string/vector.h>
#include <library/cpp/json/json_prettifier.h>
#include <ydb/public/lib/json_value/ydb_json_value.h>
-
+
namespace NYdb {
namespace NConsoleClient {
@@ -142,11 +142,11 @@ void TQueryPlanPrinter::Print(const TString& plan) {
default:
throw TMissUseException() << "This command doesn't support " << Format << " output format";
}
-}
+}
void TQueryPlanPrinter::PrintJson(const TString& plan) {
Cout << NJson::PrettifyJson(plan, true) << Endl;
-}
+}
void TQueryPlanPrinter::PrintPretty(const NJson::TJsonValue& plan) {
if (plan.GetMapSafe().contains("Plan")) {
diff --git a/ydb/public/lib/ydb_cli/common/format.h b/ydb/public/lib/ydb_cli/common/format.h
index 2d8d6a2e38b..decd9476897 100644
--- a/ydb/public/lib/ydb_cli/common/format.h
+++ b/ydb/public/lib/ydb_cli/common/format.h
@@ -6,7 +6,7 @@
#include <ydb/public/lib/json_value/ydb_json_value.h>
#include <ydb/public/sdk/cpp/client/ydb_result/result.h>
#include <ydb/public/sdk/cpp/client/ydb_types/status/status.h>
-
+
namespace NYdb {
namespace NConsoleClient {
@@ -24,7 +24,7 @@ protected:
void AddInputFormats(TClientCommand::TConfig& config, const TVector<EOutputFormat>& allowedFormats);
void AddFormats(TClientCommand::TConfig& config, const TVector<EOutputFormat>& allowedFormats);
void ParseFormats();
-
+
// Deprecated
void AddJsonOption(TClientCommand::TConfig& config,
const TString& description = "(Deprecated, will be removed soon. Use --format option instead)"
diff --git a/ydb/public/sdk/python/examples/basic_example_v1/basic_example.py b/ydb/public/sdk/python/examples/basic_example_v1/basic_example.py
index 5845a18a2e0..6fa6361e7ed 100644
--- a/ydb/public/sdk/python/examples/basic_example_v1/basic_example.py
+++ b/ydb/public/sdk/python/examples/basic_example_v1/basic_example.py
@@ -6,25 +6,25 @@ from . import basic_example_data
FillDataQuery = """PRAGMA TablePathPrefix("{}");
-DECLARE $seriesData AS List<Struct<
+DECLARE $seriesData AS List<Struct<
series_id: Uint64,
title: Utf8,
series_info: Utf8,
- release_date: Date>>;
+ release_date: Date>>;
-DECLARE $seasonsData AS List<Struct<
+DECLARE $seasonsData AS List<Struct<
series_id: Uint64,
season_id: Uint64,
title: Utf8,
first_aired: Date,
- last_aired: Date>>;
+ last_aired: Date>>;
-DECLARE $episodesData AS List<Struct<
+DECLARE $episodesData AS List<Struct<
series_id: Uint64,
season_id: Uint64,
episode_id: Uint64,
title: Utf8,
- air_date: Date>>;
+ air_date: Date>>;
REPLACE INTO series
SELECT
diff --git a/ydb/public/sdk/python/examples/basic_example_v1/basic_example_data.py b/ydb/public/sdk/python/examples/basic_example_v1/basic_example_data.py
index 967c78ead72..06c3d9608aa 100644
--- a/ydb/public/sdk/python/examples/basic_example_v1/basic_example_data.py
+++ b/ydb/public/sdk/python/examples/basic_example_v1/basic_example_data.py
@@ -120,7 +120,7 @@ def get_episodes_data_for_bulk_upsert():
Episode(2, 2, 7, "Adult Content", "2015-05-24"),
Episode(2, 2, 8, "White Hat/Black Hat", "2015-05-31"),
Episode(2, 2, 9, "Binding Arbitration", "2015-06-07"),
- Episode(2, 2, 10, "Two Days of the Condor", "2015-06-14"),
+ Episode(2, 2, 10, "Two Days of the Condor", "2015-06-14"),
Episode(2, 3, 1, "Founder Friendly", "2016-04-24"),
Episode(2, 3, 2, "Two in the Box", "2016-05-01"),
Episode(2, 3, 3, "Meinertzhagen's Haversack", "2016-05-08"),
@@ -130,7 +130,7 @@ def get_episodes_data_for_bulk_upsert():
Episode(2, 3, 7, "To Build a Better Beta", "2016-06-05"),
Episode(2, 3, 8, "Bachman's Earnings Over-Ride", "2016-06-12"),
Episode(2, 3, 9, "Daily Active Users", "2016-06-19"),
- Episode(2, 3, 10, "The Uptick", "2016-06-26"),
+ Episode(2, 3, 10, "The Uptick", "2016-06-26"),
Episode(2, 4, 1, "Success Failure", "2017-04-23"),
Episode(2, 4, 2, "Terms of Service", "2017-04-30"),
Episode(2, 4, 3, "Intellectual Property", "2017-05-07"),
@@ -140,7 +140,7 @@ def get_episodes_data_for_bulk_upsert():
Episode(2, 4, 7, "The Patent Troll", "2017-06-04"),
Episode(2, 4, 8, "The Keenan Vortex", "2017-06-11"),
Episode(2, 4, 9, "Hooli-Con", "2017-06-18"),
- Episode(2, 4, 10, "Server Error", "2017-06-25"),
+ Episode(2, 4, 10, "Server Error", "2017-06-25"),
Episode(2, 5, 1, "Grow Fast or Die Slow", "2018-03-25"),
Episode(2, 5, 2, "Reorientation", "2018-04-01"),
Episode(2, 5, 3, "Chief Operating Officer", "2018-04-08"),
diff --git a/ydb/services/datastreams/grpc_service.cpp b/ydb/services/datastreams/grpc_service.cpp
index 7a18cd4dc67..c8af620457a 100644
--- a/ydb/services/datastreams/grpc_service.cpp
+++ b/ydb/services/datastreams/grpc_service.cpp
@@ -65,7 +65,7 @@ void TGRpcDataStreamsService::SetupIncomingRequests(NGrpc::TLoggerPtr logger)
ReportGrpcReqToMon(*ActorSystem_, ctx->GetPeer()); \
ACTION; \
}, &Ydb::DataStreams::V1::DataStreamsService::AsyncService::Request ## NAME, \
- #NAME, logger, getCounterBlock("data_streams", #NAME))->Run();
+ #NAME, logger, getCounterBlock("data_streams", #NAME))->Run();
ADD_REQUEST(DescribeStream, DescribeStreamRequest, DescribeStreamResponse, {
ActorSystem_->Send(GRpcRequestProxyId_, new TEvDataStreamsDescribeStreamRequest(ctx));
diff --git a/ydb/services/persqueue_v1/persqueue.cpp b/ydb/services/persqueue_v1/persqueue.cpp
index ccacc4d43af..9e91c4d7cf9 100644
--- a/ydb/services/persqueue_v1/persqueue.cpp
+++ b/ydb/services/persqueue_v1/persqueue.cpp
@@ -86,7 +86,7 @@ void TGRpcPersQueueService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) {
[this](TIntrusivePtr<TStreamGRpcRequest::IContext> context) {
ActorSystem->Send(GRpcRequestProxy, new NKikimr::NGRpcService::TEvStreamPQWriteRequest(context));
},
- *ActorSystem, "PersQueueService/CreateWriteSession", getCounterBlock("persistent_queue", "WriteSession", true, true), nullptr
+ *ActorSystem, "PersQueueService/CreateWriteSession", getCounterBlock("persistent_queue", "WriteSession", true, true), nullptr
);
}
@@ -106,7 +106,7 @@ void TGRpcPersQueueService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) {
[this](TIntrusivePtr<TStreamGRpcRequest::IContext> context) {
ActorSystem->Send(GRpcRequestProxy, new NKikimr::NGRpcService::TEvStreamPQReadRequest(context));
},
- *ActorSystem, "PersQueueService/CreateReadSession", getCounterBlock("persistent_queue", "ReadSession", true, true), nullptr
+ *ActorSystem, "PersQueueService/CreateReadSession", getCounterBlock("persistent_queue", "ReadSession", true, true), nullptr
);
}
@@ -119,7 +119,7 @@ void TGRpcPersQueueService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) {
NGRpcService::ReportGrpcReqToMon(*ActorSystem, ctx->GetPeer()); \
ACTION; \
}, &Ydb::PersQueue::V1::SVC::AsyncService::Request ## NAME, \
- "PersQueueService/"#NAME, logger, getCounterBlock("persistent_queue", #NAME))->Run();
+ "PersQueueService/"#NAME, logger, getCounterBlock("persistent_queue", #NAME))->Run();
ADD_REQUEST(GetReadSessionsInfo, PersQueueService, ReadInfoRequest, ReadInfoResponse, {
ActorSystem->Send(GRpcRequestProxy, new NGRpcService::TEvPQReadInfoRequest(ctx));
diff --git a/ydb/services/ydb/ydb_common_ut.h b/ydb/services/ydb/ydb_common_ut.h
index c067928c785..bfc87e644f0 100644
--- a/ydb/services/ydb/ydb_common_ut.h
+++ b/ydb/services/ydb/ydb_common_ut.h
@@ -24,7 +24,7 @@ struct TKikimrTestSettings {
static constexpr bool SSL = false;
static constexpr bool AUTH = false;
static constexpr bool PrecreatePools = true;
- static constexpr bool EnableSystemViews = true;
+ static constexpr bool EnableSystemViews = true;
};
struct TKikimrTestWithAuth : TKikimrTestSettings {
@@ -35,10 +35,10 @@ struct TKikimrTestWithAuthAndSsl : TKikimrTestWithAuth {
static constexpr bool SSL = true;
};
-struct TKikimrTestNoSystemViews : TKikimrTestSettings {
- static constexpr bool EnableSystemViews = false;
-};
-
+struct TKikimrTestNoSystemViews : TKikimrTestSettings {
+ static constexpr bool EnableSystemViews = false;
+};
+
template <typename TestSettings = TKikimrTestSettings>
class TBasicKikimrWithGrpcAndRootSchema {
public:
@@ -308,6 +308,6 @@ struct TTestOlap {
using TKikimrWithGrpcAndRootSchema = TBasicKikimrWithGrpcAndRootSchema<TKikimrTestSettings>;
using TKikimrWithGrpcAndRootSchemaWithAuth = TBasicKikimrWithGrpcAndRootSchema<TKikimrTestWithAuth>;
using TKikimrWithGrpcAndRootSchemaWithAuthAndSsl = TBasicKikimrWithGrpcAndRootSchema<TKikimrTestWithAuthAndSsl>;
-using TKikimrWithGrpcAndRootSchemaNoSystemViews = TBasicKikimrWithGrpcAndRootSchema<TKikimrTestNoSystemViews>;
+using TKikimrWithGrpcAndRootSchemaNoSystemViews = TBasicKikimrWithGrpcAndRootSchema<TKikimrTestNoSystemViews>;
}
diff --git a/ydb/services/ydb/ydb_table_ut.cpp b/ydb/services/ydb/ydb_table_ut.cpp
index 4861c06b92e..e497dee61b1 100644
--- a/ydb/services/ydb/ydb_table_ut.cpp
+++ b/ydb/services/ydb/ydb_table_ut.cpp
@@ -2961,7 +2961,7 @@ R"___(<main>: Error: Transaction not found: , code: 2015
}
Y_UNIT_TEST(CopyTables) {
- TKikimrWithGrpcAndRootSchemaNoSystemViews server;
+ TKikimrWithGrpcAndRootSchemaNoSystemViews server;
server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_NOTICE);
auto connection = NYdb::TDriver(
diff --git a/ydb/services/ydb/ydb_ut.cpp b/ydb/services/ydb/ydb_ut.cpp
index 2d370630799..445ba4ec1e8 100644
--- a/ydb/services/ydb/ydb_ut.cpp
+++ b/ydb/services/ydb/ydb_ut.cpp
@@ -663,7 +663,7 @@ Y_UNIT_TEST_SUITE(TGRpcNewClient) {
}
Y_UNIT_TEST(CreateAlterUpsertDrop) {
- TKikimrWithGrpcAndRootSchemaNoSystemViews server;
+ TKikimrWithGrpcAndRootSchemaNoSystemViews server;
ui16 grpc = server.GetPort();
TString location = TStringBuilder() << "localhost:" << grpc;
@@ -891,7 +891,7 @@ Y_UNIT_TEST_SUITE(TGRpcYdbTest) {
}
Y_UNIT_TEST(MakeListRemoveDirectory) {
- TKikimrWithGrpcAndRootSchemaNoSystemViews server;
+ TKikimrWithGrpcAndRootSchemaNoSystemViews server;
ui16 grpc = server.GetPort();
std::shared_ptr<grpc::Channel> Channel_;
diff --git a/ydb/tests/functional/tenants/test_db_counters.py b/ydb/tests/functional/tenants/test_db_counters.py
index b8e3c9baa35..2451cdaab49 100644
--- a/ydb/tests/functional/tenants/test_db_counters.py
+++ b/ydb/tests/functional/tenants/test_db_counters.py
@@ -1,131 +1,131 @@
-# -*- coding: utf-8 -*-
-import logging
-import os
-import time
-import requests
-
-from hamcrest import (
- assert_that,
- equal_to,
- greater_than,
- not_none
-)
-
+# -*- coding: utf-8 -*-
+import logging
+import os
+import time
+import requests
+
+from hamcrest import (
+ assert_that,
+ equal_to,
+ greater_than,
+ not_none
+)
+
from ydb.tests.library.harness.kikimr_cluster import kikimr_cluster_factory
from ydb.tests.library.harness.kikimr_config import KikimrConfigGenerator
from ydb.tests.library.harness.util import LogLevels
import ydb
-
-logger = logging.getLogger(__name__)
-
-
-class BaseDbCounters(object):
- @classmethod
- def setup_class(cls):
- cls.cluster = kikimr_cluster_factory(
- KikimrConfigGenerator(
- additional_log_configs={
- 'SYSTEM_VIEWS': LogLevels.DEBUG
- }
- )
- )
- cls.cluster.start()
-
- @classmethod
- def teardown_class(cls):
- if hasattr(cls, 'cluster'):
- cls.cluster.stop()
-
- def setup_method(self, method=None):
- self.database = "/Root/users/{class_name}_{method_name}".format(
+
+logger = logging.getLogger(__name__)
+
+
+class BaseDbCounters(object):
+ @classmethod
+ def setup_class(cls):
+ cls.cluster = kikimr_cluster_factory(
+ KikimrConfigGenerator(
+ additional_log_configs={
+ 'SYSTEM_VIEWS': LogLevels.DEBUG
+ }
+ )
+ )
+ cls.cluster.start()
+
+ @classmethod
+ def teardown_class(cls):
+ if hasattr(cls, 'cluster'):
+ cls.cluster.stop()
+
+ def setup_method(self, method=None):
+ self.database = "/Root/users/{class_name}_{method_name}".format(
class_name=self.__class__.__name__,
method_name=method.__name__,
- )
- logger.debug("Create database %s" % self.database)
- self.cluster.create_database(
- self.database,
- storage_pool_units_count={
- 'hdd': 1
- }
- )
-
+ )
+ logger.debug("Create database %s" % self.database)
+ self.cluster.create_database(
+ self.database,
+ storage_pool_units_count={
+ 'hdd': 1
+ }
+ )
+
self.cluster.register_and_start_slots(self.database, count=1)
self.cluster.wait_tenant_up(self.database)
- def teardown_method(self, method=None):
- logger.debug("Remove database %s" % self.database)
- self.cluster.remove_database(self.database)
- self.database = None
-
- def create_table(self, driver, table):
- with ydb.SessionPool(driver, size=1) as pool:
- with pool.checkout() as session:
- session.execute_scheme(
- "create table `{}` (key Int32, value String, primary key(key));".format(
- table
- )
- )
-
- def get_db_counters(self, service, node_index=0):
- mon_port = self.cluster.slots[node_index + 1].mon_port
- counters_url = 'http://localhost:{}/counters/counters%3D{}/json'.format(mon_port, service)
- reply = requests.get(counters_url)
- if reply.status_code == 204:
- return None
-
- assert_that(reply.status_code, equal_to(200))
- ret = reply.json()
-
- assert_that(ret, not_none())
- return ret
-
- def check_db_counters(self, sensors_to_check, group):
- table = os.path.join(self.database, 'table')
-
- driver_config = ydb.DriverConfig(
- "%s:%s" % (self.cluster.nodes[1].host, self.cluster.nodes[1].port),
- self.database
- )
-
- with ydb.Driver(driver_config) as driver:
- self.create_table(driver, table)
-
- with ydb.SessionPool(driver, size=1) as pool:
- with pool.checkout() as session:
- query = "select * from `{}`".format(table)
- session.transaction().execute(query, commit_tx=True)
-
- for i in range(30):
- checked = 0
-
- counters = self.get_db_counters('db')
- if counters:
- sensors = counters['sensors']
- for sensor in sensors:
- labels = sensor['labels']
- if labels['sensor'] in sensors_to_check:
- assert_that(labels['group'], equal_to(group))
- assert_that(labels['database'], equal_to(self.database))
- assert_that(labels['host'], equal_to(''))
- assert_that(sensor['value'], greater_than(0))
- checked = checked + 1
-
- assert_that(checked, equal_to(len(sensors_to_check)))
- break
-
- if checked > 0:
- break
-
- time.sleep(5)
-
-
-class TestKqpCounters(BaseDbCounters):
- def test_case(self):
- sensors_to_check = {
- 'Requests/Bytes',
- 'Requests/QueryBytes',
- 'Requests/QueryExecute',
- 'YdbResponses/Success',
- 'Responses/Bytes',
- }
- self.check_db_counters(sensors_to_check, 'kqp')
+ def teardown_method(self, method=None):
+ logger.debug("Remove database %s" % self.database)
+ self.cluster.remove_database(self.database)
+ self.database = None
+
+ def create_table(self, driver, table):
+ with ydb.SessionPool(driver, size=1) as pool:
+ with pool.checkout() as session:
+ session.execute_scheme(
+ "create table `{}` (key Int32, value String, primary key(key));".format(
+ table
+ )
+ )
+
+ def get_db_counters(self, service, node_index=0):
+ mon_port = self.cluster.slots[node_index + 1].mon_port
+ counters_url = 'http://localhost:{}/counters/counters%3D{}/json'.format(mon_port, service)
+ reply = requests.get(counters_url)
+ if reply.status_code == 204:
+ return None
+
+ assert_that(reply.status_code, equal_to(200))
+ ret = reply.json()
+
+ assert_that(ret, not_none())
+ return ret
+
+ def check_db_counters(self, sensors_to_check, group):
+ table = os.path.join(self.database, 'table')
+
+ driver_config = ydb.DriverConfig(
+ "%s:%s" % (self.cluster.nodes[1].host, self.cluster.nodes[1].port),
+ self.database
+ )
+
+ with ydb.Driver(driver_config) as driver:
+ self.create_table(driver, table)
+
+ with ydb.SessionPool(driver, size=1) as pool:
+ with pool.checkout() as session:
+ query = "select * from `{}`".format(table)
+ session.transaction().execute(query, commit_tx=True)
+
+ for i in range(30):
+ checked = 0
+
+ counters = self.get_db_counters('db')
+ if counters:
+ sensors = counters['sensors']
+ for sensor in sensors:
+ labels = sensor['labels']
+ if labels['sensor'] in sensors_to_check:
+ assert_that(labels['group'], equal_to(group))
+ assert_that(labels['database'], equal_to(self.database))
+ assert_that(labels['host'], equal_to(''))
+ assert_that(sensor['value'], greater_than(0))
+ checked = checked + 1
+
+ assert_that(checked, equal_to(len(sensors_to_check)))
+ break
+
+ if checked > 0:
+ break
+
+ time.sleep(5)
+
+
+class TestKqpCounters(BaseDbCounters):
+ def test_case(self):
+ sensors_to_check = {
+ 'Requests/Bytes',
+ 'Requests/QueryBytes',
+ 'Requests/QueryExecute',
+ 'YdbResponses/Success',
+ 'Responses/Bytes',
+ }
+ self.check_db_counters(sensors_to_check, 'kqp')
diff --git a/ydb/tests/functional/tenants/test_system_views.py b/ydb/tests/functional/tenants/test_system_views.py
index b0fc5640f62..8c14c63d959 100644
--- a/ydb/tests/functional/tenants/test_system_views.py
+++ b/ydb/tests/functional/tenants/test_system_views.py
@@ -1,12 +1,12 @@
# -*- coding: utf-8 -*-
import logging
import os
-import time
+import time
from hamcrest import (
anything,
assert_that,
- greater_than,
+ greater_than,
has_length,
has_properties,
)
@@ -24,9 +24,9 @@ class BaseSystemViews(object):
def setup_class(cls):
cls.cluster = kikimr_cluster_factory(
KikimrConfigGenerator(
- additional_log_configs={
- 'SYSTEM_VIEWS': LogLevels.DEBUG
- }
+ additional_log_configs={
+ 'SYSTEM_VIEWS': LogLevels.DEBUG
+ }
)
)
cls.cluster.start()
@@ -60,32 +60,32 @@ class BaseSystemViews(object):
with ydb.SessionPool(driver, size=1) as pool:
with pool.checkout() as session:
session.execute_scheme(
- "create table `{}` (key Int32, value String, primary key(key));".format(
+ "create table `{}` (key Int32, value String, primary key(key));".format(
table
)
)
def read_partition_stats(self, driver, database):
table = os.path.join(database, '.sys/partition_stats')
- return self._stream_query_result(
- driver,
- "select PathId, PartIdx, Path from `{}`;".format(table)
- )
-
- def read_query_metrics(self, driver, database):
- table = os.path.join(database, '.sys/query_metrics_one_minute')
- return self._stream_query_result(
- driver,
- "select Count, SumReadBytes, SumRequestUnits from `{}`;".format(table)
- )
-
- def read_query_stats(self, driver, database, table_name):
- table = os.path.join(database, '.sys', table_name)
- return self._stream_query_result(
- driver,
- "select Duration, ReadBytes, CPUTime, RequestUnits from `{}`;".format(table)
- )
-
+ return self._stream_query_result(
+ driver,
+ "select PathId, PartIdx, Path from `{}`;".format(table)
+ )
+
+ def read_query_metrics(self, driver, database):
+ table = os.path.join(database, '.sys/query_metrics_one_minute')
+ return self._stream_query_result(
+ driver,
+ "select Count, SumReadBytes, SumRequestUnits from `{}`;".format(table)
+ )
+
+ def read_query_stats(self, driver, database, table_name):
+ table = os.path.join(database, '.sys', table_name)
+ return self._stream_query_result(
+ driver,
+ "select Duration, ReadBytes, CPUTime, RequestUnits from `{}`;".format(table)
+ )
+
def _stream_query_result(self, driver, query):
it = driver.table_client.scan_query(query)
result = []
@@ -101,61 +101,61 @@ class BaseSystemViews(object):
return result
- def check_query_metrics_and_stats(self, query_count):
- table = os.path.join(self.database, 'table')
-
- driver_config = ydb.DriverConfig(
- "%s:%s" % (self.cluster.nodes[1].host, self.cluster.nodes[1].port),
- self.database
- )
-
- with ydb.Driver(driver_config) as driver:
- driver.wait(timeout=10)
-
- self.create_table(driver, table)
-
- with ydb.SessionPool(driver, size=1) as pool:
- with pool.checkout() as session:
- for i in range(query_count):
- query = "select * from `{}` -- {}".format(table, i)
- session.transaction().execute(query, commit_tx=True)
-
- time.sleep(70)
-
- for i in range(60):
- metrics = self.read_query_metrics(driver, self.database)
- if len(metrics) == query_count:
- break
- time.sleep(5)
-
- assert_that(metrics, has_length(query_count))
- assert_that(metrics[0], has_properties({
- 'Count': 1,
- 'SumReadBytes': 0,
- 'SumRequestUnits': greater_than(0),
- }))
-
- for table_name in [
- 'top_queries_by_duration_one_minute',
- 'top_queries_by_duration_one_hour',
- 'top_queries_by_read_bytes_one_minute',
- 'top_queries_by_read_bytes_one_hour',
- 'top_queries_by_cpu_time_one_minute',
- 'top_queries_by_cpu_time_one_hour',
- 'top_queries_by_request_units_one_minute',
- 'top_queries_by_request_units_one_hour'
- ]:
- stats = self.read_query_stats(driver, self.database, table_name)
-
- assert_that(stats, has_length(min(query_count, 5)))
- assert_that(stats[0], has_properties({
- 'Duration': anything(),
- 'ReadBytes': 0,
- 'CPUTime': anything(),
- 'RequestUnits': greater_than(0)
- }))
-
-
+ def check_query_metrics_and_stats(self, query_count):
+ table = os.path.join(self.database, 'table')
+
+ driver_config = ydb.DriverConfig(
+ "%s:%s" % (self.cluster.nodes[1].host, self.cluster.nodes[1].port),
+ self.database
+ )
+
+ with ydb.Driver(driver_config) as driver:
+ driver.wait(timeout=10)
+
+ self.create_table(driver, table)
+
+ with ydb.SessionPool(driver, size=1) as pool:
+ with pool.checkout() as session:
+ for i in range(query_count):
+ query = "select * from `{}` -- {}".format(table, i)
+ session.transaction().execute(query, commit_tx=True)
+
+ time.sleep(70)
+
+ for i in range(60):
+ metrics = self.read_query_metrics(driver, self.database)
+ if len(metrics) == query_count:
+ break
+ time.sleep(5)
+
+ assert_that(metrics, has_length(query_count))
+ assert_that(metrics[0], has_properties({
+ 'Count': 1,
+ 'SumReadBytes': 0,
+ 'SumRequestUnits': greater_than(0),
+ }))
+
+ for table_name in [
+ 'top_queries_by_duration_one_minute',
+ 'top_queries_by_duration_one_hour',
+ 'top_queries_by_read_bytes_one_minute',
+ 'top_queries_by_read_bytes_one_hour',
+ 'top_queries_by_cpu_time_one_minute',
+ 'top_queries_by_cpu_time_one_hour',
+ 'top_queries_by_request_units_one_minute',
+ 'top_queries_by_request_units_one_hour'
+ ]:
+ stats = self.read_query_stats(driver, self.database, table_name)
+
+ assert_that(stats, has_length(min(query_count, 5)))
+ assert_that(stats[0], has_properties({
+ 'Duration': anything(),
+ 'ReadBytes': 0,
+ 'CPUTime': anything(),
+ 'RequestUnits': greater_than(0)
+ }))
+
+
class TestPartitionStats(BaseSystemViews):
def test_case(self):
for database in ('/Root', self.database):
@@ -178,13 +178,13 @@ class TestPartitionStats(BaseSystemViews):
'PathId': anything(),
'PartIdx': 0,
}))
-
-
-class TestQueryMetrics(BaseSystemViews):
- def test_case(self):
- self.check_query_metrics_and_stats(1)
-
-
-class TestQueryMetricsUniqueQueries(BaseSystemViews):
- def test_case(self):
- self.check_query_metrics_and_stats(10)
+
+
+class TestQueryMetrics(BaseSystemViews):
+ def test_case(self):
+ self.check_query_metrics_and_stats(1)
+
+
+class TestQueryMetricsUniqueQueries(BaseSystemViews):
+ def test_case(self):
+ self.check_query_metrics_and_stats(10)
diff --git a/ydb/tests/functional/tenants/ya.make b/ydb/tests/functional/tenants/ya.make
index dee204013e0..1e5ed022500 100644
--- a/ydb/tests/functional/tenants/ya.make
+++ b/ydb/tests/functional/tenants/ya.make
@@ -3,7 +3,7 @@ PY3TEST()
ENV(YDB_DRIVER_BINARY="ydb/apps/ydbd/ydbd")
TEST_SRCS(
common.py
- test_db_counters.py
+ test_db_counters.py
test_dynamic_tenants.py
test_tenants.py
test_storage_config.py